don't create string scanner for liquid tag

Gemfile

@@ -23,4 +23,10 @@ group :test do
   gem 'rubocop', '~> 1.61.0'
   gem 'rubocop-shopify', '~> 2.12.0', require: false
   gem 'rubocop-performance', require: false
+
+  platform :mri, :truffleruby do
+    gem 'liquid-c', github: 'Shopify/liquid-c', ref: 'main'
+  end
 end
+
+gem "strscan", ">= 3.1"

History.md

@@ -1,12 +1,5 @@
 # Liquid Change Log
 
-## 5.6.0 (unreleased)
-
-### Fixes
-
-* Fix Tokenizer to handle null source value (#1873) [Bahar Pourazar]
-
-
 ## 5.5.0 2024-03-21
 
 Please reference the GitHub release for more information.

Rakefile

@@ -43,6 +43,8 @@ task :test do
   Rake::Task['base_test'].invoke
 
   if RUBY_ENGINE == 'ruby' || RUBY_ENGINE == 'truffleruby'
+    ENV['LIQUID_C'] = '1'
+
     ENV['LIQUID_PARSER_MODE'] = 'lax'
     Rake::Task['integration_test'].reenable
     Rake::Task['integration_test'].invoke
@@ -71,7 +73,7 @@ end
 
 namespace :benchmark do
   desc "Run the liquid benchmark with lax parsing"
-  task :run do
+  task :lax do
     ruby "./performance/benchmark.rb lax"
   end
 
@@ -80,6 +82,9 @@ namespace :benchmark do
     ruby "./performance/benchmark.rb strict"
   end
 
+  desc "Run the liquid benchmark with both lax and strict parsing"
+  task run: [:lax, :strict]
+
   desc "Run unit benchmarks"
   task :unit do
     Dir["./performance/unit/*_benchmark.rb"].each do |file|

lib/liquid.rb

@@ -44,11 +44,15 @@ module Liquid
   VariableParser              = /\[(?>[^\[\]]+|\g<0>)*\]|#{VariableSegment}+\??/o
 
   RAISE_EXCEPTION_LAMBDA = ->(_e) { raise }
+
+  singleton_class.send(:attr_accessor, :cache_classes)
+  self.cache_classes = true
 end
 
 require "liquid/version"
 require "liquid/deprecations"
 require "liquid/const"
+require "liquid/template/tag_registry"
 require 'liquid/standardfilters'
 require 'liquid/file_system'
 require 'liquid/parser_switching'
@@ -68,6 +72,7 @@ require 'liquid/extensions'
 require 'liquid/errors'
 require 'liquid/interrupts'
 require 'liquid/strainer_template'
+require 'liquid/strainer_factory'
 require 'liquid/expression'
 require 'liquid/context'
 require 'liquid/tag'

lib/liquid/context.rb

@@ -19,7 +19,7 @@ module Liquid
 
     # rubocop:disable Metrics/ParameterLists
     def self.build(environment: Environment.default, environments: {}, outer_scope: {}, registers: {}, rethrow_errors: false, resource_limits: nil, static_environments: {}, &block)
-      new(environments, outer_scope, registers, rethrow_errors, resource_limits, static_environments, environment, &block)
+      new(environments, outer_scope, registers, rethrow_errors, resource_limits, static_environments, &block)
     end
 
     def initialize(environments = {}, outer_scope = {}, registers = {}, rethrow_errors = false, resource_limits = nil, static_environments = {}, environment = Environment.default)
@@ -143,7 +143,6 @@ module Liquid
       check_overflow
 
       self.class.build(
-        environment: @environment,
         resource_limits: resource_limits,
         static_environments: static_environments,
         registers: Registers.new(registers),

lib/liquid/environment.rb

@@ -41,7 +41,7 @@ module Liquid
       # @return [Environment] The new environment instance.
       def build(tags: nil, file_system: nil, error_mode: nil, exception_renderer: nil)
         ret = new
-        ret.tags = tags if tags
+        ret.tags = Template::TagRegistry.new(tags) if tags
         ret.file_system = file_system if file_system
         ret.error_mode = error_mode if error_mode
         ret.exception_renderer = exception_renderer if exception_renderer
@@ -74,7 +74,7 @@ module Liquid
     # Initializes a new environment instance.
     # @api private
     def initialize
-      @tags = Tags::STANDARD_TAGS.dup
+      @tags = Template::TagRegistry.new(Tags::STANDARD_TAGS)
       @error_mode = :lax
       @strainer_template = Class.new(StrainerTemplate).tap do |klass|
         klass.add_filter(StandardFilters)

lib/liquid/lexer.rb

@@ -92,6 +92,7 @@ module Liquid
     SINGLE_COMPARISON_TOKENS = [].tap do |table|
       table["<".ord] = COMPARISON_LESS_THAN
       table[">".ord] = COMPARISON_GREATER_THAN
+      table.freeze
     end
 
     TWO_CHARS_COMPARISON_JUMP_TABLE = [].tap do |table|
@@ -103,18 +104,17 @@ module Liquid
         sub_table["=".ord] = COMPARISION_NOT_EQUAL
         sub_table.freeze
       end
+      table.freeze
     end
 
     COMPARISON_JUMP_TABLE = [].tap do |table|
       table["<".ord] = [].tap do |sub_table|
         sub_table["=".ord] = COMPARISON_LESS_THAN_OR_EQUAL
         sub_table[">".ord] = COMPARISON_NOT_EQUAL_ALT
-        RUBY_WHITESPACE.each { |c| sub_table[c.ord] = COMPARISON_LESS_THAN }
         sub_table.freeze
       end
       table[">".ord] = [].tap do |sub_table|
         sub_table["=".ord] = COMPARISON_GREATER_THAN_OR_EQUAL
-        RUBY_WHITESPACE.each { |c| sub_table[c.ord] = COMPARISON_GREATER_THAN }
         sub_table.freeze
       end
       table.freeze
@@ -233,5 +233,6 @@ module Liquid
     end
   end
 
+  # Remove this once we can depend on strscan >= 3.1.1
   Lexer = StringScanner.instance_methods.include?(:scan_byte) ? Lexer2 : Lexer1
 end

lib/liquid/parse_tree_visitor.rb

@@ -36,7 +36,7 @@ module Liquid
     protected
 
     def children
-      @node.respond_to?(:nodelist) ? Array(@node.nodelist) : Const::EMPTY_ARRAY
+      @node.respond_to?(:nodelist) ? Array(@node.nodelist) : []
     end
   end
 end

lib/liquid/strainer_factory.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Liquid
+  # StrainerFactory is the factory for the filters system.
+  module StrainerFactory
+    extend self
+
+    def add_global_filter(filter, environment = Environment.default)
+      Deprecations.warn("StrainerFactory.add_global_filter", "Environment#register_filter")
+      environment.register_filter(filter)
+    end
+
+    def create(context, filters = Const::EMPTY_ARRAY, environment = Environment.default)
+      Deprecations.warn("StrainerFactory.create", "StrainerFactory.create_strainer")
+      environment.create_strainer(context, filters)
+    end
+
+    def global_filter_names(environment = Environment.default)
+      Deprecations.warn("StrainerFactory.global_filter_names", "Environment#filter_method_names")
+      Environment.strainer_template.filter_method_names
+    end
+  end
+end

lib/liquid/template/tag_registry.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Liquid
+  class Template
+    class TagRegistry
+      include Enumerable
+
+      def initialize(tags = nil)
+        @tags  = {}
+        @cache = {}
+        tags.each { |tag_name, klass| self[tag_name] = klass }
+        Deprecations.warn("Template::TagRegistry", "Use a Environment instance with zeitwerk")
+      end
+
+      def [](tag_name)
+        return nil unless @tags.key?(tag_name)
+        return @cache[tag_name] if Liquid.cache_classes
+
+        lookup_class(@tags[tag_name]).tap { |o| @cache[tag_name] = o }
+      end
+
+      def delete(tag_name)
+        Deprecations.warn("Template::TagRegistry#delete", "Use a Environment instance with immutable tags")
+        @tags.delete(tag_name)
+        @cache.delete(tag_name)
+      end
+
+      def []=(tag_name, klass)
+        @tags[tag_name]  = klass.name
+        @cache[tag_name] = klass
+      end
+
+      def each(&block)
+        @tags.each(&block)
+      end
+
+      private
+
+      def lookup_class(name)
+        Object.const_get(name)
+      end
+    end
+  end
+end

lib/liquid/tokenizer.rb

@@ -1,11 +1,13 @@
 # frozen_string_literal: true
 
+require "strscan"
+
 module Liquid
-  class Tokenizer
+  class Tokenizer1
     attr_reader :line_number, :for_liquid_tag
 
     def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
-      @source         = source.to_s.to_str
+      @source         = source
       @line_number    = line_number || (line_numbers ? 1 : nil)
       @for_liquid_tag = for_liquid_tag
       @offset         = 0
@@ -42,4 +44,147 @@ module Liquid
       tokens
     end
   end
+
+  class Tokenizer2
+    attr_reader :line_number, :for_liquid_tag
+
+    TAG_END = /%\}/
+    TAG_OR_VARIABLE_START = /\{[\{\%]/
+    NEWLINE = /\n/
+
+    OPEN_CURLEY = "{".ord
+    CLOSE_CURLEY = "}".ord
+    PERCENTAGE = "%".ord
+
+    def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
+      @line_number    = line_number || (line_numbers ? 1 : nil)
+      @for_liquid_tag = for_liquid_tag
+      @source         = source
+      @offset         = 0
+      @tokens         = []
+      tokenize
+    end
+
+    def shift
+      token = @tokens[@offset]
+
+      return unless token
+
+      @offset += 1
+
+      if @line_number
+        @line_number += @for_liquid_tag ? 1 : token.count("\n")
+      end
+
+      token
+    end
+
+    private
+
+    def tokenize
+      if @for_liquid_tag
+        @tokens = @source.split("\n")
+      else
+        @ss = StringScanner.new(@source)
+        @tokens << shift_normal until @ss.eos?
+      end
+
+      @ss = nil
+      @source = nil
+    end
+
+    def shift_normal
+      token = next_token
+
+      return unless token
+
+      token
+    end
+
+    def next_token
+      # possible states: :text, :tag, :variable
+      byte_a = @ss.peek_byte
+
+      if byte_a == OPEN_CURLEY
+        @ss.scan_byte
+
+        byte_b = @ss.peek_byte
+
+        if byte_b == PERCENTAGE
+          @ss.scan_byte
+          return next_tag_token
+        elsif byte_b == OPEN_CURLEY
+          @ss.scan_byte
+          return next_variable_token
+        end
+
+        @ss.pos -= 1
+      end
+
+      next_text_token
+    end
+
+    def next_text_token
+      start = @ss.pos
+
+      unless @ss.skip_until(TAG_OR_VARIABLE_START)
+        token = @ss.rest
+        @ss.terminate
+        return token
+      end
+
+      pos = @ss.pos -= 2
+      @source.byteslice(start, pos - start)
+    end
+
+    def next_variable_token
+      start = @ss.pos - 2
+
+      byte_a = byte_b = @ss.scan_byte
+
+      while byte_b
+        byte_a = @ss.scan_byte while byte_a && (byte_a != CLOSE_CURLEY && byte_a != OPEN_CURLEY)
+
+        break unless byte_a
+
+        if @ss.eos?
+          return byte_a == CLOSE_CURLEY ? @source.byteslice(start, @ss.pos - start) : "{{"
+        end
+
+        byte_b = @ss.scan_byte
+
+        if byte_a == CLOSE_CURLEY
+          if byte_b == CLOSE_CURLEY
+            return @source.byteslice(start, @ss.pos - start)
+          elsif byte_b != CLOSE_CURLEY
+            @ss.pos -= 1
+            return @source.byteslice(start, @ss.pos - start)
+          end
+        elsif byte_a == OPEN_CURLEY && byte_b == PERCENTAGE
+          return next_tag_token_with_start(start)
+        end
+
+        byte_a = byte_b
+      end
+
+      "{{"
+    end
+
+    def next_tag_token
+      start = @ss.pos - 2
+      if (len = @ss.skip_until(TAG_END))
+        @source.byteslice(start, len + 2)
+      else
+        "{%"
+      end
+    end
+
+    def next_tag_token_with_start(start)
+      @ss.skip_until(TAG_END)
+      @source.byteslice(start, @ss.pos - start)
+    end
+  end
+
+  # Remove this once we can depend on strscan >= 3.1.1
+  Tokenizer = StringScanner.instance_methods.include?(:scan_byte) ? Tokenizer2 : Tokenizer1
 end

lib/liquid/variable.rb

@@ -68,7 +68,7 @@ module Liquid
       @name = parse_context.parse_expression(p.expression)
       while p.consume?(:pipe)
         filtername = p.consume(:id)
-        filterargs = p.consume?(:colon) ? parse_filterargs(p) : Const::EMPTY_ARRAY
+        filterargs = p.consume?(:colon) ? parse_filterargs(p) : []
         @filters << parse_filter_expressions(filtername, filterargs)
       end
       p.consume(:end_of_string)
@@ -95,21 +95,15 @@ module Liquid
 
     def render_to_output_buffer(context, output)
       obj = render(context)
-      render_obj_to_output(obj, output)
-      output
-    end
 
-    def render_obj_to_output(obj, output)
-      case obj
-      when NilClass
-        # Do nothing
-      when Array
-        obj.each do |o|
-          render_obj_to_output(o, output)
-        end
-      when
+      if obj.is_a?(Array)
+        output << obj.join
+      elsif obj.nil?
+      else
         output << obj.to_s
       end
+
+      output
     end
 
     def disabled?(_context)

lib/liquid/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module Liquid
-  VERSION = "5.6.0.rc3"
+  VERSION = "5.6.0.alpha"
 end

liquid.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |s|
   s.summary     = "A secure, non-evaling end user template engine with aesthetic markup."
   s.authors     = ["Tobias Lütke"]
   s.email       = ["[email protected]"]
-  s.homepage    = "https://shopify.github.io/liquid/"
+  s.homepage    = "http://www.liquidmarkup.org"
   s.license     = "MIT"
   # s.description = "A secure, non-evaling end user template engine with aesthetic markup."
 

performance/benchmark.rb

@@ -4,19 +4,21 @@ require 'benchmark/ips'
 require_relative 'theme_runner'
 
 RubyVM::YJIT.enable if defined?(RubyVM::YJIT)
-Liquid::Environment.default.error_mode = ARGV.first.to_sym if ARGV.first
-
+Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
 profiler = ThemeRunner.new
 
 Benchmark.ips do |x|
-  x.time   = 10
-  x.warmup = 5
+  x.time   = 20
+  x.warmup = 10
 
   puts
   puts "Running benchmark for #{x.time} seconds (with #{x.warmup} seconds warmup)."
   puts
 
-  x.report("parse:") { profiler.compile }
-  x.report("render:") { profiler.render }
-  x.report("parse & render:") { profiler.run }
+  phase = ENV["PHASE"] || "all"
+
+  x.report("tokenize:") { profiler.tokenize } if phase == "all" || phase == "tokenize"
+  x.report("parse:") { profiler.compile } if phase == "all" || phase == "parse"
+  x.report("render:") { profiler.render } if phase == "all" || phase == "render"
+  x.report("parse & render:") { profiler.run } if phase == "all" || phase == "run"
 end

performance/shopify/liquid.rb

@@ -11,12 +11,11 @@ require_relative 'shop_filter'
 require_relative 'tag_filter'
 require_relative 'weight_filter'
 
-default_environment = Liquid::Environment.default
-default_environment.register_tag('paginate', Paginate)
-default_environment.register_tag('form', CommentForm)
+Liquid::Template.register_tag('paginate', Paginate)
+Liquid::Template.register_tag('form', CommentForm)
 
-default_environment.register_filter(JsonFilter)
-default_environment.register_filter(MoneyFilter)
-default_environment.register_filter(WeightFilter)
-default_environment.register_filter(ShopFilter)
-default_environment.register_filter(TagFilter)
+Liquid::Template.register_filter(JsonFilter)
+Liquid::Template.register_filter(MoneyFilter)
+Liquid::Template.register_filter(WeightFilter)
+Liquid::Template.register_filter(ShopFilter)
+Liquid::Template.register_filter(TagFilter)

performance/theme_runner.rb

@@ -48,6 +48,14 @@ class ThemeRunner
     end
   end
 
+  # `tokenize` will just test the tokenizen portion of liquid without any templates
+  def tokenize
+    @tests.each do |test_hash|
+      tokenizer = Liquid::Tokenizer.new(test_hash[:liquid], true)
+      while tokenizer.shift; end
+    end
+  end
+
   # `run` is called to benchmark rendering and compiling at the same time
   def run
     each_test do |liquid, layout, assigns, page_template, template_name|

test/integration/context_test.rb

@@ -672,21 +672,6 @@ class ContextTest < Minitest::Test
     assert_includes(result, "unscoped_products_count: 5")
   end
 
-  def test_new_isolated_context_inherits_parent_environment
-    global_environment = Liquid::Environment.build(tags: {})
-    context = Context.build(environment: global_environment)
-
-    subcontext = context.new_isolated_subcontext
-    assert_equal(global_environment, subcontext.environment)
-  end
-
-  def test_newly_built_context_inherits_parent_environment
-    global_environment = Liquid::Environment.build(tags: {})
-    context = Context.build(environment: global_environment)
-    assert_equal(global_environment, context.environment)
-    assert(context.environment.tags.each.to_a.empty?)
-  end
-
   private
 
   def assert_no_object_allocations

test/integration/error_handling_test.rb

@@ -203,19 +203,20 @@ class ErrorHandlingTest < Minitest::Test
   end
 
   def test_setting_default_exception_renderer
+    old_exception_renderer = Liquid::Template.default_exception_renderer
     exceptions = []
-    default_exception_renderer = ->(e) {
+    Liquid::Template.default_exception_renderer = ->(e) {
       exceptions << e
       ''
     }
-
-    env = Liquid::Environment.build(exception_renderer: default_exception_renderer)
-    template = Liquid::Template.parse('This is a runtime error: {{ errors.argument_error }}', environment: env)
+    template = Liquid::Template.parse('This is a runtime error: {{ errors.argument_error }}')
 
     output = template.render('errors' => ErrorDrop.new)
 
     assert_equal('This is a runtime error: ', output)
     assert_equal([Liquid::ArgumentError], template.errors.map(&:class))
+  ensure
+    Liquid::Template.default_exception_renderer = old_exception_renderer if old_exception_renderer
   end
 
   def test_setting_exception_renderer_on_environment

test/integration/parsing_quirks_test.rb

@@ -134,6 +134,14 @@ class ParsingQuirksTest < Minitest::Test
 
   def test_incomplete_expression
     with_error_mode(:lax) do
+      assert_template_result("false", "{{ false - }}")
+      assert_template_result("false", "{{ false > }}")
+      assert_template_result("false", "{{ false < }}")
+      assert_template_result("false", "{{ false = }}")
+      assert_template_result("false", "{{ false ! }}")
+      assert_template_result("false", "{{ false 1 }}")
+      assert_template_result("false", "{{ false a }}")
+
       assert_template_result("false", "{% liquid assign foo = false -\n%}{{ foo }}")
       assert_template_result("false", "{% liquid assign foo = false >\n%}{{ foo }}")
       assert_template_result("false", "{% liquid assign foo = false <\n%}{{ foo }}")

test/integration/profiler_test.rb

@@ -33,7 +33,7 @@ class ProfilerTest < Minitest::Test
   end
 
   def setup
-    Liquid::Environment.default.file_system = ProfilingFileSystem.new
+    Liquid::Template.file_system = ProfilingFileSystem.new
   end
 
   def test_template_allows_flagging_profiling

test/integration/tags/include_tag_test.rb

@@ -174,10 +174,10 @@ class IncludeTagTest < Minitest::Test
       end
     end
 
-    env = Liquid::Environment.build(file_system: infinite_file_system.new)
+    Liquid::Template.file_system = infinite_file_system.new
 
     assert_raises(Liquid::StackLevelError) do
-      Template.parse("{% include 'loop' %}", environment: env).render!
+      Template.parse("{% include 'loop' %}").render!
     end
   end
 
@@ -264,27 +264,26 @@ class IncludeTagTest < Minitest::Test
   end
 
   def test_does_not_add_error_in_strict_mode_for_missing_variable
-    env = Liquid::Environment.build(file_system: TestFileSystem.new)
+    Liquid::Template.file_system = TestFileSystem.new
 
-    a = Liquid::Template.parse(' {% include "nested_template" %}', environment: env)
+    a = Liquid::Template.parse(' {% include "nested_template" %}')
     a.render!
     assert_empty(a.errors)
   end
 
   def test_passing_options_to_included_templates
-    env = Liquid::Environment.build(file_system: TestFileSystem.new)
-
+    Liquid::Template.file_system = TestFileSystem.new
     assert_raises(Liquid::SyntaxError) do
-      Template.parse("{% include template %}", error_mode: :strict, environment: env).render!("template" => '{{ "X" || downcase }}')
+      Template.parse("{% include template %}", error_mode: :strict).render!("template" => '{{ "X" || downcase }}')
     end
     with_error_mode(:lax) do
-      assert_equal('x', Template.parse("{% include template %}", error_mode: :strict, include_options_blacklist: true, environment: env).render!("template" => '{{ "X" || downcase }}'))
+      assert_equal('x', Template.parse("{% include template %}", error_mode: :strict, include_options_blacklist: true).render!("template" => '{{ "X" || downcase }}'))
     end
     assert_raises(Liquid::SyntaxError) do
-      Template.parse("{% include template %}", error_mode: :strict, include_options_blacklist: [:locale], environment: env).render!("template" => '{{ "X" || downcase }}')
+      Template.parse("{% include template %}", error_mode: :strict, include_options_blacklist: [:locale]).render!("template" => '{{ "X" || downcase }}')
     end
     with_error_mode(:lax) do
-      assert_equal('x', Template.parse("{% include template %}", error_mode: :strict, include_options_blacklist: [:error_mode], environment: env).render!("template" => '{{ "X" || downcase }}'))
+      assert_equal('x', Template.parse("{% include template %}", error_mode: :strict, include_options_blacklist: [:error_mode]).render!("template" => '{{ "X" || downcase }}'))
     end
   end
 
@@ -335,11 +334,8 @@ class IncludeTagTest < Minitest::Test
   end
 
   def test_including_with_strict_variables
-    env = Liquid::Environment.build(
-      file_system: StubFileSystem.new('simple' => 'simple'),
-    )
-
-    template = Liquid::Template.parse("{% include 'simple' %}", error_mode: :warn, environment: env)
+    Liquid::Template.file_system = StubFileSystem.new({ "simple" => "simple" })
+    template = Liquid::Template.parse("{% include 'simple' %}", error_mode: :warn)
     template.render(nil, strict_variables: true)
 
     assert_equal([], template.errors)

test/integration/tags/raw_tag_test.rb

@@ -16,6 +16,7 @@ class RawTagTest < Minitest::Test
     assert_template_result('>{{ test }}<', '> {%- raw -%}{{ test }}{%- endraw -%} <')
     assert_template_result("> inner  <", "> {%- raw -%} inner {%- endraw %} <")
     assert_template_result("> inner <", "> {%- raw -%} inner {%- endraw -%} <")
+    assert_template_result("{Hello}", "{% raw %}{{% endraw %}Hello{% raw %}}{% endraw %}")
   end
 
   def test_open_tag_in_raw

test/integration/tags/render_tag_test.rb

@@ -82,22 +82,19 @@ class RenderTagTest < Minitest::Test
   end
 
   def test_recursively_rendered_template_does_not_produce_endless_loop
-    env = Liquid::Environment.build(
-      file_system: StubFileSystem.new('loop' => '{% render "loop" %}'),
-    )
+    Liquid::Template.file_system = StubFileSystem.new('loop' => '{% render "loop" %}')
 
     assert_raises(Liquid::StackLevelError) do
-      Template.parse('{% render "loop" %}', environment: env).render!
+      Template.parse('{% render "loop" %}').render!
     end
   end
 
   def test_sub_contexts_count_towards_the_same_recursion_limit
-    env = Liquid::Environment.build(
-      file_system: StubFileSystem.new('loop_render' => '{% render "loop_render" %}'),
+    Liquid::Template.file_system = StubFileSystem.new(
+      'loop_render' => '{% render "loop_render" %}',
     )
-
     assert_raises(Liquid::StackLevelError) do
-      Template.parse('{% render "loop_render" %}', environment: env).render!
+      Template.parse('{% render "loop_render" %}').render!
     end
   end
 

test/integration/variable_test.rb

@@ -130,10 +130,6 @@ class VariableTest < Minitest::Test
     assert_template_result('bar', '{{ foo }}', { 'foo' => :bar })
   end
 
-  def test_nested_array
-    assert_template_result('', '{{ foo }}', { 'foo' => [[nil]] })
-  end
-
   def test_dynamic_find_var
     assert_template_result('bar', '{{ [key] }}', { 'key' => 'foo', 'foo' => 'bar' })
   end

test/test_helper.rb

@@ -15,6 +15,11 @@ if (env_mode = ENV['LIQUID_PARSER_MODE'])
 end
 Liquid::Environment.default.error_mode = mode
 
+if ENV['LIQUID_C'] == '1'
+  puts "-- LIQUID C"
+  require 'liquid/c'
+end
+
 if Minitest.const_defined?('Test')
   # We're on Minitest 5+. Nothing to do here.
 else
@@ -83,11 +88,11 @@ module Minitest
     end
 
     def with_error_mode(mode)
-      old_mode = Liquid::Environment.default.error_mode
-      Liquid::Environment.default.error_mode = mode
+      old_mode = Liquid::Template.error_mode
+      Liquid::Template.error_mode = mode
       yield
     ensure
-      Liquid::Environment.default.error_mode = old_mode
+      Liquid::Template.error_mode = old_mode
     end
 
     def with_custom_tag(tag_name, tag_class, &block)

test/unit/strainer_factory_unit_test.rb

@@ -2,7 +2,7 @@
 
 require 'test_helper'
 
-class EnvironmentFilterTest < Minitest::Test
+class StrainerFactoryUnitTest < Minitest::Test
   include Liquid
 
   module AccessScopeFilters
@@ -16,6 +16,8 @@ class EnvironmentFilterTest < Minitest::Test
     private :private_filter
   end
 
+  StrainerFactory.add_global_filter(AccessScopeFilters)
+
   module LateAddedFilter
     def late_added_filter(_input)
       "filtered"
@@ -23,28 +25,24 @@ class EnvironmentFilterTest < Minitest::Test
   end
 
   def setup
-    @environment = Liquid::Environment.build do |env|
-      env.register_filter(AccessScopeFilters)
-    end
-
-    @context = Context.build(environment: @environment)
+    @context = Context.build
   end
 
   def test_strainer
-    strainer = @environment.create_strainer(@context)
+    strainer = StrainerFactory.create(@context)
     assert_equal(5, strainer.invoke('size', 'input'))
     assert_equal("public", strainer.invoke("public_filter"))
   end
 
   def test_stainer_raises_argument_error
-    strainer = @environment.create_strainer(@context)
+    strainer = StrainerFactory.create(@context)
     assert_raises(Liquid::ArgumentError) do
       strainer.invoke("public_filter", 1)
     end
   end
 
   def test_stainer_argument_error_contains_backtrace
-    strainer = @environment.create_strainer(@context)
+    strainer = StrainerFactory.create(@context)
 
     exception = assert_raises(Liquid::ArgumentError) do
       strainer.invoke("public_filter", 1)
@@ -59,7 +57,7 @@ class EnvironmentFilterTest < Minitest::Test
   end
 
   def test_strainer_only_invokes_public_filter_methods
-    strainer = @environment.create_strainer(@context)
+    strainer = StrainerFactory.create(@context)
     assert_equal(false, strainer.class.invokable?('__test__'))
     assert_equal(false, strainer.class.invokable?('test'))
     assert_equal(false, strainer.class.invokable?('instance_eval'))
@@ -68,18 +66,18 @@ class EnvironmentFilterTest < Minitest::Test
   end
 
   def test_strainer_returns_nil_if_no_filter_method_found
-    strainer = @environment.create_strainer(@context)
+    strainer = StrainerFactory.create(@context)
     assert_nil(strainer.invoke("private_filter"))
     assert_nil(strainer.invoke("undef_the_filter"))
   end
 
   def test_strainer_returns_first_argument_if_no_method_and_arguments_given
-    strainer = @environment.create_strainer(@context)
+    strainer = StrainerFactory.create(@context)
     assert_equal("password", strainer.invoke("undef_the_method", "password"))
   end
 
   def test_strainer_only_allows_methods_defined_in_filters
-    strainer = @environment.create_strainer(@context)
+    strainer = StrainerFactory.create(@context)
     assert_equal("1 + 1", strainer.invoke("instance_eval", "1 + 1"))
     assert_equal("puts",  strainer.invoke("__send__", "puts", "Hi Mom"))
     assert_equal("has_method?", strainer.invoke("invoke", "has_method?", "invoke"))
@@ -88,9 +86,7 @@ class EnvironmentFilterTest < Minitest::Test
   def test_strainer_uses_a_class_cache_to_avoid_method_cache_invalidation
     a = Module.new
     b = Module.new
-
-    strainer = @environment.create_strainer(@context, [a, b])
-
+    strainer = StrainerFactory.create(@context, [a, b])
     assert_kind_of(StrainerTemplate, strainer)
     assert_kind_of(a, strainer)
     assert_kind_of(b, strainer)
@@ -98,10 +94,8 @@ class EnvironmentFilterTest < Minitest::Test
   end
 
   def test_add_global_filter_clears_cache
-    assert_equal('input', @environment.create_strainer(@context).invoke('late_added_filter', 'input'))
-
-    @environment.register_filter(LateAddedFilter)
-
-    assert_equal('filtered', @environment.create_strainer(nil).invoke('late_added_filter', 'input'))
+    assert_equal('input', StrainerFactory.create(@context).invoke('late_added_filter', 'input'))
+    StrainerFactory.add_global_filter(LateAddedFilter)
+    assert_equal('filtered', StrainerFactory.create(nil).invoke('late_added_filter', 'input'))
   end
 end

test/unit/strainer_template_unit_test.rb

@@ -25,13 +25,11 @@ class StrainerTemplateUnitTest < Minitest::Test
   end
 
   def test_add_filter_raises_when_module_privately_overrides_registered_public_methods
-    error = assert_raises(Liquid::MethodOverrideError) do
-      Liquid::Environment.build do |env|
-        env.register_filter(PublicMethodOverrideFilter)
-        env.register_filter(PrivateMethodOverrideFilter)
-      end
-    end
+    strainer = Context.new.strainer
 
+    error = assert_raises(Liquid::MethodOverrideError) do
+      strainer.class.add_filter(PrivateMethodOverrideFilter)
+    end
     assert_equal('Liquid error: Filter overrides registered public methods as non public: public_filter', error.message)
   end
 
@@ -44,13 +42,11 @@ class StrainerTemplateUnitTest < Minitest::Test
   end
 
   def test_add_filter_raises_when_module_overrides_registered_public_method_as_protected
-    error = assert_raises(Liquid::MethodOverrideError) do
-      Liquid::Environment.build do |env|
-        env.register_filter(PublicMethodOverrideFilter)
-        env.register_filter(ProtectedMethodOverrideFilter)
-      end
-    end
+    strainer = Context.new.strainer
 
+    error = assert_raises(Liquid::MethodOverrideError) do
+      strainer.class.add_filter(ProtectedMethodOverrideFilter)
+    end
     assert_equal('Liquid error: Filter overrides registered public methods as non public: public_filter', error.message)
   end
 

test/unit/template_unit_test.rb

@@ -20,12 +20,50 @@ class TemplateUnitTest < Minitest::Test
     assert_equal(fixture("en_locale.yml"), locale.path)
   end
 
+  def test_with_cache_classes_tags_returns_the_same_class
+    original_cache_setting = Liquid.cache_classes
+    Liquid.cache_classes   = true
+
+    original_klass = Class.new
+    Object.send(:const_set, :CustomTag, original_klass)
+    Template.register_tag('custom', CustomTag)
+
+    Object.send(:remove_const, :CustomTag)
+
+    new_klass = Class.new
+    Object.send(:const_set, :CustomTag, new_klass)
+
+    assert(Template.tags['custom'].equal?(original_klass))
+  ensure
+    Object.send(:remove_const, :CustomTag)
+    Liquid.cache_classes = original_cache_setting
+  end
+
+  def test_without_cache_classes_tags_reloads_the_class
+    original_cache_setting = Liquid.cache_classes
+    Liquid.cache_classes   = false
+
+    original_klass = Class.new
+    Object.send(:const_set, :CustomTag, original_klass)
+    with_custom_tag('custom', CustomTag) do
+      Object.send(:remove_const, :CustomTag)
+
+      new_klass = Class.new
+      Object.send(:const_set, :CustomTag, new_klass)
+
+      assert(Template.tags['custom'].equal?(new_klass))
+    end
+  ensure
+    Object.send(:remove_const, :CustomTag)
+    Liquid.cache_classes = original_cache_setting
+  end
+
   class FakeTag; end
 
   def test_tags_can_be_looped_over
     with_custom_tag('fake', FakeTag) do
       result = Template.tags.map { |name, klass| [name, klass] }
-      assert(result.include?(["fake", TemplateUnitTest::FakeTag]))
+      assert(result.include?(["fake", "TemplateUnitTest::FakeTag"]))
     end
   end
 

test/unit/tokenizer_unit_test.rb

@@ -6,6 +6,7 @@ class TokenizerTest < Minitest::Test
   def test_tokenize_strings
     assert_equal([' '], tokenize(' '))
     assert_equal(['hello world'], tokenize('hello world'))
+    assert_equal(['{}'], tokenize('{}'))
   end
 
   def test_tokenize_variables
@@ -30,8 +31,17 @@ class TokenizerTest < Minitest::Test
     assert_equal([1, 1, 3], tokenize_line_numbers(" {{\n funk \n}} "))
   end
 
-  def test_tokenize_with_nil_source_returns_empty_array
-    assert_equal([], tokenize(nil))
+  def test_incomplete_curly_braces
+    assert_equal(["{{.}", " "], tokenize('{{.} '))
+    assert_equal(["{{}", "%}"], tokenize('{{}%}'))
+    assert_equal(["{{}}", "}"], tokenize('{{}}}'))
+  end
+
+  def test_unmatching_start_and_end
+    assert_equal(["{{%}"], tokenize('{{%}'))
+    assert_equal(["{{%%%}}"], tokenize('{{%%%}}'))
+    assert_equal(["{%", "}}"], tokenize('{%}}'))
+    assert_equal(["{%%}", "}"], tokenize('{%%}}'))
   end
 
   private