passing tokenizer unit test

.github/workflows/liquid.yml

@@ -13,8 +13,7 @@ jobs:
         entry:
           - { ruby: 3.0, allowed-failure: false } # minimum supported
           - { ruby: 3.2, allowed-failure: false }
-          - { ruby: 3.3, allowed-failure: false }
-          - { ruby: "3.4.0-rc1", allowed-failure: false } # latest
+          - { ruby: 3.3, allowed-failure: false } # latest
           - { ruby: ruby-head, allowed-failure: false }
     name: Test Ruby ${{ matrix.entry.ruby }}
     steps:

.ruby-version

@@ -1 +1 @@
-3.3.6
+3.3.4

Gemfile

@@ -23,4 +23,8 @@ group :test do
   gem 'rubocop', '~> 1.61.0'
   gem 'rubocop-shopify', '~> 2.12.0', require: false
   gem 'rubocop-performance', require: false
+
+  platform :mri, :truffleruby do
+    gem 'liquid-c', github: 'Shopify/liquid-c', ref: 'main'
+  end
 end

History.md

@@ -1,15 +1,5 @@
 # Liquid Change Log
 
-## 5.6.1 2025-01-07
-
-Add `omit_blank_nodes` parse option to skip blank nodes in the AST (#1870) [Michael Go]
-
-## 5.6.0 (unreleased)
-
-### Fixes
-
-* Fix Tokenizer to handle null source value (#1873) [Bahar Pourazar]
-
 ## 5.5.0 2024-03-21
 
 Please reference the GitHub release for more information.

Rakefile

@@ -43,6 +43,8 @@ task :test do
   Rake::Task['base_test'].invoke
 
   if RUBY_ENGINE == 'ruby' || RUBY_ENGINE == 'truffleruby'
+    ENV['LIQUID_C'] = '1'
+
     ENV['LIQUID_PARSER_MODE'] = 'lax'
     Rake::Task['integration_test'].reenable
     Rake::Task['integration_test'].invoke

lib/liquid.rb

@@ -44,11 +44,15 @@ module Liquid
   VariableParser              = /\[(?>[^\[\]]+|\g<0>)*\]|#{VariableSegment}+\??/o
 
   RAISE_EXCEPTION_LAMBDA = ->(_e) { raise }
+
+  singleton_class.send(:attr_accessor, :cache_classes)
+  self.cache_classes = true
 end
 
 require "liquid/version"
 require "liquid/deprecations"
 require "liquid/const"
+require "liquid/template/tag_registry"
 require 'liquid/standardfilters'
 require 'liquid/file_system'
 require 'liquid/parser_switching'
@@ -68,6 +72,7 @@ require 'liquid/extensions'
 require 'liquid/errors'
 require 'liquid/interrupts'
 require 'liquid/strainer_template'
+require 'liquid/strainer_factory'
 require 'liquid/expression'
 require 'liquid/context'
 require 'liquid/tag'

lib/liquid/block_body.rb

@@ -58,9 +58,6 @@ module Liquid
             return yield tag_name, markup
           end
           new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
-
-          next if parse_context.omit_blank_nodes && blank_node?(new_tag)
-
           @blank &&= new_tag.blank?
           @nodelist << new_tag
         end
@@ -156,9 +153,6 @@ module Liquid
             return yield tag_name, markup
           end
           new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
-
-          next if parse_context.omit_blank_nodes && blank_node?(new_tag)
-
           @blank &&= new_tag.blank?
           @nodelist << new_tag
         when token.start_with?(VARSTART)
@@ -252,17 +246,10 @@ module Liquid
     end
 
     def create_variable(token, parse_context)
-      if token.end_with?("}}")
-        i = 2
-        i = 3 if token[i] == "-"
-        parse_end = token.length - 3
-        parse_end -= 1 if token[parse_end] == "-"
-        markup_end = parse_end - i + 1
-        markup = markup_end <= 0 ? "" : token.slice(i, markup_end)
-
+      if token =~ ContentOfVariable
+        markup = Regexp.last_match(1)
         return Variable.new(markup, parse_context)
       end
-
       BlockBody.raise_missing_variable_terminator(token, parse_context)
     end
 
@@ -275,18 +262,5 @@ module Liquid
     def raise_missing_variable_terminator(token, parse_context)
       BlockBody.raise_missing_variable_terminator(token, parse_context)
     end
-
-    def blank_node?(node)
-      case node
-      when Comment
-        true
-      when BlockBody
-        true if node.nodelist.empty?
-      when Tag
-        node.nodelist.all? { |n| blank_node?(n) }
-      else
-        false
-      end
-    end
   end
 end

lib/liquid/context.rb

@@ -19,7 +19,7 @@ module Liquid
 
     # rubocop:disable Metrics/ParameterLists
     def self.build(environment: Environment.default, environments: {}, outer_scope: {}, registers: {}, rethrow_errors: false, resource_limits: nil, static_environments: {}, &block)
-      new(environments, outer_scope, registers, rethrow_errors, resource_limits, static_environments, environment, &block)
+      new(environments, outer_scope, registers, rethrow_errors, resource_limits, static_environments, &block)
     end
 
     def initialize(environments = {}, outer_scope = {}, registers = {}, rethrow_errors = false, resource_limits = nil, static_environments = {}, environment = Environment.default)
@@ -143,7 +143,6 @@ module Liquid
       check_overflow
 
       self.class.build(
-        environment: @environment,
         resource_limits: resource_limits,
         static_environments: static_environments,
         registers: Registers.new(registers),

lib/liquid/environment.rb

@@ -41,7 +41,7 @@ module Liquid
       # @return [Environment] The new environment instance.
       def build(tags: nil, file_system: nil, error_mode: nil, exception_renderer: nil)
         ret = new
-        ret.tags = tags if tags
+        ret.tags = Template::TagRegistry.new(tags) if tags
         ret.file_system = file_system if file_system
         ret.error_mode = error_mode if error_mode
         ret.exception_renderer = exception_renderer if exception_renderer
@@ -74,7 +74,7 @@ module Liquid
     # Initializes a new environment instance.
     # @api private
     def initialize
-      @tags = Tags::STANDARD_TAGS.dup
+      @tags = Template::TagRegistry.new(Tags::STANDARD_TAGS)
       @error_mode = :lax
       @strainer_template = Class.new(StrainerTemplate).tap do |klass|
         klass.add_filter(StandardFilters)

lib/liquid/lexer.rb

@@ -73,6 +73,7 @@ module Liquid
     COMPARISON_LESS_THAN = [:comparison, "<"].freeze
     COMPARISON_LESS_THAN_OR_EQUAL = [:comparison, "<="].freeze
     COMPARISON_NOT_EQUAL_ALT = [:comparison, "<>"].freeze
+    CONTAINS = /contains(?=\s)/
     DASH = [:dash, "-"].freeze
     DOT = [:dot, "."].freeze
     DOTDOT = [:dotdot, ".."].freeze
@@ -89,12 +90,7 @@ module Liquid
     SINGLE_STRING_LITERAL = /'[^\']*'/
     WHITESPACE_OR_NOTHING = /\s*/
 
-    SINGLE_COMPARISON_TOKENS = [].tap do |table|
-      table["<".ord] = COMPARISON_LESS_THAN
-      table[">".ord] = COMPARISON_GREATER_THAN
-    end
-
-    TWO_CHARS_COMPARISON_JUMP_TABLE = [].tap do |table|
+    COMPARISON_JUMP_TABLE = [].tap do |table|
       table["=".ord] = [].tap do |sub_table|
         sub_table["=".ord] = COMPARISON_EQUAL
         sub_table.freeze
@@ -103,9 +99,6 @@ module Liquid
         sub_table["=".ord] = COMPARISION_NOT_EQUAL
         sub_table.freeze
       end
-    end
-
-    COMPARISON_JUMP_TABLE = [].tap do |table|
       table["<".ord] = [].tap do |sub_table|
         sub_table["=".ord] = COMPARISON_LESS_THAN_OR_EQUAL
         sub_table[">".ord] = COMPARISON_NOT_EQUAL_ALT
@@ -170,7 +163,6 @@ module Liquid
 
         break if @ss.eos?
 
-        start_pos = @ss.pos
         peeked = @ss.peek_byte
 
         if (special = SPECIAL_TABLE[peeked])
@@ -181,7 +173,7 @@ module Liquid
             @output << DOTDOT
           elsif special == DASH
             # Special case for negative numbers
-            if (peeked_byte = @ss.peek_byte) && NUMBER_TABLE[peeked_byte]
+            if NUMBER_TABLE[@ss.peek_byte]
               @ss.pos -= 1
               @output << [:number, @ss.scan(NUMBER_LITERAL)]
             else
@@ -190,34 +182,26 @@ module Liquid
           else
             @output << special
           end
-        elsif (sub_table = TWO_CHARS_COMPARISON_JUMP_TABLE[peeked])
-          @ss.scan_byte
-          if (peeked_byte = @ss.peek_byte) && (found = sub_table[peeked_byte])
-            @output << found
-            @ss.scan_byte
-          else
-            raise_syntax_error(start_pos)
-          end
         elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
           @ss.scan_byte
-          if (peeked_byte = @ss.peek_byte) && (found = sub_table[peeked_byte])
+          if (found = sub_table[@ss.peek_byte])
             @output << found
             @ss.scan_byte
           else
-            @output << SINGLE_COMPARISON_TOKENS[peeked]
+            raise SyntaxError, "Unexpected character #{peeked.chr}"
           end
         else
           type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked]
 
           if type && (t = @ss.scan(pattern))
             # Special case for "contains"
-            @output << if type == :id && t == "contains" && @output.last&.first != :dot
+            @output << if type == :id && t == "contains"
               COMPARISON_CONTAINS
             else
               [type, t]
             end
           else
-            raise_syntax_error(start_pos)
+            raise SyntaxError, "Unexpected character #{peeked.chr}"
           end
         end
       end
@@ -225,12 +209,6 @@ module Liquid
 
       @output << EOS
     end
-
-    def raise_syntax_error(start_pos)
-      @ss.pos = start_pos
-      # the character could be a UTF-8 character, use getch to get all the bytes
-      raise SyntaxError, "Unexpected character #{@ss.getch}"
-    end
   end
 
   Lexer = StringScanner.instance_methods.include?(:scan_byte) ? Lexer2 : Lexer1

lib/liquid/parse_context.rb

@@ -3,7 +3,7 @@
 module Liquid
   class ParseContext
     attr_accessor :locale, :line_number, :trim_whitespace, :depth
-    attr_reader :partial, :warnings, :error_mode, :environment, :omit_blank_nodes
+    attr_reader :partial, :warnings, :error_mode, :environment
 
     def initialize(options = Const::EMPTY_HASH)
       @environment = options.fetch(:environment, Environment.default)
@@ -12,10 +12,6 @@ module Liquid
       @locale   = @template_options[:locale] ||= I18n.new
       @warnings = []
 
-      # remove blank nodes such as
-      # comment tags, empty if tags, etc from the AST
-      @omit_blank_nodes = options.fetch(:omit_blank_nodes, false)
-
       self.depth   = 0
       self.partial = false
     end

lib/liquid/parse_tree_visitor.rb

@@ -36,7 +36,7 @@ module Liquid
     protected
 
     def children
-      @node.respond_to?(:nodelist) ? Array(@node.nodelist) : Const::EMPTY_ARRAY
+      @node.respond_to?(:nodelist) ? Array(@node.nodelist) : []
     end
   end
 end

lib/liquid/standardfilters.rb

@@ -877,7 +877,7 @@ module Liquid
     #   - [`nil`](/docs/api/liquid/basics#nil)
     # @liquid_syntax variable | default: variable
     # @liquid_return [untyped]
-    # @liquid_optional_param allow_false: [boolean] Whether to use false values instead of the default.
+    # @liquid_optional_param allow_false [boolean] Whether to use false values instead of the default.
     def default(input, default_value = '', options = {})
       options = {} unless options.is_a?(Hash)
       false_check = options['allow_false'] ? input.nil? : !Liquid::Utils.to_liquid_value(input)

lib/liquid/strainer_factory.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Liquid
+  # StrainerFactory is the factory for the filters system.
+  module StrainerFactory
+    extend self
+
+    def add_global_filter(filter, environment = Environment.default)
+      Deprecations.warn("StrainerFactory.add_global_filter", "Environment#register_filter")
+      environment.register_filter(filter)
+    end
+
+    def create(context, filters = Const::EMPTY_ARRAY, environment = Environment.default)
+      Deprecations.warn("StrainerFactory.create", "StrainerFactory.create_strainer")
+      environment.create_strainer(context, filters)
+    end
+
+    def global_filter_names(environment = Environment.default)
+      Deprecations.warn("StrainerFactory.global_filter_names", "Environment#filter_method_names")
+      Environment.strainer_template.filter_method_names
+    end
+  end
+end

lib/liquid/template/tag_registry.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Liquid
+  class Template
+    class TagRegistry
+      include Enumerable
+
+      def initialize(tags = nil)
+        @tags  = {}
+        @cache = {}
+        tags.each { |tag_name, klass| self[tag_name] = klass }
+        Deprecations.warn("Template::TagRegistry", "Use a Environment instance with zeitwerk")
+      end
+
+      def [](tag_name)
+        return nil unless @tags.key?(tag_name)
+        return @cache[tag_name] if Liquid.cache_classes
+
+        lookup_class(@tags[tag_name]).tap { |o| @cache[tag_name] = o }
+      end
+
+      def delete(tag_name)
+        Deprecations.warn("Template::TagRegistry#delete", "Use a Environment instance with immutable tags")
+        @tags.delete(tag_name)
+        @cache.delete(tag_name)
+      end
+
+      def []=(tag_name, klass)
+        @tags[tag_name]  = klass.name
+        @cache[tag_name] = klass
+      end
+
+      def each(&block)
+        @tags.each(&block)
+      end
+
+      private
+
+      def lookup_class(name)
+        Object.const_get(name)
+      end
+    end
+  end
+end

lib/liquid/tokenizer.rb

@@ -5,21 +5,25 @@ module Liquid
     attr_reader :line_number, :for_liquid_tag
 
     def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
-      @source         = source.to_s.to_str
+      @source         = source
       @line_number    = line_number || (line_numbers ? 1 : nil)
       @for_liquid_tag = for_liquid_tag
-      @offset         = 0
-      @tokens         = tokenize
+      @offset = 0
+      @state = :outside
+      @ss = StringScanner.new(source)
     end
 
     def shift
-      token = @tokens[@offset]
-      return nil unless token
+      return nil if @ss.eos?
 
-      @offset += 1
+      token = take_new_next
 
-      if @line_number
-        @line_number += @for_liquid_tag ? 1 : token.count("\n")
+      until @state == :outside || @ss.eos?
+        token << if @state == :inside_variable || @state == :inside_tag
+          take_inside
+        else
+          take_rest
+        end
       end
 
       token
@@ -27,19 +31,65 @@ module Liquid
 
     private
 
-    def tokenize
-      return [] if @source.empty?
+    def take_rest
+      token = @ss.rest
+      @ss.terminate
+      @state = :outside
+      token
+    end
 
-      return @source.split("\n") if @for_liquid_tag
+    def take_new_next
+      start_pos = @ss.pos
 
-      tokens = @source.split(TemplateParser)
+      if (n = @ss.skip_until(VariableStart))
+        if n == 2
+          @state = :inside_variable
+          end_pos = @ss.pos
+        token = @source.byteslice(start_pos, end_pos - start_pos)
+        else
+          @ss.pos -= 2
+          token = @source.byteslice(start_pos, n - 2)
+        end
 
-      # removes the rogue empty element at the beginning of the array
-      if tokens[0]&.empty?
-        @offset += 1
+        @line_number += @for_liquid_tag ? 1 : token.count("\n") if @line_number
+        token
+      elsif (n = @ss.skip_until(TagStart))
+        if n == 2
+          @state = :inside_tag
+          end_pos = @ss.pos
+          token = @source.byteslice(start_pos, end_pos - start_pos)
+        else
+          @ss.pos -= 2
+          token = @source.byteslice(start_pos, n - 2)
+        end
+
+        @line_number += @for_liquid_tag ? 1 : token.count("\n") if @line_number
+        token
+      else
+        take_rest
       end
+    end
 
-      tokens
+    def take_inside
+      start_pos = @ss.pos
+
+      if @state == :inside_variable
+        if (n = @ss.skip_until(VariableEnd))
+          @state = :outside
+          token = @source.byteslice(start_pos, n)
+          @line_number += @for_liquid_tag ? 1 : token.count("\n") if @line_number
+          token
+        end
+      elsif @state == :inside_tag
+        if (n = @ss.skip_until(TagEnd))
+          @state = :outside
+          token = @source.byteslice(start_pos, n)
+          @line_number += @for_liquid_tag ? 1 : token.count("\n") if @line_number
+          token
+        end
+      else
+        raise SyntaxError, "Unknown state: #{@state}"
+      end
     end
   end
 end

lib/liquid/variable.rb

@@ -68,7 +68,7 @@ module Liquid
       @name = parse_context.parse_expression(p.expression)
       while p.consume?(:pipe)
         filtername = p.consume(:id)
-        filterargs = p.consume?(:colon) ? parse_filterargs(p) : Const::EMPTY_ARRAY
+        filterargs = p.consume?(:colon) ? parse_filterargs(p) : []
         @filters << parse_filter_expressions(filtername, filterargs)
       end
       p.consume(:end_of_string)
@@ -95,21 +95,15 @@ module Liquid
 
     def render_to_output_buffer(context, output)
       obj = render(context)
-      render_obj_to_output(obj, output)
-      output
-    end
 
-    def render_obj_to_output(obj, output)
-      case obj
-      when NilClass
-        # Do nothing
-      when Array
-        obj.each do |o|
-          render_obj_to_output(o, output)
-        end
-      when
+      if obj.is_a?(Array)
+        output << obj.join
+      elsif obj.nil?
+      else
         output << obj.to_s
       end
+
+      output
     end
 
     def disabled?(_context)

lib/liquid/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module Liquid
-  VERSION = "5.6.1"
+  VERSION = "5.6.0.alpha"
 end

liquid.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |s|
   s.summary     = "A secure, non-evaling end user template engine with aesthetic markup."
   s.authors     = ["Tobias Lütke"]
   s.email       = ["[email protected]"]
-  s.homepage    = "https://shopify.github.io/liquid/"
+  s.homepage    = "http://www.liquidmarkup.org"
   s.license     = "MIT"
   # s.description = "A secure, non-evaling end user template engine with aesthetic markup."
 

performance/benchmark.rb

@@ -4,8 +4,7 @@ require 'benchmark/ips'
 require_relative 'theme_runner'
 
 RubyVM::YJIT.enable if defined?(RubyVM::YJIT)
-Liquid::Environment.default.error_mode = ARGV.first.to_sym if ARGV.first
-
+Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
 profiler = ThemeRunner.new
 
 Benchmark.ips do |x|

performance/shopify/liquid.rb

@@ -11,12 +11,11 @@ require_relative 'shop_filter'
 require_relative 'tag_filter'
 require_relative 'weight_filter'
 
-default_environment = Liquid::Environment.default
-default_environment.register_tag('paginate', Paginate)
-default_environment.register_tag('form', CommentForm)
+Liquid::Template.register_tag('paginate', Paginate)
+Liquid::Template.register_tag('form', CommentForm)
 
-default_environment.register_filter(JsonFilter)
-default_environment.register_filter(MoneyFilter)
-default_environment.register_filter(WeightFilter)
-default_environment.register_filter(ShopFilter)
-default_environment.register_filter(TagFilter)
+Liquid::Template.register_filter(JsonFilter)
+Liquid::Template.register_filter(MoneyFilter)
+Liquid::Template.register_filter(WeightFilter)
+Liquid::Template.register_filter(ShopFilter)
+Liquid::Template.register_filter(TagFilter)

performance/tests/tribble/404.liquid

@@ -1,3 +1,4 @@
+  <div id="page" class="innerpage clearfix">
 
     <div id="text-page">
       <div class="entry">

test/integration/context_test.rb

@@ -672,21 +672,6 @@ class ContextTest < Minitest::Test
     assert_includes(result, "unscoped_products_count: 5")
   end
 
-  def test_new_isolated_context_inherits_parent_environment
-    global_environment = Liquid::Environment.build(tags: {})
-    context = Context.build(environment: global_environment)
-
-    subcontext = context.new_isolated_subcontext
-    assert_equal(global_environment, subcontext.environment)
-  end
-
-  def test_newly_built_context_inherits_parent_environment
-    global_environment = Liquid::Environment.build(tags: {})
-    context = Context.build(environment: global_environment)
-    assert_equal(global_environment, context.environment)
-    assert(context.environment.tags.each.to_a.empty?)
-  end
-
   private
 
   def assert_no_object_allocations

test/integration/error_handling_test.rb

@@ -203,19 +203,20 @@ class ErrorHandlingTest < Minitest::Test
   end
 
   def test_setting_default_exception_renderer
+    old_exception_renderer = Liquid::Template.default_exception_renderer
     exceptions = []
-    default_exception_renderer = ->(e) {
+    Liquid::Template.default_exception_renderer = ->(e) {
       exceptions << e
       ''
     }
-
-    env = Liquid::Environment.build(exception_renderer: default_exception_renderer)
-    template = Liquid::Template.parse('This is a runtime error: {{ errors.argument_error }}', environment: env)
+    template = Liquid::Template.parse('This is a runtime error: {{ errors.argument_error }}')
 
     output = template.render('errors' => ErrorDrop.new)
 
     assert_equal('This is a runtime error: ', output)
     assert_equal([Liquid::ArgumentError], template.errors.map(&:class))
+  ensure
+    Liquid::Template.default_exception_renderer = old_exception_renderer if old_exception_renderer
   end
 
   def test_setting_exception_renderer_on_environment

test/integration/parsing_quirks_test.rb

@@ -131,16 +131,4 @@ class ParsingQuirksTest < Minitest::Test
   def test_contains_in_id
     assert_template_result(' YES ', '{% if containsallshipments == true %} YES {% endif %}', { 'containsallshipments' => true })
   end
-
-  def test_incomplete_expression
-    with_error_mode(:lax) do
-      assert_template_result("false", "{% liquid assign foo = false -\n%}{{ foo }}")
-      assert_template_result("false", "{% liquid assign foo = false >\n%}{{ foo }}")
-      assert_template_result("false", "{% liquid assign foo = false <\n%}{{ foo }}")
-      assert_template_result("false", "{% liquid assign foo = false =\n%}{{ foo }}")
-      assert_template_result("false", "{% liquid assign foo = false !\n%}{{ foo }}")
-      assert_template_result("false", "{% liquid assign foo = false 1\n%}{{ foo }}")
-      assert_template_result("false", "{% liquid assign foo = false a\n%}{{ foo }}")
-    end
-  end
 end # ParsingQuirksTest

test/integration/profiler_test.rb

@@ -33,7 +33,7 @@ class ProfilerTest < Minitest::Test
   end
 
   def setup
-    Liquid::Environment.default.file_system = ProfilingFileSystem.new
+    Liquid::Template.file_system = ProfilingFileSystem.new
   end
 
   def test_template_allows_flagging_profiling

test/integration/tags/include_tag_test.rb

@@ -174,10 +174,10 @@ class IncludeTagTest < Minitest::Test
       end
     end
 
-    env = Liquid::Environment.build(file_system: infinite_file_system.new)
+    Liquid::Template.file_system = infinite_file_system.new
 
     assert_raises(Liquid::StackLevelError) do
-      Template.parse("{% include 'loop' %}", environment: env).render!
+      Template.parse("{% include 'loop' %}").render!
     end
   end
 
@@ -264,27 +264,26 @@ class IncludeTagTest < Minitest::Test
   end
 
   def test_does_not_add_error_in_strict_mode_for_missing_variable
-    env = Liquid::Environment.build(file_system: TestFileSystem.new)
+    Liquid::Template.file_system = TestFileSystem.new
 
-    a = Liquid::Template.parse(' {% include "nested_template" %}', environment: env)
+    a = Liquid::Template.parse(' {% include "nested_template" %}')
     a.render!
     assert_empty(a.errors)
   end
 
   def test_passing_options_to_included_templates
-    env = Liquid::Environment.build(file_system: TestFileSystem.new)
-
+    Liquid::Template.file_system = TestFileSystem.new
     assert_raises(Liquid::SyntaxError) do
-      Template.parse("{% include template %}", error_mode: :strict, environment: env).render!("template" => '{{ "X" || downcase }}')
+      Template.parse("{% include template %}", error_mode: :strict).render!("template" => '{{ "X" || downcase }}')
     end
     with_error_mode(:lax) do
-      assert_equal('x', Template.parse("{% include template %}", error_mode: :strict, include_options_blacklist: true, environment: env).render!("template" => '{{ "X" || downcase }}'))
+      assert_equal('x', Template.parse("{% include template %}", error_mode: :strict, include_options_blacklist: true).render!("template" => '{{ "X" || downcase }}'))
     end
     assert_raises(Liquid::SyntaxError) do
-      Template.parse("{% include template %}", error_mode: :strict, include_options_blacklist: [:locale], environment: env).render!("template" => '{{ "X" || downcase }}')
+      Template.parse("{% include template %}", error_mode: :strict, include_options_blacklist: [:locale]).render!("template" => '{{ "X" || downcase }}')
     end
     with_error_mode(:lax) do
-      assert_equal('x', Template.parse("{% include template %}", error_mode: :strict, include_options_blacklist: [:error_mode], environment: env).render!("template" => '{{ "X" || downcase }}'))
+      assert_equal('x', Template.parse("{% include template %}", error_mode: :strict, include_options_blacklist: [:error_mode]).render!("template" => '{{ "X" || downcase }}'))
     end
   end
 
@@ -335,11 +334,8 @@ class IncludeTagTest < Minitest::Test
   end
 
   def test_including_with_strict_variables
-    env = Liquid::Environment.build(
-      file_system: StubFileSystem.new('simple' => 'simple'),
-    )
-
-    template = Liquid::Template.parse("{% include 'simple' %}", error_mode: :warn, environment: env)
+    Liquid::Template.file_system = StubFileSystem.new({ "simple" => "simple" })
+    template = Liquid::Template.parse("{% include 'simple' %}", error_mode: :warn)
     template.render(nil, strict_variables: true)
 
     assert_equal([], template.errors)

test/integration/tags/render_tag_test.rb

@@ -82,22 +82,19 @@ class RenderTagTest < Minitest::Test
   end
 
   def test_recursively_rendered_template_does_not_produce_endless_loop
-    env = Liquid::Environment.build(
-      file_system: StubFileSystem.new('loop' => '{% render "loop" %}'),
-    )
+    Liquid::Template.file_system = StubFileSystem.new('loop' => '{% render "loop" %}')
 
     assert_raises(Liquid::StackLevelError) do
-      Template.parse('{% render "loop" %}', environment: env).render!
+      Template.parse('{% render "loop" %}').render!
     end
   end
 
   def test_sub_contexts_count_towards_the_same_recursion_limit
-    env = Liquid::Environment.build(
-      file_system: StubFileSystem.new('loop_render' => '{% render "loop_render" %}'),
+    Liquid::Template.file_system = StubFileSystem.new(
+      'loop_render' => '{% render "loop_render" %}',
     )
-
     assert_raises(Liquid::StackLevelError) do
-      Template.parse('{% render "loop_render" %}', environment: env).render!
+      Template.parse('{% render "loop_render" %}').render!
     end
   end
 

test/integration/variable_test.rb

@@ -130,10 +130,6 @@ class VariableTest < Minitest::Test
     assert_template_result('bar', '{{ foo }}', { 'foo' => :bar })
   end
 
-  def test_nested_array
-    assert_template_result('', '{{ foo }}', { 'foo' => [[nil]] })
-  end
-
   def test_dynamic_find_var
     assert_template_result('bar', '{{ [key] }}', { 'key' => 'foo', 'foo' => 'bar' })
   end

test/test_helper.rb

@@ -13,7 +13,12 @@ if (env_mode = ENV['LIQUID_PARSER_MODE'])
   puts "-- #{env_mode.upcase} ERROR MODE"
   mode = env_mode.to_sym
 end
-Liquid::Environment.default.error_mode = mode
+Liquid::Template.error_mode = mode
+
+if ENV['LIQUID_C'] == '1'
+  puts "-- LIQUID C"
+  require 'liquid/c'
+end
 
 if Minitest.const_defined?('Test')
   # We're on Minitest 5+. Nothing to do here.
@@ -83,11 +88,11 @@ module Minitest
     end
 
     def with_error_mode(mode)
-      old_mode = Liquid::Environment.default.error_mode
-      Liquid::Environment.default.error_mode = mode
+      old_mode = Liquid::Template.error_mode
+      Liquid::Template.error_mode = mode
       yield
     ensure
-      Liquid::Environment.default.error_mode = old_mode
+      Liquid::Template.error_mode = old_mode
     end
 
     def with_custom_tag(tag_name, tag_class, &block)

test/unit/block_unit_test.rb

@@ -32,12 +32,6 @@ class BlockUnitTest < Minitest::Test
     assert_equal(String, template.root.nodelist[2].class)
   end
 
-  def test_variable_with_multibyte_character
-    template = Liquid::Template.parse("{{ '❤️' }}")
-    assert_equal(1, template.root.nodelist.size)
-    assert_equal(Variable, template.root.nodelist[0].class)
-  end
-
   def test_variable_many_embedded_fragments
     template = Liquid::Template.parse("  {{funk}} {{so}} {{brother}} ")
     assert_equal(7, template.root.nodelist.size)
@@ -53,103 +47,8 @@ class BlockUnitTest < Minitest::Test
     assert_equal(3, template.root.nodelist.size)
   end
 
-  def test_remove_empty_for_blocks_with_optimization_option
-    source = <<~LIQUID.chomp
-      {% for i in (1..1000000) %}
-      {% endfor %}
-    LIQUID
-
-    assert_root_nodelist_size(source, 0, omit_blank_nodes: true)
-
-    source = <<~LIQUID.chomp
-      {% for i in (1..1000000) %}
-      {% else %}
-      {% endfor %}
-    LIQUID
-
-    assert_root_nodelist_size(source, 0, omit_blank_nodes: true)
-
-    source = <<~LIQUID.chomp
-      {% for i in list %}
-        i
-      {% endfor %}
-    LIQUID
-
-    assert_root_nodelist_size(source, 1, omit_blank_nodes: true)
-
-    source = <<~LIQUID.chomp
-      {% for i in list %}
-      {% else %}
-        1
-      {% endfor %}
-    LIQUID
-
-    assert_root_nodelist_size(source, 1, omit_blank_nodes: true)
-  end
-
-  def test_remove_comment_nodes_with_optimization_option
-    source = <<~LIQUID.chomp
-      {% comment %}
-        {% if true %}
-        {% endif %}
-      {% endcomment %}
-    LIQUID
-
-    assert_root_nodelist_size(source, 0, omit_blank_nodes: true)
-
-    source = <<~LIQUID.chomp
-      {% liquid
-        comment
-          if true
-          endif
-        endcomment
-      %}
-    LIQUID
-
-    assert_root_nodelist_size(source, 0, omit_blank_nodes: true)
-  end
-
-  def test_remove_if_nodes_with_optimization_option
-    source = <<~LIQUID.chomp
-      {% if true %}
-      {% endif %}
-    LIQUID
-
-    assert_root_nodelist_size(source, 0, omit_blank_nodes: true)
-
-    source = <<~LIQUID.chomp
-      {% unless true %}
-      {% endunless %}
-    LIQUID
-
-    assert_root_nodelist_size(source, 0, omit_blank_nodes: true)
-
-    source = <<~LIQUID.chomp
-      {% if false %}
-      {% else %}
-      {% endif %}
-    LIQUID
-
-    assert_root_nodelist_size(source, 0, omit_blank_nodes: true)
-
-    source = <<~LIQUID.chomp
-      {% if false %}
-      {% else %}
-        Hello!
-      {% endif %}
-    LIQUID
-
-    assert_root_nodelist_size(source, 1, omit_blank_nodes: true)
-  end
-
   private
 
-  def assert_root_nodelist_size(source, expected_size, parse_options = {})
-    template = Liquid::Template.parse(source, parse_options)
-
-    assert_equal(expected_size, template.root.nodelist.size)
-  end
-
   def block_types(nodelist)
     nodelist.collect(&:class)
   end

test/unit/lexer_unit_test.rb

@@ -6,134 +6,58 @@ class LexerUnitTest < Minitest::Test
   include Liquid
 
   def test_strings
-    assert_equal(
-      [[:string, %('this is a test""')], [:string, %("wat 'lol'")], [:end_of_string]],
-      tokenize(%( 'this is a test""' "wat 'lol'")),
-    )
+    tokens = Lexer.new(%( 'this is a test""' "wat 'lol'")).tokenize
+    assert_equal([[:string, %('this is a test""')], [:string, %("wat 'lol'")], [:end_of_string]], tokens)
   end
 
   def test_integer
-    assert_equal(
-      [[:id, 'hi'], [:number, '50'], [:end_of_string]],
-      tokenize('hi 50'),
-    )
+    tokens = Lexer.new('hi 50').tokenize
+    assert_equal([[:id, 'hi'], [:number, '50'], [:end_of_string]], tokens)
   end
 
   def test_float
-    assert_equal(
-      [[:id, 'hi'], [:number, '5.0'], [:end_of_string]],
-      tokenize('hi 5.0'),
-    )
+    tokens = Lexer.new('hi 5.0').tokenize
+    assert_equal([[:id, 'hi'], [:number, '5.0'], [:end_of_string]], tokens)
   end
 
   def test_comparison
-    assert_equal(
-      [[:comparison, '=='], [:comparison, '<>'], [:comparison, 'contains'], [:end_of_string]],
-      tokenize('== <> contains '),
-    )
-  end
-
-  def test_comparison_without_whitespace
-    assert_equal(
-      [[:number, '1'], [:comparison, '>'], [:number, '0'], [:end_of_string]],
-      tokenize('1>0'),
-    )
-  end
-
-  def test_comparison_with_negative_number
-    assert_equal(
-      [[:number, '1'], [:comparison, '>'], [:number, '-1'], [:end_of_string]],
-      tokenize('1>-1'),
-    )
-  end
-
-  def test_raise_for_invalid_comparison
-    assert_raises(SyntaxError) do
-      tokenize('1>!1')
-    end
-
-    assert_raises(SyntaxError) do
-      tokenize('1=<1')
-    end
-
-    assert_raises(SyntaxError) do
-      tokenize('1!!1')
-    end
+    tokens = Lexer.new('== <> contains ').tokenize
+    assert_equal([[:comparison, '=='], [:comparison, '<>'], [:comparison, 'contains'], [:end_of_string]], tokens)
   end
 
   def test_specials
-    assert_equal(
-      [[:pipe, '|'], [:dot, '.'], [:colon, ':'], [:end_of_string]],
-      tokenize('| .:'),
-    )
-
-    assert_equal(
-      [[:open_square, '['], [:comma, ','], [:close_square, ']'], [:end_of_string]],
-      tokenize('[,]'),
-    )
+    tokens = Lexer.new('| .:').tokenize
+    assert_equal([[:pipe, '|'], [:dot, '.'], [:colon, ':'], [:end_of_string]], tokens)
+    tokens = Lexer.new('[,]').tokenize
+    assert_equal([[:open_square, '['], [:comma, ','], [:close_square, ']'], [:end_of_string]], tokens)
   end
 
   def test_fancy_identifiers
-    assert_equal([[:id, 'hi'], [:id, 'five?'], [:end_of_string]], tokenize('hi five?'))
+    tokens = Lexer.new('hi five?').tokenize
+    assert_equal([[:id, 'hi'], [:id, 'five?'], [:end_of_string]], tokens)
 
-    assert_equal([[:number, '2'], [:id, 'foo'], [:end_of_string]], tokenize('2foo'))
+    tokens = Lexer.new('2foo').tokenize
+    assert_equal([[:number, '2'], [:id, 'foo'], [:end_of_string]], tokens)
   end
 
   def test_whitespace
-    assert_equal(
-      [[:id, 'five'], [:pipe, '|'], [:comparison, '=='], [:end_of_string]],
-      tokenize("five|\n\t =="),
-    )
+    tokens = Lexer.new("five|\n\t ==").tokenize
+    assert_equal([[:id, 'five'], [:pipe, '|'], [:comparison, '=='], [:end_of_string]], tokens)
   end
 
   def test_unexpected_character
     assert_raises(SyntaxError) do
-      tokenize("%")
+      Lexer.new("%").tokenize
     end
   end
 
   def test_negative_numbers
-    assert_equal(
-      [[:id, 'foo'], [:pipe, '|'], [:id, 'default'], [:colon, ":"], [:number, '-1'], [:end_of_string]],
-      tokenize("foo | default: -1"),
-    )
+    tokens = Lexer.new("foo | default: -1").tokenize
+    assert_equal([[:id, 'foo'], [:pipe, '|'], [:id, 'default'], [:colon, ":"], [:number, '-1'], [:end_of_string]], tokens)
   end
 
   def test_greater_than_two_digits
-    assert_equal(
-      [[:id, 'foo'], [:comparison, '>'], [:number, '12'], [:end_of_string]],
-      tokenize("foo > 12"),
-    )
-  end
-
-  def test_error_with_utf8_character
-    error = assert_raises(SyntaxError) do
-      tokenize("1 < 1Ø")
-    end
-
-    assert_equal(
-      'Liquid syntax error: Unexpected character Ø',
-      error.message,
-    )
-  end
-
-  def test_contains_as_attribute_name
-    assert_equal(
-      [[:id, "a"], [:dot, "."], [:id, "contains"], [:dot, "."], [:id, "b"], [:end_of_string]],
-      tokenize("a.contains.b"),
-    )
-  end
-
-  def test_tokenize_incomplete_expression
-    assert_equal([[:id, "false"], [:dash, "-"], [:end_of_string]], tokenize("false -"))
-    assert_equal([[:id, "false"], [:comparison, "<"], [:end_of_string]], tokenize("false <"))
-    assert_equal([[:id, "false"], [:comparison, ">"], [:end_of_string]], tokenize("false >"))
-    assert_equal([[:id, "false"], [:number, "1"], [:end_of_string]], tokenize("false 1"))
-  end
-
-  private
-
-  def tokenize(input)
-    Lexer.new(input).tokenize
+    tokens = Lexer.new("foo > 12").tokenize
+    assert_equal([[:id, 'foo'], [:comparison, '>'], [:number, '12'], [:end_of_string]], tokens)
   end
 end

test/unit/strainer_factory_unit_test.rb

@@ -2,7 +2,7 @@
 
 require 'test_helper'
 
-class EnvironmentFilterTest < Minitest::Test
+class StrainerFactoryUnitTest < Minitest::Test
   include Liquid
 
   module AccessScopeFilters
@@ -16,6 +16,8 @@ class EnvironmentFilterTest < Minitest::Test
     private :private_filter
   end
 
+  StrainerFactory.add_global_filter(AccessScopeFilters)
+
   module LateAddedFilter
     def late_added_filter(_input)
       "filtered"
@@ -23,28 +25,24 @@ class EnvironmentFilterTest < Minitest::Test
   end
 
   def setup
-    @environment = Liquid::Environment.build do |env|
-      env.register_filter(AccessScopeFilters)
-    end
-
-    @context = Context.build(environment: @environment)
+    @context = Context.build
   end
 
   def test_strainer
-    strainer = @environment.create_strainer(@context)
+    strainer = StrainerFactory.create(@context)
     assert_equal(5, strainer.invoke('size', 'input'))
     assert_equal("public", strainer.invoke("public_filter"))
   end
 
   def test_stainer_raises_argument_error
-    strainer = @environment.create_strainer(@context)
+    strainer = StrainerFactory.create(@context)
     assert_raises(Liquid::ArgumentError) do
       strainer.invoke("public_filter", 1)
     end
   end
 
   def test_stainer_argument_error_contains_backtrace
-    strainer = @environment.create_strainer(@context)
+    strainer = StrainerFactory.create(@context)
 
     exception = assert_raises(Liquid::ArgumentError) do
       strainer.invoke("public_filter", 1)
@@ -59,7 +57,7 @@ class EnvironmentFilterTest < Minitest::Test
   end
 
   def test_strainer_only_invokes_public_filter_methods
-    strainer = @environment.create_strainer(@context)
+    strainer = StrainerFactory.create(@context)
     assert_equal(false, strainer.class.invokable?('__test__'))
     assert_equal(false, strainer.class.invokable?('test'))
     assert_equal(false, strainer.class.invokable?('instance_eval'))
@@ -68,18 +66,18 @@ class EnvironmentFilterTest < Minitest::Test
   end
 
   def test_strainer_returns_nil_if_no_filter_method_found
-    strainer = @environment.create_strainer(@context)
+    strainer = StrainerFactory.create(@context)
     assert_nil(strainer.invoke("private_filter"))
     assert_nil(strainer.invoke("undef_the_filter"))
   end
 
   def test_strainer_returns_first_argument_if_no_method_and_arguments_given
-    strainer = @environment.create_strainer(@context)
+    strainer = StrainerFactory.create(@context)
     assert_equal("password", strainer.invoke("undef_the_method", "password"))
   end
 
   def test_strainer_only_allows_methods_defined_in_filters
-    strainer = @environment.create_strainer(@context)
+    strainer = StrainerFactory.create(@context)
     assert_equal("1 + 1", strainer.invoke("instance_eval", "1 + 1"))
     assert_equal("puts",  strainer.invoke("__send__", "puts", "Hi Mom"))
     assert_equal("has_method?", strainer.invoke("invoke", "has_method?", "invoke"))
@@ -88,9 +86,7 @@ class EnvironmentFilterTest < Minitest::Test
   def test_strainer_uses_a_class_cache_to_avoid_method_cache_invalidation
     a = Module.new
     b = Module.new
-
-    strainer = @environment.create_strainer(@context, [a, b])
-
+    strainer = StrainerFactory.create(@context, [a, b])
     assert_kind_of(StrainerTemplate, strainer)
     assert_kind_of(a, strainer)
     assert_kind_of(b, strainer)
@@ -98,10 +94,8 @@ class EnvironmentFilterTest < Minitest::Test
   end
 
   def test_add_global_filter_clears_cache
-    assert_equal('input', @environment.create_strainer(@context).invoke('late_added_filter', 'input'))
-
-    @environment.register_filter(LateAddedFilter)
-
-    assert_equal('filtered', @environment.create_strainer(nil).invoke('late_added_filter', 'input'))
+    assert_equal('input', StrainerFactory.create(@context).invoke('late_added_filter', 'input'))
+    StrainerFactory.add_global_filter(LateAddedFilter)
+    assert_equal('filtered', StrainerFactory.create(nil).invoke('late_added_filter', 'input'))
   end
 end

test/unit/strainer_template_unit_test.rb

@@ -25,13 +25,11 @@ class StrainerTemplateUnitTest < Minitest::Test
   end
 
   def test_add_filter_raises_when_module_privately_overrides_registered_public_methods
-    error = assert_raises(Liquid::MethodOverrideError) do
-      Liquid::Environment.build do |env|
-        env.register_filter(PublicMethodOverrideFilter)
-        env.register_filter(PrivateMethodOverrideFilter)
-      end
-    end
+    strainer = Context.new.strainer
 
+    error = assert_raises(Liquid::MethodOverrideError) do
+      strainer.class.add_filter(PrivateMethodOverrideFilter)
+    end
     assert_equal('Liquid error: Filter overrides registered public methods as non public: public_filter', error.message)
   end
 
@@ -44,13 +42,11 @@ class StrainerTemplateUnitTest < Minitest::Test
   end
 
   def test_add_filter_raises_when_module_overrides_registered_public_method_as_protected
-    error = assert_raises(Liquid::MethodOverrideError) do
-      Liquid::Environment.build do |env|
-        env.register_filter(PublicMethodOverrideFilter)
-        env.register_filter(ProtectedMethodOverrideFilter)
-      end
-    end
+    strainer = Context.new.strainer
 
+    error = assert_raises(Liquid::MethodOverrideError) do
+      strainer.class.add_filter(ProtectedMethodOverrideFilter)
+    end
     assert_equal('Liquid error: Filter overrides registered public methods as non public: public_filter', error.message)
   end
 

test/unit/template_unit_test.rb

@@ -7,25 +7,63 @@ class TemplateUnitTest < Minitest::Test
 
   def test_sets_default_localization_in_document
     t = Template.new
-    t.parse('{%raw%}{%endraw%}')
+    t.parse('{%comment%}{%endcomment%}')
     assert_instance_of(I18n, t.root.nodelist[0].options[:locale])
   end
 
   def test_sets_default_localization_in_context_with_quick_initialization
     t = Template.new
-    t.parse('{%raw%}{%endraw%}', locale: I18n.new(fixture("en_locale.yml")))
+    t.parse('{%comment%}{%endcomment%}', locale: I18n.new(fixture("en_locale.yml")))
 
     locale = t.root.nodelist[0].options[:locale]
     assert_instance_of(I18n, locale)
     assert_equal(fixture("en_locale.yml"), locale.path)
   end
 
+  def test_with_cache_classes_tags_returns_the_same_class
+    original_cache_setting = Liquid.cache_classes
+    Liquid.cache_classes   = true
+
+    original_klass = Class.new
+    Object.send(:const_set, :CustomTag, original_klass)
+    Template.register_tag('custom', CustomTag)
+
+    Object.send(:remove_const, :CustomTag)
+
+    new_klass = Class.new
+    Object.send(:const_set, :CustomTag, new_klass)
+
+    assert(Template.tags['custom'].equal?(original_klass))
+  ensure
+    Object.send(:remove_const, :CustomTag)
+    Liquid.cache_classes = original_cache_setting
+  end
+
+  def test_without_cache_classes_tags_reloads_the_class
+    original_cache_setting = Liquid.cache_classes
+    Liquid.cache_classes   = false
+
+    original_klass = Class.new
+    Object.send(:const_set, :CustomTag, original_klass)
+    with_custom_tag('custom', CustomTag) do
+      Object.send(:remove_const, :CustomTag)
+
+      new_klass = Class.new
+      Object.send(:const_set, :CustomTag, new_klass)
+
+      assert(Template.tags['custom'].equal?(new_klass))
+    end
+  ensure
+    Object.send(:remove_const, :CustomTag)
+    Liquid.cache_classes = original_cache_setting
+  end
+
   class FakeTag; end
 
   def test_tags_can_be_looped_over
     with_custom_tag('fake', FakeTag) do
       result = Template.tags.map { |name, klass| [name, klass] }
-      assert(result.include?(["fake", TemplateUnitTest::FakeTag]))
+      assert(result.include?(["fake", "TemplateUnitTest::FakeTag"]))
     end
   end
 

test/unit/tokenizer_unit_test.rb

@@ -30,10 +30,6 @@ class TokenizerTest < Minitest::Test
     assert_equal([1, 1, 3], tokenize_line_numbers(" {{\n funk \n}} "))
   end
 
-  def test_tokenize_with_nil_source_returns_empty_array
-    assert_equal([], tokenize(nil))
-  end
-
   private
 
   def new_tokenizer(source, parse_context: Liquid::ParseContext.new, start_line_number: nil)