WIP: more careful IF block merger

.github/workflows/liquid.yml

@@ -13,8 +13,7 @@ jobs:
         entry:
           - { ruby: 3.0, allowed-failure: false } # minimum supported
           - { ruby: 3.2, allowed-failure: false }
-          - { ruby: 3.3, allowed-failure: false }
-          - { ruby: 3.4, allowed-failure: false } # latest
+          - { ruby: 3.3, allowed-failure: false } # latest
           - { ruby: ruby-head, allowed-failure: false }
     name: Test Ruby ${{ matrix.entry.ruby }}
     steps:

.ruby-version

@@ -1 +1 @@
-3.4.1
+3.3.4

CONTRIBUTING.md

@@ -26,9 +26,3 @@
 * If it makes sense, add tests for your code and/or run a performance benchmark
 * Make sure all tests pass (`bundle exec rake`)
 * Create a pull request
-
-## Releasing
-
-* Bump the version in `lib/liquid/version.rb` and merge it on `main`
-* Create a new release using the [GitHub UI](https://github.com/Shopify/liquid/releases/new)
-

Gemfile

@@ -13,7 +13,6 @@ group :benchmark, :test do
   gem 'benchmark-ips'
   gem 'memory_profiler'
   gem 'terminal-table'
-  gem "lru_redux"
 
   install_if -> { RUBY_PLATFORM !~ /mingw|mswin|java/ && RUBY_ENGINE != 'truffleruby' } do
     gem 'stackprof'

History.md

@@ -1,12 +1,5 @@
 # Liquid Change Log
 
-## 5.6.0 (unreleased)
-
-### Fixes
-
-* Fix Tokenizer to handle null source value (#1873) [Bahar Pourazar]
-
-
 ## 5.5.0 2024-03-21
 
 Please reference the GitHub release for more information.

README.md

@@ -91,7 +91,7 @@ Liquid::Template.parse(<<~LIQUID, environment: email_environment)
 LIQUID
 ```
 
-By using Environments, you ensure that custom tags and filters are only available in the contexts where they are needed, making your Liquid templates more robust and easier to manage. For smaller projects, a global environment is available via `Liquid::Environment.default`.
+By using Environments, you ensure that custom tags and filters are only available in the contexts where they are needed, making your Liquid templates more robust and easier to manage.
 
 ### Error Modes
 

Rakefile

@@ -71,7 +71,7 @@ end
 
 namespace :benchmark do
   desc "Run the liquid benchmark with lax parsing"
-  task :lax do
+  task :run do
     ruby "./performance/benchmark.rb lax"
   end
 
@@ -80,11 +80,9 @@ namespace :benchmark do
     ruby "./performance/benchmark.rb strict"
   end
 
-  desc "Run the liquid benchmark with both lax and strict parsing"
-  task run: [:lax, :strict]
-
   desc "Run unit benchmarks"
   namespace :unit do
+    desc "Run all unit benchmarks"
     task :all do
       Dir["./performance/unit/*_benchmark.rb"].each do |file|
         puts "🧪 Running #{file}"
@@ -92,17 +90,11 @@ namespace :benchmark do
       end
     end
 
-    task :lexer do
-      Dir["./performance/unit/lexer_benchmark.rb"].each do |file|
-        puts "🧪 Running #{file}"
-        ruby file
-      end
-    end
-
-    task :expression do
-      Dir["./performance/unit/expression_benchmark.rb"].each do |file|
-        puts "🧪 Running #{file}"
-        ruby file
+    %w[lexer loom].each do |benchmark|
+      desc "Run the #{benchmark} benchmark"
+      task benchmark.to_sym do
+        puts "🧪 Running #{benchmark}"
+        ruby "./performance/unit/#{benchmark}_benchmark.rb"
       end
     end
   end

lib/liquid.rb

@@ -21,8 +21,6 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-require "strscan"
-
 module Liquid
   FilterSeparator             = /\|/
   ArgumentSeparator           = ','
@@ -46,7 +44,6 @@ module Liquid
   VariableParser              = /\[(?>[^\[\]]+|\g<0>)*\]|#{VariableSegment}+\??/o
 
   RAISE_EXCEPTION_LAMBDA = ->(_e) { raise }
-  HAS_STRING_SCANNER_SCAN_BYTE = StringScanner.instance_methods.include?(:scan_byte)
 end
 
 require "liquid/version"
@@ -71,6 +68,7 @@ require 'liquid/extensions'
 require 'liquid/errors'
 require 'liquid/interrupts'
 require 'liquid/strainer_template'
+require 'liquid/expression'
 require 'liquid/context'
 require 'liquid/tag'
 require 'liquid/block_body'
@@ -79,7 +77,6 @@ require 'liquid/variable'
 require 'liquid/variable_lookup'
 require 'liquid/range_lookup'
 require 'liquid/resource_limits'
-require 'liquid/expression'
 require 'liquid/template'
 require 'liquid/condition'
 require 'liquid/utils'
@@ -89,3 +86,4 @@ require 'liquid/partial_cache'
 require 'liquid/usage'
 require 'liquid/registers'
 require 'liquid/template_factory'
+require 'liquid/loom'

lib/liquid/block_body.rb

@@ -31,10 +31,11 @@ module Liquid
       end
     end
 
-    def freeze
-      @nodelist.freeze
-      super
-    end
+    # TODO: Freeze the nodelist after optimization
+    # def freeze
+    #   @nodelist.freeze
+    #   super
+    # end
 
     private def parse_for_liquid_tag(tokenizer, parse_context)
       while (token = tokenizer.shift)
@@ -154,6 +155,12 @@ module Liquid
           end
           new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
           @blank &&= new_tag.blank?
+
+          if parse_context.eager_optimize
+            next if new_tag.nodelist&.all? { !_1.is_a?(String) && _1.nodelist.empty? } # this is an empty block
+            next if new_tag.is_a?(If) && new_tag.blocks.empty? # this is an empty If block
+          end
+
           @nodelist << new_tag
         when token.start_with?(VARSTART)
           whitespace_handler(token, parse_context)

lib/liquid/context.rb

@@ -40,10 +40,6 @@ module Liquid
       @global_filter       = nil
       @disabled_tags       = {}
 
-      # Instead of constructing new StringScanner objects for each Expression parse,
-      # we recycle the same one.
-      @string_scanner = StringScanner.new("")
-
       @registers.static[:cached_partials] ||= {}
       @registers.static[:file_system] ||= environment.file_system
       @registers.static[:template_factory] ||= Liquid::TemplateFactory.new
@@ -180,7 +176,7 @@ module Liquid
     # Example:
     #   products == empty #=> products.empty?
     def [](expression)
-      evaluate(Expression.parse(expression, @string_scanner))
+      evaluate(Expression.parse(expression))
     end
 
     def key?(key)

lib/liquid/expression.rb

@@ -10,113 +10,37 @@ module Liquid
       'true' => true,
       'false' => false,
       'blank' => '',
-      'empty' => '',
-      # in lax mode, minus sign can be a VariableLookup
-      # For simplicity and performace, we treat it like a literal
-      '-' => VariableLookup.parse("-", nil).freeze,
+      'empty' => ''
     }.freeze
 
-    DOT = ".".ord
-    ZERO = "0".ord
-    NINE = "9".ord
-    DASH = "-".ord
+    INTEGERS_REGEX       = /\A(-?\d+)\z/
+    FLOATS_REGEX         = /\A(-?\d[\d\.]+)\z/
 
     # Use an atomic group (?>...) to avoid pathological backtracing from
     # malicious input as described in https://github.com/Shopify/liquid/issues/1357
-    RANGES_REGEX = /\A\(\s*(?>(\S+)\s*\.\.)\s*(\S+)\s*\)\z/
-    INTEGER_REGEX = /\A(-?\d+)\z/
-    FLOAT_REGEX = /\A(-?\d+)\.\d+\z/
+    RANGES_REGEX         = /\A\(\s*(?>(\S+)\s*\.\.)\s*(\S+)\s*\)\z/
 
-    class << self
-      def parse(markup, ss = StringScanner.new(""), cache = nil)
-        return unless markup
+    def self.parse(markup)
+      return nil unless markup
 
-        markup = markup.strip # markup can be a frozen string
-
-        if (markup.start_with?('"') && markup.end_with?('"')) ||
-          (markup.start_with?("'") && markup.end_with?("'"))
-          return markup[1..-2]
-        elsif LITERALS.key?(markup)
-          return LITERALS[markup]
-        end
-
-        # Cache only exists during parsing
-        if cache
-          return cache[markup] if cache.key?(markup)
-
-          cache[markup] = inner_parse(markup, ss, cache).freeze
-        else
-          inner_parse(markup, ss, nil).freeze
-        end
+      markup = markup.strip
+      if (markup.start_with?('"') && markup.end_with?('"')) ||
+         (markup.start_with?("'") && markup.end_with?("'"))
+        return markup[1..-2]
       end
 
-      def inner_parse(markup, ss, cache)
-        if (markup.start_with?("(") && markup.end_with?(")")) && markup =~ RANGES_REGEX
-          return RangeLookup.parse(
-            Regexp.last_match(1),
-            Regexp.last_match(2),
-            ss,
-            cache,
-          )
-        end
-
-        if (num = parse_number(markup, ss))
-          num
+      case markup
+      when INTEGERS_REGEX
+        Regexp.last_match(1).to_i
+      when RANGES_REGEX
+        RangeLookup.parse(Regexp.last_match(1), Regexp.last_match(2))
+      when FLOATS_REGEX
+        Regexp.last_match(1).to_f
+      else
+        if LITERALS.key?(markup)
+          LITERALS[markup]
         else
-          VariableLookup.parse(markup, ss, cache)
-        end
-      end
-
-      def parse_number(markup, ss)
-        # check if the markup is simple integer or float
-        case markup
-        when INTEGER_REGEX
-          return Integer(markup, 10)
-        when FLOAT_REGEX
-          return markup.to_f
-        end
-
-        ss.string = markup
-        # the first byte must be a digit or  a dash
-        byte = ss.scan_byte
-
-        return false if byte != DASH && (byte < ZERO || byte > NINE)
-
-        if byte == DASH
-          peek_byte = ss.peek_byte
-
-          # if it starts with a dash, the next byte must be a digit
-          return false if peek_byte.nil? || !(peek_byte >= ZERO && peek_byte <= NINE)
-        end
-
-        # The markup could be a float with multiple dots
-        first_dot_pos = nil
-        num_end_pos = nil
-
-        while (byte = ss.scan_byte)
-          return false if byte != DOT && (byte < ZERO || byte > NINE)
-
-          # we found our number and now we are just scanning the rest of the string
-          next if num_end_pos
-
-          if byte == DOT
-            if first_dot_pos.nil?
-              first_dot_pos = ss.pos
-            else
-              # we found another dot, so we know that the number ends here
-              num_end_pos = ss.pos - 1
-            end
-          end
-        end
-
-        num_end_pos = markup.length if ss.eos?
-
-        if num_end_pos
-          # number ends with a number "123.123"
-          markup.byteslice(0, num_end_pos).to_f
-        else
-          # number ends with a dot "123."
-          markup.byteslice(0, first_dot_pos).to_f
+          VariableLookup.parse(markup)
         end
       end
     end

lib/liquid/lexer.rb

@@ -1,7 +1,66 @@
 # frozen_string_literal: true
 
+require "strscan"
+
 module Liquid
-  class Lexer
+  class Lexer1
+    SPECIALS = {
+      '|' => :pipe,
+      '.' => :dot,
+      ':' => :colon,
+      ',' => :comma,
+      '[' => :open_square,
+      ']' => :close_square,
+      '(' => :open_round,
+      ')' => :close_round,
+      '?' => :question,
+      '-' => :dash,
+    }.freeze
+    IDENTIFIER            = /[a-zA-Z_][\w-]*\??/
+    SINGLE_STRING_LITERAL = /'[^\']*'/
+    DOUBLE_STRING_LITERAL = /"[^\"]*"/
+    STRING_LITERAL        = Regexp.union(SINGLE_STRING_LITERAL, DOUBLE_STRING_LITERAL)
+    NUMBER_LITERAL        = /-?\d+(\.\d+)?/
+    DOTDOT                = /\.\./
+    COMPARISON_OPERATOR   = /==|!=|<>|<=?|>=?|contains(?=\s)/
+    WHITESPACE_OR_NOTHING = /\s*/
+
+    def initialize(input)
+      @ss = StringScanner.new(input)
+    end
+
+    def tokenize
+      @output = []
+
+      until @ss.eos?
+        @ss.skip(WHITESPACE_OR_NOTHING)
+        break if @ss.eos?
+        tok      = if (t = @ss.scan(COMPARISON_OPERATOR))
+          [:comparison, t]
+        elsif (t = @ss.scan(STRING_LITERAL))
+          [:string, t]
+        elsif (t = @ss.scan(NUMBER_LITERAL))
+          [:number, t]
+        elsif (t = @ss.scan(IDENTIFIER))
+          [:id, t]
+        elsif (t = @ss.scan(DOTDOT))
+          [:dotdot, t]
+        else
+          c     = @ss.getch
+          if (s = SPECIALS[c])
+            [s, c]
+          else
+            raise SyntaxError, "Unexpected character #{c}"
+          end
+        end
+        @output << tok
+      end
+
+      @output << [:end_of_string]
+    end
+  end
+
+  class Lexer2
     CLOSE_ROUND = [:close_round, ")"].freeze
     CLOSE_SQUARE = [:close_square, "]"].freeze
     COLON = [:colon, ":"].freeze
@@ -33,7 +92,6 @@ module Liquid
     SINGLE_COMPARISON_TOKENS = [].tap do |table|
       table["<".ord] = COMPARISON_LESS_THAN
       table[">".ord] = COMPARISON_GREATER_THAN
-      table.freeze
     end
 
     TWO_CHARS_COMPARISON_JUMP_TABLE = [].tap do |table|
@@ -45,17 +103,18 @@ module Liquid
         sub_table["=".ord] = COMPARISION_NOT_EQUAL
         sub_table.freeze
       end
-      table.freeze
     end
 
     COMPARISON_JUMP_TABLE = [].tap do |table|
       table["<".ord] = [].tap do |sub_table|
         sub_table["=".ord] = COMPARISON_LESS_THAN_OR_EQUAL
         sub_table[">".ord] = COMPARISON_NOT_EQUAL_ALT
+        RUBY_WHITESPACE.each { |c| sub_table[c.ord] = COMPARISON_LESS_THAN }
         sub_table.freeze
       end
       table[">".ord] = [].tap do |sub_table|
         sub_table["=".ord] = COMPARISON_GREATER_THAN_OR_EQUAL
+        RUBY_WHITESPACE.each { |c| sub_table[c.ord] = COMPARISON_GREATER_THAN }
         sub_table.freeze
       end
       table.freeze
@@ -98,76 +157,81 @@ module Liquid
       table.freeze
     end
 
+    def initialize(input)
+      @ss = StringScanner.new(input)
+    end
+
     # rubocop:disable Metrics/BlockNesting
-    class << self
-      def tokenize(ss)
-        output = []
+    def tokenize
+      @output = []
 
-        until ss.eos?
-          ss.skip(WHITESPACE_OR_NOTHING)
+      until @ss.eos?
+        @ss.skip(WHITESPACE_OR_NOTHING)
 
-          break if ss.eos?
+        break if @ss.eos?
 
-          start_pos = ss.pos
-          peeked = ss.peek_byte
+        start_pos = @ss.pos
+        peeked = @ss.peek_byte
 
-          if (special = SPECIAL_TABLE[peeked])
-            ss.scan_byte
-            # Special case for ".."
-            if special == DOT && ss.peek_byte == DOT_ORD
-              ss.scan_byte
-              output << DOTDOT
-            elsif special == DASH
-              # Special case for negative numbers
-              if (peeked_byte = ss.peek_byte) && NUMBER_TABLE[peeked_byte]
-                ss.pos -= 1
-                output << [:number, ss.scan(NUMBER_LITERAL)]
-              else
-                output << special
-              end
+        if (special = SPECIAL_TABLE[peeked])
+          @ss.scan_byte
+          # Special case for ".."
+          if special == DOT && @ss.peek_byte == DOT_ORD
+            @ss.scan_byte
+            @output << DOTDOT
+          elsif special == DASH
+            # Special case for negative numbers
+            if (peeked_byte = @ss.peek_byte) && NUMBER_TABLE[peeked_byte]
+              @ss.pos -= 1
+              @output << [:number, @ss.scan(NUMBER_LITERAL)]
             else
-              output << special
-            end
-          elsif (sub_table = TWO_CHARS_COMPARISON_JUMP_TABLE[peeked])
-            ss.scan_byte
-            if (peeked_byte = ss.peek_byte) && (found = sub_table[peeked_byte])
-              output << found
-              ss.scan_byte
-            else
-              raise_syntax_error(start_pos, ss)
-            end
-          elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
-            ss.scan_byte
-            if (peeked_byte = ss.peek_byte) && (found = sub_table[peeked_byte])
-              output << found
-              ss.scan_byte
-            else
-              output << SINGLE_COMPARISON_TOKENS[peeked]
+              @output << special
             end
           else
-            type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked]
+            @output << special
+          end
+        elsif (sub_table = TWO_CHARS_COMPARISON_JUMP_TABLE[peeked])
+          @ss.scan_byte
+          if (peeked_byte = @ss.peek_byte) && (found = sub_table[peeked_byte])
+            @output << found
+            @ss.scan_byte
+          else
+            raise_syntax_error(start_pos)
+          end
+        elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
+          @ss.scan_byte
+          if (peeked_byte = @ss.peek_byte) && (found = sub_table[peeked_byte])
+            @output << found
+            @ss.scan_byte
+          else
+            @output << SINGLE_COMPARISON_TOKENS[peeked]
+          end
+        else
+          type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked]
 
-            if type && (t = ss.scan(pattern))
-              # Special case for "contains"
-              output << if type == :id && t == "contains" && output.last&.first != :dot
-                COMPARISON_CONTAINS
-              else
-                [type, t]
-              end
+          if type && (t = @ss.scan(pattern))
+            # Special case for "contains"
+            @output << if type == :id && t == "contains" && @output.last&.first != :dot
+              COMPARISON_CONTAINS
             else
-              raise_syntax_error(start_pos, ss)
+              [type, t]
             end
+          else
+            raise_syntax_error(start_pos)
           end
         end
-        # rubocop:enable Metrics/BlockNesting
-        output << EOS
       end
+      # rubocop:enable Metrics/BlockNesting
 
-      def raise_syntax_error(start_pos, ss)
-        ss.pos = start_pos
-        # the character could be a UTF-8 character, use getch to get all the bytes
-        raise SyntaxError, "Unexpected character #{ss.getch}"
-      end
+      @output << EOS
+    end
+
+    def raise_syntax_error(start_pos)
+      @ss.pos = start_pos
+      # the character could be a UTF-8 character, use getch to get all the bytes
+      raise SyntaxError, "Unexpected character #{@ss.getch}"
     end
   end
+
+  Lexer = StringScanner.instance_methods.include?(:scan_byte) ? Lexer2 : Lexer1
 end

lib/liquid/loom.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Liquid
+  class Loom
+    MERGABLE_IF_OPERATORS = ["==", ">", "<", "!="].freeze
+    EQUAL_OP = "==".freeze
+
+    class << self
+      def optimize(template)
+        new(template).optimize
+      end
+    end
+
+    def initialize template
+      @root = template.root
+    end
+
+    def optimize
+      merge_if_blocks
+    end
+
+    def merge_if_blocks
+      nodelist_list = [@root.nodelist]
+
+      while nodelist_list.any?
+        next_nodelist_list = []
+
+        nodelist_list.each do |nodelist|
+          i = 0
+          while i < nodelist.length
+            node = nodelist[i]
+            chain_if_blocks(nodelist, node, i) if node.is_a?(If)
+            i += 1
+          end
+        end
+
+        nodelist_list = next_nodelist_list
+      end
+    end
+
+    private
+
+    def mergable_if_blocks?(target_if, next_if)
+      target_left = target_if.blocks.first.left
+      target_right = target_if.blocks.first.right
+      next_left = next_if.blocks.first.left
+      next_right = next_if.blocks.first.right
+
+      used_variables = Hash.new { |h, k| h[k] = 0 }
+
+      [
+        target_if.blocks.first.left,
+        target_if.blocks.first.right,
+        next_if.blocks.first.left,
+        next_if.blocks.first.right
+      ].each do |var|
+        if var.is_a?(VariableLookup)
+          used_variables[var.name] += 1
+        end
+      end
+
+      return if used_variables.keys.count > 1
+
+      most_used_variable_name = used_variables.keys[0]
+
+      # TODO: I probably can't do this
+      # It might be possible to get different result between a > b and b < a
+      # Move most commonly used variable to the left side
+      if (target_left.is_a?(VariableLookup) && target_left.name != most_used_variable_name) || (target_right.is_a?(VariableLookup) && target_right.name == most_used_variable_name)
+        target_left, target_right = target_right, target_left
+      end
+
+      if (next_left.is_a?(VariableLookup) && next_left.name != most_used_variable_name) || (next_right.is_a?(VariableLookup) && next_right.name == most_used_variable_name)
+        next_left, next_right = next_right, next_left
+      end
+
+      return false unless target_left.is_a?(VariableLookup) && next_left.is_a?(VariableLookup)
+      return false if target_left.name != next_left.name
+
+      return false if target_right.nil? || next_right.nil?
+
+
+      # we need to be conversative here and only can merge ==, >, <, and != operators
+      target_operator = target_if.blocks.first.operator
+      next_operator = next_if.blocks.first.operator
+
+      return false unless MERGABLE_IF_OPERATORS.include?(target_operator) && MERGABLE_IF_OPERATORS.include?(next_operator)
+
+      return false if target_operator == next_operator && target_right == next_right
+
+      return false if target_right.is_a?(VariableLookup) || next_right.is_a?(VariableLookup)
+
+      true
+    end
+
+    def chain_if_blocks(nodelist, first_if_node, first_if_index)
+      used_variables = Set.new
+
+      # only check the top level Condition (ignore children conditions for now)
+      first_if_node.blocks.each do |condition|
+        used_variables << condition.left
+        used_variables << condition.right if condition.right
+      end
+
+      if_blocks = []
+
+      nodelist[first_if_index + 1..-1].each do |node|
+        break unless node.is_a?(If)
+
+        # check if the variables used in the current block are used in the previous block
+        break unless mergable_if_blocks?(first_if_node, node)
+
+        if_blocks << node
+      end
+
+      nodelist.delete_if { |node| if_blocks.include?(node) }
+
+      if_blocks.each do |if_block|
+        first_if_node.blocks << if_block.blocks.first
+      end
+    end
+  end
+end

lib/liquid/parse_context.rb

@@ -3,7 +3,7 @@
 module Liquid
   class ParseContext
     attr_accessor :locale, :line_number, :trim_whitespace, :depth
-    attr_reader :partial, :warnings, :error_mode, :environment
+    attr_reader :partial, :warnings, :error_mode, :environment, :eager_optimize
 
     def initialize(options = Const::EMPTY_HASH)
       @environment = options.fetch(:environment, Environment.default)
@@ -11,18 +11,7 @@ module Liquid
 
       @locale   = @template_options[:locale] ||= I18n.new
       @warnings = []
-
-      # constructing new StringScanner in Lexer, Tokenizer, etc is expensive
-      # This StringScanner will be shared by all of them
-      @string_scanner = StringScanner.new("")
-
-      @expression_cache = if options[:expression_cache].nil?
-        {}
-      elsif options[:expression_cache].respond_to?(:[]) && options[:expression_cache].respond_to?(:[]=)
-        options[:expression_cache]
-      elsif options[:expression_cache]
-        {}
-      end
+      @eager_optimize = options.fetch(:eager_optimize, ENV["OPTIMIZE"] == "true")
 
       self.depth   = 0
       self.partial = false
@@ -36,22 +25,12 @@ module Liquid
       Liquid::BlockBody.new
     end
 
-    def new_parser(input)
-      @string_scanner.string = input
-      Parser.new(@string_scanner)
-    end
-
-    def new_tokenizer(source, start_line_number: nil, for_liquid_tag: false)
-      Tokenizer.new(
-        source: source,
-        string_scanner: @string_scanner,
-        line_number: start_line_number,
-        for_liquid_tag: for_liquid_tag,
-      )
+    def new_tokenizer(markup, start_line_number: nil, for_liquid_tag: false)
+      Tokenizer.new(markup, line_number: start_line_number, for_liquid_tag: for_liquid_tag)
     end
 
     def parse_expression(markup)
-      Expression.parse(markup, @string_scanner, @expression_cache)
+      Expression.parse(markup)
     end
 
     def partial=(value)

lib/liquid/parser.rb

@@ -3,8 +3,8 @@
 module Liquid
   class Parser
     def initialize(input)
-      ss = input.is_a?(StringScanner) ? input : StringScanner.new(input)
-      @tokens = Lexer.tokenize(ss)
+      l       = Lexer.new(input)
+      @tokens = l.tokenize
       @p      = 0 # pointer to current location
     end
 

lib/liquid/range_lookup.rb

@@ -2,9 +2,9 @@
 
 module Liquid
   class RangeLookup
-    def self.parse(start_markup, end_markup, string_scanner, cache = nil)
-      start_obj = Expression.parse(start_markup, string_scanner, cache)
-      end_obj   = Expression.parse(end_markup, string_scanner, cache)
+    def self.parse(start_markup, end_markup)
+      start_obj = Expression.parse(start_markup)
+      end_obj   = Expression.parse(end_markup)
       if start_obj.respond_to?(:evaluate) || end_obj.respond_to?(:evaluate)
         new(start_obj, end_obj)
       else

lib/liquid/standardfilters.rb

@@ -64,7 +64,7 @@ module Liquid
     # @liquid_syntax string | downcase
     # @liquid_return [string]
     def downcase(input)
-      Utils.to_s(input).downcase
+      input.to_s.downcase
     end
 
     # @liquid_public_docs
@@ -75,7 +75,7 @@ module Liquid
     # @liquid_syntax string | upcase
     # @liquid_return [string]
     def upcase(input)
-      Utils.to_s(input).upcase
+      input.to_s.upcase
     end
 
     # @liquid_public_docs
@@ -86,7 +86,7 @@ module Liquid
     # @liquid_syntax string | capitalize
     # @liquid_return [string]
     def capitalize(input)
-      Utils.to_s(input).capitalize
+      input.to_s.capitalize
     end
 
     # @liquid_public_docs
@@ -97,7 +97,7 @@ module Liquid
     # @liquid_syntax string | escape
     # @liquid_return [string]
     def escape(input)
-      CGI.escapeHTML(Utils.to_s(input)) unless input.nil?
+      CGI.escapeHTML(input.to_s) unless input.nil?
     end
     alias_method :h, :escape
 
@@ -109,7 +109,7 @@ module Liquid
     # @liquid_syntax string | escape_once
     # @liquid_return [string]
     def escape_once(input)
-      Utils.to_s(input).gsub(HTML_ESCAPE_ONCE_REGEXP, HTML_ESCAPE)
+      input.to_s.gsub(HTML_ESCAPE_ONCE_REGEXP, HTML_ESCAPE)
     end
 
     # @liquid_public_docs
@@ -124,7 +124,7 @@ module Liquid
     # @liquid_syntax string | url_encode
     # @liquid_return [string]
     def url_encode(input)
-      CGI.escape(Utils.to_s(input)) unless input.nil?
+      CGI.escape(input.to_s) unless input.nil?
     end
 
     # @liquid_public_docs
@@ -138,7 +138,7 @@ module Liquid
     def url_decode(input)
       return if input.nil?
 
-      result = CGI.unescape(Utils.to_s(input))
+      result = CGI.unescape(input.to_s)
       raise Liquid::ArgumentError, "invalid byte sequence in #{result.encoding}" unless result.valid_encoding?
 
       result
@@ -152,7 +152,7 @@ module Liquid
     # @liquid_syntax string | base64_encode
     # @liquid_return [string]
     def base64_encode(input)
-      Base64.strict_encode64(Utils.to_s(input))
+      Base64.strict_encode64(input.to_s)
     end
 
     # @liquid_public_docs
@@ -163,7 +163,7 @@ module Liquid
     # @liquid_syntax string | base64_decode
     # @liquid_return [string]
     def base64_decode(input)
-      input = Utils.to_s(input)
+      input = input.to_s
       StandardFilters.try_coerce_encoding(Base64.strict_decode64(input), encoding: input.encoding)
     rescue ::ArgumentError
       raise Liquid::ArgumentError, "invalid base64 provided to base64_decode"
@@ -177,7 +177,7 @@ module Liquid
     # @liquid_syntax string | base64_url_safe_encode
     # @liquid_return [string]
     def base64_url_safe_encode(input)
-      Base64.urlsafe_encode64(Utils.to_s(input))
+      Base64.urlsafe_encode64(input.to_s)
     end
 
     # @liquid_public_docs
@@ -188,7 +188,7 @@ module Liquid
     # @liquid_syntax string | base64_url_safe_decode
     # @liquid_return [string]
     def base64_url_safe_decode(input)
-      input = Utils.to_s(input)
+      input = input.to_s
       StandardFilters.try_coerce_encoding(Base64.urlsafe_decode64(input), encoding: input.encoding)
     rescue ::ArgumentError
       raise Liquid::ArgumentError, "invalid base64 provided to base64_url_safe_decode"
@@ -212,7 +212,7 @@ module Liquid
         if input.is_a?(Array)
           input.slice(offset, length) || []
         else
-          Utils.to_s(input).slice(offset, length) || ''
+          input.to_s.slice(offset, length) || ''
         end
       rescue RangeError
         if I64_RANGE.cover?(length) && I64_RANGE.cover?(offset)
@@ -236,10 +236,10 @@ module Liquid
     # @liquid_return [string]
     def truncate(input, length = 50, truncate_string = "...")
       return if input.nil?
-      input_str = Utils.to_s(input)
+      input_str = input.to_s
       length    = Utils.to_integer(length)
 
-      truncate_string_str = Utils.to_s(truncate_string)
+      truncate_string_str = truncate_string.to_s
 
       l = length - truncate_string_str.length
       l = 0 if l < 0
@@ -263,7 +263,7 @@ module Liquid
     # @liquid_return [string]
     def truncatewords(input, words = 15, truncate_string = "...")
       return if input.nil?
-      input = Utils.to_s(input)
+      input = input.to_s
       words = Utils.to_integer(words)
       words = 1 if words <= 0
 
@@ -277,8 +277,7 @@ module Liquid
       return input if wordlist.length <= words
 
       wordlist.pop
-      truncate_string = Utils.to_s(truncate_string)
-      wordlist.join(" ").concat(truncate_string)
+      wordlist.join(" ").concat(truncate_string.to_s)
     end
 
     # @liquid_public_docs
@@ -289,9 +288,7 @@ module Liquid
     # @liquid_syntax string | split: string
     # @liquid_return [array[string]]
     def split(input, pattern)
-      pattern = Utils.to_s(pattern)
-      input = Utils.to_s(input)
-      input.split(pattern)
+      input.to_s.split(pattern.to_s)
     end
 
     # @liquid_public_docs
@@ -302,8 +299,7 @@ module Liquid
     # @liquid_syntax string | strip
     # @liquid_return [string]
     def strip(input)
-      input = Utils.to_s(input)
-      input.strip
+      input.to_s.strip
     end
 
     # @liquid_public_docs
@@ -314,8 +310,7 @@ module Liquid
     # @liquid_syntax string | lstrip
     # @liquid_return [string]
     def lstrip(input)
-      input = Utils.to_s(input)
-      input.lstrip
+      input.to_s.lstrip
     end
 
     # @liquid_public_docs
@@ -326,8 +321,7 @@ module Liquid
     # @liquid_syntax string | rstrip
     # @liquid_return [string]
     def rstrip(input)
-      input = Utils.to_s(input)
-      input.rstrip
+      input.to_s.rstrip
     end
 
     # @liquid_public_docs
@@ -338,9 +332,8 @@ module Liquid
     # @liquid_syntax string | strip_html
     # @liquid_return [string]
     def strip_html(input)
-      input = Utils.to_s(input)
       empty  = ''
-      result = input.gsub(STRIP_HTML_BLOCKS, empty)
+      result = input.to_s.gsub(STRIP_HTML_BLOCKS, empty)
       result.gsub!(STRIP_HTML_TAGS, empty)
       result
     end
@@ -353,8 +346,7 @@ module Liquid
     # @liquid_syntax string | strip_newlines
     # @liquid_return [string]
     def strip_newlines(input)
-      input = Utils.to_s(input)
-      input.gsub(/\r?\n/, '')
+      input.to_s.gsub(/\r?\n/, '')
     end
 
     # @liquid_public_docs
@@ -365,7 +357,6 @@ module Liquid
     # @liquid_syntax array | join
     # @liquid_return [string]
     def join(input, glue = ' ')
-      glue = Utils.to_s(glue)
       InputIterator.new(input, context).join(glue)
     end
 
@@ -387,7 +378,7 @@ module Liquid
         end
       elsif ary.all? { |el| el.respond_to?(:[]) }
         begin
-          ary.sort { |a, b| nil_safe_compare(fetch_property(a, property), fetch_property(b, property)) }
+          ary.sort { |a, b| nil_safe_compare(a[property], b[property]) }
         rescue TypeError
           raise_property_error(property)
         end
@@ -416,7 +407,7 @@ module Liquid
         end
       elsif ary.all? { |el| el.respond_to?(:[]) }
         begin
-          ary.sort { |a, b| nil_safe_casecmp(fetch_property(a, property), fetch_property(b, property)) }
+          ary.sort { |a, b| nil_safe_casecmp(a[property], b[property]) }
         rescue TypeError
           raise_property_error(property)
         end
@@ -433,59 +424,29 @@ module Liquid
     # @liquid_syntax array | where: string, string
     # @liquid_return [array[untyped]]
     def where(input, property, target_value = nil)
-      filter_array(input, property, target_value) { |ary, &block| ary.select(&block) }
-    end
+      ary = InputIterator.new(input, context)
 
-    # @liquid_public_docs
-    # @liquid_type filter
-    # @liquid_category array
-    # @liquid_summary
-    #   Filters an array to exclude items with a specific property value.
-    # @liquid_description
-    #   This requires you to provide both the property name and the associated value.
-    # @liquid_syntax array | reject: string, string
-    # @liquid_return [array[untyped]]
-    def reject(input, property, target_value = nil)
-      filter_array(input, property, target_value) { |ary, &block| ary.reject(&block) }
-    end
-
-    # @liquid_public_docs
-    # @liquid_type filter
-    # @liquid_category array
-    # @liquid_summary
-    #   Tests if any item in an array has a specific property value.
-    # @liquid_description
-    #   This requires you to provide both the property name and the associated value.
-    # @liquid_syntax array | some: string, string
-    # @liquid_return [boolean]
-    def has(input, property, target_value = nil)
-      filter_array(input, property, target_value) { |ary, &block| ary.any?(&block) }
-    end
-
-    # @liquid_public_docs
-    # @liquid_type filter
-    # @liquid_category array
-    # @liquid_summary
-    #   Returns the first item in an array with a specific property value.
-    # @liquid_description
-    #   This requires you to provide both the property name and the associated value.
-    # @liquid_syntax array | find: string, string
-    # @liquid_return [untyped]
-    def find(input, property, target_value = nil)
-      filter_array(input, property, target_value) { |ary, &block| ary.find(&block) }
-    end
-
-    # @liquid_public_docs
-    # @liquid_type filter
-    # @liquid_category array
-    # @liquid_summary
-    #   Returns the index of the first item in an array with a specific property value.
-    # @liquid_description
-    #   This requires you to provide both the property name and the associated value.
-    # @liquid_syntax array | find_index: string, string
-    # @liquid_return [number]
-    def find_index(input, property, target_value = nil)
-      filter_array(input, property, target_value) { |ary, &block| ary.find_index(&block) }
+      if ary.empty?
+        []
+      elsif target_value.nil?
+        ary.select do |item|
+          item[property]
+        rescue TypeError
+          raise_property_error(property)
+        rescue NoMethodError
+          return nil unless item.respond_to?(:[])
+          raise
+        end
+      else
+        ary.select do |item|
+          item[property] == target_value
+        rescue TypeError
+          raise_property_error(property)
+        rescue NoMethodError
+          return nil unless item.respond_to?(:[])
+          raise
+        end
+      end
     end
 
     # @liquid_public_docs
@@ -504,7 +465,7 @@ module Liquid
         []
       else
         ary.uniq do |item|
-          fetch_property(item, property)
+          item[property]
         rescue TypeError
           raise_property_error(property)
         rescue NoMethodError
@@ -540,7 +501,7 @@ module Liquid
         if property == "to_liquid"
           e
         elsif e.respond_to?(:[])
-          r = fetch_property(e, property)
+          r = e[property]
           r.is_a?(Proc) ? r.call : r
         end
       end
@@ -564,7 +525,7 @@ module Liquid
         []
       else
         ary.reject do |item|
-          fetch_property(item, property).nil?
+          item[property].nil?
         rescue TypeError
           raise_property_error(property)
         rescue NoMethodError
@@ -582,10 +543,7 @@ module Liquid
     # @liquid_syntax string | replace: string, string
     # @liquid_return [string]
     def replace(input, string, replacement = '')
-      string = Utils.to_s(string)
-      replacement = Utils.to_s(replacement)
-      input = Utils.to_s(input)
-      input.gsub(string, replacement)
+      input.to_s.gsub(string.to_s, replacement.to_s)
     end
 
     # @liquid_public_docs
@@ -596,10 +554,7 @@ module Liquid
     # @liquid_syntax string | replace_first: string, string
     # @liquid_return [string]
     def replace_first(input, string, replacement = '')
-      string = Utils.to_s(string)
-      replacement = Utils.to_s(replacement)
-      input = Utils.to_s(input)
-      input.sub(string, replacement)
+      input.to_s.sub(string.to_s, replacement.to_s)
     end
 
     # @liquid_public_docs
@@ -610,9 +565,9 @@ module Liquid
     # @liquid_syntax string | replace_last: string, string
     # @liquid_return [string]
     def replace_last(input, string, replacement)
-      input = Utils.to_s(input)
-      string = Utils.to_s(string)
-      replacement = Utils.to_s(replacement)
+      input = input.to_s
+      string = string.to_s
+      replacement = replacement.to_s
 
       start_index = input.rindex(string)
 
@@ -664,9 +619,7 @@ module Liquid
     # @liquid_syntax string | append: string
     # @liquid_return [string]
     def append(input, string)
-      input = Utils.to_s(input)
-      string = Utils.to_s(string)
-      input + string
+      input.to_s + string.to_s
     end
 
     # @liquid_public_docs
@@ -695,9 +648,7 @@ module Liquid
     # @liquid_syntax string | prepend: string
     # @liquid_return [string]
     def prepend(input, string)
-      input = Utils.to_s(input)
-      string = Utils.to_s(string)
-      string + input
+      string.to_s + input.to_s
     end
 
     # @liquid_public_docs
@@ -708,8 +659,7 @@ module Liquid
     # @liquid_syntax string | newline_to_br
     # @liquid_return [string]
     def newline_to_br(input)
-      input = Utils.to_s(input)
-      input.gsub(/\r?\n/, "<br />\n")
+      input.to_s.gsub(/\r?\n/, "<br />\n")
     end
 
     # Reformat a date using Ruby's core Time#strftime( string ) -> string
@@ -744,12 +694,11 @@ module Liquid
     #
     #   See also: http://www.ruby-doc.org/core/Time.html#method-i-strftime
     def date(input, format)
-      str_format = Utils.to_s(format)
-      return input if str_format.empty?
+      return input if format.to_s.empty?
 
       return input unless (date = Utils.to_date(input))
 
-      date.strftime(str_format)
+      date.strftime(format.to_s)
     end
 
     # @liquid_public_docs
@@ -950,7 +899,7 @@ module Liquid
         if property.nil?
           item
         elsif item.respond_to?(:[])
-          fetch_property(item, property)
+          item[property]
         else
           0
         end
@@ -969,50 +918,6 @@ module Liquid
 
     attr_reader :context
 
-    def filter_array(input, property, target_value, &block)
-      ary = InputIterator.new(input, context)
-
-      return [] if ary.empty?
-
-      block.call(ary) do |item|
-        if target_value.nil?
-          fetch_property(item, property)
-        else
-          fetch_property(item, property) == target_value
-        end
-      rescue TypeError
-        raise_property_error(property)
-      rescue NoMethodError
-        return nil unless item.respond_to?(:[])
-        raise
-      end
-    end
-
-    def fetch_property(drop, property_or_keys)
-      ##
-      # This keeps backward compatibility by supporting properties containing
-      # dots. This is valid in Liquid syntax and used in some runtimes, such as
-      # Shopify with metafields.
-      #
-      # Using this approach, properties like 'price.value' can be accessed in
-      # both of the following examples:
-      #
-      # ```
-      # [
-      #   { 'name' => 'Item 1', 'price.price' => 40000 },
-      #   { 'name' => 'Item 2', 'price' => { 'value' => 39900 } }
-      # ]
-      # ```
-      value = drop[property_or_keys]
-
-      return value if !value.nil? || !property_or_keys.is_a?(String)
-
-      keys = property_or_keys.split('.')
-      keys.reduce(drop) do |drop, key|
-        drop.respond_to?(:[]) ? drop[key] : drop
-      end
-    end
-
     def raise_property_error(property)
       raise Liquid::ArgumentError, "cannot select the property '#{property}'"
     end
@@ -1063,18 +968,7 @@ module Liquid
       end
 
       def join(glue)
-        first = true
-        output = +""
-        @input.each do |item|
-          if first
-            first = false
-          else
-            output << glue
-          end
-
-          output << Liquid::Utils.to_s(item)
-        end
-        output
+        to_a.join(glue.to_s)
       end
 
       def concat(args)

lib/liquid/tags/cycle.rb

@@ -68,13 +68,7 @@ module Liquid
     def variables_from_string(markup)
       markup.split(',').collect do |var|
         var =~ /\s*(#{QuotedFragment})\s*/o
-        next unless Regexp.last_match(1)
-
-        # Expression Parser returns cached objects, and we need to dup them to
-        # start the cycle over for each new cycle call.
-        # Liquid-C does not have a cache, so we don't need to dup the object.
-        var = parse_expression(Regexp.last_match(1))
-        var.is_a?(VariableLookup) ? var.dup : var
+        Regexp.last_match(1) ? parse_expression(Regexp.last_match(1)) : nil
       end.compact
     end
 

lib/liquid/tags/for.rb

@@ -88,7 +88,7 @@ module Liquid
     end
 
     def strict_parse(markup)
-      p = @parse_context.new_parser(markup)
+      p = Parser.new(markup)
       @variable_name = p.consume(:id)
       raise SyntaxError, options[:locale].t("errors.syntax.for_invalid_in") unless p.id?('in')
 

lib/liquid/tags/if.rb

@@ -23,6 +23,7 @@ module Liquid
     def initialize(tag_name, markup, options)
       super
       @blocks = []
+      @has_else_block = false
       push_block('if', markup)
     end
 
@@ -33,17 +34,44 @@ module Liquid
     def parse(tokens)
       while parse_body(@blocks.last.attachment, tokens)
       end
-      @blocks.reverse_each do |block|
-        block.attachment.remove_blank_strings if blank?
-        block.attachment.freeze
+
+      if parse_context.eager_optimize && definitive_false_statement?
+        @blocks.clear
+      else
+        @blocks.reverse_each do |block|
+          block.attachment.remove_blank_strings if blank?
+          block.attachment.freeze
+        end
       end
     end
 
+    def definitive_false_statement?
+      # check if any blocks have variable lookups
+      @blocks.each do |condition|
+        return false if condition.left.is_a?(VariableLookup) || condition.right&.is_a?(VariableLookup)
+
+        child_condition = condition.child_condition
+
+        while child_condition
+          return false if child_condition&.left.is_a?(VariableLookup) || child_condition&.right&.is_a?(VariableLookup)
+          child_condition = child_condition.child_condition
+        end
+      end
+
+      # check if all blocks are false
+      @blocks.each do |condition|
+        return false if condition.evaluate
+      end
+
+      true
+    end
+
     ELSE_TAG_NAMES = ['elsif', 'else'].freeze
     private_constant :ELSE_TAG_NAMES
 
     def unknown_tag(tag, markup, tokens)
       if ELSE_TAG_NAMES.include?(tag)
+        @has_else_block = true
         push_block(tag, markup)
       else
         super
@@ -102,7 +130,7 @@ module Liquid
     end
 
     def strict_parse(markup)
-      p = @parse_context.new_parser(markup)
+      p = Parser.new(markup)
       condition = parse_binary_comparisons(p)
       p.consume(:end_of_string)
       condition

lib/liquid/template.rb

@@ -82,7 +82,11 @@ module Liquid
       # See Liquid::Profiler for more information
       def parse(source, options = {})
         environment = options[:environment] || Environment.default
-        new(environment: environment).parse(source, options)
+        template = new(environment: environment).parse(source, options)
+
+        Loom.optimize(template) if options[:eager_optimize]
+
+        template
       end
     end
 

lib/liquid/tokenizer.rb

@@ -1,43 +1,20 @@
 # frozen_string_literal: true
 
-require "strscan"
-
 module Liquid
   class Tokenizer
     attr_reader :line_number, :for_liquid_tag
 
-    TAG_END = /%\}/
-    TAG_OR_VARIABLE_START = /\{[\{\%]/
-    NEWLINE = /\n/
-
-    OPEN_CURLEY = "{".ord
-    CLOSE_CURLEY = "}".ord
-    PERCENTAGE = "%".ord
-
-    def initialize(
-      source:,
-      string_scanner:,
-      line_numbers: false,
-      line_number: nil,
-      for_liquid_tag: false
-    )
-      @line_number = line_number || (line_numbers ? 1 : nil)
+    def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
+      @source         = source
+      @line_number    = line_number || (line_numbers ? 1 : nil)
       @for_liquid_tag = for_liquid_tag
-      @source = source.to_s.to_str
-      @offset = 0
-      @tokens = []
-
-      if @source
-        @ss = string_scanner
-        @ss.string = @source
-        tokenize
-      end
+      @offset         = 0
+      @tokens         = tokenize
     end
 
     def shift
       token = @tokens[@offset]
-
-      return unless token
+      return nil unless token
 
       @offset += 1
 
@@ -51,105 +28,18 @@ module Liquid
     private
 
     def tokenize
-      if @for_liquid_tag
-        @tokens = @source.split("\n")
-      else
-        @tokens << shift_normal until @ss.eos?
+      return [] if @source.empty?
+
+      return @source.split("\n") if @for_liquid_tag
+
+      tokens = @source.split(TemplateParser)
+
+      # removes the rogue empty element at the beginning of the array
+      if tokens[0]&.empty?
+        @offset += 1
       end
 
-      @source = nil
-      @ss = nil
-    end
-
-    def shift_normal
-      token = next_token
-
-      return unless token
-
-      token
-    end
-
-    def next_token
-      # possible states: :text, :tag, :variable
-      byte_a = @ss.peek_byte
-
-      if byte_a == OPEN_CURLEY
-        @ss.scan_byte
-
-        byte_b = @ss.peek_byte
-
-        if byte_b == PERCENTAGE
-          @ss.scan_byte
-          return next_tag_token
-        elsif byte_b == OPEN_CURLEY
-          @ss.scan_byte
-          return next_variable_token
-        end
-
-        @ss.pos -= 1
-      end
-
-      next_text_token
-    end
-
-    def next_text_token
-      start = @ss.pos
-
-      unless @ss.skip_until(TAG_OR_VARIABLE_START)
-        token = @ss.rest
-        @ss.terminate
-        return token
-      end
-
-      pos = @ss.pos -= 2
-      @source.byteslice(start, pos - start)
-    end
-
-    def next_variable_token
-      start = @ss.pos - 2
-
-      byte_a = byte_b = @ss.scan_byte
-
-      while byte_b
-        byte_a = @ss.scan_byte while byte_a && (byte_a != CLOSE_CURLEY && byte_a != OPEN_CURLEY)
-
-        break unless byte_a
-
-        if @ss.eos?
-          return byte_a == CLOSE_CURLEY ? @source.byteslice(start, @ss.pos - start) : "{{"
-        end
-
-        byte_b = @ss.scan_byte
-
-        if byte_a == CLOSE_CURLEY
-          if byte_b == CLOSE_CURLEY
-            return @source.byteslice(start, @ss.pos - start)
-          elsif byte_b != CLOSE_CURLEY
-            @ss.pos -= 1
-            return @source.byteslice(start, @ss.pos - start)
-          end
-        elsif byte_a == OPEN_CURLEY && byte_b == PERCENTAGE
-          return next_tag_token_with_start(start)
-        end
-
-        byte_a = byte_b
-      end
-
-      "{{"
-    end
-
-    def next_tag_token
-      start = @ss.pos - 2
-      if (len = @ss.skip_until(TAG_END))
-        @source.byteslice(start, len + 2)
-      else
-        "{%"
-      end
-    end
-
-    def next_tag_token_with_start(start)
-      @ss.skip_until(TAG_END)
-      @source.byteslice(start, @ss.pos - start)
+      tokens
     end
   end
 end

lib/liquid/utils.rb

@@ -89,101 +89,5 @@ module Liquid
       # Otherwise return the object itself
       obj
     end
-
-    def self.to_s(obj, seen = {})
-      case obj
-      when Hash
-        # If the custom hash implementation overrides `#to_s`, use their
-        # custom implementation. Otherwise we use Liquid's default
-        # implementation.
-        if obj.class.instance_method(:to_s) == HASH_TO_S_METHOD
-          hash_inspect(obj, seen)
-        else
-          obj.to_s
-        end
-      when Array
-        array_inspect(obj, seen)
-      else
-        obj.to_s
-      end
-    end
-
-    def self.inspect(obj, seen = {})
-      case obj
-      when Hash
-        # If the custom hash implementation overrides `#inspect`, use their
-        # custom implementation. Otherwise we use Liquid's default
-        # implementation.
-        if obj.class.instance_method(:inspect) == HASH_INSPECT_METHOD
-          hash_inspect(obj, seen)
-        else
-          obj.inspect
-        end
-      when Array
-        array_inspect(obj, seen)
-      else
-        obj.inspect
-      end
-    end
-
-    def self.array_inspect(arr, seen = {})
-      if seen[arr.object_id]
-        return "[...]"
-      end
-
-      seen[arr.object_id] = true
-      str = +"["
-      cursor = 0
-      len = arr.length
-
-      while cursor < len
-        if cursor > 0
-          str << ", "
-        end
-
-        item_str = inspect(arr[cursor], seen)
-        str << item_str
-        cursor += 1
-      end
-
-      str << "]"
-      str
-    ensure
-      seen.delete(arr.object_id)
-    end
-
-    def self.hash_inspect(hash, seen = {})
-      if seen[hash.object_id]
-        return "{...}"
-      end
-      seen[hash.object_id] = true
-
-      str = +"{"
-      first = true
-      hash.each do |key, value|
-        if first
-          first = false
-        else
-          str << ", "
-        end
-
-        key_str = inspect(key, seen)
-        str << key_str
-        str << "=>"
-
-        value_str = inspect(value, seen)
-        str << value_str
-      end
-      str << "}"
-      str
-    ensure
-      seen.delete(hash.object_id)
-    end
-
-    HASH_TO_S_METHOD = Hash.instance_method(:to_s)
-    private_constant :HASH_TO_S_METHOD
-
-    HASH_INSPECT_METHOD = Hash.instance_method(:inspect)
-    private_constant :HASH_INSPECT_METHOD
   end
 end

lib/liquid/variable.rb

@@ -61,7 +61,7 @@ module Liquid
 
     def strict_parse(markup)
       @filters = []
-      p = @parse_context.new_parser(markup)
+      p = Parser.new(markup)
 
       return if p.look(:end_of_string)
 
@@ -107,8 +107,8 @@ module Liquid
         obj.each do |o|
           render_obj_to_output(o, output)
         end
-      else
-        output << Liquid::Utils.to_s(obj)
+      when
+        output << obj.to_s
       end
     end
 

lib/liquid/variable_lookup.rb

@@ -6,20 +6,16 @@ module Liquid
 
     attr_reader :name, :lookups
 
-    def self.parse(markup, string_scanner = StringScanner.new(""), cache = nil)
-      new(markup, string_scanner, cache)
+    def self.parse(markup)
+      new(markup)
     end
 
-    def initialize(markup, string_scanner = StringScanner.new(""), cache = nil)
+    def initialize(markup)
       lookups = markup.scan(VariableParser)
 
       name = lookups.shift
       if name&.start_with?('[') && name&.end_with?(']')
-        name = Expression.parse(
-          name[1..-2],
-          string_scanner,
-          cache,
-        )
+        name = Expression.parse(name[1..-2])
       end
       @name = name
 
@@ -29,11 +25,7 @@ module Liquid
       @lookups.each_index do |i|
         lookup = lookups[i]
         if lookup&.start_with?('[') && lookup&.end_with?(']')
-          lookups[i] = Expression.parse(
-            lookup[1..-2],
-            string_scanner,
-            cache,
-          )
+          lookups[i] = Expression.parse(lookup[1..-2])
         elsif COMMAND_METHODS.include?(lookup)
           @command_flags |= 1 << i
         end

lib/liquid/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module Liquid
-  VERSION = "5.6.5"
+  VERSION = "5.6.0.rc2"
 end

liquid.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |s|
 
   s.require_path = "lib"
 
-  s.add_dependency("strscan", ">= 3.1.1")
+  s.add_dependency("strscan")
   s.add_dependency("bigdecimal")
 
   s.add_development_dependency('rake', '~> 13.0')

performance/benchmark.rb

@@ -9,17 +9,14 @@ Liquid::Environment.default.error_mode = ARGV.first.to_sym if ARGV.first
 profiler = ThemeRunner.new
 
 Benchmark.ips do |x|
-  x.time   = 20
-  x.warmup = 10
+  x.time   = 10
+  x.warmup = 5
 
   puts
   puts "Running benchmark for #{x.time} seconds (with #{x.warmup} seconds warmup)."
   puts
 
-  phase = ENV["PHASE"] || "all"
-
-  x.report("tokenize:") { profiler.tokenize } if phase == "all" || phase == "tokenize"
-  x.report("parse:") { profiler.compile } if phase == "all" || phase == "parse"
-  x.report("render:") { profiler.render } if phase == "all" || phase == "render"
-  x.report("parse & render:") { profiler.run } if phase == "all" || phase == "run"
+  x.report("parse:") { profiler.compile }
+  x.report("render:") { profiler.render }
+  x.report("parse & render:") { profiler.run }
 end

performance/theme_runner.rb

@@ -48,19 +48,6 @@ class ThemeRunner
     end
   end
 
-  # `tokenize` will just test the tokenizen portion of liquid without any templates
-  def tokenize
-    ss = StringScanner.new("")
-    @tests.each do |test_hash|
-      tokenizer = Liquid::Tokenizer.new(
-        source: test_hash[:liquid],
-        string_scanner: ss,
-        line_numbers: true,
-      )
-      while tokenizer.shift; end
-    end
-  end
-
   # `run` is called to benchmark rendering and compiling at the same time
   def run
     each_test do |liquid, layout, assigns, page_template, template_name|

performance/unit/expression_benchmark.rb

@@ -1,94 +0,0 @@
-# frozen_string_literal: true
-
-require "benchmark/ips"
-
-# benchmark liquid lexing
-
-require 'liquid'
-
-RubyVM::YJIT.enable
-
-STRING_MARKUPS = [
-  "\"foo\"",
-  "\"fooooooooooo\"",
-  "\"foooooooooooooooooooooooooooooo\"",
-  "'foo'",
-  "'fooooooooooo'",
-  "'foooooooooooooooooooooooooooooo'",
-]
-
-VARIABLE_MARKUPS = [
-  "article",
-  "article.title",
-  "article.title.size",
-  "very_long_variable_name_2024_11_05",
-  "very_long_variable_name_2024_11_05.size",
-]
-
-NUMBER_MARKUPS = [
-  "0",
-  "35",
-  "1241891024912849",
-  "3.5",
-  "3.51214128409128",
-  "12381902839.123819283910283",
-  "123.456.789",
-  "-123",
-  "-12.33",
-  "-405.231",
-  "-0",
-  "0",
-  "0.0",
-  "0.0000000000000000000000",
-  "0.00000000001",
-]
-
-RANGE_MARKUPS = [
-  "(1..30)",
-  "(1...30)",
-  "(1..30..5)",
-  "(1.0...30.0)",
-  "(1.........30)",
-  "(1..foo)",
-  "(foo..30)",
-  "(foo..bar)",
-  "(foo...bar...100)",
-  "(foo...bar...100.0)",
-]
-
-LITERAL_MARKUPS = [
-  nil,
-  'nil',
-  'null',
-  '',
-  'true',
-  'false',
-  'blank',
-  'empty',
-]
-
-MARKUPS = {
-  "string" => STRING_MARKUPS,
-  "literal" => LITERAL_MARKUPS,
-  "variable" => VARIABLE_MARKUPS,
-  "number" => NUMBER_MARKUPS,
-  "range" => RANGE_MARKUPS,
-}
-
-Benchmark.ips do |x|
-  x.config(time: 5, warmup: 5)
-
-  MARKUPS.each do |type, markups|
-    x.report("Liquid::Expression#parse: #{type}") do
-      markups.each do |markup|
-        Liquid::Expression.parse(markup)
-      end
-    end
-  end
-
-  x.report("Liquid::Expression#parse: all") do
-    MARKUPS.values.flatten.each do |markup|
-      Liquid::Expression.parse(markup)
-    end
-  end
-end

performance/unit/lexer_benchmark.rb

@@ -29,12 +29,31 @@ EXPRESSIONS = [
   "foo | default: -1",
 ]
 
+EXPRESSIONS.each do |expr|
+  lexer_1_result = Liquid::Lexer1.new(expr).tokenize
+  lexer_2_result = Liquid::Lexer2.new(expr).tokenize
+
+  next if lexer_1_result == lexer_2_result
+
+  warn "Lexer1 and Lexer2 results are different for expression: #{expr}"
+  warn "expected: #{lexer_1_result}"
+  warn "got: #{lexer_2_result}"
+  abort
+end
+
 Benchmark.ips do |x|
   x.config(time: 10, warmup: 5)
 
-  x.report("Liquid::Lexer#tokenize") do
+  x.report("Liquid::Lexer1#tokenize") do
     EXPRESSIONS.each do |expr|
-      l = Liquid::Lexer.new(expr)
+      l = Liquid::Lexer1.new(expr)
+      l.tokenize
+    end
+  end
+
+  x.report("Liquid::Lexer2#tokenize") do
+    EXPRESSIONS.each do |expr|
+      l = Liquid::Lexer2.new(expr)
       l.tokenize
     end
   end

performance/unit/loom_benchmark.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "benchmark/ips"
+require 'liquid'
+
+RubyVM::YJIT.enable
+
+TEMPLATE = <<~LIQUID
+{% if false %}
+  {% for i in (1..1000000) %}
+    {{ "Hello world!" }}
+  {% endfor %}
+{% endif %}
+
+{% assign result = 1 %}
+{% if foo == 1 %}{% assign result = 1 %}{% endif %}{% if foo == 2 %}{% assign result = 2 %}{% endif %}{% if foo == 3 %}{% assign result = 3 %}{% endif %}
+Result: {{ result }}
+LIQUID
+
+baseline_template = Liquid::Template.parse(TEMPLATE, eager_optimize: false)
+optimized_template = Liquid::Template.parse(TEMPLATE, eager_optimize: true)
+
+[nil, 1, 2, 3].each do |foo|
+  baseline_output = baseline_template.render('foo' => foo)
+  optimized_output = optimized_template.render('foo' => foo)
+
+  if baseline_output != optimized_output
+    puts "WARNING! Baseline and optimized templates render differently for foo=#{foo}"
+    puts "Baseline: #{baseline_output}"
+    puts "Optimized: #{optimized_output}"
+
+    raise
+  end
+end
+
+def render(template, foo)
+  template.render('foo' => foo)
+end
+
+Benchmark.ips do |x|
+  x.config(time: 20, warmup: 3)
+
+  x.report("baseline") do
+    [nil, 1, 2, 3].each do |foo|
+      render(baseline_template, foo)
+    end
+  end
+
+  x.report("optimized") do
+    [nil, 1, 2, 3].each do |foo|
+      render(optimized_template, foo)
+    end
+  end
+
+  x.compare!
+end

test/integration/expression_test.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require 'test_helper'
-require 'lru_redux'
 
 class ExpressionTest < Minitest::Test
   def test_keyword_literals
@@ -14,7 +13,6 @@ class ExpressionTest < Minitest::Test
     assert_template_result("double quoted", '{{"double quoted"}}')
     assert_template_result("spaced", "{{ 'spaced' }}")
     assert_template_result("spaced2", "{{ 'spaced2' }}")
-    assert_template_result("emoji🔥", "{{ 'emoji🔥' }}")
   end
 
   def test_int
@@ -24,18 +22,8 @@ class ExpressionTest < Minitest::Test
   end
 
   def test_float
-    assert_template_result("-17.42", "{{ -17.42 }}")
     assert_template_result("2.5", "{{ 2.5 }}")
-    assert_expression_result(0.0, "0.....5")
-    assert_expression_result(0.0, "-0..1")
     assert_expression_result(1.5, "1.5")
-
-    # this is a unfortunate quirky behavior of Liquid
-    result = Expression.parse(".5")
-    assert_kind_of(Liquid::VariableLookup, result)
-
-    result = Expression.parse("-.5")
-    assert_kind_of(Liquid::VariableLookup, result)
   end
 
   def test_range
@@ -52,101 +40,6 @@ class ExpressionTest < Minitest::Test
     )
   end
 
-  def test_quirky_negative_sign_expression_markup
-    result = Expression.parse("-", nil)
-    assert(result.is_a?(VariableLookup))
-    assert_equal("-", result.name)
-
-    # for this template, the expression markup is "-"
-    assert_template_result(
-      "",
-      "{{ - 'theme.css' - }}",
-    )
-  end
-
-  def test_expression_cache
-    skip("Liquid-C does not support Expression caching") if defined?(Liquid::C) && Liquid::C.enabled
-
-    cache = {}
-    template = <<~LIQUID
-      {% assign x = 1 %}
-      {{ x }}
-      {% assign x = 2 %}
-      {{ x }}
-      {% assign y = 1 %}
-      {{ y }}
-    LIQUID
-
-    Liquid::Template.parse(template, expression_cache: cache).render
-
-    assert_equal(
-      ["1", "2", "x", "y"],
-      cache.to_a.map { _1[0] }.sort,
-    )
-  end
-
-  def test_expression_cache_with_true_boolean
-    skip("Liquid-C does not support Expression caching") if defined?(Liquid::C) && Liquid::C.enabled
-
-    template = <<~LIQUID
-      {% assign x = 1 %}
-      {{ x }}
-      {% assign x = 2 %}
-      {{ x }}
-      {% assign y = 1 %}
-      {{ y }}
-    LIQUID
-
-    parse_context = ParseContext.new(expression_cache: true)
-
-    Liquid::Template.parse(template, parse_context).render
-
-    cache = parse_context.instance_variable_get(:@expression_cache)
-
-    assert_equal(
-      ["1", "2", "x", "y"],
-      cache.to_a.map { _1[0] }.sort,
-    )
-  end
-
-  def test_expression_cache_with_lru_redux
-    skip("Liquid-C does not support Expression caching") if defined?(Liquid::C) && Liquid::C.enabled
-
-    cache = LruRedux::Cache.new(10)
-    template = <<~LIQUID
-      {% assign x = 1 %}
-      {{ x }}
-      {% assign x = 2 %}
-      {{ x }}
-      {% assign y = 1 %}
-      {{ y }}
-    LIQUID
-
-    Liquid::Template.parse(template, expression_cache: cache).render
-
-    assert_equal(
-      ["1", "2", "x", "y"],
-      cache.to_a.map { _1[0] }.sort,
-    )
-  end
-
-  def test_disable_expression_cache
-    skip("Liquid-C does not support Expression caching") if defined?(Liquid::C) && Liquid::C.enabled
-
-    template = <<~LIQUID
-      {% assign x = 1 %}
-      {{ x }}
-      {% assign x = 2 %}
-      {{ x }}
-      {% assign y = 1 %}
-      {{ y }}
-    LIQUID
-
-    parse_context = Liquid::ParseContext.new(expression_cache: false)
-    Liquid::Template.parse(template, parse_context).render
-    assert(parse_context.instance_variable_get(:@expression_cache).nil?)
-  end
-
   private
 
   def assert_expression_result(expect, markup, **assigns)

test/integration/hash_rendering_test.rb

@@ -1,106 +0,0 @@
-# frozen_string_literal: true
-
-require 'test_helper'
-
-class HashRenderingTest < Minitest::Test
-  def test_render_empty_hash
-    assert_template_result("{}", "{{ my_hash }}", { "my_hash" => {} })
-  end
-
-  def test_render_hash_with_string_keys_and_values
-    assert_template_result("{\"key1\"=>\"value1\", \"key2\"=>\"value2\"}", "{{ my_hash }}", { "my_hash" => { "key1" => "value1", "key2" => "value2" } })
-  end
-
-  def test_render_hash_with_symbol_keys_and_integer_values
-    assert_template_result("{:key1=>1, :key2=>2}", "{{ my_hash }}", { "my_hash" => { key1: 1, key2: 2 } })
-  end
-
-  def test_render_nested_hash
-    assert_template_result("{\"outer\"=>{\"inner\"=>\"value\"}}", "{{ my_hash }}", { "my_hash" => { "outer" => { "inner" => "value" } } })
-  end
-
-  def test_render_hash_with_array_values
-    assert_template_result("{\"numbers\"=>[1, 2, 3]}", "{{ my_hash }}", { "my_hash" => { "numbers" => [1, 2, 3] } })
-  end
-
-  def test_render_recursive_hash
-    recursive_hash = { "self" => {} }
-    recursive_hash["self"]["self"] = recursive_hash
-    assert_template_result("{\"self\"=>{\"self\"=>{...}}}", "{{ my_hash }}", { "my_hash" => recursive_hash })
-  end
-
-  def test_hash_with_downcase_filter
-    assert_template_result("{\"key\"=>\"value\", \"anotherkey\"=>\"anothervalue\"}", "{{ my_hash | downcase }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
-  end
-
-  def test_hash_with_upcase_filter
-    assert_template_result("{\"KEY\"=>\"VALUE\", \"ANOTHERKEY\"=>\"ANOTHERVALUE\"}", "{{ my_hash | upcase }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
-  end
-
-  def test_hash_with_strip_filter
-    assert_template_result("{\"Key\"=>\"Value\", \"AnotherKey\"=>\"AnotherValue\"}", "{{ my_hash | strip }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
-  end
-
-  def test_hash_with_escape_filter
-    assert_template_result("{&quot;Key&quot;=&gt;&quot;Value&quot;, &quot;AnotherKey&quot;=&gt;&quot;AnotherValue&quot;}", "{{ my_hash | escape }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
-  end
-
-  def test_hash_with_url_encode_filter
-    assert_template_result("%7B%22Key%22%3D%3E%22Value%22%2C+%22AnotherKey%22%3D%3E%22AnotherValue%22%7D", "{{ my_hash | url_encode }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
-  end
-
-  def test_hash_with_strip_html_filter
-    assert_template_result("{\"Key\"=>\"Value\", \"AnotherKey\"=>\"AnotherValue\"}", "{{ my_hash | strip_html }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
-  end
-
-  def test_hash_with_truncate__20_filter
-    assert_template_result("{\"Key\"=>\"Value\", ...", "{{ my_hash | truncate: 20 }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
-  end
-
-  def test_hash_with_replace___key____replaced_key__filter
-    assert_template_result("{\"Key\"=>\"Value\", \"AnotherKey\"=>\"AnotherValue\"}", "{{ my_hash | replace: 'key', 'replaced_key' }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
-  end
-
-  def test_hash_with_append____appended_text__filter
-    assert_template_result("{\"Key\"=>\"Value\", \"AnotherKey\"=>\"AnotherValue\"} appended text", "{{ my_hash | append: ' appended text' }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
-  end
-
-  def test_hash_with_prepend___prepended_text___filter
-    assert_template_result("prepended text {\"Key\"=>\"Value\", \"AnotherKey\"=>\"AnotherValue\"}", "{{ my_hash | prepend: 'prepended text ' }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
-  end
-
-  def test_render_hash_with_array_values_empty
-    assert_template_result("{\"numbers\"=>[]}", "{{ my_hash }}", { "my_hash" => { "numbers" => [] } })
-  end
-
-  def test_render_hash_with_array_values_hash
-    assert_template_result("{\"numbers\"=>[{:foo=>42}]}", "{{ my_hash }}", { "my_hash" => { "numbers" => [{ foo: 42 }] } })
-  end
-
-  def test_render_hash_with_hash_key
-    assert_template_result("{{\"foo\"=>\"bar\"}=>42}", "{{ my_hash }}", { "my_hash" => { Hash["foo" => "bar"] => 42 } })
-  end
-
-  def test_join_filter_with_hash
-    array = [{ "key1" => "value1" }, { "key2" => "value2" }]
-    glue = { "lol" => "wut" }
-    assert_template_result("{\"key1\"=>\"value1\"}{\"lol\"=>\"wut\"}{\"key2\"=>\"value2\"}", "{{ my_array | join: glue }}", { "my_array" => array, "glue" => glue })
-  end
-
-  def test_rendering_hash_with_custom_to_s_method_uses_custom_to_s
-    my_hash = Class.new(Hash) do
-      def to_s
-        "kewl"
-      end
-    end.new
-
-    assert_template_result("kewl", "{{ my_hash }}", { "my_hash" => my_hash })
-  end
-
-  def test_rendering_hash_without_custom_to_s_uses_default_inspect
-    my_hash = Class.new(Hash).new
-    my_hash[:foo] = :bar
-
-    assert_template_result("{:foo=>:bar}", "{{ my_hash }}", { "my_hash" => my_hash })
-  end
-end

test/integration/parsing_quirks_test.rb

@@ -134,14 +134,6 @@ class ParsingQuirksTest < Minitest::Test
 
   def test_incomplete_expression
     with_error_mode(:lax) do
-      assert_template_result("false", "{{ false - }}")
-      assert_template_result("false", "{{ false > }}")
-      assert_template_result("false", "{{ false < }}")
-      assert_template_result("false", "{{ false = }}")
-      assert_template_result("false", "{{ false ! }}")
-      assert_template_result("false", "{{ false 1 }}")
-      assert_template_result("false", "{{ false a }}")
-
       assert_template_result("false", "{% liquid assign foo = false -\n%}{{ foo }}")
       assert_template_result("false", "{% liquid assign foo = false >\n%}{{ foo }}")
       assert_template_result("false", "{% liquid assign foo = false <\n%}{{ foo }}")

test/integration/standard_filter_test.rb

@@ -54,30 +54,6 @@ class TestEnumerable < Liquid::Drop
   end
 end
 
-class TestDeepEnumerable < Liquid::Drop
-  include Enumerable
-
-  class Product < Liquid::Drop
-    attr_reader :title, :price, :premium
-
-    def initialize(title:, price:, premium: nil)
-      @title = { "content" => title, "language" => "en" }
-      @price = { "value" => price, "unit" => "USD" }
-      @premium = { "category" => premium } if premium
-    end
-  end
-
-  def each(&block)
-    [
-      Product.new(title: "Pro goggles",    price: 1299),
-      Product.new(title: "Thermal gloves", price: 1299),
-      Product.new(title: "Alpine jacket",  price: 3999, premium: 'Basic'),
-      Product.new(title: "Mountain boots", price: 3899, premium: 'Pro'),
-      Product.new(title: "Safety helmet",  price: 1999)
-    ].each(&block)
-  end
-end
-
 class NumberLikeThing < Liquid::Drop
   def initialize(amount)
     @amount = amount
@@ -416,15 +392,6 @@ class StandardFiltersTest < Minitest::Test
     end
   end
 
-  def test_sort_natural_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | sort_natural: 'title.content' | map: 'title.content' | join: ', ' -}}
-    LIQUID
-    expected_output = "Alpine jacket, Mountain boots, Pro goggles, Safety helmet, Thermal gloves"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_legacy_sort_hash
     assert_equal([{ a: 1, b: 2 }], @filters.sort(a: 1, b: 2))
   end
@@ -461,15 +428,6 @@ class StandardFiltersTest < Minitest::Test
     end
   end
 
-  def test_uniq_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | uniq: 'price.value' | map: "title.content" | join: ', ' -}}
-    LIQUID
-    expected_output = "Pro goggles, Alpine jacket, Mountain boots, Safety helmet"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_compact_empty_array
     assert_equal([], @filters.compact([], "a"))
   end
@@ -486,15 +444,6 @@ class StandardFiltersTest < Minitest::Test
     end
   end
 
-  def test_compact_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | compact: 'premium.category' | map: 'title.content' | join: ', ' -}}
-    LIQUID
-    expected_output = "Alpine jacket, Mountain boots"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_reverse
     assert_equal([4, 3, 2, 1], @filters.reverse([1, 2, 3, 4]))
   end
@@ -604,15 +553,6 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result("213", '{{ foo | sort: "bar" | map: "foo" }}', { "foo" => TestEnumerable.new })
   end
 
-  def test_sort_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | sort: 'price.value' | map: 'title.content' | join: ', ' -}}
-    LIQUID
-    expected_output = "Pro goggles, Thermal gloves, Safety helmet, Mountain boots, Alpine jacket"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_first_and_last_call_to_liquid
     assert_template_result('foobar', '{{ foo | first }}', { 'foo' => [ThingWithToLiquid.new] })
     assert_template_result('foobar', '{{ foo | last }}', { 'foo' => [ThingWithToLiquid.new] })
@@ -887,219 +827,21 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result('abc', "{{ 'abc' | date: '%D' }}")
   end
 
-  def test_reject
-    array = [
-      { "handle" => "alpha", "ok" => true },
-      { "handle" => "beta", "ok" => false },
-      { "handle" => "gamma", "ok" => false },
-      { "handle" => "delta", "ok" => true },
-    ]
-
-    template = "{{ array | reject: 'ok' | map: 'handle' | join: ' ' }}"
-    expected_output = "beta gamma"
-
-    assert_template_result(expected_output, template, { "array" => array })
-  end
-
-  def test_reject_with_value
-    array = [
-      { "handle" => "alpha", "ok" => true },
-      { "handle" => "beta", "ok" => false },
-      { "handle" => "gamma", "ok" => false },
-      { "handle" => "delta", "ok" => true },
-    ]
-
-    template = "{{ array | reject: 'ok', true | map: 'handle' | join: ' ' }}"
-    expected_output = "beta gamma"
-
-    assert_template_result(expected_output, template, { "array" => array })
-  end
-
-  def test_reject_with_false_value
-    array = [
-      { "handle" => "alpha", "ok" => true },
-      { "handle" => "beta", "ok" => false },
-      { "handle" => "gamma", "ok" => false },
-      { "handle" => "delta", "ok" => true },
-    ]
-
-    template = "{{ array | reject: 'ok', false | map: 'handle' | join: ' ' }}"
-    expected_output = "alpha delta"
-
-    assert_template_result(expected_output, template, { "array" => array })
-  end
-
-  def test_reject_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | reject: 'title.content', 'Pro goggles' | map: 'price.value' | join: ', ' -}}
-    LIQUID
-    expected_output = "1299, 3999, 3899, 1999"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
-  def test_has
-    array = [
-      { "handle" => "alpha", "ok" => true },
-      { "handle" => "beta", "ok" => false },
-      { "handle" => "gamma", "ok" => false },
-      { "handle" => "delta", "ok" => false },
-    ]
-
-    expected_output = "true"
-
-    assert_template_result(expected_output, "{{ array | has: 'ok' }}", { "array" => array })
-    assert_template_result(expected_output, "{{ array | has: 'ok', true }}", { "array" => array })
-  end
-
-  def test_has_when_does_not_have_it
-    array = [
-      { "handle" => "alpha", "ok" => false },
-      { "handle" => "beta", "ok" => false },
-      { "handle" => "gamma", "ok" => false },
-      { "handle" => "delta", "ok" => false },
-    ]
-
-    expected_output = "false"
-
-    assert_template_result(expected_output, "{{ array | has: 'ok' }}", { "array" => array })
-    assert_template_result(expected_output, "{{ array | has: 'ok', true }}", { "array" => array })
-  end
-
-  def test_has_with_false_value
-    array = [
-      { "handle" => "alpha", "ok" => true },
-      { "handle" => "beta", "ok" => false },
-      { "handle" => "gamma", "ok" => false },
-      { "handle" => "delta", "ok" => true },
-    ]
-
-    template = "{{ array | has: 'ok', false }}"
-    expected_output = "true"
-
-    assert_template_result(expected_output, template, { "array" => array })
-  end
-
-  def test_has_with_false_value_when_does_not_have_it
-    array = [
-      { "handle" => "alpha", "ok" => true },
-      { "handle" => "beta", "ok" => true },
-      { "handle" => "gamma", "ok" => true },
-      { "handle" => "delta", "ok" => true },
-    ]
-
-    template = "{{ array | has: 'ok', false }}"
-    expected_output = "false"
-
-    assert_template_result(expected_output, template, { "array" => array })
-  end
-
-  def test_has_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | has: 'title.content', 'Pro goggles' -}},
-      {{- products | has: 'title.content', 'foo' -}}
-    LIQUID
-    expected_output = "true,false"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
-  def test_find_with_value
-    products = [
-      { "title" => "Pro goggles",    "price" => 1299 },
-      { "title" => "Thermal gloves", "price" => 1499 },
-      { "title" => "Alpine jacket",  "price" => 3999 },
-      { "title" => "Mountain boots", "price" => 3899 },
-      { "title" => "Safety helmet",  "price" => 1999 }
-    ]
-
-    template = <<~LIQUID
-      {%- assign product = products | find: 'price', 3999 -%}
-      {{- product.title -}}
-    LIQUID
-    expected_output = "Alpine jacket"
-
-    assert_template_result(expected_output, template, { "products" => products })
-  end
-
-  def test_find_with_deep_enumerables
-    template = <<~LIQUID
-      {%- assign product = products | find: 'title.content', 'Pro goggles' -%}
-      {{- product.title.content -}}
-    LIQUID
-    expected_output = "Pro goggles"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
-  def test_find_index_with_value
-    products = [
-      { "title" => "Pro goggles",    "price" => 1299 },
-      { "title" => "Thermal gloves", "price" => 1499 },
-      { "title" => "Alpine jacket",  "price" => 3999 },
-      { "title" => "Mountain boots", "price" => 3899 },
-      { "title" => "Safety helmet",  "price" => 1999 }
-    ]
-
-    template = <<~LIQUID
-      {%- assign index = products | find_index: 'price', 3999 -%}
-      {{- index -}}
-    LIQUID
-    expected_output = "2"
-
-    assert_template_result(expected_output, template, { "products" => products })
-  end
-
-  def test_find_index_with_deep_enumerables
-    template = <<~LIQUID
-      {%- assign index = products | find_index: 'title.content', 'Alpine jacket' -%}
-      {{- index -}}
-    LIQUID
-    expected_output = "2"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_where
-    array = [
+    input = [
       { "handle" => "alpha", "ok" => true },
       { "handle" => "beta", "ok" => false },
       { "handle" => "gamma", "ok" => false },
       { "handle" => "delta", "ok" => true },
     ]
 
-    template = "{{ array | where: 'ok' | map: 'handle' | join: ' ' }}"
-    expected_output = "alpha delta"
-
-    assert_template_result(expected_output, template, { "array" => array })
-  end
-
-  def test_where_with_value
-    array = [
+    expectation = [
       { "handle" => "alpha", "ok" => true },
-      { "handle" => "beta", "ok" => false },
-      { "handle" => "gamma", "ok" => false },
       { "handle" => "delta", "ok" => true },
     ]
 
-    template = "{{ array | where: 'ok', true | map: 'handle' | join: ' ' }}"
-    expected_output = "alpha delta"
-
-    assert_template_result(expected_output, template, { "array" => array })
-  end
-
-  def test_where_with_false_value
-    array = [
-      { "handle" => "alpha", "ok" => true },
-      { "handle" => "beta", "ok" => false },
-      { "handle" => "gamma", "ok" => false },
-      { "handle" => "delta", "ok" => true },
-    ]
-
-    template = "{{ array | where: 'ok', false | map: 'handle' | join: ' ' }}"
-    expected_output = "beta gamma"
-
-    assert_template_result(expected_output, template, { "array" => array })
+    assert_equal(expectation, @filters.where(input, "ok", true))
+    assert_equal(expectation, @filters.where(input, "ok"))
   end
 
   def test_where_string_keys
@@ -1158,15 +900,6 @@ class StandardFiltersTest < Minitest::Test
     assert_nil(@filters.where([nil], "ok"))
   end
 
-  def test_where_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | where: 'title.content', 'Pro goggles' | map: 'price.value' -}}
-    LIQUID
-    expected_output = "1299"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_all_filters_never_raise_non_liquid_exception
     test_drop = TestDrop.new(value: "test")
     test_drop.context = Context.new
@@ -1318,15 +1051,6 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result("0", "{{ input | sum: 'subtotal' }}", { "input" => input })
   end
 
-  def test_sum_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | sum: 'price.value' -}}
-    LIQUID
-    expected_output = "12495"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   private
 
   def with_timezone(tz)

test/integration/tags/cycle_tag_test.rb

@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-require 'test_helper'
-
-class CycleTagTest < Minitest::Test
-  def test_simple_cycle
-    template = <<~LIQUID
-      {%- cycle '1', '2', '3' -%}
-      {%- cycle '1', '2', '3' -%}
-      {%- cycle '1', '2', '3' -%}
-    LIQUID
-
-    assert_template_result("123", template)
-  end
-
-  def test_simple_cycle_inside_for_loop
-    template = <<~LIQUID
-      {%- for i in (1..3) -%}
-        {% cycle '1', '2', '3' %}
-      {%- endfor -%}
-    LIQUID
-
-    assert_template_result("123", template)
-  end
-
-  def test_cycle_with_variables_inside_for_loop
-    template = <<~LIQUID
-      {%- assign a = 1 -%}
-      {%- assign b = 2 -%}
-      {%- assign c = 3 -%}
-      {%- for i in (1..3) -%}
-        {% cycle a, b, c %}
-      {%- endfor -%}
-    LIQUID
-
-    assert_template_result("123", template)
-  end
-
-  def test_cycle_tag_always_resets_cycle
-    template = <<~LIQUID
-      {%- assign a = "1" -%}
-      {%- cycle a, "2" -%}
-      {%- cycle a, "2" -%}
-    LIQUID
-
-    assert_template_result("11", template)
-  end
-end

test/integration/tags/raw_tag_test.rb

@@ -16,7 +16,6 @@ class RawTagTest < Minitest::Test
     assert_template_result('>{{ test }}<', '> {%- raw -%}{{ test }}{%- endraw -%} <')
     assert_template_result("> inner  <", "> {%- raw -%} inner {%- endraw %} <")
     assert_template_result("> inner <", "> {%- raw -%} inner {%- endraw -%} <")
-    assert_template_result("{Hello}", "{% raw %}{{% endraw %}Hello{% raw %}}{% endraw %}")
   end
 
   def test_open_tag_in_raw

test/unit/eager_optimize_test.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class EagerOptimizeTest < Minitest::Test
+  include Liquid
+
+  def test_remove_empty_blocks
+    source = <<~LIQUID.gsub(/\n/, '')
+      {% for i in (1..1000000) %}
+      {% endfor %}
+    LIQUID
+
+    template = Liquid::Template.parse(source, eager_optimize: true)
+    assert_equal(0, total_node_count(template))
+  end
+
+  def test_remove_false_if_block
+    source = <<~LIQUID.gsub(/\n/, '')
+      {% if false %}
+        {% if true %}
+          {% if true %}
+            {% if true %}
+              {{ "Hello world!" }}
+            {% endif %}
+          {% endif %}
+        {% endif %}
+      {% endif %}
+    LIQUID
+
+    template = Liquid::Template.parse(source, eager_optimize: true)
+    assert_equal(0, total_node_count(template))
+
+    source = <<~LIQUID.gsub(/\n/, '')
+      {% if false %}
+        {% for i in (1..1000000) %}
+          {{ "Hello world!" }}
+        {% endfor %}
+      {% endif %}
+    LIQUID
+
+    template = Liquid::Template.parse(source, eager_optimize: true)
+    assert_equal(0, total_node_count(template))
+  end
+
+  def test_remove_multiple_false_if_block
+    source = <<~LIQUID.gsub(/\n/, '')
+      {% if false %}
+        {% if true %}
+          {% if true %}
+            {% if true %}
+              {{ "Hello world!" }}
+            {% endif %}
+          {% endif %}
+        {% endif %}
+      {% endif %}
+    LIQUID
+
+    template = Liquid::Template.parse(source, eager_optimize: true)
+    assert_equal(0, total_node_count(template))
+  end
+
+  def test_merge_if_blocks
+    # for now, work with consecutive if blocks without any String nodes in between
+    source = <<~LIQUID.gsub(/\n/, '')
+      {% if foo == 1 %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if foo == 2 %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if foo == 3 %}
+        foo: {{ foo }}
+      {% endif %}
+    LIQUID
+
+    assert_optimization([Liquid::If], source, { "foo" => nil })
+    assert_optimization([Liquid::If], source, { "foo" => 1 })
+    assert_optimization([Liquid::If], source, { "foo" => 2 })
+    assert_optimization([Liquid::If], source, { "foo" => 5 })
+
+    source = <<~LIQUID.gsub(/\n/, '')
+      {% assign bar = "application" %}
+      {% if foo == 1 %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if foo == 2 and bar contains "app" %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if 3 == foo and bar == "application" %}
+        foo: {{ foo }}
+      {% endif %}
+    LIQUID
+
+    assert_optimization([Liquid::Assign, Liquid::If], source)
+  end
+
+  def test_does_not_merge_if_blocks
+    assert_optimization([Liquid::If, Liquid::If], <<~LIQUID.gsub(/\n/, ''))
+      {% if foo == 1 %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if k == 1 %}
+        foo: {{ foo }}
+      {% endif %}
+    LIQUID
+
+    assert_optimization([Liquid::If, Liquid::If], <<~LIQUID.gsub(/\n/, ''))
+      {% if foo == 1 %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if foo == 1 %}
+        foo: {{ foo }}
+      {% endif %}
+    LIQUID
+
+    assert_optimization([Liquid::If, Liquid::If], <<~LIQUID.gsub(/\n/, ''))
+      {% if foo == 1 %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if a == foo %}
+        foo: {{ foo }}
+      {% endif %}
+    LIQUID
+
+    assert_optimization([Liquid::If, Liquid::If], <<~LIQUID.gsub(/\n/, ''))
+      {% if foo %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if foo %}
+        foo: {{ foo }}
+      {% endif %}
+    LIQUID
+
+    assert_optimization([Liquid::If, Liquid::If], <<~LIQUID.gsub(/\n/, ''))
+      {% if foo == 1 %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if foo >= 1 %}
+        foo: {{ foo }}
+      {% endif %}
+    LIQUID
+
+    assert_optimization([Liquid::If, Liquid::If], <<~LIQUID.gsub(/\n/, ''))
+      {% if foo == 1 %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if 1  %}
+        foo: {{ foo }}
+      {% endif %}
+    LIQUID
+  end
+
+  private
+
+  def assert_optimization(expected, source, context = { "foo" => 1 })
+    template = Template.parse(source, eager_optimize: true)
+    assert_equal(expected, template.root.nodelist.map(&:class),)
+
+    baseline_template = Template.parse(source, eager_optimize: false)
+
+    assert_equal(
+      baseline_template.render(context),
+      template.render(context),
+    )
+  end
+
+  def total_node_count(template)
+    root = template.root
+    children = root.nodelist
+    count = 0
+
+    while children.any?
+      next_children = []
+
+      children.each do |node|
+        count += 1 unless node.is_a?(Liquid::BlockBody)
+        next_children.concat(node.nodelist) if node.respond_to?(:nodelist) && node.nodelist
+      end
+
+      children = next_children
+    end
+
+    count
+  end
+end

test/unit/environment_filter_test.rb

@@ -36,14 +36,14 @@ class EnvironmentFilterTest < Minitest::Test
     assert_equal("public", strainer.invoke("public_filter"))
   end
 
-  def test_strainer_raises_argument_error
+  def test_stainer_raises_argument_error
     strainer = @environment.create_strainer(@context)
     assert_raises(Liquid::ArgumentError) do
       strainer.invoke("public_filter", 1)
     end
   end
 
-  def test_strainer_argument_error_contains_backtrace
+  def test_stainer_argument_error_contains_backtrace
     strainer = @environment.create_strainer(@context)
 
     exception = assert_raises(Liquid::ArgumentError) do
@@ -54,9 +54,8 @@ class EnvironmentFilterTest < Minitest::Test
       /\ALiquid error: wrong number of arguments \((1 for 0|given 1, expected 0)\)\z/,
       exception.message,
     )
-
     source = AccessScopeFilters.instance_method(:public_filter).source_location
-    assert_equal(source[0..1].map(&:to_s), exception.backtrace[0].split(':')[0..1])
+    assert_equal(source.map(&:to_s), exception.backtrace[0].split(':')[0..1])
   end
 
   def test_strainer_only_invokes_public_filter_methods

test/unit/lexer_unit_test.rb

@@ -134,6 +134,6 @@ class LexerUnitTest < Minitest::Test
   private
 
   def tokenize(input)
-    Lexer.tokenize(StringScanner.new(input))
+    Lexer.new(input).tokenize
   end
 end

test/unit/parser_unit_test.rb

@@ -6,20 +6,20 @@ class ParserUnitTest < Minitest::Test
   include Liquid
 
   def test_consume
-    p = new_parser("wat: 7")
+    p = Parser.new("wat: 7")
     assert_equal('wat', p.consume(:id))
     assert_equal(':', p.consume(:colon))
     assert_equal('7', p.consume(:number))
   end
 
   def test_jump
-    p = new_parser("wat: 7")
+    p = Parser.new("wat: 7")
     p.jump(2)
     assert_equal('7', p.consume(:number))
   end
 
   def test_consume?
-    p = new_parser("wat: 7")
+    p = Parser.new("wat: 7")
     assert_equal('wat', p.consume?(:id))
     assert_equal(false, p.consume?(:dot))
     assert_equal(':', p.consume(:colon))
@@ -27,7 +27,7 @@ class ParserUnitTest < Minitest::Test
   end
 
   def test_id?
-    p = new_parser("wat 6 Peter Hegemon")
+    p = Parser.new("wat 6 Peter Hegemon")
     assert_equal('wat', p.id?('wat'))
     assert_equal(false, p.id?('endgame'))
     assert_equal('6', p.consume(:number))
@@ -36,7 +36,7 @@ class ParserUnitTest < Minitest::Test
   end
 
   def test_look
-    p = new_parser("wat 6 Peter Hegemon")
+    p = Parser.new("wat 6 Peter Hegemon")
     assert_equal(true, p.look(:id))
     assert_equal('wat', p.consume(:id))
     assert_equal(false, p.look(:comparison))
@@ -46,12 +46,12 @@ class ParserUnitTest < Minitest::Test
   end
 
   def test_expressions
-    p = new_parser("hi.there hi?[5].there? hi.there.bob")
+    p = Parser.new("hi.there hi?[5].there? hi.there.bob")
     assert_equal('hi.there', p.expression)
     assert_equal('hi?[5].there?', p.expression)
     assert_equal('hi.there.bob', p.expression)
 
-    p = new_parser("567 6.0 'lol' \"wut\"")
+    p = Parser.new("567 6.0 'lol' \"wut\"")
     assert_equal('567', p.expression)
     assert_equal('6.0', p.expression)
     assert_equal("'lol'", p.expression)
@@ -59,7 +59,7 @@ class ParserUnitTest < Minitest::Test
   end
 
   def test_ranges
-    p = new_parser("(5..7) (1.5..9.6) (young..old) (hi[5].wat..old)")
+    p = Parser.new("(5..7) (1.5..9.6) (young..old) (hi[5].wat..old)")
     assert_equal('(5..7)', p.expression)
     assert_equal('(1.5..9.6)', p.expression)
     assert_equal('(young..old)', p.expression)
@@ -67,7 +67,7 @@ class ParserUnitTest < Minitest::Test
   end
 
   def test_arguments
-    p = new_parser("filter: hi.there[5], keyarg: 7")
+    p = Parser.new("filter: hi.there[5], keyarg: 7")
     assert_equal('filter', p.consume(:id))
     assert_equal(':', p.consume(:colon))
     assert_equal('hi.there[5]', p.argument)
@@ -77,14 +77,8 @@ class ParserUnitTest < Minitest::Test
 
   def test_invalid_expression
     assert_raises(SyntaxError) do
-      p = new_parser("==")
+      p = Parser.new("==")
       p.expression
     end
   end
-
-  private
-
-  def new_parser(str)
-    Parser.new(StringScanner.new(str))
-  end
 end

test/unit/tag_unit_test.rb

@@ -6,18 +6,18 @@ class TagUnitTest < Minitest::Test
   include Liquid
 
   def test_tag
-    tag = Tag.parse('tag', "", new_tokenizer, ParseContext.new)
+    tag = Tag.parse('tag', "", Tokenizer.new(""), ParseContext.new)
     assert_equal('liquid::tag', tag.name)
     assert_equal('', tag.render(Context.new))
   end
 
   def test_return_raw_text_of_tag
-    tag = Tag.parse("long_tag", "param1, param2, param3", new_tokenizer, ParseContext.new)
+    tag = Tag.parse("long_tag", "param1, param2, param3", Tokenizer.new(""), ParseContext.new)
     assert_equal("long_tag param1, param2, param3", tag.raw)
   end
 
   def test_tag_name_should_return_name_of_the_tag
-    tag = Tag.parse("some_tag", "", new_tokenizer, ParseContext.new)
+    tag = Tag.parse("some_tag", "", Tokenizer.new(""), ParseContext.new)
     assert_equal('some_tag', tag.tag_name)
   end
 
@@ -26,16 +26,7 @@ class TagUnitTest < Minitest::Test
   end
 
   def test_tag_render_to_output_buffer_nil_value
-    custom_tag = CustomTag.parse("some_tag", "", new_tokenizer, ParseContext.new)
+    custom_tag = CustomTag.parse("some_tag", "", Tokenizer.new(""), ParseContext.new)
     assert_equal('some string', custom_tag.render_to_output_buffer(Context.new, "some string"))
   end
-
-  private
-
-  def new_tokenizer
-    Tokenizer.new(
-      source: "",
-      string_scanner: StringScanner.new(""),
-    )
-  end
 end

test/unit/tokenizer_unit_test.rb

@@ -6,7 +6,6 @@ class TokenizerTest < Minitest::Test
   def test_tokenize_strings
     assert_equal([' '], tokenize(' '))
     assert_equal(['hello world'], tokenize('hello world'))
-    assert_equal(['{}'], tokenize('{}'))
   end
 
   def test_tokenize_variables
@@ -31,23 +30,6 @@ class TokenizerTest < Minitest::Test
     assert_equal([1, 1, 3], tokenize_line_numbers(" {{\n funk \n}} "))
   end
 
-  def test_tokenize_with_nil_source_returns_empty_array
-    assert_equal([], tokenize(nil))
-  end
-
-  def test_incomplete_curly_braces
-    assert_equal(["{{.}", " "], tokenize('{{.} '))
-    assert_equal(["{{}", "%}"], tokenize('{{}%}'))
-    assert_equal(["{{}}", "}"], tokenize('{{}}}'))
-  end
-
-  def test_unmatching_start_and_end
-    assert_equal(["{{%}"], tokenize('{{%}'))
-    assert_equal(["{{%%%}}"], tokenize('{{%%%}}'))
-    assert_equal(["{%", "}}"], tokenize('{%}}'))
-    assert_equal(["{%%}", "}"], tokenize('{%%}}'))
-  end
-
   private
 
   def new_tokenizer(source, parse_context: Liquid::ParseContext.new, start_line_number: nil)