WIP: more careful IF block merger

.github/workflows/liquid.yml

@@ -13,8 +13,7 @@ jobs:
         entry:
           - { ruby: 3.0, allowed-failure: false } # minimum supported
           - { ruby: 3.2, allowed-failure: false }
-          - { ruby: 3.3, allowed-failure: false }
-          - { ruby: "3.4.0-rc1", allowed-failure: false } # latest
+          - { ruby: 3.3, allowed-failure: false } # latest
           - { ruby: ruby-head, allowed-failure: false }
     name: Test Ruby ${{ matrix.entry.ruby }}
     steps:

.ruby-version

@@ -1 +1 @@
-3.3.6
+3.3.4

Gemfile

@@ -13,7 +13,6 @@ group :benchmark, :test do
   gem 'benchmark-ips'
   gem 'memory_profiler'
   gem 'terminal-table'
-  gem "lru_redux"
 
   install_if -> { RUBY_PLATFORM !~ /mingw|mswin|java/ && RUBY_ENGINE != 'truffleruby' } do
     gem 'stackprof'

History.md

@@ -1,12 +1,5 @@
 # Liquid Change Log
 
-## 5.6.0 (unreleased)
-
-### Fixes
-
-* Fix Tokenizer to handle null source value (#1873) [Bahar Pourazar]
-
-
 ## 5.5.0 2024-03-21
 
 Please reference the GitHub release for more information.

Rakefile

@@ -71,7 +71,7 @@ end
 
 namespace :benchmark do
   desc "Run the liquid benchmark with lax parsing"
-  task :lax do
+  task :run do
     ruby "./performance/benchmark.rb lax"
   end
 
@@ -80,11 +80,9 @@ namespace :benchmark do
     ruby "./performance/benchmark.rb strict"
   end
 
-  desc "Run the liquid benchmark with both lax and strict parsing"
-  task run: [:lax, :strict]
-
   desc "Run unit benchmarks"
   namespace :unit do
+    desc "Run all unit benchmarks"
     task :all do
       Dir["./performance/unit/*_benchmark.rb"].each do |file|
         puts "🧪 Running #{file}"
@@ -92,17 +90,11 @@ namespace :benchmark do
       end
     end
 
-    task :lexer do
-      Dir["./performance/unit/lexer_benchmark.rb"].each do |file|
-        puts "🧪 Running #{file}"
-        ruby file
-      end
-    end
-
-    task :expression do
-      Dir["./performance/unit/expression_benchmark.rb"].each do |file|
-        puts "🧪 Running #{file}"
-        ruby file
+    %w[lexer loom].each do |benchmark|
+      desc "Run the #{benchmark} benchmark"
+      task benchmark.to_sym do
+        puts "🧪 Running #{benchmark}"
+        ruby "./performance/unit/#{benchmark}_benchmark.rb"
       end
     end
   end

lib/liquid.rb

@@ -21,8 +21,6 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-require "strscan"
-
 module Liquid
   FilterSeparator             = /\|/
   ArgumentSeparator           = ','
@@ -46,7 +44,6 @@ module Liquid
   VariableParser              = /\[(?>[^\[\]]+|\g<0>)*\]|#{VariableSegment}+\??/o
 
   RAISE_EXCEPTION_LAMBDA = ->(_e) { raise }
-  HAS_STRING_SCANNER_SCAN_BYTE = StringScanner.instance_methods.include?(:scan_byte)
 end
 
 require "liquid/version"
@@ -71,6 +68,7 @@ require 'liquid/extensions'
 require 'liquid/errors'
 require 'liquid/interrupts'
 require 'liquid/strainer_template'
+require 'liquid/expression'
 require 'liquid/context'
 require 'liquid/tag'
 require 'liquid/block_body'
@@ -79,7 +77,6 @@ require 'liquid/variable'
 require 'liquid/variable_lookup'
 require 'liquid/range_lookup'
 require 'liquid/resource_limits'
-require 'liquid/expression'
 require 'liquid/template'
 require 'liquid/condition'
 require 'liquid/utils'
@@ -89,3 +86,4 @@ require 'liquid/partial_cache'
 require 'liquid/usage'
 require 'liquid/registers'
 require 'liquid/template_factory'
+require 'liquid/loom'

lib/liquid/block_body.rb

@@ -31,10 +31,11 @@ module Liquid
       end
     end
 
-    def freeze
-      @nodelist.freeze
-      super
-    end
+    # TODO: Freeze the nodelist after optimization
+    # def freeze
+    #   @nodelist.freeze
+    #   super
+    # end
 
     private def parse_for_liquid_tag(tokenizer, parse_context)
       while (token = tokenizer.shift)
@@ -154,6 +155,12 @@ module Liquid
           end
           new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
           @blank &&= new_tag.blank?
+
+          if parse_context.eager_optimize
+            next if new_tag.nodelist&.all? { !_1.is_a?(String) && _1.nodelist.empty? } # this is an empty block
+            next if new_tag.is_a?(If) && new_tag.blocks.empty? # this is an empty If block
+          end
+
           @nodelist << new_tag
         when token.start_with?(VARSTART)
           whitespace_handler(token, parse_context)

lib/liquid/context.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "lru_redux"
-
 module Liquid
   # Context keeps the variable stack and resolves variables, as well as keywords
   #
@@ -41,11 +39,6 @@ module Liquid
       @filters             = []
       @global_filter       = nil
       @disabled_tags       = {}
-      @expression_cache    = LruRedux::ThreadSafeCache.new(1000)
-
-      # Instead of constructing new StringScanner objects for each Expression parse,
-      # we recycle the same one.
-      @string_scanner = StringScanner.new("")
 
       @registers.static[:cached_partials] ||= {}
       @registers.static[:file_system] ||= environment.file_system
@@ -183,7 +176,7 @@ module Liquid
     # Example:
     #   products == empty #=> products.empty?
     def [](expression)
-      evaluate(Expression.parse(expression, @string_scanner, @expression_cache))
+      evaluate(Expression.parse(expression))
     end
 
     def key?(key)

lib/liquid/expression.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "lru_redux"
-
 module Liquid
   class Expression
     LITERALS = {
@@ -12,106 +10,37 @@ module Liquid
       'true' => true,
       'false' => false,
       'blank' => '',
-      'empty' => '',
-      # in lax mode, minus sign can be a VariableLookup
-      # For simplicity and performace, we treat it like a literal
-      '-' => VariableLookup.parse("-", nil).freeze,
+      'empty' => ''
     }.freeze
 
-    DOT = ".".ord
-    ZERO = "0".ord
-    NINE = "9".ord
-    DASH = "-".ord
+    INTEGERS_REGEX       = /\A(-?\d+)\z/
+    FLOATS_REGEX         = /\A(-?\d[\d\.]+)\z/
 
     # Use an atomic group (?>...) to avoid pathological backtracing from
     # malicious input as described in https://github.com/Shopify/liquid/issues/1357
-    RANGES_REGEX = /\A\(\s*(?>(\S+)\s*\.\.)\s*(\S+)\s*\)\z/
-    INTEGER_REGEX = /\A(-?\d+)\z/
-    FLOAT_REGEX = /\A(-?\d+)\.\d+\z/
+    RANGES_REGEX         = /\A\(\s*(?>(\S+)\s*\.\.)\s*(\S+)\s*\)\z/
 
-    class << self
-      def parse(markup, ss = StringScanner.new(""), cache = nil)
-        return unless markup
+    def self.parse(markup)
+      return nil unless markup
 
-        markup = markup.strip # markup can be a frozen string
-
-        if (markup.start_with?('"') && markup.end_with?('"')) ||
-          (markup.start_with?("'") && markup.end_with?("'"))
-          return markup[1..-2]
-        elsif LITERALS.key?(markup)
-          return LITERALS[markup]
-        end
-
-        # Cache only exists during parsing
-        if cache
-          return cache[markup] if cache.key?(markup)
-
-          cache[markup] = inner_parse(markup, ss, cache).freeze
-        else
-          inner_parse(markup, ss, nil).freeze
-        end
+      markup = markup.strip
+      if (markup.start_with?('"') && markup.end_with?('"')) ||
+         (markup.start_with?("'") && markup.end_with?("'"))
+        return markup[1..-2]
       end
 
-      def inner_parse(markup, ss, cache)
-        if (markup.start_with?("(") && markup.end_with?(")")) && markup =~ RANGES_REGEX
-          return RangeLookup.parse(
-            Regexp.last_match(1),
-            Regexp.last_match(2),
-            ss,
-            cache,
-          )
-        end
-
-        if (num = parse_number(markup, ss))
-          num
+      case markup
+      when INTEGERS_REGEX
+        Regexp.last_match(1).to_i
+      when RANGES_REGEX
+        RangeLookup.parse(Regexp.last_match(1), Regexp.last_match(2))
+      when FLOATS_REGEX
+        Regexp.last_match(1).to_f
+      else
+        if LITERALS.key?(markup)
+          LITERALS[markup]
         else
-          VariableLookup.parse(markup, ss, cache)
-        end
-      end
-
-      def parse_number(markup, ss)
-        # check if the markup is simple integer or float
-        case markup
-        when INTEGER_REGEX
-          return Integer(markup, 10)
-        when FLOAT_REGEX
-          return markup.to_f
-        end
-
-        ss.string = markup
-        # the first byte must be a digit, a period, or  a dash
-        byte = ss.scan_byte
-
-        return false if byte != DASH && byte != DOT && (byte < ZERO || byte > NINE)
-
-        # The markup could be a float with multiple dots
-        first_dot_pos = nil
-        num_end_pos = nil
-
-        while (byte = ss.scan_byte)
-          return false if byte != DOT && (byte < ZERO || byte > NINE)
-
-          # we found our number and now we are just scanning the rest of the string
-          next if num_end_pos
-
-          if byte == DOT
-            if first_dot_pos.nil?
-              first_dot_pos = ss.pos
-            else
-              # we found another dot, so we know that the number ends here
-              num_end_pos = ss.pos - 1
-            end
-          end
-        end
-
-        num_end_pos = markup.length if ss.eos?
-
-        if num_end_pos
-          # number ends with a number "123.123"
-          markup.byteslice(0, num_end_pos).to_f
-        else
-          # number ends with a dot "123."
-          markup.byteslice(0, first_dot_pos).to_f
+          VariableLookup.parse(markup)
         end
       end
     end

lib/liquid/lexer.rb

@@ -1,7 +1,66 @@
 # frozen_string_literal: true
 
+require "strscan"
+
 module Liquid
-  class Lexer
+  class Lexer1
+    SPECIALS = {
+      '|' => :pipe,
+      '.' => :dot,
+      ':' => :colon,
+      ',' => :comma,
+      '[' => :open_square,
+      ']' => :close_square,
+      '(' => :open_round,
+      ')' => :close_round,
+      '?' => :question,
+      '-' => :dash,
+    }.freeze
+    IDENTIFIER            = /[a-zA-Z_][\w-]*\??/
+    SINGLE_STRING_LITERAL = /'[^\']*'/
+    DOUBLE_STRING_LITERAL = /"[^\"]*"/
+    STRING_LITERAL        = Regexp.union(SINGLE_STRING_LITERAL, DOUBLE_STRING_LITERAL)
+    NUMBER_LITERAL        = /-?\d+(\.\d+)?/
+    DOTDOT                = /\.\./
+    COMPARISON_OPERATOR   = /==|!=|<>|<=?|>=?|contains(?=\s)/
+    WHITESPACE_OR_NOTHING = /\s*/
+
+    def initialize(input)
+      @ss = StringScanner.new(input)
+    end
+
+    def tokenize
+      @output = []
+
+      until @ss.eos?
+        @ss.skip(WHITESPACE_OR_NOTHING)
+        break if @ss.eos?
+        tok      = if (t = @ss.scan(COMPARISON_OPERATOR))
+          [:comparison, t]
+        elsif (t = @ss.scan(STRING_LITERAL))
+          [:string, t]
+        elsif (t = @ss.scan(NUMBER_LITERAL))
+          [:number, t]
+        elsif (t = @ss.scan(IDENTIFIER))
+          [:id, t]
+        elsif (t = @ss.scan(DOTDOT))
+          [:dotdot, t]
+        else
+          c     = @ss.getch
+          if (s = SPECIALS[c])
+            [s, c]
+          else
+            raise SyntaxError, "Unexpected character #{c}"
+          end
+        end
+        @output << tok
+      end
+
+      @output << [:end_of_string]
+    end
+  end
+
+  class Lexer2
     CLOSE_ROUND = [:close_round, ")"].freeze
     CLOSE_SQUARE = [:close_square, "]"].freeze
     COLON = [:colon, ":"].freeze
@@ -33,7 +92,6 @@ module Liquid
     SINGLE_COMPARISON_TOKENS = [].tap do |table|
       table["<".ord] = COMPARISON_LESS_THAN
       table[">".ord] = COMPARISON_GREATER_THAN
-      table.freeze
     end
 
     TWO_CHARS_COMPARISON_JUMP_TABLE = [].tap do |table|
@@ -45,17 +103,18 @@ module Liquid
         sub_table["=".ord] = COMPARISION_NOT_EQUAL
         sub_table.freeze
       end
-      table.freeze
     end
 
     COMPARISON_JUMP_TABLE = [].tap do |table|
       table["<".ord] = [].tap do |sub_table|
         sub_table["=".ord] = COMPARISON_LESS_THAN_OR_EQUAL
         sub_table[">".ord] = COMPARISON_NOT_EQUAL_ALT
+        RUBY_WHITESPACE.each { |c| sub_table[c.ord] = COMPARISON_LESS_THAN }
         sub_table.freeze
       end
       table[">".ord] = [].tap do |sub_table|
         sub_table["=".ord] = COMPARISON_GREATER_THAN_OR_EQUAL
+        RUBY_WHITESPACE.each { |c| sub_table[c.ord] = COMPARISON_GREATER_THAN }
         sub_table.freeze
       end
       table.freeze
@@ -98,76 +157,81 @@ module Liquid
       table.freeze
     end
 
+    def initialize(input)
+      @ss = StringScanner.new(input)
+    end
+
     # rubocop:disable Metrics/BlockNesting
-    class << self
-      def tokenize(ss)
-        output = []
+    def tokenize
+      @output = []
 
-        until ss.eos?
-          ss.skip(WHITESPACE_OR_NOTHING)
+      until @ss.eos?
+        @ss.skip(WHITESPACE_OR_NOTHING)
 
-          break if ss.eos?
+        break if @ss.eos?
 
-          start_pos = ss.pos
-          peeked = ss.peek_byte
+        start_pos = @ss.pos
+        peeked = @ss.peek_byte
 
-          if (special = SPECIAL_TABLE[peeked])
-            ss.scan_byte
-            # Special case for ".."
-            if special == DOT && ss.peek_byte == DOT_ORD
-              ss.scan_byte
-              output << DOTDOT
-            elsif special == DASH
-              # Special case for negative numbers
-              if (peeked_byte = ss.peek_byte) && NUMBER_TABLE[peeked_byte]
-                ss.pos -= 1
-                output << [:number, ss.scan(NUMBER_LITERAL)]
-              else
-                output << special
-              end
+        if (special = SPECIAL_TABLE[peeked])
+          @ss.scan_byte
+          # Special case for ".."
+          if special == DOT && @ss.peek_byte == DOT_ORD
+            @ss.scan_byte
+            @output << DOTDOT
+          elsif special == DASH
+            # Special case for negative numbers
+            if (peeked_byte = @ss.peek_byte) && NUMBER_TABLE[peeked_byte]
+              @ss.pos -= 1
+              @output << [:number, @ss.scan(NUMBER_LITERAL)]
             else
-              output << special
-            end
-          elsif (sub_table = TWO_CHARS_COMPARISON_JUMP_TABLE[peeked])
-            ss.scan_byte
-            if (peeked_byte = ss.peek_byte) && (found = sub_table[peeked_byte])
-              output << found
-              ss.scan_byte
-            else
-              raise_syntax_error(start_pos, ss)
-            end
-          elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
-            ss.scan_byte
-            if (peeked_byte = ss.peek_byte) && (found = sub_table[peeked_byte])
-              output << found
-              ss.scan_byte
-            else
-              output << SINGLE_COMPARISON_TOKENS[peeked]
+              @output << special
             end
           else
-            type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked]
+            @output << special
+          end
+        elsif (sub_table = TWO_CHARS_COMPARISON_JUMP_TABLE[peeked])
+          @ss.scan_byte
+          if (peeked_byte = @ss.peek_byte) && (found = sub_table[peeked_byte])
+            @output << found
+            @ss.scan_byte
+          else
+            raise_syntax_error(start_pos)
+          end
+        elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
+          @ss.scan_byte
+          if (peeked_byte = @ss.peek_byte) && (found = sub_table[peeked_byte])
+            @output << found
+            @ss.scan_byte
+          else
+            @output << SINGLE_COMPARISON_TOKENS[peeked]
+          end
+        else
+          type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked]
 
-            if type && (t = ss.scan(pattern))
-              # Special case for "contains"
-              output << if type == :id && t == "contains" && output.last&.first != :dot
-                COMPARISON_CONTAINS
-              else
-                [type, t]
-              end
+          if type && (t = @ss.scan(pattern))
+            # Special case for "contains"
+            @output << if type == :id && t == "contains" && @output.last&.first != :dot
+              COMPARISON_CONTAINS
             else
-              raise_syntax_error(start_pos, ss)
+              [type, t]
             end
+          else
+            raise_syntax_error(start_pos)
           end
         end
-        # rubocop:enable Metrics/BlockNesting
-        output << EOS
       end
+      # rubocop:enable Metrics/BlockNesting
 
-      def raise_syntax_error(start_pos, ss)
-        ss.pos = start_pos
-        # the character could be a UTF-8 character, use getch to get all the bytes
-        raise SyntaxError, "Unexpected character #{ss.getch}"
-      end
+      @output << EOS
+    end
+
+    def raise_syntax_error(start_pos)
+      @ss.pos = start_pos
+      # the character could be a UTF-8 character, use getch to get all the bytes
+      raise SyntaxError, "Unexpected character #{@ss.getch}"
     end
   end
+
+  Lexer = StringScanner.instance_methods.include?(:scan_byte) ? Lexer2 : Lexer1
 end

lib/liquid/loom.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Liquid
+  class Loom
+    MERGABLE_IF_OPERATORS = ["==", ">", "<", "!="].freeze
+    EQUAL_OP = "==".freeze
+
+    class << self
+      def optimize(template)
+        new(template).optimize
+      end
+    end
+
+    def initialize template
+      @root = template.root
+    end
+
+    def optimize
+      merge_if_blocks
+    end
+
+    def merge_if_blocks
+      nodelist_list = [@root.nodelist]
+
+      while nodelist_list.any?
+        next_nodelist_list = []
+
+        nodelist_list.each do |nodelist|
+          i = 0
+          while i < nodelist.length
+            node = nodelist[i]
+            chain_if_blocks(nodelist, node, i) if node.is_a?(If)
+            i += 1
+          end
+        end
+
+        nodelist_list = next_nodelist_list
+      end
+    end
+
+    private
+
+    def mergable_if_blocks?(target_if, next_if)
+      target_left = target_if.blocks.first.left
+      target_right = target_if.blocks.first.right
+      next_left = next_if.blocks.first.left
+      next_right = next_if.blocks.first.right
+
+      used_variables = Hash.new { |h, k| h[k] = 0 }
+
+      [
+        target_if.blocks.first.left,
+        target_if.blocks.first.right,
+        next_if.blocks.first.left,
+        next_if.blocks.first.right
+      ].each do |var|
+        if var.is_a?(VariableLookup)
+          used_variables[var.name] += 1
+        end
+      end
+
+      return if used_variables.keys.count > 1
+
+      most_used_variable_name = used_variables.keys[0]
+
+      # TODO: I probably can't do this
+      # It might be possible to get different result between a > b and b < a
+      # Move most commonly used variable to the left side
+      if (target_left.is_a?(VariableLookup) && target_left.name != most_used_variable_name) || (target_right.is_a?(VariableLookup) && target_right.name == most_used_variable_name)
+        target_left, target_right = target_right, target_left
+      end
+
+      if (next_left.is_a?(VariableLookup) && next_left.name != most_used_variable_name) || (next_right.is_a?(VariableLookup) && next_right.name == most_used_variable_name)
+        next_left, next_right = next_right, next_left
+      end
+
+      return false unless target_left.is_a?(VariableLookup) && next_left.is_a?(VariableLookup)
+      return false if target_left.name != next_left.name
+
+      return false if target_right.nil? || next_right.nil?
+
+
+      # we need to be conversative here and only can merge ==, >, <, and != operators
+      target_operator = target_if.blocks.first.operator
+      next_operator = next_if.blocks.first.operator
+
+      return false unless MERGABLE_IF_OPERATORS.include?(target_operator) && MERGABLE_IF_OPERATORS.include?(next_operator)
+
+      return false if target_operator == next_operator && target_right == next_right
+
+      return false if target_right.is_a?(VariableLookup) || next_right.is_a?(VariableLookup)
+
+      true
+    end
+
+    def chain_if_blocks(nodelist, first_if_node, first_if_index)
+      used_variables = Set.new
+
+      # only check the top level Condition (ignore children conditions for now)
+      first_if_node.blocks.each do |condition|
+        used_variables << condition.left
+        used_variables << condition.right if condition.right
+      end
+
+      if_blocks = []
+
+      nodelist[first_if_index + 1..-1].each do |node|
+        break unless node.is_a?(If)
+
+        # check if the variables used in the current block are used in the previous block
+        break unless mergable_if_blocks?(first_if_node, node)
+
+        if_blocks << node
+      end
+
+      nodelist.delete_if { |node| if_blocks.include?(node) }
+
+      if_blocks.each do |if_block|
+        first_if_node.blocks << if_block.blocks.first
+      end
+    end
+  end
+end

lib/liquid/parse_context.rb

@@ -3,7 +3,7 @@
 module Liquid
   class ParseContext
     attr_accessor :locale, :line_number, :trim_whitespace, :depth
-    attr_reader :partial, :warnings, :error_mode, :environment
+    attr_reader :partial, :warnings, :error_mode, :environment, :eager_optimize
 
     def initialize(options = Const::EMPTY_HASH)
       @environment = options.fetch(:environment, Environment.default)
@@ -11,18 +11,7 @@ module Liquid
 
       @locale   = @template_options[:locale] ||= I18n.new
       @warnings = []
-
-      # constructing new StringScanner in Lexer, Tokenizer, etc is expensive
-      # This StringScanner will be shared by all of them
-      @string_scanner = StringScanner.new("")
-
-      @expression_cache = if options[:expression_cache].nil?
-        {}
-      elsif options[:expression_cache].respond_to?(:[]) && options[:expression_cache].respond_to?(:[]=)
-        options[:expression_cache]
-      elsif options[:expression_cache]
-        {}
-      end
+      @eager_optimize = options.fetch(:eager_optimize, ENV["OPTIMIZE"] == "true")
 
       self.depth   = 0
       self.partial = false
@@ -36,22 +25,12 @@ module Liquid
       Liquid::BlockBody.new
     end
 
-    def new_parser(input)
-      @string_scanner.string = input
-      Parser.new(@string_scanner)
-    end
-
-    def new_tokenizer(source, start_line_number: nil, for_liquid_tag: false)
-      Tokenizer.new(
-        source: source,
-        string_scanner: @string_scanner,
-        line_number: start_line_number,
-        for_liquid_tag: for_liquid_tag,
-      )
+    def new_tokenizer(markup, start_line_number: nil, for_liquid_tag: false)
+      Tokenizer.new(markup, line_number: start_line_number, for_liquid_tag: for_liquid_tag)
     end
 
     def parse_expression(markup)
-      Expression.parse(markup, @string_scanner, @expression_cache)
+      Expression.parse(markup)
     end
 
     def partial=(value)

lib/liquid/parser.rb

@@ -3,8 +3,8 @@
 module Liquid
   class Parser
     def initialize(input)
-      ss = input.is_a?(StringScanner) ? input : StringScanner.new(input)
-      @tokens = Lexer.tokenize(ss)
+      l       = Lexer.new(input)
+      @tokens = l.tokenize
       @p      = 0 # pointer to current location
     end
 

lib/liquid/range_lookup.rb

@@ -2,9 +2,9 @@
 
 module Liquid
   class RangeLookup
-    def self.parse(start_markup, end_markup, string_scanner, cache = nil)
-      start_obj = Expression.parse(start_markup, string_scanner, cache)
-      end_obj   = Expression.parse(end_markup, string_scanner, cache)
+    def self.parse(start_markup, end_markup)
+      start_obj = Expression.parse(start_markup)
+      end_obj   = Expression.parse(end_markup)
       if start_obj.respond_to?(:evaluate) || end_obj.respond_to?(:evaluate)
         new(start_obj, end_obj)
       else

lib/liquid/tags/cycle.rb

@@ -68,13 +68,7 @@ module Liquid
     def variables_from_string(markup)
       markup.split(',').collect do |var|
         var =~ /\s*(#{QuotedFragment})\s*/o
-        next unless Regexp.last_match(1)
-
-        # Expression Parser returns cached objects, and we need to dup them to
-        # start the cycle over for each new cycle call.
-        # Liquid-C does not have a cache, so we don't need to dup the object.
-        var = parse_expression(Regexp.last_match(1))
-        var.is_a?(VariableLookup) ? var.dup : var
+        Regexp.last_match(1) ? parse_expression(Regexp.last_match(1)) : nil
       end.compact
     end
 

lib/liquid/tags/for.rb

@@ -88,7 +88,7 @@ module Liquid
     end
 
     def strict_parse(markup)
-      p = @parse_context.new_parser(markup)
+      p = Parser.new(markup)
       @variable_name = p.consume(:id)
       raise SyntaxError, options[:locale].t("errors.syntax.for_invalid_in") unless p.id?('in')
 

lib/liquid/tags/if.rb

@@ -23,6 +23,7 @@ module Liquid
     def initialize(tag_name, markup, options)
       super
       @blocks = []
+      @has_else_block = false
       push_block('if', markup)
     end
 
@@ -33,17 +34,44 @@ module Liquid
     def parse(tokens)
       while parse_body(@blocks.last.attachment, tokens)
       end
-      @blocks.reverse_each do |block|
-        block.attachment.remove_blank_strings if blank?
-        block.attachment.freeze
+
+      if parse_context.eager_optimize && definitive_false_statement?
+        @blocks.clear
+      else
+        @blocks.reverse_each do |block|
+          block.attachment.remove_blank_strings if blank?
+          block.attachment.freeze
+        end
       end
     end
 
+    def definitive_false_statement?
+      # check if any blocks have variable lookups
+      @blocks.each do |condition|
+        return false if condition.left.is_a?(VariableLookup) || condition.right&.is_a?(VariableLookup)
+
+        child_condition = condition.child_condition
+
+        while child_condition
+          return false if child_condition&.left.is_a?(VariableLookup) || child_condition&.right&.is_a?(VariableLookup)
+          child_condition = child_condition.child_condition
+        end
+      end
+
+      # check if all blocks are false
+      @blocks.each do |condition|
+        return false if condition.evaluate
+      end
+
+      true
+    end
+
     ELSE_TAG_NAMES = ['elsif', 'else'].freeze
     private_constant :ELSE_TAG_NAMES
 
     def unknown_tag(tag, markup, tokens)
       if ELSE_TAG_NAMES.include?(tag)
+        @has_else_block = true
         push_block(tag, markup)
       else
         super
@@ -102,7 +130,7 @@ module Liquid
     end
 
     def strict_parse(markup)
-      p = @parse_context.new_parser(markup)
+      p = Parser.new(markup)
       condition = parse_binary_comparisons(p)
       p.consume(:end_of_string)
       condition

lib/liquid/template.rb

@@ -82,7 +82,11 @@ module Liquid
       # See Liquid::Profiler for more information
       def parse(source, options = {})
         environment = options[:environment] || Environment.default
-        new(environment: environment).parse(source, options)
+        template = new(environment: environment).parse(source, options)
+
+        Loom.optimize(template) if options[:eager_optimize]
+
+        template
       end
     end
 

lib/liquid/tokenizer.rb

@@ -1,43 +1,20 @@
 # frozen_string_literal: true
 
-require "strscan"
-
 module Liquid
   class Tokenizer
     attr_reader :line_number, :for_liquid_tag
 
-    TAG_END = /%\}/
-    TAG_OR_VARIABLE_START = /\{[\{\%]/
-    NEWLINE = /\n/
-
-    OPEN_CURLEY = "{".ord
-    CLOSE_CURLEY = "}".ord
-    PERCENTAGE = "%".ord
-
-    def initialize(
-      source:,
-      string_scanner:,
-      line_numbers: false,
-      line_number: nil,
-      for_liquid_tag: false
-    )
-      @line_number = line_number || (line_numbers ? 1 : nil)
+    def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
+      @source         = source
+      @line_number    = line_number || (line_numbers ? 1 : nil)
       @for_liquid_tag = for_liquid_tag
-      @source = source.to_s.to_str
-      @offset = 0
-      @tokens = []
-
-      if @source
-        @ss = string_scanner
-        @ss.string = @source
-        tokenize
-      end
+      @offset         = 0
+      @tokens         = tokenize
     end
 
     def shift
       token = @tokens[@offset]
-
-      return unless token
+      return nil unless token
 
       @offset += 1
 
@@ -51,105 +28,18 @@ module Liquid
     private
 
     def tokenize
-      if @for_liquid_tag
-        @tokens = @source.split("\n")
-      else
-        @tokens << shift_normal until @ss.eos?
+      return [] if @source.empty?
+
+      return @source.split("\n") if @for_liquid_tag
+
+      tokens = @source.split(TemplateParser)
+
+      # removes the rogue empty element at the beginning of the array
+      if tokens[0]&.empty?
+        @offset += 1
       end
 
-      @source = nil
-      @ss = nil
-    end
-
-    def shift_normal
-      token = next_token
-
-      return unless token
-
-      token
-    end
-
-    def next_token
-      # possible states: :text, :tag, :variable
-      byte_a = @ss.peek_byte
-
-      if byte_a == OPEN_CURLEY
-        @ss.scan_byte
-
-        byte_b = @ss.peek_byte
-
-        if byte_b == PERCENTAGE
-          @ss.scan_byte
-          return next_tag_token
-        elsif byte_b == OPEN_CURLEY
-          @ss.scan_byte
-          return next_variable_token
-        end
-
-        @ss.pos -= 1
-      end
-
-      next_text_token
-    end
-
-    def next_text_token
-      start = @ss.pos
-
-      unless @ss.skip_until(TAG_OR_VARIABLE_START)
-        token = @ss.rest
-        @ss.terminate
-        return token
-      end
-
-      pos = @ss.pos -= 2
-      @source.byteslice(start, pos - start)
-    end
-
-    def next_variable_token
-      start = @ss.pos - 2
-
-      byte_a = byte_b = @ss.scan_byte
-
-      while byte_b
-        byte_a = @ss.scan_byte while byte_a && (byte_a != CLOSE_CURLEY && byte_a != OPEN_CURLEY)
-
-        break unless byte_a
-
-        if @ss.eos?
-          return byte_a == CLOSE_CURLEY ? @source.byteslice(start, @ss.pos - start) : "{{"
-        end
-
-        byte_b = @ss.scan_byte
-
-        if byte_a == CLOSE_CURLEY
-          if byte_b == CLOSE_CURLEY
-            return @source.byteslice(start, @ss.pos - start)
-          elsif byte_b != CLOSE_CURLEY
-            @ss.pos -= 1
-            return @source.byteslice(start, @ss.pos - start)
-          end
-        elsif byte_a == OPEN_CURLEY && byte_b == PERCENTAGE
-          return next_tag_token_with_start(start)
-        end
-
-        byte_a = byte_b
-      end
-
-      "{{"
-    end
-
-    def next_tag_token
-      start = @ss.pos - 2
-      if (len = @ss.skip_until(TAG_END))
-        @source.byteslice(start, len + 2)
-      else
-        "{%"
-      end
-    end
-
-    def next_tag_token_with_start(start)
-      @ss.skip_until(TAG_END)
-      @source.byteslice(start, @ss.pos - start)
+      tokens
     end
   end
 end

lib/liquid/variable.rb

@@ -61,7 +61,7 @@ module Liquid
 
     def strict_parse(markup)
       @filters = []
-      p = @parse_context.new_parser(markup)
+      p = Parser.new(markup)
 
       return if p.look(:end_of_string)
 

lib/liquid/variable_lookup.rb

@@ -6,20 +6,16 @@ module Liquid
 
     attr_reader :name, :lookups
 
-    def self.parse(markup, string_scanner, cache = nil)
-      new(markup, string_scanner, cache)
+    def self.parse(markup)
+      new(markup)
     end
 
-    def initialize(markup, string_scanner = StringScanner.new(""), cache = nil)
+    def initialize(markup)
       lookups = markup.scan(VariableParser)
 
       name = lookups.shift
       if name&.start_with?('[') && name&.end_with?(']')
-        name = Expression.parse(
-          name[1..-2],
-          string_scanner,
-          cache,
-        )
+        name = Expression.parse(name[1..-2])
       end
       @name = name
 
@@ -29,11 +25,7 @@ module Liquid
       @lookups.each_index do |i|
         lookup = lookups[i]
         if lookup&.start_with?('[') && lookup&.end_with?(']')
-          lookups[i] = Expression.parse(
-            lookup[1..-2],
-            string_scanner,
-            cache,
-          )
+          lookups[i] = Expression.parse(lookup[1..-2])
         elsif COMMAND_METHODS.include?(lookup)
           @command_flags |= 1 << i
         end

lib/liquid/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module Liquid
-  VERSION = "5.6.1"
+  VERSION = "5.6.0.rc2"
 end

liquid.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |s|
 
   s.require_path = "lib"
 
-  s.add_dependency("strscan", ">= 3.1.1")
+  s.add_dependency("strscan")
   s.add_dependency("bigdecimal")
 
   s.add_development_dependency('rake', '~> 13.0')

performance/benchmark.rb

@@ -9,17 +9,14 @@ Liquid::Environment.default.error_mode = ARGV.first.to_sym if ARGV.first
 profiler = ThemeRunner.new
 
 Benchmark.ips do |x|
-  x.time   = 20
-  x.warmup = 10
+  x.time   = 10
+  x.warmup = 5
 
   puts
   puts "Running benchmark for #{x.time} seconds (with #{x.warmup} seconds warmup)."
   puts
 
-  phase = ENV["PHASE"] || "all"
-
-  x.report("tokenize:") { profiler.tokenize } if phase == "all" || phase == "tokenize"
-  x.report("parse:") { profiler.compile } if phase == "all" || phase == "parse"
-  x.report("render:") { profiler.render } if phase == "all" || phase == "render"
-  x.report("parse & render:") { profiler.run } if phase == "all" || phase == "run"
+  x.report("parse:") { profiler.compile }
+  x.report("render:") { profiler.render }
+  x.report("parse & render:") { profiler.run }
 end

performance/theme_runner.rb

@@ -48,19 +48,6 @@ class ThemeRunner
     end
   end
 
-  # `tokenize` will just test the tokenizen portion of liquid without any templates
-  def tokenize
-    ss = StringScanner.new("")
-    @tests.each do |test_hash|
-      tokenizer = Liquid::Tokenizer.new(
-        source: test_hash[:liquid],
-        string_scanner: ss,
-        line_numbers: true,
-      )
-      while tokenizer.shift; end
-    end
-  end
-
   # `run` is called to benchmark rendering and compiling at the same time
   def run
     each_test do |liquid, layout, assigns, page_template, template_name|

performance/unit/expression_benchmark.rb

@@ -1,94 +0,0 @@
-# frozen_string_literal: true
-
-require "benchmark/ips"
-
-# benchmark liquid lexing
-
-require 'liquid'
-
-RubyVM::YJIT.enable
-
-STRING_MARKUPS = [
-  "\"foo\"",
-  "\"fooooooooooo\"",
-  "\"foooooooooooooooooooooooooooooo\"",
-  "'foo'",
-  "'fooooooooooo'",
-  "'foooooooooooooooooooooooooooooo'",
-]
-
-VARIABLE_MARKUPS = [
-  "article",
-  "article.title",
-  "article.title.size",
-  "very_long_variable_name_2024_11_05",
-  "very_long_variable_name_2024_11_05.size",
-]
-
-NUMBER_MARKUPS = [
-  "0",
-  "35",
-  "1241891024912849",
-  "3.5",
-  "3.51214128409128",
-  "12381902839.123819283910283",
-  "123.456.789",
-  "-123",
-  "-12.33",
-  "-405.231",
-  "-0",
-  "0",
-  "0.0",
-  "0.0000000000000000000000",
-  "0.00000000001",
-]
-
-RANGE_MARKUPS = [
-  "(1..30)",
-  "(1...30)",
-  "(1..30..5)",
-  "(1.0...30.0)",
-  "(1.........30)",
-  "(1..foo)",
-  "(foo..30)",
-  "(foo..bar)",
-  "(foo...bar...100)",
-  "(foo...bar...100.0)",
-]
-
-LITERAL_MARKUPS = [
-  nil,
-  'nil',
-  'null',
-  '',
-  'true',
-  'false',
-  'blank',
-  'empty',
-]
-
-MARKUPS = {
-  "string" => STRING_MARKUPS,
-  "literal" => LITERAL_MARKUPS,
-  "variable" => VARIABLE_MARKUPS,
-  "number" => NUMBER_MARKUPS,
-  "range" => RANGE_MARKUPS,
-}
-
-Benchmark.ips do |x|
-  x.config(time: 5, warmup: 5)
-
-  MARKUPS.each do |type, markups|
-    x.report("Liquid::Expression#parse: #{type}") do
-      markups.each do |markup|
-        Liquid::Expression.parse(markup)
-      end
-    end
-  end
-
-  x.report("Liquid::Expression#parse: all") do
-    MARKUPS.values.flatten.each do |markup|
-      Liquid::Expression.parse(markup)
-    end
-  end
-end

performance/unit/lexer_benchmark.rb

@@ -29,12 +29,31 @@ EXPRESSIONS = [
   "foo | default: -1",
 ]
 
+EXPRESSIONS.each do |expr|
+  lexer_1_result = Liquid::Lexer1.new(expr).tokenize
+  lexer_2_result = Liquid::Lexer2.new(expr).tokenize
+
+  next if lexer_1_result == lexer_2_result
+
+  warn "Lexer1 and Lexer2 results are different for expression: #{expr}"
+  warn "expected: #{lexer_1_result}"
+  warn "got: #{lexer_2_result}"
+  abort
+end
+
 Benchmark.ips do |x|
   x.config(time: 10, warmup: 5)
 
-  x.report("Liquid::Lexer#tokenize") do
+  x.report("Liquid::Lexer1#tokenize") do
     EXPRESSIONS.each do |expr|
-      l = Liquid::Lexer.new(expr)
+      l = Liquid::Lexer1.new(expr)
+      l.tokenize
+    end
+  end
+
+  x.report("Liquid::Lexer2#tokenize") do
+    EXPRESSIONS.each do |expr|
+      l = Liquid::Lexer2.new(expr)
       l.tokenize
     end
   end

performance/unit/loom_benchmark.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "benchmark/ips"
+require 'liquid'
+
+RubyVM::YJIT.enable
+
+TEMPLATE = <<~LIQUID
+{% if false %}
+  {% for i in (1..1000000) %}
+    {{ "Hello world!" }}
+  {% endfor %}
+{% endif %}
+
+{% assign result = 1 %}
+{% if foo == 1 %}{% assign result = 1 %}{% endif %}{% if foo == 2 %}{% assign result = 2 %}{% endif %}{% if foo == 3 %}{% assign result = 3 %}{% endif %}
+Result: {{ result }}
+LIQUID
+
+baseline_template = Liquid::Template.parse(TEMPLATE, eager_optimize: false)
+optimized_template = Liquid::Template.parse(TEMPLATE, eager_optimize: true)
+
+[nil, 1, 2, 3].each do |foo|
+  baseline_output = baseline_template.render('foo' => foo)
+  optimized_output = optimized_template.render('foo' => foo)
+
+  if baseline_output != optimized_output
+    puts "WARNING! Baseline and optimized templates render differently for foo=#{foo}"
+    puts "Baseline: #{baseline_output}"
+    puts "Optimized: #{optimized_output}"
+
+    raise
+  end
+end
+
+def render(template, foo)
+  template.render('foo' => foo)
+end
+
+Benchmark.ips do |x|
+  x.config(time: 20, warmup: 3)
+
+  x.report("baseline") do
+    [nil, 1, 2, 3].each do |foo|
+      render(baseline_template, foo)
+    end
+  end
+
+  x.report("optimized") do
+    [nil, 1, 2, 3].each do |foo|
+      render(optimized_template, foo)
+    end
+  end
+
+  x.compare!
+end

test/integration/expression_test.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require 'test_helper'
-require 'lru_redux'
 
 class ExpressionTest < Minitest::Test
   def test_keyword_literals
@@ -14,7 +13,6 @@ class ExpressionTest < Minitest::Test
     assert_template_result("double quoted", '{{"double quoted"}}')
     assert_template_result("spaced", "{{ 'spaced' }}")
     assert_template_result("spaced2", "{{ 'spaced2' }}")
-    assert_template_result("emoji🔥", "{{ 'emoji🔥' }}")
   end
 
   def test_int
@@ -24,7 +22,6 @@ class ExpressionTest < Minitest::Test
   end
 
   def test_float
-    assert_template_result("-17.42", "{{ -17.42 }}")
     assert_template_result("2.5", "{{ 2.5 }}")
     assert_expression_result(1.5, "1.5")
   end
@@ -43,101 +40,6 @@ class ExpressionTest < Minitest::Test
     )
   end
 
-  def test_quirky_negative_sign_expression_markup
-    result = Expression.parse("-", nil)
-    assert(result.is_a?(VariableLookup))
-    assert_equal("-", result.name)
-
-    # for this template, the expression markup is "-"
-    assert_template_result(
-      "",
-      "{{ - 'theme.css' - }}",
-    )
-  end
-
-  def test_expression_cache
-    skip("Liquid-C does not support Expression caching") if defined?(Liquid::C) && Liquid::C.enabled
-
-    cache = {}
-    template = <<~LIQUID
-      {% assign x = 1 %}
-      {{ x }}
-      {% assign x = 2 %}
-      {{ x }}
-      {% assign y = 1 %}
-      {{ y }}
-    LIQUID
-
-    Liquid::Template.parse(template, expression_cache: cache).render
-
-    assert_equal(
-      ["1", "2", "x", "y"],
-      cache.to_a.map { _1[0] }.sort,
-    )
-  end
-
-  def test_expression_cache_with_true_boolean
-    skip("Liquid-C does not support Expression caching") if defined?(Liquid::C) && Liquid::C.enabled
-
-    template = <<~LIQUID
-      {% assign x = 1 %}
-      {{ x }}
-      {% assign x = 2 %}
-      {{ x }}
-      {% assign y = 1 %}
-      {{ y }}
-    LIQUID
-
-    parse_context = ParseContext.new(expression_cache: true)
-
-    Liquid::Template.parse(template, parse_context).render
-
-    cache = parse_context.instance_variable_get(:@expression_cache)
-
-    assert_equal(
-      ["1", "2", "x", "y"],
-      cache.to_a.map { _1[0] }.sort,
-    )
-  end
-
-  def test_expression_cache_with_lru_redux
-    skip("Liquid-C does not support Expression caching") if defined?(Liquid::C) && Liquid::C.enabled
-
-    cache = LruRedux::Cache.new(10)
-    template = <<~LIQUID
-      {% assign x = 1 %}
-      {{ x }}
-      {% assign x = 2 %}
-      {{ x }}
-      {% assign y = 1 %}
-      {{ y }}
-    LIQUID
-
-    Liquid::Template.parse(template, expression_cache: cache).render
-
-    assert_equal(
-      ["1", "2", "x", "y"],
-      cache.to_a.map { _1[0] }.sort,
-    )
-  end
-
-  def test_disable_expression_cache
-    skip("Liquid-C does not support Expression caching") if defined?(Liquid::C) && Liquid::C.enabled
-
-    template = <<~LIQUID
-      {% assign x = 1 %}
-      {{ x }}
-      {% assign x = 2 %}
-      {{ x }}
-      {% assign y = 1 %}
-      {{ y }}
-    LIQUID
-
-    parse_context = Liquid::ParseContext.new(expression_cache: false)
-    Liquid::Template.parse(template, parse_context).render
-    assert(parse_context.instance_variable_get(:@expression_cache).nil?)
-  end
-
   private
 
   def assert_expression_result(expect, markup, **assigns)

test/integration/parsing_quirks_test.rb

@@ -134,14 +134,6 @@ class ParsingQuirksTest < Minitest::Test
 
   def test_incomplete_expression
     with_error_mode(:lax) do
-      assert_template_result("false", "{{ false - }}")
-      assert_template_result("false", "{{ false > }}")
-      assert_template_result("false", "{{ false < }}")
-      assert_template_result("false", "{{ false = }}")
-      assert_template_result("false", "{{ false ! }}")
-      assert_template_result("false", "{{ false 1 }}")
-      assert_template_result("false", "{{ false a }}")
-
       assert_template_result("false", "{% liquid assign foo = false -\n%}{{ foo }}")
       assert_template_result("false", "{% liquid assign foo = false >\n%}{{ foo }}")
       assert_template_result("false", "{% liquid assign foo = false <\n%}{{ foo }}")

test/integration/tags/cycle_tag_test.rb

@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-require 'test_helper'
-
-class CycleTagTest < Minitest::Test
-  def test_simple_cycle
-    template = <<~LIQUID
-      {%- cycle '1', '2', '3' -%}
-      {%- cycle '1', '2', '3' -%}
-      {%- cycle '1', '2', '3' -%}
-    LIQUID
-
-    assert_template_result("123", template)
-  end
-
-  def test_simple_cycle_inside_for_loop
-    template = <<~LIQUID
-      {%- for i in (1..3) -%}
-        {% cycle '1', '2', '3' %}
-      {%- endfor -%}
-    LIQUID
-
-    assert_template_result("123", template)
-  end
-
-  def test_cycle_with_variables_inside_for_loop
-    template = <<~LIQUID
-      {%- assign a = 1 -%}
-      {%- assign b = 2 -%}
-      {%- assign c = 3 -%}
-      {%- for i in (1..3) -%}
-        {% cycle a, b, c %}
-      {%- endfor -%}
-    LIQUID
-
-    assert_template_result("123", template)
-  end
-
-  def test_cycle_tag_always_resets_cycle
-    template = <<~LIQUID
-      {%- assign a = "1" -%}
-      {%- cycle a, "2" -%}
-      {%- cycle a, "2" -%}
-    LIQUID
-
-    assert_template_result("11", template)
-  end
-end

test/integration/tags/raw_tag_test.rb

@@ -16,7 +16,6 @@ class RawTagTest < Minitest::Test
     assert_template_result('>{{ test }}<', '> {%- raw -%}{{ test }}{%- endraw -%} <')
     assert_template_result("> inner  <", "> {%- raw -%} inner {%- endraw %} <")
     assert_template_result("> inner <", "> {%- raw -%} inner {%- endraw -%} <")
-    assert_template_result("{Hello}", "{% raw %}{{% endraw %}Hello{% raw %}}{% endraw %}")
   end
 
   def test_open_tag_in_raw

test/unit/eager_optimize_test.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class EagerOptimizeTest < Minitest::Test
+  include Liquid
+
+  def test_remove_empty_blocks
+    source = <<~LIQUID.gsub(/\n/, '')
+      {% for i in (1..1000000) %}
+      {% endfor %}
+    LIQUID
+
+    template = Liquid::Template.parse(source, eager_optimize: true)
+    assert_equal(0, total_node_count(template))
+  end
+
+  def test_remove_false_if_block
+    source = <<~LIQUID.gsub(/\n/, '')
+      {% if false %}
+        {% if true %}
+          {% if true %}
+            {% if true %}
+              {{ "Hello world!" }}
+            {% endif %}
+          {% endif %}
+        {% endif %}
+      {% endif %}
+    LIQUID
+
+    template = Liquid::Template.parse(source, eager_optimize: true)
+    assert_equal(0, total_node_count(template))
+
+    source = <<~LIQUID.gsub(/\n/, '')
+      {% if false %}
+        {% for i in (1..1000000) %}
+          {{ "Hello world!" }}
+        {% endfor %}
+      {% endif %}
+    LIQUID
+
+    template = Liquid::Template.parse(source, eager_optimize: true)
+    assert_equal(0, total_node_count(template))
+  end
+
+  def test_remove_multiple_false_if_block
+    source = <<~LIQUID.gsub(/\n/, '')
+      {% if false %}
+        {% if true %}
+          {% if true %}
+            {% if true %}
+              {{ "Hello world!" }}
+            {% endif %}
+          {% endif %}
+        {% endif %}
+      {% endif %}
+    LIQUID
+
+    template = Liquid::Template.parse(source, eager_optimize: true)
+    assert_equal(0, total_node_count(template))
+  end
+
+  def test_merge_if_blocks
+    # for now, work with consecutive if blocks without any String nodes in between
+    source = <<~LIQUID.gsub(/\n/, '')
+      {% if foo == 1 %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if foo == 2 %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if foo == 3 %}
+        foo: {{ foo }}
+      {% endif %}
+    LIQUID
+
+    assert_optimization([Liquid::If], source, { "foo" => nil })
+    assert_optimization([Liquid::If], source, { "foo" => 1 })
+    assert_optimization([Liquid::If], source, { "foo" => 2 })
+    assert_optimization([Liquid::If], source, { "foo" => 5 })
+
+    source = <<~LIQUID.gsub(/\n/, '')
+      {% assign bar = "application" %}
+      {% if foo == 1 %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if foo == 2 and bar contains "app" %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if 3 == foo and bar == "application" %}
+        foo: {{ foo }}
+      {% endif %}
+    LIQUID
+
+    assert_optimization([Liquid::Assign, Liquid::If], source)
+  end
+
+  def test_does_not_merge_if_blocks
+    assert_optimization([Liquid::If, Liquid::If], <<~LIQUID.gsub(/\n/, ''))
+      {% if foo == 1 %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if k == 1 %}
+        foo: {{ foo }}
+      {% endif %}
+    LIQUID
+
+    assert_optimization([Liquid::If, Liquid::If], <<~LIQUID.gsub(/\n/, ''))
+      {% if foo == 1 %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if foo == 1 %}
+        foo: {{ foo }}
+      {% endif %}
+    LIQUID
+
+    assert_optimization([Liquid::If, Liquid::If], <<~LIQUID.gsub(/\n/, ''))
+      {% if foo == 1 %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if a == foo %}
+        foo: {{ foo }}
+      {% endif %}
+    LIQUID
+
+    assert_optimization([Liquid::If, Liquid::If], <<~LIQUID.gsub(/\n/, ''))
+      {% if foo %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if foo %}
+        foo: {{ foo }}
+      {% endif %}
+    LIQUID
+
+    assert_optimization([Liquid::If, Liquid::If], <<~LIQUID.gsub(/\n/, ''))
+      {% if foo == 1 %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if foo >= 1 %}
+        foo: {{ foo }}
+      {% endif %}
+    LIQUID
+
+    assert_optimization([Liquid::If, Liquid::If], <<~LIQUID.gsub(/\n/, ''))
+      {% if foo == 1 %}
+        foo: {{ foo }}
+      {% endif %}
+      {% if 1  %}
+        foo: {{ foo }}
+      {% endif %}
+    LIQUID
+  end
+
+  private
+
+  def assert_optimization(expected, source, context = { "foo" => 1 })
+    template = Template.parse(source, eager_optimize: true)
+    assert_equal(expected, template.root.nodelist.map(&:class),)
+
+    baseline_template = Template.parse(source, eager_optimize: false)
+
+    assert_equal(
+      baseline_template.render(context),
+      template.render(context),
+    )
+  end
+
+  def total_node_count(template)
+    root = template.root
+    children = root.nodelist
+    count = 0
+
+    while children.any?
+      next_children = []
+
+      children.each do |node|
+        count += 1 unless node.is_a?(Liquid::BlockBody)
+        next_children.concat(node.nodelist) if node.respond_to?(:nodelist) && node.nodelist
+      end
+
+      children = next_children
+    end
+
+    count
+  end
+end

test/unit/environment_filter_test.rb

@@ -36,14 +36,14 @@ class EnvironmentFilterTest < Minitest::Test
     assert_equal("public", strainer.invoke("public_filter"))
   end
 
-  def test_strainer_raises_argument_error
+  def test_stainer_raises_argument_error
     strainer = @environment.create_strainer(@context)
     assert_raises(Liquid::ArgumentError) do
       strainer.invoke("public_filter", 1)
     end
   end
 
-  def test_strainer_argument_error_contains_backtrace
+  def test_stainer_argument_error_contains_backtrace
     strainer = @environment.create_strainer(@context)
 
     exception = assert_raises(Liquid::ArgumentError) do
@@ -54,9 +54,8 @@ class EnvironmentFilterTest < Minitest::Test
       /\ALiquid error: wrong number of arguments \((1 for 0|given 1, expected 0)\)\z/,
       exception.message,
     )
-
     source = AccessScopeFilters.instance_method(:public_filter).source_location
-    assert_equal(source[0..1].map(&:to_s), exception.backtrace[0].split(':')[0..1])
+    assert_equal(source.map(&:to_s), exception.backtrace[0].split(':')[0..1])
   end
 
   def test_strainer_only_invokes_public_filter_methods

test/unit/lexer_unit_test.rb

@@ -134,6 +134,6 @@ class LexerUnitTest < Minitest::Test
   private
 
   def tokenize(input)
-    Lexer.tokenize(StringScanner.new(input))
+    Lexer.new(input).tokenize
   end
 end

test/unit/parser_unit_test.rb

@@ -6,20 +6,20 @@ class ParserUnitTest < Minitest::Test
   include Liquid
 
   def test_consume
-    p = new_parser("wat: 7")
+    p = Parser.new("wat: 7")
     assert_equal('wat', p.consume(:id))
     assert_equal(':', p.consume(:colon))
     assert_equal('7', p.consume(:number))
   end
 
   def test_jump
-    p = new_parser("wat: 7")
+    p = Parser.new("wat: 7")
     p.jump(2)
     assert_equal('7', p.consume(:number))
   end
 
   def test_consume?
-    p = new_parser("wat: 7")
+    p = Parser.new("wat: 7")
     assert_equal('wat', p.consume?(:id))
     assert_equal(false, p.consume?(:dot))
     assert_equal(':', p.consume(:colon))
@@ -27,7 +27,7 @@ class ParserUnitTest < Minitest::Test
   end
 
   def test_id?
-    p = new_parser("wat 6 Peter Hegemon")
+    p = Parser.new("wat 6 Peter Hegemon")
     assert_equal('wat', p.id?('wat'))
     assert_equal(false, p.id?('endgame'))
     assert_equal('6', p.consume(:number))
@@ -36,7 +36,7 @@ class ParserUnitTest < Minitest::Test
   end
 
   def test_look
-    p = new_parser("wat 6 Peter Hegemon")
+    p = Parser.new("wat 6 Peter Hegemon")
     assert_equal(true, p.look(:id))
     assert_equal('wat', p.consume(:id))
     assert_equal(false, p.look(:comparison))
@@ -46,12 +46,12 @@ class ParserUnitTest < Minitest::Test
   end
 
   def test_expressions
-    p = new_parser("hi.there hi?[5].there? hi.there.bob")
+    p = Parser.new("hi.there hi?[5].there? hi.there.bob")
     assert_equal('hi.there', p.expression)
     assert_equal('hi?[5].there?', p.expression)
     assert_equal('hi.there.bob', p.expression)
 
-    p = new_parser("567 6.0 'lol' \"wut\"")
+    p = Parser.new("567 6.0 'lol' \"wut\"")
     assert_equal('567', p.expression)
     assert_equal('6.0', p.expression)
     assert_equal("'lol'", p.expression)
@@ -59,7 +59,7 @@ class ParserUnitTest < Minitest::Test
   end
 
   def test_ranges
-    p = new_parser("(5..7) (1.5..9.6) (young..old) (hi[5].wat..old)")
+    p = Parser.new("(5..7) (1.5..9.6) (young..old) (hi[5].wat..old)")
     assert_equal('(5..7)', p.expression)
     assert_equal('(1.5..9.6)', p.expression)
     assert_equal('(young..old)', p.expression)
@@ -67,7 +67,7 @@ class ParserUnitTest < Minitest::Test
   end
 
   def test_arguments
-    p = new_parser("filter: hi.there[5], keyarg: 7")
+    p = Parser.new("filter: hi.there[5], keyarg: 7")
     assert_equal('filter', p.consume(:id))
     assert_equal(':', p.consume(:colon))
     assert_equal('hi.there[5]', p.argument)
@@ -77,14 +77,8 @@ class ParserUnitTest < Minitest::Test
 
   def test_invalid_expression
     assert_raises(SyntaxError) do
-      p = new_parser("==")
+      p = Parser.new("==")
       p.expression
     end
   end
-
-  private
-
-  def new_parser(str)
-    Parser.new(StringScanner.new(str))
-  end
 end

test/unit/tag_unit_test.rb

@@ -6,18 +6,18 @@ class TagUnitTest < Minitest::Test
   include Liquid
 
   def test_tag
-    tag = Tag.parse('tag', "", new_tokenizer, ParseContext.new)
+    tag = Tag.parse('tag', "", Tokenizer.new(""), ParseContext.new)
     assert_equal('liquid::tag', tag.name)
     assert_equal('', tag.render(Context.new))
   end
 
   def test_return_raw_text_of_tag
-    tag = Tag.parse("long_tag", "param1, param2, param3", new_tokenizer, ParseContext.new)
+    tag = Tag.parse("long_tag", "param1, param2, param3", Tokenizer.new(""), ParseContext.new)
     assert_equal("long_tag param1, param2, param3", tag.raw)
   end
 
   def test_tag_name_should_return_name_of_the_tag
-    tag = Tag.parse("some_tag", "", new_tokenizer, ParseContext.new)
+    tag = Tag.parse("some_tag", "", Tokenizer.new(""), ParseContext.new)
     assert_equal('some_tag', tag.tag_name)
   end
 
@@ -26,16 +26,7 @@ class TagUnitTest < Minitest::Test
   end
 
   def test_tag_render_to_output_buffer_nil_value
-    custom_tag = CustomTag.parse("some_tag", "", new_tokenizer, ParseContext.new)
+    custom_tag = CustomTag.parse("some_tag", "", Tokenizer.new(""), ParseContext.new)
     assert_equal('some string', custom_tag.render_to_output_buffer(Context.new, "some string"))
   end
-
-  private
-
-  def new_tokenizer
-    Tokenizer.new(
-      source: "",
-      string_scanner: StringScanner.new(""),
-    )
-  end
 end

test/unit/tokenizer_unit_test.rb

@@ -6,7 +6,6 @@ class TokenizerTest < Minitest::Test
   def test_tokenize_strings
     assert_equal([' '], tokenize(' '))
     assert_equal(['hello world'], tokenize('hello world'))
-    assert_equal(['{}'], tokenize('{}'))
   end
 
   def test_tokenize_variables
@@ -31,23 +30,6 @@ class TokenizerTest < Minitest::Test
     assert_equal([1, 1, 3], tokenize_line_numbers(" {{\n funk \n}} "))
   end
 
-  def test_tokenize_with_nil_source_returns_empty_array
-    assert_equal([], tokenize(nil))
-  end
-
-  def test_incomplete_curly_braces
-    assert_equal(["{{.}", " "], tokenize('{{.} '))
-    assert_equal(["{{}", "%}"], tokenize('{{}%}'))
-    assert_equal(["{{}}", "}"], tokenize('{{}}}'))
-  end
-
-  def test_unmatching_start_and_end
-    assert_equal(["{{%}"], tokenize('{{%}'))
-    assert_equal(["{{%%%}}"], tokenize('{{%%%}}'))
-    assert_equal(["{%", "}}"], tokenize('{%}}'))
-    assert_equal(["{%%}", "}"], tokenize('{%%}}'))
-  end
-
   private
 
   def new_tokenizer(source, parse_context: Liquid::ParseContext.new, start_line_number: nil)