fix lexer parsing comparison without whitespaces

Faster Variable BlockBody Matching

lib/liquid/block_body.rb

@@ -246,10 +246,17 @@ module Liquid
     end
 
     def create_variable(token, parse_context)
-      if token =~ ContentOfVariable
-        markup = Regexp.last_match(1)
+      if token.end_with?("}}")
+        i = 2
+        i = 3 if token[i] == "-"
+        parse_end = token.length - 3
+        parse_end -= 1 if token[parse_end] == "-"
+        markup_end = parse_end - i + 1
+        markup = markup_end <= 0 ? "" : token.byteslice(i, markup_end)
+
         return Variable.new(markup, parse_context)
       end
+
       BlockBody.raise_missing_variable_terminator(token, parse_context)
     end
 

lib/liquid/lexer.rb

@@ -90,7 +90,12 @@ module Liquid
     SINGLE_STRING_LITERAL = /'[^\']*'/
     WHITESPACE_OR_NOTHING = /\s*/
 
-    COMPARISON_JUMP_TABLE = [].tap do |table|
+    SINGLE_COMPARISON_TOKENS = [].tap do |table|
+      table["<".ord] = COMPARISON_LESS_THAN
+      table[">".ord] = COMPARISON_GREATER_THAN
+    end
+
+    TWO_CHARS_COMPARISON_JUMP_TABLE = [].tap do |table|
       table["=".ord] = [].tap do |sub_table|
         sub_table["=".ord] = COMPARISON_EQUAL
         sub_table.freeze
@@ -99,6 +104,9 @@ module Liquid
         sub_table["=".ord] = COMPARISION_NOT_EQUAL
         sub_table.freeze
       end
+    end
+
+    COMPARISON_JUMP_TABLE = [].tap do |table|
       table["<".ord] = [].tap do |sub_table|
         sub_table["=".ord] = COMPARISON_LESS_THAN_OR_EQUAL
         sub_table[">".ord] = COMPARISON_NOT_EQUAL_ALT
@@ -182,7 +190,7 @@ module Liquid
           else
             @output << special
           end
-        elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
+        elsif (sub_table = TWO_CHARS_COMPARISON_JUMP_TABLE[peeked])
           @ss.scan_byte
           if (found = sub_table[@ss.peek_byte])
             @output << found
@@ -190,6 +198,14 @@ module Liquid
           else
             raise SyntaxError, "Unexpected character #{peeked.chr}"
           end
+        elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
+          @ss.scan_byte
+          if (found = sub_table[@ss.peek_byte])
+            @output << found
+            @ss.scan_byte
+          else
+            @output << SINGLE_COMPARISON_TOKENS[peeked]
+          end
         else
           type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked]
 

lib/liquid/standardfilters.rb

@@ -877,7 +877,7 @@ module Liquid
     #   - [`nil`](/docs/api/liquid/basics#nil)
     # @liquid_syntax variable | default: variable
     # @liquid_return [untyped]
-    # @liquid_optional_param allow_false [boolean] Whether to use false values instead of the default.
+    # @liquid_optional_param allow_false: [boolean] Whether to use false values instead of the default.
     def default(input, default_value = '', options = {})
       options = {} unless options.is_a?(Hash)
       false_check = options['allow_false'] ? input.nil? : !Liquid::Utils.to_liquid_value(input)

lib/liquid/tokenizer.rb

@@ -1,40 +1,23 @@
 # frozen_string_literal: true
 
-require "strscan"
-
 module Liquid
   class Tokenizer
     attr_reader :line_number, :for_liquid_tag
 
-    TAG_END = /%\}/
-    TAG_OR_VARIABLE_START = /\{[\{\%}]/
-    NEWLINE = /\n/
-
-    OPEN_CURLEY = 123
-    CLOSE_CURLEY = 125
-    PERCENTAGE = 37
-
     def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
-      @line_number = line_number || (line_numbers ? 1 : nil)
-
+      @source         = source
+      @line_number    = line_number || (line_numbers ? 1 : nil)
       @for_liquid_tag = for_liquid_tag
-      @tokens = []
-      tokenize(source)
-    end
-
-    private def tokenize(source)
-      ss = StringScanner.new(source)
-
-      while token = t_shift(ss)
-        @tokens.push(token)
-      end
+      @offset         = 0
+      @tokens         = tokenize
     end
 
     def shift
-      token = @tokens.shift
-
+      token = @tokens[@offset]
       return nil unless token
 
+      @offset += 1
+
       if @line_number
         @line_number += @for_liquid_tag ? 1 : token.count("\n")
       end
@@ -42,105 +25,21 @@ module Liquid
       token
     end
 
-    private def t_shift(ss)
-      return nil if ss.eos?
-
-      @for_liquid_tag ? next_liquid_token(ss) : next_token(ss)
-    end
-
     private
 
-    def next_liquid_token(ss)
-      # read until we find a \n
-      start = ss.pos
-      if ss.scan_until(NEWLINE).nil?
-        token = ss.rest
-        ss.terminate
-        return token
+    def tokenize
+      return [] if @source.empty?
+
+      return @source.split("\n") if @for_liquid_tag
+
+      tokens = @source.split(TemplateParser)
+
+      # removes the rogue empty element at the beginning of the array
+      if tokens[0]&.empty?
+        @offset += 1
       end
 
-      ss.string.byteslice(start, ss.pos - start - 1)
-    end
-
-    def next_token(ss)
-      if ss.pos == 0
-        if ss.string.start_with?("{{")
-          ss.pos = 2
-          return next_variable_token(ss)
-        elsif ss.string.start_with?("{%")
-          ss.pos = 2
-          return next_tag_token(ss)
-        else
-          return next_text_token(ss)
-        end
-      end
-
-      # possible states: :text, :tag, :variable
-      byte_a = ss.scan_byte
-
-      if byte_a == OPEN_CURLEY
-        byte_b = ss.scan_byte
-
-        if byte_b == PERCENTAGE
-          return next_tag_token(ss)
-        elsif byte_b == OPEN_CURLEY
-          return next_variable_token(ss)
-        end
-
-        ss.pos -= 1
-      end
-
-      ss.pos -= 1
-      next_text_token(ss)
-    end
-
-    def next_text_token(ss)
-      start = ss.pos
-
-      unless ss.skip_until(TAG_OR_VARIABLE_START)
-        token = ss.rest
-        ss.terminate
-        return token
-      end
-
-      ss.pos -= 2
-      ss.string.byteslice(start, ss.pos - start)
-    end
-
-    def next_variable_token(ss)
-      start = ss.pos - 2
-
-      # it is possible to see a {% before a }} so we need to check for that
-      byte_a = ss.scan_byte
-
-      until ss.eos?
-        while ss.eos? == false && byte_a != CLOSE_CURLEY && byte_a != OPEN_CURLEY
-          byte_a = ss.scan_byte
-        end
-
-        break if ss.eos?
-
-        byte_b = ss.scan_byte
-
-        if byte_b != CLOSE_CURLEY && byte_b != PERCENTAGE
-          byte_a = byte_b
-          next
-        elsif byte_a == CLOSE_CURLEY && byte_b == CLOSE_CURLEY
-          return ss.string.byteslice(start, ss.pos - start)
-        elsif byte_a == OPEN_CURLEY && byte_b == PERCENTAGE
-          return next_tag_token(ss, start)
-        end
-      end
-
-      return "{{"
-    end
-
-    def next_tag_token(ss, start = nil)
-      start ||= ss.pos - 2
-
-      ss.scan_until(TAG_END)
-
-      ss.string.byteslice(start, ss.pos - start)
+      tokens
     end
   end
 end

test/unit/lexer_unit_test.rb

@@ -25,6 +25,30 @@ class LexerUnitTest < Minitest::Test
     assert_equal([[:comparison, '=='], [:comparison, '<>'], [:comparison, 'contains'], [:end_of_string]], tokens)
   end
 
+  def test_comparison_without_whitespace
+    tokens = Lexer.new('1>0').tokenize
+    assert_equal([[:number, '1'], [:comparison, '>'], [:number, '0'], [:end_of_string]], tokens)
+  end
+
+  def test_comparison_with_negative_number
+    tokens = Lexer.new('1>-1').tokenize
+    assert_equal([[:number, '1'], [:comparison, '>'], [:number, '-1'], [:end_of_string]], tokens)
+  end
+
+  def test_raise_for_invalid_comparison
+    assert_raises(SyntaxError) do
+      Lexer.new('1>!1').tokenize
+    end
+
+    assert_raises(SyntaxError) do
+      Lexer.new('1=<1').tokenize
+    end
+
+    assert_raises(SyntaxError) do
+      Lexer.new('1!!1').tokenize
+    end
+  end
+
   def test_specials
     tokens = Lexer.new('| .:').tokenize
     assert_equal([[:pipe, '|'], [:dot, '.'], [:colon, ':'], [:end_of_string]], tokens)