fix lexer parsing ID 'contains' as comparison

raise syntax error from lexer parser with UTF-8 character

lib/liquid/lexer.rb

@@ -73,7 +73,6 @@ module Liquid
     COMPARISON_LESS_THAN = [:comparison, "<"].freeze
     COMPARISON_LESS_THAN_OR_EQUAL = [:comparison, "<="].freeze
     COMPARISON_NOT_EQUAL_ALT = [:comparison, "<>"].freeze
-    CONTAINS = /contains(?=\s)/
     DASH = [:dash, "-"].freeze
     DOT = [:dot, "."].freeze
     DOTDOT = [:dotdot, ".."].freeze
@@ -90,7 +89,12 @@ module Liquid
     SINGLE_STRING_LITERAL = /'[^\']*'/
     WHITESPACE_OR_NOTHING = /\s*/
 
-    COMPARISON_JUMP_TABLE = [].tap do |table|
+    SINGLE_COMPARISON_TOKENS = [].tap do |table|
+      table["<".ord] = COMPARISON_LESS_THAN
+      table[">".ord] = COMPARISON_GREATER_THAN
+    end
+
+    TWO_CHARS_COMPARISON_JUMP_TABLE = [].tap do |table|
       table["=".ord] = [].tap do |sub_table|
         sub_table["=".ord] = COMPARISON_EQUAL
         sub_table.freeze
@@ -99,6 +103,9 @@ module Liquid
         sub_table["=".ord] = COMPARISION_NOT_EQUAL
         sub_table.freeze
       end
+    end
+
+    COMPARISON_JUMP_TABLE = [].tap do |table|
       table["<".ord] = [].tap do |sub_table|
         sub_table["=".ord] = COMPARISON_LESS_THAN_OR_EQUAL
         sub_table[">".ord] = COMPARISON_NOT_EQUAL_ALT
@@ -163,6 +170,7 @@ module Liquid
 
         break if @ss.eos?
 
+        start_pos = @ss.pos
         peeked = @ss.peek_byte
 
         if (special = SPECIAL_TABLE[peeked])
@@ -182,26 +190,34 @@ module Liquid
           else
             @output << special
           end
+        elsif (sub_table = TWO_CHARS_COMPARISON_JUMP_TABLE[peeked])
+          @ss.scan_byte
+          if (found = sub_table[@ss.peek_byte])
+            @output << found
+            @ss.scan_byte
+          else
+            raise_syntax_error(start_pos)
+          end
         elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
           @ss.scan_byte
           if (found = sub_table[@ss.peek_byte])
             @output << found
             @ss.scan_byte
           else
-            raise SyntaxError, "Unexpected character #{peeked.chr}"
+            @output << SINGLE_COMPARISON_TOKENS[peeked]
           end
         else
           type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked]
 
           if type && (t = @ss.scan(pattern))
             # Special case for "contains"
-            @output << if type == :id && t == "contains"
+            @output << if type == :id && t == "contains" && @output.last&.first != :dot
               COMPARISON_CONTAINS
             else
               [type, t]
             end
           else
-            raise SyntaxError, "Unexpected character #{peeked.chr}"
+            raise_syntax_error(start_pos)
           end
         end
       end
@@ -209,6 +225,12 @@ module Liquid
 
       @output << EOS
     end
+
+    def raise_syntax_error(start_pos)
+      @ss.pos = start_pos
+      # the character could be a UTF-8 character, use getch to get all the bytes
+      raise SyntaxError, "Unexpected character #{@ss.getch}"
+    end
   end
 
   Lexer = StringScanner.instance_methods.include?(:scan_byte) ? Lexer2 : Lexer1

test/unit/lexer_unit_test.rb

@@ -25,6 +25,30 @@ class LexerUnitTest < Minitest::Test
     assert_equal([[:comparison, '=='], [:comparison, '<>'], [:comparison, 'contains'], [:end_of_string]], tokens)
   end
 
+  def test_comparison_without_whitespace
+    tokens = Lexer.new('1>0').tokenize
+    assert_equal([[:number, '1'], [:comparison, '>'], [:number, '0'], [:end_of_string]], tokens)
+  end
+
+  def test_comparison_with_negative_number
+    tokens = Lexer.new('1>-1').tokenize
+    assert_equal([[:number, '1'], [:comparison, '>'], [:number, '-1'], [:end_of_string]], tokens)
+  end
+
+  def test_raise_for_invalid_comparison
+    assert_raises(SyntaxError) do
+      Lexer.new('1>!1').tokenize
+    end
+
+    assert_raises(SyntaxError) do
+      Lexer.new('1=<1').tokenize
+    end
+
+    assert_raises(SyntaxError) do
+      Lexer.new('1!!1').tokenize
+    end
+  end
+
   def test_specials
     tokens = Lexer.new('| .:').tokenize
     assert_equal([[:pipe, '|'], [:dot, '.'], [:colon, ':'], [:end_of_string]], tokens)
@@ -60,4 +84,22 @@ class LexerUnitTest < Minitest::Test
     tokens = Lexer.new("foo > 12").tokenize
     assert_equal([[:id, 'foo'], [:comparison, '>'], [:number, '12'], [:end_of_string]], tokens)
   end
+
+  def test_error_with_utf8_character
+    error = assert_raises(SyntaxError) do
+      Lexer.new("1 < 1Ø").tokenize
+    end
+
+    assert_equal(
+      'Liquid syntax error: Unexpected character Ø',
+      error.message,
+    )
+  end
+
+  def test_contains_as_attribute_name
+    assert_equal(
+      [[:id, "a"], [:dot, "."], [:id, "contains"], [:dot, "."], [:id, "b"], [:end_of_string]],
+      Lexer.new("a.contains.b").tokenize,
+    )
+  end
 end