fix lexer parsing comparison without whitespaces

lib/liquid/block_body.rb

@@ -252,7 +252,7 @@ module Liquid
         parse_end = token.length - 3
         parse_end -= 1 if token[parse_end] == "-"
         markup_end = parse_end - i + 1
-        markup = markup_end <= 0 ? "" : token.slice(i, markup_end)
+        markup = markup_end <= 0 ? "" : token.byteslice(i, markup_end)
 
         return Variable.new(markup, parse_context)
       end

lib/liquid/lexer.rb

@@ -90,7 +90,12 @@ module Liquid
     SINGLE_STRING_LITERAL = /'[^\']*'/
     WHITESPACE_OR_NOTHING = /\s*/
 
-    COMPARISON_JUMP_TABLE = [].tap do |table|
+    SINGLE_COMPARISON_TOKENS = [].tap do |table|
+      table["<".ord] = COMPARISON_LESS_THAN
+      table[">".ord] = COMPARISON_GREATER_THAN
+    end
+
+    TWO_CHARS_COMPARISON_JUMP_TABLE = [].tap do |table|
       table["=".ord] = [].tap do |sub_table|
         sub_table["=".ord] = COMPARISON_EQUAL
         sub_table.freeze
@@ -99,6 +104,9 @@ module Liquid
         sub_table["=".ord] = COMPARISION_NOT_EQUAL
         sub_table.freeze
       end
+    end
+
+    COMPARISON_JUMP_TABLE = [].tap do |table|
       table["<".ord] = [].tap do |sub_table|
         sub_table["=".ord] = COMPARISON_LESS_THAN_OR_EQUAL
         sub_table[">".ord] = COMPARISON_NOT_EQUAL_ALT
@@ -182,7 +190,7 @@ module Liquid
           else
             @output << special
           end
-        elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
+        elsif (sub_table = TWO_CHARS_COMPARISON_JUMP_TABLE[peeked])
           @ss.scan_byte
           if (found = sub_table[@ss.peek_byte])
             @output << found
@@ -190,6 +198,14 @@ module Liquid
           else
             raise SyntaxError, "Unexpected character #{peeked.chr}"
           end
+        elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
+          @ss.scan_byte
+          if (found = sub_table[@ss.peek_byte])
+            @output << found
+            @ss.scan_byte
+          else
+            @output << SINGLE_COMPARISON_TOKENS[peeked]
+          end
         else
           type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked]
 

test/unit/block_unit_test.rb

@@ -32,12 +32,6 @@ class BlockUnitTest < Minitest::Test
     assert_equal(String, template.root.nodelist[2].class)
   end
 
-  def test_variable_with_multibyte_character
-    template = Liquid::Template.parse("{{ '❤️' }}")
-    assert_equal(1, template.root.nodelist.size)
-    assert_equal(Variable, template.root.nodelist[0].class)
-  end
-
   def test_variable_many_embedded_fragments
     template = Liquid::Template.parse("  {{funk}} {{so}} {{brother}} ")
     assert_equal(7, template.root.nodelist.size)

test/unit/lexer_unit_test.rb

@@ -25,6 +25,30 @@ class LexerUnitTest < Minitest::Test
     assert_equal([[:comparison, '=='], [:comparison, '<>'], [:comparison, 'contains'], [:end_of_string]], tokens)
   end
 
+  def test_comparison_without_whitespace
+    tokens = Lexer.new('1>0').tokenize
+    assert_equal([[:number, '1'], [:comparison, '>'], [:number, '0'], [:end_of_string]], tokens)
+  end
+
+  def test_comparison_with_negative_number
+    tokens = Lexer.new('1>-1').tokenize
+    assert_equal([[:number, '1'], [:comparison, '>'], [:number, '-1'], [:end_of_string]], tokens)
+  end
+
+  def test_raise_for_invalid_comparison
+    assert_raises(SyntaxError) do
+      Lexer.new('1>!1').tokenize
+    end
+
+    assert_raises(SyntaxError) do
+      Lexer.new('1=<1').tokenize
+    end
+
+    assert_raises(SyntaxError) do
+      Lexer.new('1!!1').tokenize
+    end
+  end
+
   def test_specials
     tokens = Lexer.new('| .:').tokenize
     assert_equal([[:pipe, '|'], [:dot, '.'], [:colon, ':'], [:end_of_string]], tokens)