raise syntax error from lexer parser with utf8 character

lib/liquid/lexer.rb

@@ -171,6 +171,7 @@ module Liquid
 
         break if @ss.eos?
 
+        start_pos = @ss.pos
         peeked = @ss.peek_byte
 
         if (special = SPECIAL_TABLE[peeked])
@@ -196,7 +197,7 @@ module Liquid
             @output << found
             @ss.scan_byte
           else
-            raise SyntaxError, "Unexpected character #{peeked.chr}"
+            raise_syntax_error(start_pos)
           end
         elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
           @ss.scan_byte
@@ -217,7 +218,7 @@ module Liquid
               [type, t]
             end
           else
-            raise SyntaxError, "Unexpected character #{peeked.chr}"
+            raise_syntax_error(start_pos)
           end
         end
       end
@@ -225,6 +226,12 @@ module Liquid
 
       @output << EOS
     end
+
+    def raise_syntax_error(start_pos)
+      @ss.pos = start_pos
+      # the character could be a UTF-8 character, use getch to get all the bytes
+      raise SyntaxError, "Unexpected character #{@ss.getch}"
+    end
   end
 
   Lexer = StringScanner.instance_methods.include?(:scan_byte) ? Lexer2 : Lexer1

test/unit/lexer_unit_test.rb

@@ -84,4 +84,15 @@ class LexerUnitTest < Minitest::Test
     tokens = Lexer.new("foo > 12").tokenize
     assert_equal([[:id, 'foo'], [:comparison, '>'], [:number, '12'], [:end_of_string]], tokens)
   end
+
+  def test_error_with_utf8_character
+    error = assert_raises(SyntaxError) do
+      Lexer.new("1 < 1Ø").tokenize
+    end
+
+    assert_equal(
+      'Liquid syntax error: Unexpected character Ø',
+      error.message,
+    )
+  end
 end