diff --git a/History.md b/History.md index fe1e3819..c03e2d95 100644 --- a/History.md +++ b/History.md @@ -2,6 +2,8 @@ ## 5.7.3 (unreleased) +* Raise Liquid::SyntaxError when parsing invalidly encoded strings + ## 5.7.2 2025-01-31 * Fix array filters to not support nested properties diff --git a/lib/liquid/lexer.rb b/lib/liquid/lexer.rb index b28e8784..f1740dba 100644 --- a/lib/liquid/lexer.rb +++ b/lib/liquid/lexer.rb @@ -161,6 +161,12 @@ module Liquid end # rubocop:enable Metrics/BlockNesting output << EOS + rescue ::ArgumentError => e + if e.message == "invalid byte sequence in #{ss.string.encoding}" + raise SyntaxError, "Invalid byte sequence in #{ss.string.encoding}" + else + raise + end end def raise_syntax_error(start_pos, ss) diff --git a/lib/liquid/tokenizer.rb b/lib/liquid/tokenizer.rb index 008ec070..56dd218e 100644 --- a/lib/liquid/tokenizer.rb +++ b/lib/liquid/tokenizer.rb @@ -103,6 +103,12 @@ module Liquid pos = @ss.pos -= 2 @source.byteslice(start, pos - start) + rescue ::ArgumentError => e + if e.message == "invalid byte sequence in #{@ss.string.encoding}" + raise SyntaxError, "Invalid byte sequence in #{@ss.string.encoding}" + else + raise + end end def next_variable_token diff --git a/test/unit/lexer_unit_test.rb b/test/unit/lexer_unit_test.rb index 18e1e439..73eeb739 100644 --- a/test/unit/lexer_unit_test.rb +++ b/test/unit/lexer_unit_test.rb @@ -131,6 +131,16 @@ class LexerUnitTest < Minitest::Test assert_equal([[:id, "false"], [:number, "1"], [:end_of_string]], tokenize("false 1")) end + def test_error_with_invalid_utf8 + error = assert_raises(SyntaxError) do + tokenize("\x00\xff") + end + assert_equal( + 'Liquid syntax error: Invalid byte sequence in UTF-8', + error.message, + ) + end + private def tokenize(input) diff --git a/test/unit/template_unit_test.rb b/test/unit/template_unit_test.rb index e9de2d04..1f349657 100644 --- a/test/unit/template_unit_test.rb +++ b/test/unit/template_unit_test.rb @@ -35,4 +35,15 @@ class TemplateUnitTest < Minitest::Test def test_template_inheritance assert_equal("foo", TemplateSubclass.parse("foo").render) end + + def test_invalid_utf8 + input = "\xff\x00" + error = assert_raises(SyntaxError) do + Liquid::Tokenizer.new(source: input, string_scanner: StringScanner.new(input)) + end + assert_equal( + 'Liquid syntax error: Invalid byte sequence in UTF-8', + error.message, + ) + end end