diff --git a/lib/liquid/tokenizer.rb b/lib/liquid/tokenizer.rb index bc33adcb..bc26814b 100644 --- a/lib/liquid/tokenizer.rb +++ b/lib/liquid/tokenizer.rb @@ -14,9 +14,6 @@ module Liquid CLOSE_CURLEY = "}".ord PERCENTAGE = "%".ord - CLOSE_CURLEY_FOLLOWED_BY_CLOSE_CURLEY = (CLOSE_CURLEY << 8) | CLOSE_CURLEY - OPEN_CURLEY_FOLLOWED_BY_PERCENTAGE = (OPEN_CURLEY << 8) | PERCENTAGE - def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false) @line_number = line_number || (line_numbers ? 1 : nil) @for_liquid_tag = for_liquid_tag @@ -107,7 +104,6 @@ module Liquid def next_variable_token start = @ss.pos - 2 - # it is possible to see a {% before a }} so we need to check for that byte_a = @ss.scan_byte byte_b = byte_a @@ -116,19 +112,28 @@ module Liquid break unless byte_a + if @ss.eos? + if byte_a == CLOSE_CURLEY + return @source.byteslice(start, @ss.pos - start) + else + break + end + end + byte_b = @ss.scan_byte - if byte_b > CLOSE_CURLEY || (byte_b != CLOSE_CURLEY && byte_b != PERCENTAGE) - byte_a = byte_b - next - end - - val = (byte_a << 8) | byte_b - if val == CLOSE_CURLEY_FOLLOWED_BY_CLOSE_CURLEY - return @source.byteslice(start, @ss.pos - start) - elsif val == OPEN_CURLEY_FOLLOWED_BY_PERCENTAGE + if byte_a == CLOSE_CURLEY + if byte_b == CLOSE_CURLEY + return @source.byteslice(start, @ss.pos - start) + elsif byte_b != CLOSE_CURLEY + @ss.pos -= 1 + return @source.byteslice(start, @ss.pos - start) + end + elsif byte_a == OPEN_CURLEY && byte_b == PERCENTAGE return next_tag_token_with_start(start) end + + byte_a = byte_b end "{{" diff --git a/test/integration/tags/raw_tag_test.rb b/test/integration/tags/raw_tag_test.rb index 3629f39c..2ca43d3b 100644 --- a/test/integration/tags/raw_tag_test.rb +++ b/test/integration/tags/raw_tag_test.rb @@ -16,6 +16,7 @@ class RawTagTest < Minitest::Test assert_template_result('>{{ test }}<', '> {%- raw -%}{{ test }}{%- endraw -%} <') assert_template_result("> inner <", "> {%- raw -%} inner {%- endraw %} <") assert_template_result("> inner <", "> {%- raw -%} inner {%- endraw -%} <") + assert_template_result("{Hello}", "{% raw %}{{% endraw %}Hello{% raw %}}{% endraw %}") end def test_open_tag_in_raw diff --git a/test/unit/tokenizer_unit_test.rb b/test/unit/tokenizer_unit_test.rb index a50b76ce..5a881afe 100644 --- a/test/unit/tokenizer_unit_test.rb +++ b/test/unit/tokenizer_unit_test.rb @@ -31,6 +31,19 @@ class TokenizerTest < Minitest::Test assert_equal([1, 1, 3], tokenize_line_numbers(" {{\n funk \n}} ")) end + def test_incomplete_curly_braces + assert_equal(["{{.}", " "], tokenize('{{.} ')) + assert_equal(["{{}", "%}"], tokenize('{{}%}')) + assert_equal(["{{}}", "}"], tokenize('{{}}}')) + end + + def test_unmatching_start_and_end + assert_equal(["{{%}"], tokenize('{{%}')) + assert_equal(["{{%%%}}"], tokenize('{{%%%}}')) + assert_equal(["{%", "}}"], tokenize('{%}}')) + assert_equal(["{%%}", "}"], tokenize('{%%}}')) + end + private def new_tokenizer(source, parse_context: Liquid::ParseContext.new, start_line_number: nil)