diff --git a/lib/liquid/tokenizer.rb b/lib/liquid/tokenizer.rb index 59b4c47e..5bbceaa7 100644 --- a/lib/liquid/tokenizer.rb +++ b/lib/liquid/tokenizer.rb @@ -146,24 +146,18 @@ module Liquid pos = idx + 2 end else - # { followed by something else — it's text - # Keep scanning from after this { - # Find next { that could be {% or {{ - next_open = idx + 1 - while next_open < len - ni = src.byteindex('{', next_open) - unless ni - @tokens << src.byteslice(pos, len - pos) - pos = len - break - end - nb = ni + 1 < len ? src.getbyte(ni + 1) : nil - if nb == PERCENTAGE || nb == OPEN_CURLEY - @tokens << src.byteslice(pos, ni - pos) - pos = ni - break - end - next_open = ni + 1 + # Lone '{' — not the start of a tag or variable. + # Find the next '{{' or '{%' to know where this text token ends. + # Using two byteindex calls avoids a nested loop and is always O(n). + tag_start = src.byteindex('{%', idx + 1) + var_start = src.byteindex('{{', idx + 1) + next_token = [tag_start, var_start].compact.min + if next_token + @tokens << src.byteslice(pos, next_token - pos) + pos = next_token + else + @tokens << src.byteslice(pos, len - pos) + pos = len end end end diff --git a/test/unit/tokenizer_unit_test.rb b/test/unit/tokenizer_unit_test.rb index 76d379d3..b3da5e4d 100644 --- a/test/unit/tokenizer_unit_test.rb +++ b/test/unit/tokenizer_unit_test.rb @@ -48,6 +48,33 @@ class TokenizerTest < Minitest::Test assert_equal(["{%%}", "}"], tokenize('{%%}}')) end + # Regression: lone '{' at or near end of string previously caused an infinite + # loop. The stray-{ else branch left `pos` unchanged when no further '{{' or + # '{%' existed, so the outer loop found the same '{' on every iteration. + def test_lone_brace_does_not_loop + assert_equal(["{"], tokenize('{')) + assert_equal(["a{"], tokenize('a{')) + assert_equal(["hello { world {"], tokenize('hello { world {')) + assert_equal(["{ world"], tokenize('{ world')) + assert_equal(["x{y"], tokenize('x{y')) + assert_equal(["{b{c"], tokenize('{b{c')) + end + + def test_lone_brace_before_real_token + assert_equal( + ["a { b ", "{% if x %}", "yes", "{% endif %}", " c"], + tokenize('a { b {% if x %}yes{% endif %} c'), + ) + assert_equal( + ["x { ", "{{ var }}", " y"], + tokenize('x { {{ var }} y'), + ) + assert_equal( + ["{ ", "{{ var }}"], + tokenize('{ {{ var }}'), + ) + end + private def new_tokenizer(source, parse_context: Liquid::ParseContext.new, start_line_number: nil)