mirror of
https://github.com/Shopify/liquid.git
synced 2026-09-12 23:40:45 -07:00
Chris Pak
d9c42fd2eb
The stray-{ else branch in tokenize_fast had a nested while loop that could
exit via its condition (when next_open >= len) without advancing pos.
The outer while pos < len loop would then find the same { again forever.
Reproduced by: Liquid::Template.parse('a{') -- hangs indefinitely.
Replaces the nested scan loop with two String#byteindex calls to find the
next '{%' and '{{' directly, then takes the minimum. Always O(n), eliminates
the nested loop entirely, and impossible to leave pos stranded.
Adds regression tests covering the three inputs that previously hung
plus adjacent stray-brace cases.
108 lines
3.5 KiB
Ruby
108 lines
3.5 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require 'test_helper'
|
|
|
|
class TokenizerTest < Minitest::Test
|
|
def test_tokenize_strings
|
|
assert_equal([' '], tokenize(' '))
|
|
assert_equal(['hello world'], tokenize('hello world'))
|
|
assert_equal(['{}'], tokenize('{}'))
|
|
end
|
|
|
|
def test_tokenize_variables
|
|
assert_equal(['{{funk}}'], tokenize('{{funk}}'))
|
|
assert_equal([' ', '{{funk}}', ' '], tokenize(' {{funk}} '))
|
|
assert_equal([' ', '{{funk}}', ' ', '{{so}}', ' ', '{{brother}}', ' '], tokenize(' {{funk}} {{so}} {{brother}} '))
|
|
assert_equal([' ', '{{ funk }}', ' '], tokenize(' {{ funk }} '))
|
|
end
|
|
|
|
def test_tokenize_blocks
|
|
assert_equal(['{%comment%}'], tokenize('{%comment%}'))
|
|
assert_equal([' ', '{%comment%}', ' '], tokenize(' {%comment%} '))
|
|
|
|
assert_equal([' ', '{%comment%}', ' ', '{%endcomment%}', ' '], tokenize(' {%comment%} {%endcomment%} '))
|
|
assert_equal([' ', '{% comment %}', ' ', '{% endcomment %}', ' '], tokenize(" {% comment %} {% endcomment %} "))
|
|
end
|
|
|
|
def test_calculate_line_numbers_per_token_with_profiling
|
|
assert_equal([1], tokenize_line_numbers("{{funk}}"))
|
|
assert_equal([1, 1, 1], tokenize_line_numbers(" {{funk}} "))
|
|
assert_equal([1, 2, 2], tokenize_line_numbers("\n{{funk}}\n"))
|
|
assert_equal([1, 1, 3], tokenize_line_numbers(" {{\n funk \n}} "))
|
|
end
|
|
|
|
def test_tokenize_with_nil_source_returns_empty_array
|
|
assert_equal([], tokenize(nil))
|
|
end
|
|
|
|
def test_incomplete_curly_braces
|
|
assert_equal(["{{.}", " "], tokenize('{{.} '))
|
|
assert_equal(["{{}", "%}"], tokenize('{{}%}'))
|
|
assert_equal(["{{}}", "}"], tokenize('{{}}}'))
|
|
end
|
|
|
|
def test_unmatching_start_and_end
|
|
assert_equal(["{{%}"], tokenize('{{%}'))
|
|
assert_equal(["{{%%%}}"], tokenize('{{%%%}}'))
|
|
assert_equal(["{%", "}}"], tokenize('{%}}'))
|
|
assert_equal(["{%%}", "}"], tokenize('{%%}}'))
|
|
end
|
|
|
|
# Regression: lone '{' at or near end of string previously caused an infinite
|
|
# loop. The stray-{ else branch left `pos` unchanged when no further '{{' or
|
|
# '{%' existed, so the outer loop found the same '{' on every iteration.
|
|
def test_lone_brace_does_not_loop
|
|
assert_equal(["{"], tokenize('{'))
|
|
assert_equal(["a{"], tokenize('a{'))
|
|
assert_equal(["hello { world {"], tokenize('hello { world {'))
|
|
assert_equal(["{ world"], tokenize('{ world'))
|
|
assert_equal(["x{y"], tokenize('x{y'))
|
|
assert_equal(["{b{c"], tokenize('{b{c'))
|
|
end
|
|
|
|
def test_lone_brace_before_real_token
|
|
assert_equal(
|
|
["a { b ", "{% if x %}", "yes", "{% endif %}", " c"],
|
|
tokenize('a { b {% if x %}yes{% endif %} c'),
|
|
)
|
|
assert_equal(
|
|
["x { ", "{{ var }}", " y"],
|
|
tokenize('x { {{ var }} y'),
|
|
)
|
|
assert_equal(
|
|
["{ ", "{{ var }}"],
|
|
tokenize('{ {{ var }}'),
|
|
)
|
|
end
|
|
|
|
private
|
|
|
|
def new_tokenizer(source, parse_context: Liquid::ParseContext.new, start_line_number: nil)
|
|
parse_context.new_tokenizer(source, start_line_number: start_line_number)
|
|
end
|
|
|
|
def tokenize(source)
|
|
tokenizer = new_tokenizer(source)
|
|
tokens = []
|
|
# shift is private in Liquid::C::Tokenizer, since it is only for unit testing
|
|
while (t = tokenizer.send(:shift))
|
|
tokens << t
|
|
end
|
|
tokens
|
|
end
|
|
|
|
def tokenize_line_numbers(source)
|
|
tokenizer = new_tokenizer(source, start_line_number: 1)
|
|
line_numbers = []
|
|
loop do
|
|
line_number = tokenizer.line_number
|
|
if tokenizer.send(:shift)
|
|
line_numbers << line_number
|
|
else
|
|
break
|
|
end
|
|
end
|
|
line_numbers
|
|
end
|
|
end
|