mirror of
https://github.com/Shopify/liquid.git
synced 2026-09-16 09:20:42 -07:00
Replace StringScanner tokenizer with String#byteindex — 12% faster parse, no regex overhead for delimiter finding\n\nResult: {"status":"keep","combined_µs":3556,"parse_µs":2388,"render_µs":1168,"allocations":24882}
This commit is contained in:
Tobi Lutke
+105
-86
@@ -54,108 +54,127 @@ module Liquid
|
||||
if @for_liquid_tag
|
||||
@tokens = @source.split("\n")
|
||||
else
|
||||
@tokens << shift_normal until @ss.eos?
|
||||
tokenize_fast
|
||||
end
|
||||
|
||||
@source = nil
|
||||
@ss = nil
|
||||
end
|
||||
|
||||
def shift_normal
|
||||
token = next_token
|
||||
# Fast tokenizer using String#index instead of StringScanner regex.
|
||||
# String#index is ~40% faster for finding { delimiters.
|
||||
def tokenize_fast
|
||||
src = @source
|
||||
unless src.valid_encoding?
|
||||
raise SyntaxError, "Invalid byte sequence in #{src.encoding}"
|
||||
end
|
||||
|
||||
return unless token
|
||||
len = src.bytesize
|
||||
pos = 0
|
||||
|
||||
token
|
||||
end
|
||||
while pos < len
|
||||
# Find next { which could start a tag or variable
|
||||
idx = src.byteindex('{', pos)
|
||||
|
||||
def next_token
|
||||
# possible states: :text, :tag, :variable
|
||||
byte_a = @ss.peek_byte
|
||||
|
||||
if byte_a == OPEN_CURLEY
|
||||
@ss.scan_byte
|
||||
|
||||
byte_b = @ss.peek_byte
|
||||
|
||||
if byte_b == PERCENTAGE
|
||||
@ss.scan_byte
|
||||
return next_tag_token
|
||||
elsif byte_b == OPEN_CURLEY
|
||||
@ss.scan_byte
|
||||
return next_variable_token
|
||||
unless idx
|
||||
# No more tags/variables — rest is text
|
||||
@tokens << src.byteslice(pos, len - pos) if pos < len
|
||||
break
|
||||
end
|
||||
|
||||
@ss.pos -= 1
|
||||
end
|
||||
next_byte = idx + 1 < len ? src.getbyte(idx + 1) : nil
|
||||
|
||||
next_text_token
|
||||
end
|
||||
if next_byte == PERCENTAGE # {%
|
||||
# Emit text before tag
|
||||
@tokens << src.byteslice(pos, idx - pos) if idx > pos
|
||||
|
||||
def next_text_token
|
||||
start = @ss.pos
|
||||
|
||||
unless @ss.skip_until(TAG_OR_VARIABLE_START)
|
||||
token = @ss.rest
|
||||
@ss.terminate
|
||||
return token
|
||||
end
|
||||
|
||||
pos = @ss.pos -= 2
|
||||
@source.byteslice(start, pos - start)
|
||||
rescue ::ArgumentError => e
|
||||
if e.message == "invalid byte sequence in #{@ss.string.encoding}"
|
||||
raise SyntaxError, "Invalid byte sequence in #{@ss.string.encoding}"
|
||||
else
|
||||
raise
|
||||
end
|
||||
end
|
||||
|
||||
def next_variable_token
|
||||
start = @ss.pos - 2
|
||||
|
||||
byte_a = byte_b = @ss.scan_byte
|
||||
|
||||
while byte_b
|
||||
byte_a = @ss.scan_byte while byte_a && byte_a != CLOSE_CURLEY && byte_a != OPEN_CURLEY
|
||||
|
||||
break unless byte_a
|
||||
|
||||
if @ss.eos?
|
||||
return byte_a == CLOSE_CURLEY ? @source.byteslice(start, @ss.pos - start) : "{{"
|
||||
end
|
||||
|
||||
byte_b = @ss.scan_byte
|
||||
|
||||
if byte_a == CLOSE_CURLEY
|
||||
if byte_b == CLOSE_CURLEY
|
||||
return @source.byteslice(start, @ss.pos - start)
|
||||
elsif byte_b != CLOSE_CURLEY
|
||||
@ss.pos -= 1
|
||||
return @source.byteslice(start, @ss.pos - start)
|
||||
# Find %} to close the tag
|
||||
close = src.byteindex('%}', idx + 2)
|
||||
if close
|
||||
@tokens << src.byteslice(idx, close + 2 - idx)
|
||||
pos = close + 2
|
||||
else
|
||||
@tokens << "{%"
|
||||
pos = idx + 2
|
||||
end
|
||||
elsif next_byte == OPEN_CURLEY # {{
|
||||
# Emit text before variable
|
||||
@tokens << src.byteslice(pos, idx - pos) if idx > pos
|
||||
|
||||
# Scan variable token — matches original tokenizer's byte-by-byte logic:
|
||||
# Find } or {, then check next byte for }}/{% nesting
|
||||
scan_pos = idx + 2
|
||||
found = false
|
||||
while scan_pos < len
|
||||
b = src.getbyte(scan_pos)
|
||||
if b == CLOSE_CURLEY # }
|
||||
if scan_pos + 1 >= len
|
||||
# } at end of string — emit token up to here
|
||||
@tokens << src.byteslice(idx, scan_pos + 1 - idx)
|
||||
pos = scan_pos + 1
|
||||
found = true
|
||||
break
|
||||
end
|
||||
b2 = src.getbyte(scan_pos + 1)
|
||||
if b2 == CLOSE_CURLEY
|
||||
# Found }} — close variable
|
||||
@tokens << src.byteslice(idx, scan_pos + 2 - idx)
|
||||
pos = scan_pos + 2
|
||||
found = true
|
||||
break
|
||||
else
|
||||
# } followed by non-} — emit token up to here (matches original: @ss.pos -= 1)
|
||||
@tokens << src.byteslice(idx, scan_pos + 1 - idx)
|
||||
pos = scan_pos + 1
|
||||
found = true
|
||||
break
|
||||
end
|
||||
elsif b == OPEN_CURLEY
|
||||
if scan_pos + 1 < len && src.getbyte(scan_pos + 1) == PERCENTAGE
|
||||
# Found {% inside {{ — scan to %} and emit as one token
|
||||
close = src.byteindex('%}', scan_pos + 2)
|
||||
if close
|
||||
@tokens << src.byteslice(idx, close + 2 - idx)
|
||||
pos = close + 2
|
||||
else
|
||||
@tokens << src.byteslice(idx, len - idx)
|
||||
pos = len
|
||||
end
|
||||
found = true
|
||||
break
|
||||
end
|
||||
scan_pos += 1
|
||||
else
|
||||
scan_pos += 1
|
||||
end
|
||||
end
|
||||
|
||||
unless found
|
||||
@tokens << "{{"
|
||||
pos = idx + 2
|
||||
end
|
||||
else
|
||||
# { followed by something else — it's text
|
||||
# Keep scanning from after this {
|
||||
# Find next { that could be {% or {{
|
||||
next_open = idx + 1
|
||||
while next_open < len
|
||||
ni = src.byteindex('{', next_open)
|
||||
unless ni
|
||||
@tokens << src.byteslice(pos, len - pos)
|
||||
pos = len
|
||||
break
|
||||
end
|
||||
nb = ni + 1 < len ? src.getbyte(ni + 1) : nil
|
||||
if nb == PERCENTAGE || nb == OPEN_CURLEY
|
||||
@tokens << src.byteslice(pos, ni - pos)
|
||||
pos = ni
|
||||
break
|
||||
end
|
||||
next_open = ni + 1
|
||||
end
|
||||
elsif byte_a == OPEN_CURLEY && byte_b == PERCENTAGE
|
||||
return next_tag_token_with_start(start)
|
||||
end
|
||||
|
||||
byte_a = byte_b
|
||||
end
|
||||
|
||||
"{{"
|
||||
end
|
||||
|
||||
def next_tag_token
|
||||
start = @ss.pos - 2
|
||||
if (len = @ss.skip_until(TAG_END))
|
||||
@source.byteslice(start, len + 2)
|
||||
else
|
||||
"{%"
|
||||
end
|
||||
end
|
||||
|
||||
def next_tag_token_with_start(start)
|
||||
@ss.skip_until(TAG_END)
|
||||
@source.byteslice(start, @ss.pos - start)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user