Labyricorn/liquid
1
0
Fork 0
mirror of https://github.com/Shopify/liquid.git synced 2026-09-16 09:20:42 -07:00
Code Issues Packages Projects Releases Wiki Activity

return [tag_name, markup, newlines] from parse_tag_token: avoid 2 whitespace string allocs

Browse Source
This commit is contained in:
Tobi Lutke
2026-03-11 07:45:15 -04:00
parent 82407092cc
commit 58d2514521
1 changed files with 27 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/liquid/block_body.rb
+27 -16
View File
@@ -14,22 +14,31 @@ module Liquid
# Fast manual tag token parser - avoids regex MatchData allocation # Fast manual tag token parser - avoids regex MatchData allocation
# Parses "{%[-] tag_name markup [-]%}" and returns [pre_ws, tag_name, post_ws, markup] or nil # Parses "{%[-] tag_name markup [-]%}" and returns [pre_ws, tag_name, post_ws, markup] or nil
NEWLINE_BYTE = 10 # "\n".ord
# Fast manual tag token parser - avoids regex MatchData allocation
# Parses "{%[-] tag_name markup [-]%}" and returns [tag_name, markup, newline_count] or nil
def self.parse_tag_token(token) def self.parse_tag_token(token)
# token starts with "{%" # token starts with "{%"
pos = 2 pos = 2
len = token.length len = token.length
newlines = 0
# skip optional whitespace control '-' # skip optional whitespace control '-'
pos += 1 if pos < len && token.getbyte(pos) == 45 # '-' pos += 1 if pos < len && token.getbyte(pos) == 45 # '-'
# capture pre-whitespace (for line number counting) # skip pre-whitespace, counting newlines
ws_start = pos
while pos < len while pos < len
b = token.getbyte(pos) b = token.getbyte(pos)
break unless b == 32 || b == 9 || b == 10 || b == 13 # space, tab, \n, \r if b == NEWLINE_BYTE
pos += 1 newlines += 1
pos += 1
elsif b == 32 || b == 9 || b == 13 # space, tab, \r
pos += 1
else
break
end
end end
pre_ws = token.byteslice(ws_start, pos - ws_start)
# parse tag name: # or \w+ # parse tag name: # or \w+
name_start = pos name_start = pos
@@ -45,14 +54,18 @@ module Liquid
return nil if pos == name_start # no tag name found return nil if pos == name_start # no tag name found
tag_name = token.byteslice(name_start, pos - name_start) tag_name = token.byteslice(name_start, pos - name_start)
# capture post-whitespace # skip post-whitespace, counting newlines
post_ws_start = pos
while pos < len while pos < len
b = token.getbyte(pos) b = token.getbyte(pos)
break unless b == 32 || b == 9 || b == 10 || b == 13 if b == NEWLINE_BYTE
pos += 1 newlines += 1
pos += 1
elsif b == 32 || b == 9 || b == 13
pos += 1
else
break
end
end end
post_ws = token.byteslice(post_ws_start, pos - post_ws_start)
# the rest is markup, up to optional '-' and '%}' # the rest is markup, up to optional '-' and '%}'
# token ends with '%}' (guaranteed by tokenizer) # token ends with '%}' (guaranteed by tokenizer)
@@ -60,7 +73,7 @@ module Liquid
markup_end -= 1 if markup_end > pos && token.getbyte(markup_end - 1) == 45 # trailing '-' markup_end -= 1 if markup_end > pos && token.getbyte(markup_end - 1) == 45 # trailing '-'
markup = pos >= markup_end ? "" : token.byteslice(pos, markup_end - pos) markup = pos >= markup_end ? "" : token.byteslice(pos, markup_end - pos)
[pre_ws, tag_name, post_ws, markup] [tag_name, markup, newlines]
end end
attr_reader :nodelist attr_reader :nodelist
@@ -191,12 +204,10 @@ module Liquid
unless parsed unless parsed
return handle_invalid_tag_token(token, parse_context, &block) return handle_invalid_tag_token(token, parse_context, &block)
end end
pre_ws, tag_name, post_ws, markup = parsed tag_name, markup, newlines = parsed
if parse_context.line_number if parse_context.line_number && newlines > 0
# newlines inside the tag should increase the line number, parse_context.line_number += newlines
# particularly important for multiline {% liquid %} tags
parse_context.line_number += pre_ws.count("\n") + post_ws.count("\n")
end end
if tag_name == 'liquid' if tag_name == 'liquid'