mirror of
https://github.com/Shopify/liquid.git
synced 2026-09-12 23:40:45 -07:00
introduce Cursor class: centralize byte-level scanning for tag/variable/condition parsing
This commit is contained in:
Tobi Lutke
@@ -83,6 +83,7 @@ require 'liquid/expression'
|
||||
require 'liquid/template'
|
||||
require 'liquid/condition'
|
||||
require 'liquid/utils'
|
||||
require 'liquid/cursor'
|
||||
require 'liquid/tokenizer'
|
||||
require 'liquid/parse_context'
|
||||
require 'liquid/partial_cache'
|
||||
|
||||
@@ -220,14 +220,15 @@ module Liquid
|
||||
second_byte = token.getbyte(1)
|
||||
if second_byte == PERCENT_BYTE
|
||||
whitespace_handler(token, parse_context)
|
||||
tag_name = BlockBody.parse_tag_token(token)
|
||||
cursor = parse_context.cursor
|
||||
tag_name = cursor.parse_tag_token(token)
|
||||
unless tag_name
|
||||
return handle_invalid_tag_token(token, parse_context, &block)
|
||||
end
|
||||
markup = BlockBody._last_markup
|
||||
markup = cursor.tag_markup
|
||||
|
||||
if parse_context.line_number
|
||||
newlines = BlockBody._last_newlines
|
||||
newlines = cursor.tag_newlines
|
||||
parse_context.line_number += newlines if newlines > 0
|
||||
end
|
||||
|
||||
@@ -351,13 +352,7 @@ module Liquid
|
||||
def create_variable(token, parse_context)
|
||||
len = token.bytesize
|
||||
if len >= 4 && token.getbyte(len - 1) == CLOSE_CURLEY_BYTE && token.getbyte(len - 2) == CLOSE_CURLEY_BYTE
|
||||
i = 2
|
||||
i = 3 if token.getbyte(i) == DASH_BYTE
|
||||
parse_end = len - 3
|
||||
parse_end -= 1 if token.getbyte(parse_end) == DASH_BYTE
|
||||
markup_end = parse_end - i + 1
|
||||
markup = markup_end <= 0 ? "" : token.byteslice(i, markup_end)
|
||||
|
||||
markup = parse_context.cursor.parse_variable_token(token)
|
||||
return Variable.new(markup, parse_context)
|
||||
end
|
||||
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require "strscan"
|
||||
|
||||
module Liquid
|
||||
# Single-pass forward-only scanner for Liquid parsing.
|
||||
# Wraps StringScanner with higher-level methods for common Liquid constructs.
|
||||
# One Cursor per template parse — threaded through all parsing code.
|
||||
class Cursor
|
||||
# Byte constants
|
||||
SPACE = 32
|
||||
TAB = 9
|
||||
NL = 10
|
||||
CR = 13
|
||||
FF = 12
|
||||
DASH = 45 # '-'
|
||||
DOT = 46 # '.'
|
||||
COLON = 58 # ':'
|
||||
PIPE = 124 # '|'
|
||||
QUOTE_S = 39 # "'"
|
||||
QUOTE_D = 34 # '"'
|
||||
LBRACK = 91 # '['
|
||||
RBRACK = 93 # ']'
|
||||
LPAREN = 40 # '('
|
||||
RPAREN = 41 # ')'
|
||||
QMARK = 63 # '?'
|
||||
HASH = 35 # '#'
|
||||
USCORE = 95 # '_'
|
||||
COMMA = 44
|
||||
ZERO = 48
|
||||
NINE = 57
|
||||
PCT = 37 # '%'
|
||||
LCURLY = 123 # '{'
|
||||
RCURLY = 125 # '}'
|
||||
|
||||
attr_reader :ss
|
||||
|
||||
def initialize(source)
|
||||
@source = source
|
||||
@ss = StringScanner.new(source)
|
||||
end
|
||||
|
||||
# ── Position ────────────────────────────────────────────────────
|
||||
def pos; @ss.pos; end
|
||||
def pos=(n); @ss.pos = n; end
|
||||
def eos?; @ss.eos?; end
|
||||
def peek_byte; @ss.peek_byte; end
|
||||
def scan_byte; @ss.scan_byte; end
|
||||
|
||||
# Reset scanner to a new string (for reuse on sub-markup)
|
||||
def reset(source)
|
||||
@source = source
|
||||
@ss.string = source
|
||||
end
|
||||
|
||||
# ── Whitespace ──────────────────────────────────────────────────
|
||||
# Skip spaces/tabs/newlines/cr, return count of newlines skipped
|
||||
def skip_ws
|
||||
nl = 0
|
||||
while (b = @ss.peek_byte)
|
||||
case b
|
||||
when SPACE, TAB, CR, FF then @ss.scan_byte
|
||||
when NL then @ss.scan_byte; nl += 1
|
||||
else break
|
||||
end
|
||||
end
|
||||
nl
|
||||
end
|
||||
|
||||
# Check if remaining bytes are all whitespace
|
||||
def rest_blank?
|
||||
p = @ss.pos
|
||||
len = @source.bytesize
|
||||
while p < len
|
||||
b = @source.getbyte(p)
|
||||
return false unless b == SPACE || b == TAB || b == NL || b == CR || b == FF
|
||||
p += 1
|
||||
end
|
||||
true
|
||||
end
|
||||
|
||||
# ── Identifiers ─────────────────────────────────────────────────
|
||||
# Scan a single identifier: [a-zA-Z_][\w-]*\??
|
||||
# Returns the string or nil if not at an identifier
|
||||
def scan_id
|
||||
start = @ss.pos
|
||||
b = @ss.peek_byte
|
||||
return nil unless b && ((b >= 97 && b <= 122) || (b >= 65 && b <= 90) || b == USCORE)
|
||||
@ss.scan_byte
|
||||
while (b = @ss.peek_byte)
|
||||
break unless (b >= 97 && b <= 122) || (b >= 65 && b <= 90) ||
|
||||
(b >= 48 && b <= 57) || b == USCORE || b == DASH
|
||||
@ss.scan_byte
|
||||
end
|
||||
@ss.scan_byte if @ss.peek_byte == QMARK
|
||||
@source.byteslice(start, @ss.pos - start)
|
||||
end
|
||||
|
||||
# Scan a tag name: '#' or \w+
|
||||
def scan_tag_name
|
||||
if @ss.peek_byte == HASH
|
||||
@ss.scan_byte
|
||||
"#"
|
||||
else
|
||||
scan_id
|
||||
end
|
||||
end
|
||||
|
||||
# ── Numbers ─────────────────────────────────────────────────────
|
||||
# Try to scan an integer or float. Returns the number or nil.
|
||||
def scan_number
|
||||
start = @ss.pos
|
||||
b = @ss.peek_byte
|
||||
return nil unless b
|
||||
|
||||
if b == DASH
|
||||
@ss.scan_byte
|
||||
b = @ss.peek_byte
|
||||
unless b && b >= ZERO && b <= NINE
|
||||
@ss.pos = start
|
||||
return nil
|
||||
end
|
||||
elsif b >= ZERO && b <= NINE
|
||||
# ok
|
||||
else
|
||||
return nil
|
||||
end
|
||||
|
||||
# Scan digits
|
||||
@ss.scan_byte
|
||||
@ss.scan_byte while (b = @ss.peek_byte) && b >= ZERO && b <= NINE
|
||||
|
||||
if @ss.peek_byte == DOT
|
||||
@ss.scan_byte
|
||||
# Must have digit after dot for float
|
||||
if (b = @ss.peek_byte) && b >= ZERO && b <= NINE
|
||||
@ss.scan_byte
|
||||
@ss.scan_byte while (b = @ss.peek_byte) && b >= ZERO && b <= NINE
|
||||
return @source.byteslice(start, @ss.pos - start).to_f
|
||||
else
|
||||
# "123." — integer portion only, rewind past dot
|
||||
@ss.pos -= 1
|
||||
end
|
||||
end
|
||||
|
||||
Integer(@source.byteslice(start, @ss.pos - start), 10)
|
||||
end
|
||||
|
||||
# ── Strings ─────────────────────────────────────────────────────
|
||||
# Scan a quoted string ('...' or "..."). Returns the content without quotes, or nil.
|
||||
def scan_quoted_string
|
||||
b = @ss.peek_byte
|
||||
return nil unless b == QUOTE_S || b == QUOTE_D
|
||||
quote = b
|
||||
@ss.scan_byte
|
||||
start = @ss.pos
|
||||
@ss.scan_byte while (b = @ss.peek_byte) && b != quote
|
||||
content = @source.byteslice(start, @ss.pos - start)
|
||||
@ss.scan_byte if @ss.peek_byte == quote # consume closing quote
|
||||
content
|
||||
end
|
||||
|
||||
# Scan a quoted string including quotes. Returns the full "..." or '...' string, or nil.
|
||||
def scan_quoted_string_raw
|
||||
b = @ss.peek_byte
|
||||
return nil unless b == QUOTE_S || b == QUOTE_D
|
||||
quote = b
|
||||
start = @ss.pos
|
||||
@ss.scan_byte
|
||||
@ss.scan_byte while (b = @ss.peek_byte) && b != quote
|
||||
@ss.scan_byte if @ss.peek_byte == quote
|
||||
@source.byteslice(start, @ss.pos - start)
|
||||
end
|
||||
|
||||
# ── Expressions ─────────────────────────────────────────────────
|
||||
# Scan a simple variable lookup: name(.name)* — no brackets, no filters
|
||||
# Returns the string or nil
|
||||
def scan_dotted_id
|
||||
start = @ss.pos
|
||||
return nil unless scan_id
|
||||
while @ss.peek_byte == DOT
|
||||
@ss.scan_byte
|
||||
unless scan_id
|
||||
@ss.pos -= 1 # rewind the dot
|
||||
break
|
||||
end
|
||||
end
|
||||
@source.byteslice(start, @ss.pos - start)
|
||||
end
|
||||
|
||||
# Scan a "QuotedFragment" — a quoted string or non-whitespace/comma/pipe run
|
||||
def scan_fragment
|
||||
b = @ss.peek_byte
|
||||
return nil unless b
|
||||
if b == QUOTE_S || b == QUOTE_D
|
||||
scan_quoted_string_raw
|
||||
else
|
||||
start = @ss.pos
|
||||
while (b = @ss.peek_byte)
|
||||
break if b == SPACE || b == TAB || b == NL || b == CR || b == COMMA || b == PIPE
|
||||
@ss.scan_byte
|
||||
end
|
||||
len = @ss.pos - start
|
||||
len > 0 ? @source.byteslice(start, len) : nil
|
||||
end
|
||||
end
|
||||
|
||||
# ── Comparison operators ────────────────────────────────────────
|
||||
COMPARISON_OPS = {
|
||||
'==' => '==', '!=' => '!=', '<>' => '<>',
|
||||
'<=' => '<=', '>=' => '>=', '<' => '<', '>' => '>',
|
||||
'contains' => 'contains',
|
||||
}.freeze
|
||||
|
||||
# Scan a comparison operator. Returns frozen string or nil.
|
||||
def scan_comparison_op
|
||||
start = @ss.pos
|
||||
b = @ss.peek_byte
|
||||
case b
|
||||
when 61, 33, 60, 62 # = ! < >
|
||||
@ss.scan_byte
|
||||
b2 = @ss.peek_byte
|
||||
if b2 == 61 || b2 == 62 # second char of ==, !=, <=, >=, <>
|
||||
@ss.scan_byte
|
||||
end
|
||||
when 99 # 'c' for contains
|
||||
id = scan_id
|
||||
return nil unless id == "contains"
|
||||
return COMPARISON_OPS['contains']
|
||||
else
|
||||
return nil
|
||||
end
|
||||
op_str = @source.byteslice(start, @ss.pos - start)
|
||||
COMPARISON_OPS[op_str] || (@ss.pos = start; nil)
|
||||
end
|
||||
|
||||
# ── Tag parsing helpers ─────────────────────────────────────────
|
||||
# Results from last parse_tag_token call (avoids array allocation)
|
||||
attr_reader :tag_markup, :tag_newlines
|
||||
|
||||
# Parse the interior of a tag token: "{%[-] tag_name markup [-]%}"
|
||||
# Caller provides the full token string. Sets cursor to the token.
|
||||
# Returns tag_name string or nil. Sets tag_markup and tag_newlines.
|
||||
def parse_tag_token(token)
|
||||
reset(token)
|
||||
@ss.pos = 2 # skip "{%"
|
||||
@ss.scan_byte if peek_byte == DASH # skip whitespace control '-'
|
||||
nl = skip_ws
|
||||
tag_name = scan_tag_name
|
||||
return nil unless tag_name
|
||||
nl += skip_ws
|
||||
|
||||
# markup is everything up to optional '-' before '%}'
|
||||
markup_end = token.bytesize - 2
|
||||
markup_end -= 1 if markup_end > @ss.pos && token.getbyte(markup_end - 1) == DASH
|
||||
@tag_markup = @ss.pos >= markup_end ? "" : token.byteslice(@ss.pos, markup_end - @ss.pos)
|
||||
@tag_newlines = nl
|
||||
|
||||
tag_name
|
||||
end
|
||||
|
||||
# Parse variable token interior: extract markup from "{{[-] ... [-]}}"
|
||||
def parse_variable_token(token)
|
||||
len = token.bytesize
|
||||
return nil if len < 4
|
||||
i = 2
|
||||
i = 3 if token.getbyte(i) == DASH
|
||||
parse_end = len - 3
|
||||
parse_end -= 1 if token.getbyte(parse_end) == DASH
|
||||
markup_len = parse_end - i + 1
|
||||
markup_len <= 0 ? "" : token.byteslice(i, markup_len)
|
||||
end
|
||||
|
||||
# ── Simple condition parser ─────────────────────────────────────
|
||||
# Results from last parse_simple_condition call
|
||||
attr_reader :cond_left, :cond_op, :cond_right
|
||||
|
||||
# Parse "expr [op expr]" from current position to end.
|
||||
# Returns true on success, nil on failure. Sets cond_left, cond_op, cond_right.
|
||||
def parse_simple_condition
|
||||
skip_ws
|
||||
@cond_left = scan_fragment
|
||||
return nil unless @cond_left
|
||||
|
||||
skip_ws
|
||||
if eos?
|
||||
@cond_op = nil
|
||||
@cond_right = nil
|
||||
return true
|
||||
end
|
||||
|
||||
@cond_op = scan_comparison_op
|
||||
return nil unless @cond_op
|
||||
|
||||
skip_ws
|
||||
@cond_right = scan_fragment
|
||||
return nil unless @cond_right
|
||||
|
||||
skip_ws
|
||||
return nil unless eos? # trailing junk
|
||||
true
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -3,7 +3,7 @@
|
||||
module Liquid
|
||||
class ParseContext
|
||||
attr_accessor :locale, :line_number, :trim_whitespace, :depth
|
||||
attr_reader :partial, :warnings, :error_mode, :environment, :expression_cache, :string_scanner
|
||||
attr_reader :partial, :warnings, :error_mode, :environment, :expression_cache, :string_scanner, :cursor
|
||||
|
||||
def initialize(options = Const::EMPTY_HASH)
|
||||
@environment = options.fetch(:environment, Environment.default)
|
||||
@@ -24,6 +24,8 @@ module Liquid
|
||||
{}
|
||||
end
|
||||
|
||||
@cursor = Cursor.new("")
|
||||
|
||||
self.depth = 0
|
||||
self.partial = false
|
||||
end
|
||||
|
||||
+7
-105
@@ -88,119 +88,21 @@ module Liquid
|
||||
# Fast path regex for simple conditions: "expr", "expr op expr" (no and/or)
|
||||
SIMPLE_CONDITION = /\A\s*(#{QuotedFragment})\s*(?:([=!<>a-z_]+)\s*(#{QuotedFragment}))?\s*\z/o
|
||||
|
||||
# Operators indexed by first byte for fast lookup
|
||||
COMPARISON_OPS = {
|
||||
'==' => '==', '!=' => '!=', '<>' => '<>',
|
||||
'<=' => '<=', '>=' => '>=', '<' => '<', '>' => '>',
|
||||
'contains' => 'contains',
|
||||
}.freeze
|
||||
|
||||
# Parse a simple condition "expr [op expr]" without regex.
|
||||
# Returns [left, op, right] or nil if not parseable.
|
||||
def self.parse_simple_condition(markup)
|
||||
len = markup.bytesize
|
||||
pos = 0
|
||||
|
||||
# Skip leading whitespace
|
||||
pos += 1 while pos < len && (b = markup.getbyte(pos)) && (b == 32 || b == 9 || b == 10 || b == 13)
|
||||
return nil if pos >= len
|
||||
|
||||
# Scan left expression (QuotedFragment): quoted string or non-whitespace/comma/pipe sequence
|
||||
left_start = pos
|
||||
b = markup.getbyte(pos)
|
||||
if b == 34 || b == 39 # quoted string
|
||||
quote = b
|
||||
pos += 1
|
||||
pos += 1 while pos < len && markup.getbyte(pos) != quote
|
||||
pos += 1 if pos < len # closing quote
|
||||
else
|
||||
# Non-whitespace, non-comma, non-pipe chars (QuotedFragment without quotes)
|
||||
while pos < len
|
||||
b = markup.getbyte(pos)
|
||||
break if b == 32 || b == 9 || b == 10 || b == 13 || b == 44 || b == 124 # space, tab, \n, \r, comma, pipe
|
||||
pos += 1
|
||||
end
|
||||
end
|
||||
left_end = pos
|
||||
|
||||
return nil if left_start == left_end
|
||||
|
||||
# Skip whitespace
|
||||
pos += 1 while pos < len && (b = markup.getbyte(pos)) && (b == 32 || b == 9 || b == 10 || b == 13)
|
||||
|
||||
# End of markup? Simple truthiness
|
||||
if pos >= len
|
||||
left = markup.byteslice(left_start, left_end - left_start)
|
||||
return [left, nil, nil]
|
||||
end
|
||||
|
||||
# Scan operator
|
||||
op_start = pos
|
||||
b = markup.getbyte(pos)
|
||||
if b == 61 || b == 33 || b == 60 || b == 62 # =, !, <, >
|
||||
pos += 1
|
||||
b2 = markup.getbyte(pos)
|
||||
pos += 1 if b2 && (b2 == 61 || b2 == 62) # second char of ==, !=, <=, >=, <>
|
||||
elsif b == 99 # 'c' for 'contains'
|
||||
while pos < len
|
||||
b = markup.getbyte(pos)
|
||||
break unless (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || b == 95
|
||||
pos += 1
|
||||
end
|
||||
else
|
||||
return nil # unknown operator start
|
||||
end
|
||||
op = markup.byteslice(op_start, pos - op_start)
|
||||
return nil unless COMPARISON_OPS.key?(op)
|
||||
op = COMPARISON_OPS[op] # use frozen string
|
||||
|
||||
# Skip whitespace
|
||||
pos += 1 while pos < len && (b = markup.getbyte(pos)) && (b == 32 || b == 9 || b == 10 || b == 13)
|
||||
return nil if pos >= len # op without right operand
|
||||
|
||||
# Scan right expression
|
||||
right_start = pos
|
||||
b = markup.getbyte(pos)
|
||||
if b == 34 || b == 39
|
||||
quote = b
|
||||
pos += 1
|
||||
pos += 1 while pos < len && markup.getbyte(pos) != quote
|
||||
pos += 1 if pos < len
|
||||
else
|
||||
while pos < len
|
||||
b = markup.getbyte(pos)
|
||||
break if b == 32 || b == 9 || b == 10 || b == 13 || b == 44 || b == 124
|
||||
pos += 1
|
||||
end
|
||||
end
|
||||
right_end = pos
|
||||
|
||||
return nil if right_start == right_end
|
||||
|
||||
# Skip trailing whitespace
|
||||
pos += 1 while pos < len && (b = markup.getbyte(pos)) && (b == 32 || b == 9 || b == 10 || b == 13)
|
||||
return nil unless pos >= len # extra stuff after right expr
|
||||
|
||||
left = markup.byteslice(left_start, left_end - left_start)
|
||||
right = markup.byteslice(right_start, right_end - right_start)
|
||||
[left, op, right]
|
||||
end
|
||||
|
||||
def lax_parse(markup)
|
||||
# Fastest path: simple identifier truthiness like "product.available" or "forloop.first"
|
||||
if (simple = Variable.simple_variable_markup(markup))
|
||||
return Condition.new(parse_expression(simple))
|
||||
end
|
||||
|
||||
# Fast path: simple condition without and/or — manual byte parser
|
||||
# Fast path: simple condition without and/or — use Cursor
|
||||
if !markup.include?(' and ') && !markup.include?(' or ')
|
||||
parsed = If.parse_simple_condition(markup)
|
||||
if parsed
|
||||
left, op, right = parsed
|
||||
cursor = @parse_context.cursor
|
||||
cursor.reset(markup)
|
||||
if cursor.parse_simple_condition
|
||||
return Condition.new(
|
||||
parse_expression(left),
|
||||
op,
|
||||
right ? parse_expression(right) : nil,
|
||||
parse_expression(cursor.cond_left),
|
||||
cursor.cond_op,
|
||||
cursor.cond_right ? parse_expression(cursor.cond_right) : nil,
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user