Decomposes, extracts, and names the important concepts

Decomposes the 211-line try_fast_parse monolith in variable.rb into four
named private methods -- fast_scan_name, fast_resolve_name, fast_scan_filters,
fast_scan_filter_args -- and simplifies simple_variable_markup from a 55-line
byte scanner to a 10-line regex with fast pre-checks.

Generates invoke_single/invoke_two via module_eval in strainer_template.rb
and context.rb instead of duplicating near-identical method bodies. The two
arity variants differ only in parameter lists; generating them makes the
pattern explicit and eliminates copy-paste drift.

Extracts identical 4-line text-token handling block in block_body.rb into
private append_text_token(token, parse_context). The block appeared in
both the stray-{ fallback and the plain-text branch of parse_for_document.

Extracts accessible?(object, key) predicate from the 4-line inline ternary
in variable_lookup.rb that checked hash/array key presence; extracts
liquidize(object, context) from the duplicated to_liquid + context= wiring
that appeared in both the key-found and command-method branches.

Extracts find_in_envs(envs, key, raise_on_not_found:) from
try_variable_find_in_environments in context.rb, which looped over
@environments then @static_environments with identical loop bodies.

Adds 28 fast-path equivalence tests for Variable (variable_fast_parse_test.rb).
This commit is contained in:
Chris Pak
2026-04-04 22:13:43 -07:00
parent 03e5e29b0b
commit ba11b85875
6 changed files with 458 additions and 291 deletions
+62 -45
View File
@@ -52,8 +52,7 @@ module Liquid
end end
unless (tag = parse_context.environment.tag_for_name(tag_name)) unless (tag = parse_context.environment.tag_for_name(tag_name))
# end parsing if we reach an unknown tag and let the caller decide # end parsing if we reach an unknown tag; let the caller determine how to proceed
# determine how to proceed
return yield tag_name, markup return yield tag_name, markup
end end
new_tag = tag.parse(tag_name, markup, tokenizer, parse_context) new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
@@ -124,11 +123,9 @@ module Liquid
end end
# Fast check if string is whitespace-only (replaces WhitespaceOrNothing regex)
BLANK_STRING_REGEX = /\A\s*\z/
def self.blank_string?(str) def self.blank_string?(str)
str.match?(BLANK_STRING_REGEX) str.match?(WhitespaceOrNothing)
end end
private def parse_for_document(tokenizer, parse_context, &block) private def parse_for_document(tokenizer, parse_context, &block)
@@ -139,52 +136,26 @@ module Liquid
if first_byte == Cursor::LCURLY if first_byte == Cursor::LCURLY
second_byte = token.getbyte(1) second_byte = token.getbyte(1)
if second_byte == Cursor::PCT if second_byte == Cursor::PCT
whitespace_handler(token, parse_context) # handle_tag_token returns:
cursor = parse_context.cursor # nil — tag parsed normally, continue (update line number)
tag_name = cursor.parse_tag_token(token) # :next — 'liquid' inline tag; skip line number update
unless tag_name # :unknown — end tag or unknown tag; yield to caller and return
return handle_invalid_tag_token(token, parse_context, &block) # :invalid — malformed tag token; delegate to handle_invalid_tag_token
end result = handle_tag_token(token, parse_context, tokenizer)
markup = cursor.tag_markup next unless result # nil: normal
next if result == :next # :next: 'liquid'
if parse_context.line_number return yield(@_unknown_tag_name, parse_context.cursor.tag_markup) if result == :unknown
newlines = cursor.tag_newlines return handle_invalid_tag_token(token, parse_context, &block) # :invalid
parse_context.line_number += newlines if newlines > 0
end
if tag_name == 'liquid'
parse_liquid_tag(markup, parse_context)
next
end
unless (tag = parse_context.environment.tag_for_name(tag_name))
# end parsing if we reach an unknown tag and let the caller decide
# determine how to proceed
return yield tag_name, markup
end
new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
@blank &&= new_tag.blank?
@nodelist << new_tag
elsif second_byte == Cursor::LCURLY elsif second_byte == Cursor::LCURLY
whitespace_handler(token, parse_context) whitespace_handler(token, parse_context)
@nodelist << create_variable(token, parse_context) @nodelist << create_variable(token, parse_context)
@blank = false @blank = false
else else
# Fallback: text token starting with '{' # Fallback: text token starting with '{'
if parse_context.trim_whitespace append_text_token(token, parse_context)
token.lstrip!
end
parse_context.trim_whitespace = false
@nodelist << token
@blank &&= BlockBody.blank_string?(token)
end end
else else
if parse_context.trim_whitespace append_text_token(token, parse_context)
token.lstrip!
end
parse_context.trim_whitespace = false
@nodelist << token
@blank &&= BlockBody.blank_string?(token)
end end
parse_context.line_number = tokenizer.line_number parse_context.line_number = tokenizer.line_number
end end
@@ -192,8 +163,53 @@ module Liquid
yield nil, nil yield nil, nil
end end
# Handles a {%...%} tag token. Does not receive the outer block — callers handle
# yield/block passing themselves, keeping the Proc off the hot path.
# Returns:
# nil — tag parsed, caller continues the loop
# :next — 'liquid' inline tag; caller skips line number update
# :unknown — unknown/end tag; @_unknown_tag_name holds the tag name;
# markup is in parse_context.cursor.tag_markup
# :invalid — malformed token; caller delegates to handle_invalid_tag_token
private def handle_tag_token(token, parse_context, tokenizer)
whitespace_handler(token, parse_context)
cursor = parse_context.cursor
tag_name = cursor.parse_tag_token(token)
return :invalid unless tag_name
def whitespace_handler(token, parse_context) markup = cursor.tag_markup
if parse_context.line_number
newlines = cursor.tag_newlines
parse_context.line_number += newlines if newlines > 0
end
if tag_name == 'liquid'
parse_liquid_tag(markup, parse_context)
return :next
end
tag = parse_context.environment.tag_for_name(tag_name)
unless tag
# end parsing if we reach an unknown tag; let the caller determine how to proceed
@_unknown_tag_name = tag_name
return :unknown
end
new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
@blank &&= new_tag.blank?
@nodelist << new_tag
nil
end
def append_text_token(token, parse_context)
token.lstrip! if parse_context.trim_whitespace
parse_context.trim_whitespace = false
@nodelist << token
@blank &&= BlockBody.blank_string?(token)
end
private :append_text_token
private def whitespace_handler(token, parse_context)
if token.getbyte(2) == Cursor::DASH if token.getbyte(2) == Cursor::DASH
previous_token = @nodelist.last previous_token = @nodelist.last
if previous_token.is_a?(String) if previous_token.is_a?(String)
@@ -236,7 +252,7 @@ module Liquid
end end
def render_to_output_buffer(context, output) def render_to_output_buffer(context, output)
freeze unless frozen? freeze
resource_limits = context.resource_limits resource_limits = context.resource_limits
resource_limits.increment_render_score(@nodelist.length) resource_limits.increment_render_score(@nodelist.length)
@@ -262,6 +278,7 @@ module Liquid
private private
# Indirection allows subclasses to intercept per-node rendering.
def render_node(context, output, node) def render_node(context, output, node)
BlockBody.render_node(context, output, node) BlockBody.render_node(context, output, node)
end end
+27 -27
View File
@@ -77,7 +77,7 @@ module Liquid
# Note that this does not register the filters with the main Template object. see <tt>Template.register_filter</tt> # Note that this does not register the filters with the main Template object. see <tt>Template.register_filter</tt>
# for that # for that
def add_filters(filters) def add_filters(filters)
filters = [filters].flatten.compact filters = Array(filters).flatten.compact
@filters += filters @filters += filters
@strainer = nil @strainer = nil
end end
@@ -88,7 +88,7 @@ module Liquid
# are there any not handled interrupts? # are there any not handled interrupts?
def interrupt? def interrupt?
!@interrupts.frozen? && !@interrupts.empty? !@interrupts.equal?(Const::EMPTY_ARRAY) && @interrupts.any?
end end
# push an interrupt to the stack. this interrupt is considered not handled. # push an interrupt to the stack. this interrupt is considered not handled.
@@ -114,15 +114,18 @@ module Liquid
strainer.invoke(method, *args).to_liquid strainer.invoke(method, *args).to_liquid
end end
# Fast path for single-argument filter invocation (the most common case: # Arity-specialized filter delegation — generated to match StrainerTemplate's specializations.
# {{ value | filter }}) — avoids *args splat allocation. # The pattern (avoid *args splat) is the same for each arity; generating makes it explicit.
def invoke_single(method, input) {
strainer.invoke_single(method, input).to_liquid invoke_single: ['input'],
invoke_two: ['input', 'arg1'],
}.each do |method_name, params|
all_params = (["method"] + params).join(", ")
module_eval(<<~RUBY, __FILE__, __LINE__ + 1)
def #{method_name}(#{all_params})
strainer.#{method_name}(#{all_params}).to_liquid
end end
RUBY
# Fast path for two-argument filter invocation (e.g. {{ value | default: 'x' }})
def invoke_two(method, input, arg1)
strainer.invoke_two(method, input, arg1).to_liquid
end end
# Push new local scope on the stack. use <tt>Context#stack</tt> instead # Push new local scope on the stack. use <tt>Context#stack</tt> instead
@@ -200,7 +203,7 @@ module Liquid
end end
def key?(key) def key?(key)
find_variable(key, raise_on_not_found: false) != nil !find_variable(key, raise_on_not_found: false).nil?
end end
def evaluate(object) def evaluate(object)
@@ -218,10 +221,10 @@ module Liquid
variable = try_variable_find_in_environments(key, raise_on_not_found: raise_on_not_found) variable = try_variable_find_in_environments(key, raise_on_not_found: raise_on_not_found)
else else
# Multiple scopes — search through all of them # Multiple scopes — search through all of them
index = @scopes.find_index { |s| s.key?(key) } scope = @scopes.find { |s| s.key?(key) }
variable = if index variable = if scope
lookup_and_evaluate(@scopes[index], key, raise_on_not_found: raise_on_not_found) lookup_and_evaluate(scope, key, raise_on_not_found: raise_on_not_found)
else else
try_variable_find_in_environments(key, raise_on_not_found: raise_on_not_found) try_variable_find_in_environments(key, raise_on_not_found: raise_on_not_found)
end end
@@ -230,9 +233,7 @@ module Liquid
# update variable's context before invoking #to_liquid # update variable's context before invoking #to_liquid
# Fast path: primitive types don't need context= or to_liquid conversion # Fast path: primitive types don't need context= or to_liquid conversion
case variable case variable
when String, Integer, Float, NilClass, TrueClass, FalseClass when String, Integer, Float, NilClass, TrueClass, FalseClass, Array, Hash, Time
return variable
when Array, Hash, Time
return variable return variable
end end
@@ -286,17 +287,16 @@ module Liquid
attr_reader :base_scope_depth attr_reader :base_scope_depth
def try_variable_find_in_environments(key, raise_on_not_found:) def try_variable_find_in_environments(key, raise_on_not_found:)
@environments.each do |environment| found = find_in_envs(@environments, key, raise_on_not_found: raise_on_not_found)
return found unless found.nil? && !(@strict_variables && raise_on_not_found)
find_in_envs(@static_environments, key, raise_on_not_found: raise_on_not_found)
end
def find_in_envs(envs, key, raise_on_not_found:)
envs.each do |environment|
found_variable = lookup_and_evaluate(environment, key, raise_on_not_found: raise_on_not_found) found_variable = lookup_and_evaluate(environment, key, raise_on_not_found: raise_on_not_found)
if !found_variable.nil? || @strict_variables && raise_on_not_found return found_variable if !found_variable.nil? || (@strict_variables && raise_on_not_found)
return found_variable
end
end
@static_environments.each do |environment|
found_variable = lookup_and_evaluate(environment, key, raise_on_not_found: raise_on_not_found)
if !found_variable.nil? || @strict_variables && raise_on_not_found
return found_variable
end
end end
nil nil
end end
+16 -17
View File
@@ -59,31 +59,30 @@ module Liquid
raise Liquid::ArgumentError, e.message, e.backtrace raise Liquid::ArgumentError, e.message, e.backtrace
end end
# Fast path for single-argument (no extra args) filter invocation. # Arity-specialized filter invocation.
# Avoids *args splat allocation for the common {{ value | filter }} case. # Avoids *args splat allocation for the common 0-arg and 1-arg cases.
def invoke_single(method, input) # `invoke` (general case) still uses *args for 2+ extra arguments.
{
invoke_single: ['input'],
invoke_two: ['input', 'arg1'],
}.each do |method_name, params|
all_params = (["method"] + params).join(", ")
send_params = params.join(", ")
# __LINE__ + 1 is a parse-time constant; both generated methods will report
# the same file:line in backtraces. The method name in the trace distinguishes them.
module_eval(<<~RUBY, __FILE__, __LINE__ + 1)
def #{method_name}(#{all_params})
if self.class.invokable?(method) if self.class.invokable?(method)
send(method, input) send(method, #{send_params})
elsif @context.strict_filters elsif @context.strict_filters
raise Liquid::UndefinedFilter, "undefined filter #{method}" raise Liquid::UndefinedFilter, "undefined filter \#{method}"
else else
input input
end end
rescue ::ArgumentError => e rescue ::ArgumentError => e
raise Liquid::ArgumentError, e.message, e.backtrace raise Liquid::ArgumentError, e.message, e.backtrace
end end
RUBY
# Fast path for two-argument filter invocation (input + one arg).
def invoke_two(method, input, arg1)
if self.class.invokable?(method)
send(method, input, arg1)
elsif @context.strict_filters
raise Liquid::UndefinedFilter, "undefined filter #{method}"
else
input
end
rescue ::ArgumentError => e
raise Liquid::ArgumentError, e.message, e.backtrace
end end
end end
end end
+185 -174
View File
@@ -14,66 +14,24 @@ module Liquid
class Variable class Variable
# Checks if markup is a simple "name.lookup.chain" with no filters/brackets/quotes. # Checks if markup is a simple "name.lookup.chain" with no filters/brackets/quotes.
# Returns the trimmed markup string, or nil if not simple. # Returns the trimmed markup string, or nil if not simple.
# Avoids regex MatchData allocation.
def self.simple_variable_markup(markup) def self.simple_variable_markup(markup)
len = markup.bytesize return if markup.empty?
return if len == 0 return unless markup.match?(SIMPLE_VARIABLE_RE)
# Avoid allocation when there's no surrounding whitespace (the common case)
# Skip leading whitespace first = markup.getbyte(0)
pos = 0 last = markup.getbyte(markup.bytesize - 1)
while pos < len needs_strip = first == Cursor::SPACE || first == Cursor::TAB || first == Cursor::NL || first == Cursor::CR ||
b = markup.getbyte(pos) last == Cursor::SPACE || last == Cursor::TAB || last == Cursor::NL || last == Cursor::CR
break unless b == 32 || b == 9 || b == 10 || b == 13 needs_strip ? markup.strip : markup
pos += 1
end
return if pos >= len
start = pos
# First char must be [a-zA-Z_]
b = markup.getbyte(pos)
return unless (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || b == 95
pos += 1
# Scan segments: [\w-]* (. [\w-]*)*
while pos < len
b = markup.getbyte(pos)
if (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || (b >= 48 && b <= 57) || b == 95 || b == 45
pos += 1
elsif b == 46 # '.'
pos += 1
# After dot, must have [a-zA-Z_]
return if pos >= len
b = markup.getbyte(pos)
return unless (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || b == 95
pos += 1
else
break
end
end
content_end = pos
# Skip trailing whitespace
while pos < len
b = markup.getbyte(pos)
return unless b == 32 || b == 9 || b == 10 || b == 13
pos += 1
end
# Must have consumed everything
return unless pos == len
if start == 0 && content_end == len
markup
else
markup.byteslice(start, content_end - start)
end
end end
# Cache for [filtername, EMPTY_ARRAY] tuples — avoids repeated array creation # Cache for [filtername, EMPTY_ARRAY] tuples — avoids repeated array creation
NO_ARG_FILTER_CACHE = Hash.new { |h, k| h[k] = [k, Const::EMPTY_ARRAY].freeze } NO_ARG_FILTER_CACHE = Hash.new { |h, k| h[k] = [k, Const::EMPTY_ARRAY].freeze }
# Regex for a simple variable lookup with optional surrounding whitespace.
# Shares the identifier grammar with VariableLookup::SIMPLE_LOOKUP_RE.
SIMPLE_VARIABLE_RE = /\A\s*[\w-]+\??(?:\.[\w-]+\??)*\s*\z/
FilterMarkupRegex = /#{FilterSeparator}\s*(.*)/om FilterMarkupRegex = /#{FilterSeparator}\s*(.*)/om
FilterParser = /(?:\s+|#{QuotedFragment}|#{ArgumentSeparator})+/o FilterParser = /(?:\s+|#{QuotedFragment}|#{ArgumentSeparator})+/o
FilterArgsRegex = /(?:#{FilterArgumentSeparator}|#{ArgumentSeparator})\s*((?:\w+\s*\:\s*)?#{QuotedFragment})/o FilterArgsRegex = /(?:#{FilterArgumentSeparator}|#{ArgumentSeparator})\s*((?:\w+\s*\:\s*)?#{QuotedFragment})/o
@@ -102,6 +60,34 @@ module Liquid
end end
private def try_fast_parse(markup, parse_context) private def try_fast_parse(markup, parse_context)
pos = fast_scan_name(markup)
return false unless pos
# fast_resolve_name calls VariableLookup.parse_simple / Expression::LITERALS — the
# only sites that can raise SyntaxError on malformed input. The byte scanners return
# false instead of raising.
begin
fast_resolve_name(markup, parse_context)
rescue SyntaxError
return false
end
# End of markup — no filters
if pos >= markup.bytesize
@filters = Const::EMPTY_ARRAY
return true
end
# Must be followed by a pipe filter separator
return false unless markup.getbyte(pos) == Cursor::PIPE
fast_scan_filters(markup, pos, parse_context)
end
# Scan the variable name (quoted string or identifier chain) at the start of markup.
# Returns the position after the name + trailing whitespace, or false on failure.
# Sets @_fast_name_start and @_fast_name_end for fast_resolve_name.
private def fast_scan_name(markup)
len = markup.bytesize len = markup.bytesize
return false if len == 0 return false if len == 0
@@ -109,40 +95,40 @@ module Liquid
pos = 0 pos = 0
while pos < len while pos < len
b = markup.getbyte(pos) b = markup.getbyte(pos)
break unless b == 32 || b == 9 || b == 10 || b == 13 break unless b == Cursor::SPACE || b == Cursor::TAB || b == Cursor::NL || b == Cursor::CR
pos += 1 pos += 1
end end
return false if pos >= len return false if pos >= len
b = markup.getbyte(pos) b = markup.getbyte(pos)
if b == 39 || b == 34 # single or double quote if b == Cursor::QUOTE_S || b == Cursor::QUOTE_D
# Quoted string literal: scan to matching close quote # Quoted string literal: scan to matching close quote
quote = b quote = b
name_start = pos @_fast_name_start = pos
pos += 1 pos += 1
pos += 1 while pos < len && markup.getbyte(pos) != quote pos += 1 while pos < len && markup.getbyte(pos) != quote
pos += 1 if pos < len # skip closing quote pos += 1 if pos < len # skip closing quote
name_end = pos @_fast_name_end = pos
elsif (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || b == 95 elsif ByteTables::IDENT_START[b]
# Identifier: scan [\w-]*(\.[\w-]*)* # Identifier chain: [a-zA-Z_][a-zA-Z0-9_-]*(.[a-zA-Z_][a-zA-Z0-9_-]*)*
name_start = pos @_fast_name_start = pos
pos += 1 pos += 1
while pos < len while pos < len
b = markup.getbyte(pos) b = markup.getbyte(pos)
if (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || (b >= 48 && b <= 57) || b == 95 || b == 45 if ByteTables::IDENT_CONT[b]
pos += 1 pos += 1
elsif b == 46 # '.' elsif b == Cursor::DOT
pos += 1 pos += 1
return false if pos >= len return false if pos >= len
b = markup.getbyte(pos) b = markup.getbyte(pos)
return false unless (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || b == 95 return false unless ByteTables::IDENT_START[b]
pos += 1 pos += 1
else else
break break
end end
end end
name_end = pos @_fast_name_end = pos
else else
return false return false
end end
@@ -150,22 +136,30 @@ module Liquid
# Skip whitespace after name # Skip whitespace after name
while pos < len while pos < len
b = markup.getbyte(pos) b = markup.getbyte(pos)
break unless b == 32 || b == 9 || b == 10 || b == 13 break unless b == Cursor::SPACE || b == Cursor::TAB || b == Cursor::NL || b == Cursor::CR
pos += 1 pos += 1
end end
# Resolve the name expression — avoid byteslice when markup is already the name pos
expr_markup = if name_start == 0 && name_end == len
markup # no whitespace, no filters — reuse the string
else
markup.byteslice(name_start, name_end - name_start)
end end
# Resolve the scanned name bytes to a Liquid expression object.
# Reads @_fast_name_start / @_fast_name_end set by fast_scan_name.
# Sets @name. May raise SyntaxError (rescued in try_fast_parse).
private def fast_resolve_name(markup, parse_context)
name_start = @_fast_name_start
name_end = @_fast_name_end
len = markup.bytesize
# Avoid byteslice when the name spans the whole markup (no surrounding whitespace/filters)
expr_markup = name_start == 0 && name_end == len ? markup : markup.byteslice(name_start, name_end - name_start)
cache = parse_context.expression_cache cache = parse_context.expression_cache
ss = parse_context.string_scanner ss = parse_context.string_scanner
first_byte = expr_markup.getbyte(0) first_byte = expr_markup.getbyte(0)
@name = if first_byte == 39 || first_byte == 34 # quoted string @name = if first_byte == Cursor::QUOTE_S || first_byte == Cursor::QUOTE_D
# Strip quotes for string literal # String literal — strip enclosing quotes
expr_markup.byteslice(1, expr_markup.bytesize - 2) expr_markup.byteslice(1, expr_markup.bytesize - 2)
elsif Expression::LITERALS.key?(expr_markup) elsif Expression::LITERALS.key?(expr_markup)
Expression::LITERALS[expr_markup] Expression::LITERALS[expr_markup]
@@ -174,111 +168,81 @@ module Liquid
else else
VariableLookup.parse_simple(expr_markup, ss || StringScanner.new(""), nil).freeze VariableLookup.parse_simple(expr_markup, ss || StringScanner.new(""), nil).freeze
end end
# End of markup? No filters.
if pos >= len
@filters = Const::EMPTY_ARRAY
return true
end end
# Must be a pipe for filters # Scan the filter chain starting at `pos` (the first '|').
return false unless markup.getbyte(pos) == 124 # '|' # Returns true on success (sets @filters), false to fall back to the Lexer.
# Rescues SyntaxError from Expression.parse inside fast_scan_filter_args.
# Try fast filter scanning first — handles no-arg and simple-arg filters private def fast_scan_filters(markup, pos, parse_context)
# Falls through to Lexer-based parsing for complex cases len = markup.bytesize
@filters = [] @filters = []
filter_pos = pos filter_pos = pos
while filter_pos < len && markup.getbyte(filter_pos) == 124 # '|' while filter_pos < len && markup.getbyte(filter_pos) == Cursor::PIPE
filter_pos += 1 filter_pos += 1
# Skip whitespace # Skip spaces after pipe (tabs/newlines handled in the between-filters skip below)
filter_pos += 1 while filter_pos < len && markup.getbyte(filter_pos) == 32 filter_pos += 1 while filter_pos < len && markup.getbyte(filter_pos) == Cursor::SPACE
# Scan filter name # Scan filter name: must start with [a-zA-Z_]
fname_start = filter_pos fname_start = filter_pos
b = filter_pos < len ? markup.getbyte(filter_pos) : nil b = filter_pos < len ? markup.getbyte(filter_pos) : nil
break unless b && ((b >= 97 && b <= 122) || (b >= 65 && b <= 90) || b == 95) break unless b && ByteTables::IDENT_START[b]
filter_pos += 1 filter_pos += 1
while filter_pos < len while filter_pos < len
b = markup.getbyte(filter_pos) b = markup.getbyte(filter_pos)
break unless (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || (b >= 48 && b <= 57) || b == 95 || b == 45 break unless ByteTables::IDENT_CONT[b]
filter_pos += 1 filter_pos += 1
end end
filtername = markup.byteslice(fname_start, filter_pos - fname_start) filtername = markup.byteslice(fname_start, filter_pos - fname_start)
# Skip whitespace # Skip whitespace after filter name
filter_pos += 1 while filter_pos < len && markup.getbyte(filter_pos) == 32 filter_pos += 1 while filter_pos < len && markup.getbyte(filter_pos) == Cursor::SPACE
# Has arguments — try fast scanning for positional args if filter_pos < len && markup.getbyte(filter_pos) == Cursor::COLON
if filter_pos < len && markup.getbyte(filter_pos) == 58 # ':' # Has arguments — fast-scan positional args; fall to Lexer on keyword args
filter_pos += 1 # skip ':' filter_pos += 1 # skip ':'
filter_pos += 1 while filter_pos < len && markup.getbyte(filter_pos) == 32 filter_pos += 1 while filter_pos < len && markup.getbyte(filter_pos) == Cursor::SPACE
filter_args = [] result = fast_scan_filter_args(markup, filter_pos, parse_context)
fall_to_lexer = false return fall_to_lexer_filters(markup, pos, fname_start, len, parse_context) if result == :fall_to_lexer
loop do filter_args, filter_pos = result
arg_start = filter_pos @filters << [filtername, filter_args]
b = filter_pos < len ? markup.getbyte(filter_pos) : nil
if b == 39 || b == 34 # quoted string
quote = b
filter_pos += 1
filter_pos += 1 while filter_pos < len && markup.getbyte(filter_pos) != quote
filter_pos += 1 if filter_pos < len # skip closing quote
filter_args << markup.byteslice(arg_start + 1, filter_pos - arg_start - 2)
elsif b && ((b >= 48 && b <= 57) || (b == 45 && filter_pos + 1 < len && markup.getbyte(filter_pos + 1) >= 48 && markup.getbyte(filter_pos + 1) <= 57))
# Number
filter_pos += 1 if b == 45
filter_pos += 1 while filter_pos < len && markup.getbyte(filter_pos) >= 48 && markup.getbyte(filter_pos) <= 57
if filter_pos < len && markup.getbyte(filter_pos) == 46 # float
filter_pos += 1
filter_pos += 1 while filter_pos < len && markup.getbyte(filter_pos) >= 48 && markup.getbyte(filter_pos) <= 57
end
num_str = markup.byteslice(arg_start, filter_pos - arg_start)
filter_args << (num_str.include?('.') ? num_str.to_f : num_str.to_i)
elsif b && ((b >= 97 && b <= 122) || (b >= 65 && b <= 90) || b == 95)
# Identifier
id_start = filter_pos
filter_pos += 1
while filter_pos < len
b2 = markup.getbyte(filter_pos)
break unless (b2 >= 97 && b2 <= 122) || (b2 >= 65 && b2 <= 90) || (b2 >= 48 && b2 <= 57) || b2 == 95 || b2 == 45 || b2 == 46
filter_pos += 1
end
filter_pos += 1 if filter_pos < len && markup.getbyte(filter_pos) == 63
# Check if keyword arg (id followed by ':')
kw_check = filter_pos
kw_check += 1 while kw_check < len && markup.getbyte(kw_check) == 32
if kw_check < len && markup.getbyte(kw_check) == 58
fall_to_lexer = true
break
end
id_markup = markup.byteslice(id_start, filter_pos - id_start)
filter_args << Expression.parse(id_markup, parse_context.string_scanner, parse_context.expression_cache)
else else
fall_to_lexer = true # No-arg filter — reuse the cached [name, EMPTY_ARRAY] tuple
break @filters << NO_ARG_FILTER_CACHE[filtername]
end end
# Skip whitespace after arg # Skip whitespace (including tabs and newlines) between filters
filter_pos += 1 while filter_pos < len && markup.getbyte(filter_pos) == 32 filter_pos += 1 while filter_pos < len && (
markup.getbyte(filter_pos) == Cursor::SPACE ||
# Comma = more args; pipe/end = done markup.getbyte(filter_pos) == Cursor::TAB ||
if filter_pos < len && markup.getbyte(filter_pos) == 44 markup.getbyte(filter_pos) == Cursor::NL ||
filter_pos += 1 markup.getbyte(filter_pos) == Cursor::CR
filter_pos += 1 while filter_pos < len && markup.getbyte(filter_pos) == 32 )
else
break
end
end end
if fall_to_lexer # Trailing bytes that aren't a pipe mean something the fast path doesn't handle
# Complex filter — fall to Lexer for this and remaining filters return false if filter_pos < len
@filters = Const::EMPTY_ARRAY if @filters.empty?
true
rescue SyntaxError
# Expression.parse (called inside fast_scan_filter_args for identifier args) can
# raise SyntaxError on malformed input. Fall back to full Lexer parse.
@name = nil
@filters = nil
false
end
# Called when fast_scan_filter_args encounters keyword args or an unrecognised
# token. Hands the remaining filter chain (from the pipe before fname_start)
# to the full Lexer-based parser, merges results into @filters, and returns true.
private def fall_to_lexer_filters(markup, pos, fname_start, len, parse_context)
# Walk back from fname_start to find the pipe that opened this filter.
# Equivalent to: markup.rindex('|', fname_start), bounded by pos.
rest_start = fname_start rest_start = fname_start
rest_start -= 1 while rest_start > pos && markup.getbyte(rest_start) != 124 rest_start -= 1 while rest_start > pos && markup.getbyte(rest_start) != Cursor::PIPE
rest_markup = markup.byteslice(rest_start, len - rest_start) rest_markup = markup.byteslice(rest_start, len - rest_start)
p = parse_context.new_parser(rest_markup) p = parse_context.new_parser(rest_markup)
while p.consume?(:pipe) while p.consume?(:pipe)
@@ -287,30 +251,77 @@ module Liquid
@filters << lax_parse_filter_expressions(fn, fa) @filters << lax_parse_filter_expressions(fn, fa)
end end
p.consume(:end_of_string) p.consume(:end_of_string)
@filters = Const::EMPTY_ARRAY if @filters.empty?
return true
end
@filters << [filtername, filter_args]
else
# No args — add as simple filter
@filters << NO_ARG_FILTER_CACHE[filtername]
end
# Skip whitespace between filters
filter_pos += 1 while filter_pos < len && (markup.getbyte(filter_pos) == 32 || markup.getbyte(filter_pos) == 9 || markup.getbyte(filter_pos) == 10 || markup.getbyte(filter_pos) == 13)
end
# Must have consumed everything
return false if filter_pos < len
@filters = Const::EMPTY_ARRAY if @filters.empty? @filters = Const::EMPTY_ARRAY if @filters.empty?
true true
rescue SyntaxError end
# If fast parse fails, fall back to full parse
@name = nil # Scan positional filter arguments starting at `filter_pos`.
@filters = nil # Returns [filter_args_array, new_filter_pos] on success, or :fall_to_lexer when
false # keyword args or unrecognised tokens are encountered.
private def fast_scan_filter_args(markup, filter_pos, parse_context)
len = markup.bytesize
filter_args = []
loop do
arg_start = filter_pos
b = filter_pos < len ? markup.getbyte(filter_pos) : nil
if b == Cursor::QUOTE_S || b == Cursor::QUOTE_D
# Quoted string argument
quote = b
filter_pos += 1
filter_pos += 1 while filter_pos < len && markup.getbyte(filter_pos) != quote
filter_pos += 1 if filter_pos < len # skip closing quote
filter_args << markup.byteslice(arg_start + 1, filter_pos - arg_start - 2)
elsif b && (ByteTables::DIGIT[b] ||
(b == Cursor::DASH && filter_pos + 1 < len && ByteTables::DIGIT[markup.getbyte(filter_pos + 1)]))
# Numeric argument (integer or float, optionally negative)
filter_pos += 1 if b == Cursor::DASH
filter_pos += 1 while filter_pos < len && ByteTables::DIGIT[markup.getbyte(filter_pos)]
if filter_pos < len && markup.getbyte(filter_pos) == Cursor::DOT # float
filter_pos += 1
filter_pos += 1 while filter_pos < len && ByteTables::DIGIT[markup.getbyte(filter_pos)]
end
num_str = markup.byteslice(arg_start, filter_pos - arg_start)
filter_args << (num_str.include?('.') ? num_str.to_f : num_str.to_i)
elsif b && ByteTables::IDENT_START[b]
# Identifier argument — may be a variable lookup or keyword arg
id_start = filter_pos
filter_pos += 1
while filter_pos < len
b2 = markup.getbyte(filter_pos)
break unless ByteTables::IDENT_CONT[b2] || b2 == Cursor::DOT
filter_pos += 1
end
filter_pos += 1 if filter_pos < len && markup.getbyte(filter_pos) == Cursor::QMARK
# Peek past whitespace: if followed by ':', this is a keyword arg → fall to Lexer
kw_check = filter_pos
kw_check += 1 while kw_check < len && markup.getbyte(kw_check) == Cursor::SPACE
return :fall_to_lexer if kw_check < len && markup.getbyte(kw_check) == Cursor::COLON
id_markup = markup.byteslice(id_start, filter_pos - id_start)
filter_args << Expression.parse(id_markup, parse_context.string_scanner, parse_context.expression_cache)
else
return :fall_to_lexer
end
# Skip whitespace after argument
filter_pos += 1 while filter_pos < len && markup.getbyte(filter_pos) == Cursor::SPACE
# Comma: more arguments follow; anything else: done with this filter's args
if filter_pos < len && markup.getbyte(filter_pos) == Cursor::COMMA
filter_pos += 1
filter_pos += 1 while filter_pos < len && markup.getbyte(filter_pos) == Cursor::SPACE
else
break
end
end
[filter_args, filter_pos]
end end
def raw def raw
+29 -15
View File
@@ -33,9 +33,9 @@ module Liquid
pos += 1 pos += 1
while pos < len && depth > 0 while pos < len && depth > 0
b = markup.getbyte(pos) b = markup.getbyte(pos)
if b == 91 if b == 91 # rubocop:disable Metrics/BlockNesting
depth += 1 depth += 1
elsif b == 93 elsif b == 93 # rubocop:disable Metrics/BlockNesting
depth -= 1 depth -= 1
end end
pos += 1 pos += 1
@@ -114,7 +114,7 @@ module Liquid
lookups = self.class.scan_variable(markup) lookups = self.class.scan_variable(markup)
name = lookups.shift name = lookups.shift
if name&.start_with?('[') && name&.end_with?(']') if name&.start_with?('[') && name.end_with?(']')
name = Expression.parse( name = Expression.parse(
name[1..-2], name[1..-2],
string_scanner, string_scanner,
@@ -126,9 +126,8 @@ module Liquid
@lookups = lookups @lookups = lookups
@command_flags = 0 @command_flags = 0
@lookups.each_index do |i| @lookups.each_with_index do |lookup, i|
lookup = lookups[i] if lookup&.start_with?('[') && lookup.end_with?(']')
if lookup&.start_with?('[') && lookup&.end_with?(']')
lookups[i] = Expression.parse( lookups[i] = Expression.parse(
lookup[1..-2], lookup[1..-2],
string_scanner, string_scanner,
@@ -160,18 +159,13 @@ module Liquid
# If object is a hash- or array-like object we look for the # If object is a hash- or array-like object we look for the
# presence of the key and if its available we return it # presence of the key and if its available we return it
if object.instance_of?(Hash) ? object.key?(key) : if accessible?(object, key)
(object.respond_to?(:[]) &&
((object.respond_to?(:key?) && object.key?(key)) ||
(object.respond_to?(:fetch) && key.is_a?(Integer))))
# if its a proc we will replace the entry with the proc # if its a proc we will replace the entry with the proc
object = context.lookup_and_evaluate(object, key) object = context.lookup_and_evaluate(object, key)
# Skip to_liquid for common primitive types (they return self) # Skip to_liquid for common primitive types (they return self)
unless object.instance_of?(String) || object.instance_of?(Integer) || object.instance_of?(Float) || unless object.instance_of?(String) || object.instance_of?(Integer) || object.instance_of?(Float) ||
object.instance_of?(Array) || object.instance_of?(Hash) || object.nil? object.instance_of?(Array) || object.instance_of?(Hash) || object.nil?
object = object.to_liquid object = liquidize(object, context)
object.context = context if object.respond_to?(:context=)
end end
# Some special cases. If the part wasn't in square brackets and # Some special cases. If the part wasn't in square brackets and
@@ -180,8 +174,7 @@ module Liquid
elsif lookup_command?(i) && object.respond_to?(key) elsif lookup_command?(i) && object.respond_to?(key)
object = object.send(key) object = object.send(key)
unless object.instance_of?(String) || object.instance_of?(Integer) || object.instance_of?(Array) || object.nil? unless object.instance_of?(String) || object.instance_of?(Integer) || object.instance_of?(Array) || object.nil?
object = object.to_liquid object = liquidize(object, context)
object.context = context if object.respond_to?(:context=)
end end
# Handle string first/last like ActiveSupport does (returns first/last character) # Handle string first/last like ActiveSupport does (returns first/last character)
@@ -205,6 +198,27 @@ module Liquid
self.class == other.class && state == other.state self.class == other.class && state == other.state
end end
private
# Returns true if +object+ has +key+ accessible via [] lookup.
def accessible?(object, key)
if object.instance_of?(Hash)
object.key?(key)
else
object.respond_to?(:[]) &&
((object.respond_to?(:key?) && object.key?(key)) ||
(object.respond_to?(:fetch) && key.is_a?(Integer)))
end
end
# Calls to_liquid on +object+ and wires up the context reference if needed.
# Skipped for primitive types that return self from to_liquid.
def liquidize(object, context)
object = object.to_liquid
object.context = context if object.respond_to?(:context=)
object
end
protected protected
def state def state
+126
View File
@@ -0,0 +1,126 @@
# frozen_string_literal: true
require 'test_helper'
# Tests that the fast-path parser (try_fast_parse) produces the same result as the
# full Lexer → Parser pipeline for every input we expect it to handle.
#
# This protects against silent regressions where a change to try_fast_parse causes it
# to produce different output from the slow path (the existing test suite would still
# pass because the slow path catches it, but correctness would be silently lost).
class VariableFastParseTest < Minitest::Test
include Liquid
EQUIVALENCE_CASES = [
# Simple lookups
"product",
"product.title",
"product.variants.first.title",
# Quoted string literals
"'hello'",
'"hello"',
# Variables with no-arg filters
"product | upcase",
"product | upcase | downcase",
"product | strip | upcase | downcase",
# Variables with single-arg filters
"product | truncate: 50",
"product | plus: 1",
"product | plus: -3",
"product | round: 2",
"product | append: ' world'",
# Variables with multi-arg filters
"product | replace: 'a', 'b'",
"product | pluralize: 'item', 'items'",
"product | slice: 0, 5",
# Chained mixed filters
"product.title | truncate: 50",
"'hello' | append: ' world' | upcase",
"name | prepend: 'Dr. ' | append: ' PhD' | upcase",
# Numeric args
"count | plus: 1.5",
"price | minus: 0.99",
# No whitespace around pipe
"x|upcase",
"x|replace:'a','b'|upcase",
# Leading/trailing whitespace
" product ",
" product.title | upcase ",
].freeze
EQUIVALENCE_CASES.each_with_index do |markup, i|
define_method(:"test_fast_parse_equivalence_#{i.to_s.rjust(2, "0")}") do
lax_ctx = Liquid::ParseContext.new(error_mode: :lax)
strict_ctx = Liquid::ParseContext.new(error_mode: :strict)
lax_var = Liquid::Variable.new(markup, lax_ctx)
strict_var = Liquid::Variable.new(markup, strict_ctx)
assert_equal strict_var.name,
lax_var.name,
"Name mismatch for #{markup.inspect}: " \
"lax=#{lax_var.name.inspect} strict=#{strict_var.name.inspect}"
assert_equal strict_var.filters.length,
lax_var.filters.length,
"Filter count mismatch for #{markup.inspect}: " \
"lax=#{lax_var.filters.inspect} strict=#{strict_var.filters.inspect}"
strict_var.filters.each_with_index do |(s_name, *), i|
l_name = lax_var.filters[i][0]
assert_equal s_name,
l_name,
"Filter name mismatch at index #{i} for #{markup.inspect}"
end
end
end
# Verify the fast path is actually taken for simple variables (i.e. filters is the
# shared frozen EMPTY_ARRAY, not a newly allocated array).
def test_fast_path_taken_for_simple_variable
ctx = Liquid::ParseContext.new(error_mode: :lax)
var = Liquid::Variable.new("product.title", ctx)
assert_same(
Liquid::Const::EMPTY_ARRAY,
var.filters,
"Expected fast path (frozen EMPTY_ARRAY) for simple variable",
)
end
def test_fast_path_taken_for_no_arg_filter
ctx = Liquid::ParseContext.new(error_mode: :lax)
var = Liquid::Variable.new("product | upcase", ctx)
assert_equal(1, var.filters.length)
assert_equal("upcase", var.filters[0][0])
# The no-arg filter tuple should come from NO_ARG_FILTER_CACHE (frozen)
assert_predicate(var.filters[0], :frozen?)
end
def test_fast_path_taken_for_single_arg_filter
ctx = Liquid::ParseContext.new(error_mode: :lax)
var = Liquid::Variable.new("product | truncate: 50", ctx)
assert_equal(1, var.filters.length)
assert_equal("truncate", var.filters[0][0])
assert_equal([50], var.filters[0][1])
end
# Keyword args must fall through to the Lexer — verify the result is still correct.
def test_keyword_arg_falls_to_lexer_and_parses_correctly
ctx = Liquid::ParseContext.new(error_mode: :lax)
var = Liquid::Variable.new("img | img_tag: class: 'hero'", ctx)
assert_equal(1, var.filters.length)
assert_equal("img_tag", var.filters[0][0])
end
# Numeric filter arguments: integers and floats
def test_numeric_filter_args
ctx = Liquid::ParseContext.new(error_mode: :lax)
int_var = Liquid::Variable.new("price | plus: 3", ctx)
assert_equal([3], int_var.filters[0][1])
neg_var = Liquid::Variable.new("price | minus: -1", ctx)
assert_equal([-1], neg_var.filters[0][1])
float_var = Liquid::Variable.new("price | round: 2.5", ctx)
assert_equal([2.5], float_var.filters[0][1])
end
end