Add ExpressionParser and ExpressionConsumer

This commit is contained in:
Guilherme Carreiro
2025-10-27 16:33:31 +01:00
committed by Guilherme Carreiro
parent 6a9e46dd19
commit 1bff382ebc
7 changed files with 773 additions and 1 deletions
+2
View File
@@ -80,6 +80,8 @@ require 'liquid/variable_lookup'
require 'liquid/range_lookup'
require 'liquid/resource_limits'
require 'liquid/expression'
require 'liquid/expression_consumer'
require 'liquid/expression_parser'
require 'liquid/template'
require 'liquid/condition'
require 'liquid/utils'
+1 -1
View File
@@ -34,7 +34,7 @@ module Liquid
# @param file_system The default file system that is used
# to load templates from.
# @param error_mode [Symbol] The default error mode for all templates
# (either :strict, :warn, :lax, or :rigid).
# (either :rigid, :strict, :warn, or :lax).
# @param exception_renderer [Proc] The exception renderer that is used to
# render exceptions.
# @yieldparam environment [Environment] The environment instance that is being built.
+182
View File
@@ -0,0 +1,182 @@
# frozen_string_literal: true
module Liquid
module ExpressionConsumer
INTEGER_REGEX = /\A(-?\d+)\z/
FLOAT_REGEX = /\A(-?\d+)\.\d+\z/
LITERALS = {
'nil' => nil,
'null' => nil,
'true' => true,
'false' => false,
'blank' => '',
'empty' => '',
}.freeze
MINUS_VARIABLE_LOOKUP = VariableLookup.parse("-", nil).freeze
class << self
# Consumes tokens from a Parser instance to build an Expression
# object.
#
# This method reads tokens from the current parser position,
# consuming exactly one complete expression. The parser position is
# advanced past the consumed tokens.
#
# Unlike ExpressionParser.parse, this method does NOT validate that
# all tokens are consumed. It stops after consuming a complete
# expression, leaving any remaining tokens for the caller to handle.
#
# This is the efficient low-level method used by tags that manage
# their own Parser instances and need to consume multiple
# expressions from a single token stream.
#
# Returns an Expression object appropriate for the token type:
# - Literals (nil, true, false, numbers, strings)
# - VariableLookup (variables with optional property/index access)
# - RangeLookup or Range (for range expressions like (1..10))
#
# Raises SyntaxError if invalid token encountered.
#
# Examples:
# parser = parse_context.new_parser("product.title | upcase")
# expr = ExpressionConsumer.consume(parser, parse_context)
# #=> #<VariableLookup @name="product" @lookups=["title"]>
# # Parser is now positioned at the pipe token
#
# parser = parse_context.new_parser("42")
# ExpressionConsumer.consume(parser, parse_context)
# #=> 42
#
# parser = parse_context.new_parser("items[0]")
# ExpressionConsumer.consume(parser, parse_context)
# #=> #<VariableLookup @name="items" @lookups=[0]>
def consume(parser, parse_context)
token = parser.tokens[parser.point]
case token[0]
when :string
str = parser.consume(:string)
parse_string(str)
when :number
num_str = parser.consume(:number)
parse_number(num_str)
when :id
parse_id(parser, parse_context)
when :open_square
# Bracket notation: [expression]
parser.consume(:open_square)
inner = consume(parser, parse_context)
parser.consume(:close_square)
lookups = parse_variable_lookups(parser, parse_context)
build_variable_lookup(inner, lookups)
when :open_round
# Range notation: (start..end)
parser.consume(:open_round)
start_obj = consume(parser, parse_context)
parser.consume(:dotdot)
end_obj = consume(parser, parse_context)
parser.consume(:close_round)
build_range_lookup(start_obj, end_obj)
else
raise SyntaxError, "#{token} is not a valid expression"
end
end
private
def parse_string(str)
str[1..-2]
end
def parse_number(num_str)
case num_str
when INTEGER_REGEX then Integer(num_str, 10)
when FLOAT_REGEX then num_str.to_f
else
raise Liquid::SyntaxError, "Invalid expression type in number expression"
end
end
def parse_id(parser, parse_context)
id_value = parser.consume(:id)
lookups = parse_variable_lookups(parser, parse_context)
if LITERALS.key?(id_value)
# Case: nil
return LITERALS[id_value] if lookups.empty?
# Case: nil.size
return build_variable_lookup(id_value, lookups)
end
if id_value == '-'
# Case (backwards compatibility): -
return MINUS_VARIABLE_LOOKUP if lookups.empty?
# Case: -var
return build_variable_lookup('-', lookups)
end
# Case: var
build_variable_lookup(id_value, lookups)
end
def parse_variable_lookups(parser, parse_context)
lookups = []
loop do
if parser.look(:open_square)
parser.consume(:open_square)
lookup = consume(parser, parse_context)
parser.consume(:close_square)
lookups << lookup
next
end
if parser.look(:dot)
parser.consume(:dot)
id = parser.consume(:id)
lookups << id
next
end
break
end
lookups
end
# todo(guilherme): avoid allocate, simplify this
def build_variable_lookup(name, lookups)
lookup = VariableLookup.allocate
lookup.instance_variable_set(:@name, name)
lookup.instance_variable_set(:@lookups, lookups)
command_flags = 0
lookups.each_with_index do |lookup_item, i|
if lookup_item.is_a?(String) && VariableLookup::COMMAND_METHODS.include?(lookup_item)
command_flags |= 1 << i
end
end
lookup.instance_variable_set(:@command_flags, command_flags)
lookup
end
# todo(guilherme): use RangeLookup.parse logic, simplify this
def build_range_lookup(start_obj, end_obj)
if !start_obj.respond_to?(:evaluate) && !end_obj.respond_to?(:evaluate)
begin
start_obj.to_i..end_obj.to_i
rescue NoMethodError
raise Liquid::SyntaxError, "Invalid expression type in range expression"
end
else
RangeLookup.new(start_obj, end_obj)
end
end
end
end
end
+51
View File
@@ -0,0 +1,51 @@
# frozen_string_literal: true
module Liquid
module ExpressionParser
class << self
# Parses a Liquid expression string into an Expression object using
# strict token-based validation.
#
# This method tokenizes the markup, validates that the expression
# consumes all available tokens (no trailing garbage), and builds
# an appropriate Expression object (literal, VariableLookup, or
# RangeLookup).
#
# Returns nil if the markup is empty or contains only whitespace.
#
# Raises SyntaxError if:
# - Invalid syntax is encountered
# - Extra tokens remain after the expression
# (e.g., "product title" instead of "product.title")
#
# Examples:
# ExpressionParser.parse("product.title", ctx)
# #=> #<VariableLookup @name="product" @lookups=["title"]>
#
# ExpressionParser.parse("42", ctx)
# #=> 42
#
# ExpressionParser.parse("(1..10)", ctx)
# #=> 1..10
#
# ExpressionParser.parse("", ctx)
# #=> nil
#
# ExpressionParser.parse("product title", ctx)
# #=> raises SyntaxError (extra token "title")
def parse(markup, parse_context)
parser = parse_context.new_parser(markup)
# Whitespaces only.
return if parser.look(:end_of_string)
result = ExpressionConsumer.consume(parser, parse_context)
# Extra tokens after the expression.
parser.consume(:end_of_string) unless parser.look(:end_of_string)
result
end
end
end
end
+6
View File
@@ -2,6 +2,8 @@
module Liquid
class Parser
attr_reader :tokens
def initialize(input)
ss = input.is_a?(StringScanner) ? input : StringScanner.new(input)
@tokens = Lexer.tokenize(ss)
@@ -82,6 +84,10 @@ module Liquid
str
end
def point
@p
end
def variable_lookups
str = +""
loop do
+429
View File
@@ -0,0 +1,429 @@
# frozen_string_literal: true
require 'test_helper'
class ExpressionConsumerTest < Minitest::Test
include Liquid
def test_consume_string_literal_with_double_quotes
parser = parse_context.new_parser('"hello"')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal('hello', result)
end
def test_consume_string_literal_with_single_quotes
parser = parse_context.new_parser("'world'")
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal('world', result)
end
def test_consume_string_with_empty_content
parser = parse_context.new_parser('""')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal('', result)
end
def test_consume_single_quote_empty_string
parser = parse_context.new_parser("''")
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal('', result)
end
def test_consume_integer_literal
parser = parse_context.new_parser('42')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal(42, result)
assert_kind_of(Integer, result)
end
def test_consume_negative_integer_literal
parser = parse_context.new_parser('-42')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal(-42, result)
end
def test_consume_float_literal
parser = parse_context.new_parser('3.14')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal(3.14, result)
assert_kind_of(Float, result)
end
def test_consume_negative_float_literal
parser = parse_context.new_parser('-3.14')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal(-3.14, result)
end
def test_consume_zero_as_integer
parser = parse_context.new_parser('0')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal(0, result)
assert_kind_of(Integer, result)
end
def test_consume_zero_as_float
parser = parse_context.new_parser('0.0')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal(0.0, result)
assert_kind_of(Float, result)
end
def test_consume_nil_literal
parser = parse_context.new_parser('nil')
result = ExpressionConsumer.consume(parser, parse_context)
assert_nil(result)
end
def test_consume_null_literal
parser = parse_context.new_parser('null')
result = ExpressionConsumer.consume(parser, parse_context)
assert_nil(result)
end
def test_consume_true_literal
parser = parse_context.new_parser('true')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal(true, result)
end
def test_consume_false_literal
parser = parse_context.new_parser('false')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal(false, result)
end
def test_consume_blank_literal
parser = parse_context.new_parser('blank')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal('', result)
end
def test_consume_empty_literal
parser = parse_context.new_parser('empty')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal('', result)
end
def test_consume_nil_literal_with_lookups
parser = parse_context.new_parser('nil.size')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(VariableLookup, result)
assert_equal('nil', result.name)
assert_equal(['size'], result.lookups)
end
def test_consume_true_literal_with_lookups
parser = parse_context.new_parser('true.size')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(VariableLookup, result)
assert_equal('true', result.name)
assert_equal(['size'], result.lookups)
end
def test_consume_negative_number_parses_as_number
parser = parse_context.new_parser('-5')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal(-5, result)
end
def test_consume_simple_variable
parser = parse_context.new_parser('product')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(VariableLookup, result)
assert_equal('product', result.name)
assert_equal([], result.lookups)
end
def test_consume_variable_with_dot_lookup
parser = parse_context.new_parser('product.title')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(VariableLookup, result)
assert_equal('product', result.name)
assert_equal(['title'], result.lookups)
end
def test_consume_variable_with_multiple_dot_lookups
parser = parse_context.new_parser('product.variants.first')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(VariableLookup, result)
assert_equal('product', result.name)
assert_equal(['variants', 'first'], result.lookups)
end
def test_consume_variable_with_bracket_lookup
parser = parse_context.new_parser('items[0]')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(VariableLookup, result)
assert_equal('items', result.name)
assert_equal(0, result.lookups[0])
end
def test_consume_variable_with_bracket_string_lookup
parser = parse_context.new_parser('items["key"]')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(VariableLookup, result)
assert_equal('items', result.name)
assert_equal('key', result.lookups[0])
end
def test_consume_variable_with_bracket_variable_lookup
parser = parse_context.new_parser('items[index]')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(VariableLookup, result)
assert_equal('index', result.lookups[0].name)
end
def test_consume_variable_with_mixed_lookups
parser = parse_context.new_parser('product.variants[0].title')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(VariableLookup, result)
assert_equal('product', result.name)
assert_equal(3, result.lookups.length)
assert_equal('variants', result.lookups[0])
assert_equal(0, result.lookups[1])
assert_equal('title', result.lookups[2])
end
def test_consume_bracket_notation_without_variable
parser = parse_context.new_parser('[0]')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(VariableLookup, result)
assert_equal(0, result.name)
end
def test_consume_bracket_notation_with_lookups
parser = parse_context.new_parser('[0].title')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(VariableLookup, result)
assert_equal(0, result.name)
assert_equal(['title'], result.lookups)
end
def test_consume_bracket_notation_with_bracket_lookups
parser = parse_context.new_parser('[0][1]')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(VariableLookup, result)
assert_equal(0, result.name)
assert_equal(1, result.lookups[0])
end
def test_consume_range_with_integer_literals
parser = parse_context.new_parser('(1..5)')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(Range, result)
assert_equal(1..5, result)
end
def test_consume_range_with_negative_integers
parser = parse_context.new_parser('(-5..-1)')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(Range, result)
assert_equal(-5..-1, result)
end
def test_consume_range_with_variable_start
parser = parse_context.new_parser('(start..10)')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(RangeLookup, result)
assert_kind_of(VariableLookup, result.start_obj)
assert_equal(10, result.end_obj)
end
def test_consume_range_with_variable_end
parser = parse_context.new_parser('(1..end)')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(RangeLookup, result)
assert_equal(1, result.start_obj)
assert_kind_of(VariableLookup, result.end_obj)
end
def test_consume_range_with_both_variables
parser = parse_context.new_parser('(start..end)')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(RangeLookup, result)
assert_kind_of(VariableLookup, result.start_obj)
assert_kind_of(VariableLookup, result.end_obj)
end
def test_consume_range_with_variable_lookups
parser = parse_context.new_parser('(start.value..end.value)')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(RangeLookup, result)
assert_equal(['value'], result.start_obj.lookups)
end
def test_consume_command_method_size_sets_flag
parser = parse_context.new_parser('items.size')
result = ExpressionConsumer.consume(parser, parse_context)
assert(result.lookup_command?(0))
end
def test_consume_command_method_first_sets_flag
parser = parse_context.new_parser('items.first')
result = ExpressionConsumer.consume(parser, parse_context)
assert(result.lookup_command?(0))
end
def test_consume_command_method_last_sets_flag
parser = parse_context.new_parser('items.last')
result = ExpressionConsumer.consume(parser, parse_context)
assert(result.lookup_command?(0))
end
def test_consume_non_command_method_does_not_set_flag
parser = parse_context.new_parser('items.title')
result = ExpressionConsumer.consume(parser, parse_context)
refute(result.lookup_command?(0))
end
def test_consume_advances_parser_position
parser = parse_context.new_parser('foo.bar')
ExpressionConsumer.consume(parser, parse_context)
assert(parser.look(:end_of_string))
end
def test_consume_stops_before_extra_tokens
parser = parse_context.new_parser('foo bar')
ExpressionConsumer.consume(parser, parse_context)
refute(parser.look(:end_of_string))
end
def test_consume_with_nested_brackets
parser = parse_context.new_parser('items[items[0]]')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(VariableLookup, result)
assert_kind_of(VariableLookup, result.lookups[0])
end
def test_consume_bracket_with_range
parser = parse_context.new_parser('items[(1..3)]')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(Range, result.lookups[0])
assert_equal(1..3, result.lookups[0])
end
def test_consume_raises_on_invalid_token_type
parser = parse_context.new_parser('|')
error = assert_raises(SyntaxError) do
ExpressionConsumer.consume(parser, parse_context)
end
assert_match(/is not a valid expression/, error.message)
end
def test_consume_range_with_string_literals
parser = parse_context.new_parser('("a".."z")')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(Range, result)
assert_equal(0..0, result)
end
def test_consume_multiple_command_methods
parser = parse_context.new_parser('items.first.size.last')
result = ExpressionConsumer.consume(parser, parse_context)
assert(result.lookup_command?(0))
assert(result.lookup_command?(1))
assert(result.lookup_command?(2))
end
def test_consume_dot_after_bracket
parser = parse_context.new_parser('items[0].title')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal(0, result.lookups[0])
assert_equal('title', result.lookups[1])
end
def test_consume_bracket_after_dot
parser = parse_context.new_parser('product.variants[0]')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal('variants', result.lookups[0])
assert_equal(0, result.lookups[1])
end
def test_consume_multiple_brackets_with_different_types
parser = parse_context.new_parser('a[0]["key"][var]')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal(0, result.lookups[0])
assert_equal('key', result.lookups[1])
assert_kind_of(VariableLookup, result.lookups[2])
end
def test_consume_deep_nested_brackets
parser = parse_context.new_parser('a[b[c[d]]]')
result = ExpressionConsumer.consume(parser, parse_context)
inner1 = result.lookups[0]
assert_kind_of(VariableLookup, inner1)
inner2 = inner1.lookups[0]
assert_kind_of(VariableLookup, inner2)
end
def test_consume_with_spaces_in_range
parser = parse_context.new_parser('( 1 .. 10 )')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal(1..10, result)
end
def test_consume_starting_with_bracket_then_dots
parser = parse_context.new_parser('[0].first.last')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal(0, result.name)
assert(result.lookup_command?(0))
assert(result.lookup_command?(1))
end
def test_consume_only_dot_lookups
parser = parse_context.new_parser('a.b.c.d')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal(['b', 'c', 'd'], result.lookups)
end
def test_consume_only_bracket_lookups
parser = parse_context.new_parser('a[0][1][2]')
result = ExpressionConsumer.consume(parser, parse_context)
assert_equal(3, result.lookups.length)
assert_equal(0, result.lookups[0])
assert_equal(1, result.lookups[1])
assert_equal(2, result.lookups[2])
end
def test_consume_complex_nested_expression
parser = parse_context.new_parser('product.variants[index].title')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(VariableLookup, result.lookups[1])
assert_equal('index', result.lookups[1].name)
end
def test_consume_range_with_bracketed_variables
parser = parse_context.new_parser('(items[0]..items[1])')
result = ExpressionConsumer.consume(parser, parse_context)
assert_kind_of(RangeLookup, result)
assert_kind_of(VariableLookup, result.start_obj)
end
def test_consume_evaluates_correctly
parser = parse_context.new_parser('product')
result = ExpressionConsumer.consume(parser, parse_context)
context = Context.new({ 'product' => 'Test Product' })
assert_equal('Test Product', context.evaluate(result))
end
def test_consume_with_lookups_evaluates_correctly
parser = parse_context.new_parser('product.title')
result = ExpressionConsumer.consume(parser, parse_context)
context = Context.new({ 'product' => { 'title' => 'My Title' } })
assert_equal('My Title', context.evaluate(result))
end
def test_consume_range_evaluates_correctly
parser = parse_context.new_parser('(start..end)')
result = ExpressionConsumer.consume(parser, parse_context)
context = Context.new({ 'start' => 1, 'end' => 5 })
assert_equal(1..5, context.evaluate(result))
end
private
def parse_context
@parse_context ||= ParseContext.new(environment: Environment.build(error_mode: :rigid))
end
end
+102
View File
@@ -0,0 +1,102 @@
# frozen_string_literal: true
require 'test_helper'
class ExpressionParserTest < Minitest::Test
include Liquid
def test_parse_returns_nil_for_empty_string
result = ExpressionParser.parse('', parse_context)
assert_nil(result)
end
def test_parse_returns_nil_for_whitespace_only
result = ExpressionParser.parse(' ', parse_context)
assert_nil(result)
end
def test_parse_raises_on_extra_tokens_after_expression
error = assert_raises(SyntaxError) do
ExpressionParser.parse('foo bar', parse_context)
end
assert_match(/Expected end_of_string but found id/, error.message)
end
def test_parse_string_literal_with_double_quotes
result = ExpressionParser.parse('"hello"', parse_context)
assert_equal('hello', result)
end
def test_parse_string_literal_with_single_quotes
result = ExpressionParser.parse("'world'", parse_context)
assert_equal('world', result)
end
def test_parse_integer_literal
result = ExpressionParser.parse('42', parse_context)
assert_equal(42, result)
end
def test_parse_float_literal
result = ExpressionParser.parse('3.14', parse_context)
assert_equal(3.14, result)
end
def test_parse_nil_literal
result = ExpressionParser.parse('nil', parse_context)
assert_nil(result)
end
def test_parse_true_literal
result = ExpressionParser.parse('true', parse_context)
assert_equal(true, result)
end
def test_parse_simple_variable
result = ExpressionParser.parse('product', parse_context)
assert_kind_of(VariableLookup, result)
assert_equal('product', result.name)
end
def test_parse_variable_with_dot_lookup
result = ExpressionParser.parse('product.title', parse_context)
assert_kind_of(VariableLookup, result)
assert_equal('product', result.name)
assert_equal(['title'], result.lookups)
end
def test_parse_variable_with_bracket_lookup
result = ExpressionParser.parse('items[0]', parse_context)
assert_kind_of(VariableLookup, result)
assert_equal('items', result.name)
assert_equal(0, result.lookups[0])
end
def test_parse_range_with_integer_literals
result = ExpressionParser.parse('(1..5)', parse_context)
assert_kind_of(Range, result)
assert_equal(1..5, result)
end
def test_parse_range_with_variables
result = ExpressionParser.parse('(start..end)', parse_context)
assert_kind_of(RangeLookup, result)
end
def test_parse_validates_end_of_string
result = ExpressionParser.parse('foo', parse_context)
assert_kind_of(VariableLookup, result)
end
def test_parse_evaluates_correctly
result = ExpressionParser.parse('product.title', parse_context)
context = Context.new({ 'product' => { 'title' => 'My Title' } })
assert_equal('My Title', context.evaluate(result))
end
private
def parse_context
@parse_context ||= ParseContext.new(environment: Environment.build(error_mode: :rigid))
end
end