diff --git a/lib/liquid.rb b/lib/liquid.rb index 4d0a71a6..14b02d26 100644 --- a/lib/liquid.rb +++ b/lib/liquid.rb @@ -83,6 +83,7 @@ require 'liquid/expression' require 'liquid/template' require 'liquid/condition' require 'liquid/utils' +require 'liquid/cursor' require 'liquid/tokenizer' require 'liquid/parse_context' require 'liquid/partial_cache' diff --git a/lib/liquid/block_body.rb b/lib/liquid/block_body.rb index a47f71fe..f971b0f7 100644 --- a/lib/liquid/block_body.rb +++ b/lib/liquid/block_body.rb @@ -220,14 +220,15 @@ module Liquid second_byte = token.getbyte(1) if second_byte == PERCENT_BYTE whitespace_handler(token, parse_context) - tag_name = BlockBody.parse_tag_token(token) + cursor = parse_context.cursor + tag_name = cursor.parse_tag_token(token) unless tag_name return handle_invalid_tag_token(token, parse_context, &block) end - markup = BlockBody._last_markup + markup = cursor.tag_markup if parse_context.line_number - newlines = BlockBody._last_newlines + newlines = cursor.tag_newlines parse_context.line_number += newlines if newlines > 0 end @@ -351,13 +352,7 @@ module Liquid def create_variable(token, parse_context) len = token.bytesize if len >= 4 && token.getbyte(len - 1) == CLOSE_CURLEY_BYTE && token.getbyte(len - 2) == CLOSE_CURLEY_BYTE - i = 2 - i = 3 if token.getbyte(i) == DASH_BYTE - parse_end = len - 3 - parse_end -= 1 if token.getbyte(parse_end) == DASH_BYTE - markup_end = parse_end - i + 1 - markup = markup_end <= 0 ? "" : token.byteslice(i, markup_end) - + markup = parse_context.cursor.parse_variable_token(token) return Variable.new(markup, parse_context) end diff --git a/lib/liquid/cursor.rb b/lib/liquid/cursor.rb new file mode 100644 index 00000000..13eba7c2 --- /dev/null +++ b/lib/liquid/cursor.rb @@ -0,0 +1,304 @@ +# frozen_string_literal: true + +require "strscan" + +module Liquid + # Single-pass forward-only scanner for Liquid parsing. + # Wraps StringScanner with higher-level methods for common Liquid constructs. + # One Cursor per template parse — threaded through all parsing code. + class Cursor + # Byte constants + SPACE = 32 + TAB = 9 + NL = 10 + CR = 13 + FF = 12 + DASH = 45 # '-' + DOT = 46 # '.' + COLON = 58 # ':' + PIPE = 124 # '|' + QUOTE_S = 39 # "'" + QUOTE_D = 34 # '"' + LBRACK = 91 # '[' + RBRACK = 93 # ']' + LPAREN = 40 # '(' + RPAREN = 41 # ')' + QMARK = 63 # '?' + HASH = 35 # '#' + USCORE = 95 # '_' + COMMA = 44 + ZERO = 48 + NINE = 57 + PCT = 37 # '%' + LCURLY = 123 # '{' + RCURLY = 125 # '}' + + attr_reader :ss + + def initialize(source) + @source = source + @ss = StringScanner.new(source) + end + + # ── Position ──────────────────────────────────────────────────── + def pos; @ss.pos; end + def pos=(n); @ss.pos = n; end + def eos?; @ss.eos?; end + def peek_byte; @ss.peek_byte; end + def scan_byte; @ss.scan_byte; end + + # Reset scanner to a new string (for reuse on sub-markup) + def reset(source) + @source = source + @ss.string = source + end + + # ── Whitespace ────────────────────────────────────────────────── + # Skip spaces/tabs/newlines/cr, return count of newlines skipped + def skip_ws + nl = 0 + while (b = @ss.peek_byte) + case b + when SPACE, TAB, CR, FF then @ss.scan_byte + when NL then @ss.scan_byte; nl += 1 + else break + end + end + nl + end + + # Check if remaining bytes are all whitespace + def rest_blank? + p = @ss.pos + len = @source.bytesize + while p < len + b = @source.getbyte(p) + return false unless b == SPACE || b == TAB || b == NL || b == CR || b == FF + p += 1 + end + true + end + + # ── Identifiers ───────────────────────────────────────────────── + # Scan a single identifier: [a-zA-Z_][\w-]*\?? + # Returns the string or nil if not at an identifier + def scan_id + start = @ss.pos + b = @ss.peek_byte + return nil unless b && ((b >= 97 && b <= 122) || (b >= 65 && b <= 90) || b == USCORE) + @ss.scan_byte + while (b = @ss.peek_byte) + break unless (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || + (b >= 48 && b <= 57) || b == USCORE || b == DASH + @ss.scan_byte + end + @ss.scan_byte if @ss.peek_byte == QMARK + @source.byteslice(start, @ss.pos - start) + end + + # Scan a tag name: '#' or \w+ + def scan_tag_name + if @ss.peek_byte == HASH + @ss.scan_byte + "#" + else + scan_id + end + end + + # ── Numbers ───────────────────────────────────────────────────── + # Try to scan an integer or float. Returns the number or nil. + def scan_number + start = @ss.pos + b = @ss.peek_byte + return nil unless b + + if b == DASH + @ss.scan_byte + b = @ss.peek_byte + unless b && b >= ZERO && b <= NINE + @ss.pos = start + return nil + end + elsif b >= ZERO && b <= NINE + # ok + else + return nil + end + + # Scan digits + @ss.scan_byte + @ss.scan_byte while (b = @ss.peek_byte) && b >= ZERO && b <= NINE + + if @ss.peek_byte == DOT + @ss.scan_byte + # Must have digit after dot for float + if (b = @ss.peek_byte) && b >= ZERO && b <= NINE + @ss.scan_byte + @ss.scan_byte while (b = @ss.peek_byte) && b >= ZERO && b <= NINE + return @source.byteslice(start, @ss.pos - start).to_f + else + # "123." — integer portion only, rewind past dot + @ss.pos -= 1 + end + end + + Integer(@source.byteslice(start, @ss.pos - start), 10) + end + + # ── Strings ───────────────────────────────────────────────────── + # Scan a quoted string ('...' or "..."). Returns the content without quotes, or nil. + def scan_quoted_string + b = @ss.peek_byte + return nil unless b == QUOTE_S || b == QUOTE_D + quote = b + @ss.scan_byte + start = @ss.pos + @ss.scan_byte while (b = @ss.peek_byte) && b != quote + content = @source.byteslice(start, @ss.pos - start) + @ss.scan_byte if @ss.peek_byte == quote # consume closing quote + content + end + + # Scan a quoted string including quotes. Returns the full "..." or '...' string, or nil. + def scan_quoted_string_raw + b = @ss.peek_byte + return nil unless b == QUOTE_S || b == QUOTE_D + quote = b + start = @ss.pos + @ss.scan_byte + @ss.scan_byte while (b = @ss.peek_byte) && b != quote + @ss.scan_byte if @ss.peek_byte == quote + @source.byteslice(start, @ss.pos - start) + end + + # ── Expressions ───────────────────────────────────────────────── + # Scan a simple variable lookup: name(.name)* — no brackets, no filters + # Returns the string or nil + def scan_dotted_id + start = @ss.pos + return nil unless scan_id + while @ss.peek_byte == DOT + @ss.scan_byte + unless scan_id + @ss.pos -= 1 # rewind the dot + break + end + end + @source.byteslice(start, @ss.pos - start) + end + + # Scan a "QuotedFragment" — a quoted string or non-whitespace/comma/pipe run + def scan_fragment + b = @ss.peek_byte + return nil unless b + if b == QUOTE_S || b == QUOTE_D + scan_quoted_string_raw + else + start = @ss.pos + while (b = @ss.peek_byte) + break if b == SPACE || b == TAB || b == NL || b == CR || b == COMMA || b == PIPE + @ss.scan_byte + end + len = @ss.pos - start + len > 0 ? @source.byteslice(start, len) : nil + end + end + + # ── Comparison operators ──────────────────────────────────────── + COMPARISON_OPS = { + '==' => '==', '!=' => '!=', '<>' => '<>', + '<=' => '<=', '>=' => '>=', '<' => '<', '>' => '>', + 'contains' => 'contains', + }.freeze + + # Scan a comparison operator. Returns frozen string or nil. + def scan_comparison_op + start = @ss.pos + b = @ss.peek_byte + case b + when 61, 33, 60, 62 # = ! < > + @ss.scan_byte + b2 = @ss.peek_byte + if b2 == 61 || b2 == 62 # second char of ==, !=, <=, >=, <> + @ss.scan_byte + end + when 99 # 'c' for contains + id = scan_id + return nil unless id == "contains" + return COMPARISON_OPS['contains'] + else + return nil + end + op_str = @source.byteslice(start, @ss.pos - start) + COMPARISON_OPS[op_str] || (@ss.pos = start; nil) + end + + # ── Tag parsing helpers ───────────────────────────────────────── + # Results from last parse_tag_token call (avoids array allocation) + attr_reader :tag_markup, :tag_newlines + + # Parse the interior of a tag token: "{%[-] tag_name markup [-]%}" + # Caller provides the full token string. Sets cursor to the token. + # Returns tag_name string or nil. Sets tag_markup and tag_newlines. + def parse_tag_token(token) + reset(token) + @ss.pos = 2 # skip "{%" + @ss.scan_byte if peek_byte == DASH # skip whitespace control '-' + nl = skip_ws + tag_name = scan_tag_name + return nil unless tag_name + nl += skip_ws + + # markup is everything up to optional '-' before '%}' + markup_end = token.bytesize - 2 + markup_end -= 1 if markup_end > @ss.pos && token.getbyte(markup_end - 1) == DASH + @tag_markup = @ss.pos >= markup_end ? "" : token.byteslice(@ss.pos, markup_end - @ss.pos) + @tag_newlines = nl + + tag_name + end + + # Parse variable token interior: extract markup from "{{[-] ... [-]}}" + def parse_variable_token(token) + len = token.bytesize + return nil if len < 4 + i = 2 + i = 3 if token.getbyte(i) == DASH + parse_end = len - 3 + parse_end -= 1 if token.getbyte(parse_end) == DASH + markup_len = parse_end - i + 1 + markup_len <= 0 ? "" : token.byteslice(i, markup_len) + end + + # ── Simple condition parser ───────────────────────────────────── + # Results from last parse_simple_condition call + attr_reader :cond_left, :cond_op, :cond_right + + # Parse "expr [op expr]" from current position to end. + # Returns true on success, nil on failure. Sets cond_left, cond_op, cond_right. + def parse_simple_condition + skip_ws + @cond_left = scan_fragment + return nil unless @cond_left + + skip_ws + if eos? + @cond_op = nil + @cond_right = nil + return true + end + + @cond_op = scan_comparison_op + return nil unless @cond_op + + skip_ws + @cond_right = scan_fragment + return nil unless @cond_right + + skip_ws + return nil unless eos? # trailing junk + true + end + end +end diff --git a/lib/liquid/parse_context.rb b/lib/liquid/parse_context.rb index 4bec4c87..d736319e 100644 --- a/lib/liquid/parse_context.rb +++ b/lib/liquid/parse_context.rb @@ -3,7 +3,7 @@ module Liquid class ParseContext attr_accessor :locale, :line_number, :trim_whitespace, :depth - attr_reader :partial, :warnings, :error_mode, :environment, :expression_cache, :string_scanner + attr_reader :partial, :warnings, :error_mode, :environment, :expression_cache, :string_scanner, :cursor def initialize(options = Const::EMPTY_HASH) @environment = options.fetch(:environment, Environment.default) @@ -24,6 +24,8 @@ module Liquid {} end + @cursor = Cursor.new("") + self.depth = 0 self.partial = false end diff --git a/lib/liquid/tags/if.rb b/lib/liquid/tags/if.rb index d05b1ef2..242141be 100644 --- a/lib/liquid/tags/if.rb +++ b/lib/liquid/tags/if.rb @@ -88,119 +88,21 @@ module Liquid # Fast path regex for simple conditions: "expr", "expr op expr" (no and/or) SIMPLE_CONDITION = /\A\s*(#{QuotedFragment})\s*(?:([=!<>a-z_]+)\s*(#{QuotedFragment}))?\s*\z/o - # Operators indexed by first byte for fast lookup - COMPARISON_OPS = { - '==' => '==', '!=' => '!=', '<>' => '<>', - '<=' => '<=', '>=' => '>=', '<' => '<', '>' => '>', - 'contains' => 'contains', - }.freeze - - # Parse a simple condition "expr [op expr]" without regex. - # Returns [left, op, right] or nil if not parseable. - def self.parse_simple_condition(markup) - len = markup.bytesize - pos = 0 - - # Skip leading whitespace - pos += 1 while pos < len && (b = markup.getbyte(pos)) && (b == 32 || b == 9 || b == 10 || b == 13) - return nil if pos >= len - - # Scan left expression (QuotedFragment): quoted string or non-whitespace/comma/pipe sequence - left_start = pos - b = markup.getbyte(pos) - if b == 34 || b == 39 # quoted string - quote = b - pos += 1 - pos += 1 while pos < len && markup.getbyte(pos) != quote - pos += 1 if pos < len # closing quote - else - # Non-whitespace, non-comma, non-pipe chars (QuotedFragment without quotes) - while pos < len - b = markup.getbyte(pos) - break if b == 32 || b == 9 || b == 10 || b == 13 || b == 44 || b == 124 # space, tab, \n, \r, comma, pipe - pos += 1 - end - end - left_end = pos - - return nil if left_start == left_end - - # Skip whitespace - pos += 1 while pos < len && (b = markup.getbyte(pos)) && (b == 32 || b == 9 || b == 10 || b == 13) - - # End of markup? Simple truthiness - if pos >= len - left = markup.byteslice(left_start, left_end - left_start) - return [left, nil, nil] - end - - # Scan operator - op_start = pos - b = markup.getbyte(pos) - if b == 61 || b == 33 || b == 60 || b == 62 # =, !, <, > - pos += 1 - b2 = markup.getbyte(pos) - pos += 1 if b2 && (b2 == 61 || b2 == 62) # second char of ==, !=, <=, >=, <> - elsif b == 99 # 'c' for 'contains' - while pos < len - b = markup.getbyte(pos) - break unless (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || b == 95 - pos += 1 - end - else - return nil # unknown operator start - end - op = markup.byteslice(op_start, pos - op_start) - return nil unless COMPARISON_OPS.key?(op) - op = COMPARISON_OPS[op] # use frozen string - - # Skip whitespace - pos += 1 while pos < len && (b = markup.getbyte(pos)) && (b == 32 || b == 9 || b == 10 || b == 13) - return nil if pos >= len # op without right operand - - # Scan right expression - right_start = pos - b = markup.getbyte(pos) - if b == 34 || b == 39 - quote = b - pos += 1 - pos += 1 while pos < len && markup.getbyte(pos) != quote - pos += 1 if pos < len - else - while pos < len - b = markup.getbyte(pos) - break if b == 32 || b == 9 || b == 10 || b == 13 || b == 44 || b == 124 - pos += 1 - end - end - right_end = pos - - return nil if right_start == right_end - - # Skip trailing whitespace - pos += 1 while pos < len && (b = markup.getbyte(pos)) && (b == 32 || b == 9 || b == 10 || b == 13) - return nil unless pos >= len # extra stuff after right expr - - left = markup.byteslice(left_start, left_end - left_start) - right = markup.byteslice(right_start, right_end - right_start) - [left, op, right] - end - def lax_parse(markup) # Fastest path: simple identifier truthiness like "product.available" or "forloop.first" if (simple = Variable.simple_variable_markup(markup)) return Condition.new(parse_expression(simple)) end - # Fast path: simple condition without and/or — manual byte parser + # Fast path: simple condition without and/or — use Cursor if !markup.include?(' and ') && !markup.include?(' or ') - parsed = If.parse_simple_condition(markup) - if parsed - left, op, right = parsed + cursor = @parse_context.cursor + cursor.reset(markup) + if cursor.parse_simple_condition return Condition.new( - parse_expression(left), - op, - right ? parse_expression(right) : nil, + parse_expression(cursor.cond_left), + cursor.cond_op, + cursor.cond_right ? parse_expression(cursor.cond_right) : nil, ) end end