introduce Cursor class: centralize byte-level scanning for tag/variable/condition parsing

lib/liquid.rb

@@ -83,6 +83,7 @@ require 'liquid/expression'
 require 'liquid/template'
 require 'liquid/condition'
 require 'liquid/utils'
+require 'liquid/cursor'
 require 'liquid/tokenizer'
 require 'liquid/parse_context'
 require 'liquid/partial_cache'

lib/liquid/block_body.rb

@@ -220,14 +220,15 @@ module Liquid
           second_byte = token.getbyte(1)
           if second_byte == PERCENT_BYTE
             whitespace_handler(token, parse_context)
-            tag_name = BlockBody.parse_tag_token(token)
+            cursor = parse_context.cursor
+            tag_name = cursor.parse_tag_token(token)
             unless tag_name
               return handle_invalid_tag_token(token, parse_context, &block)
             end
-            markup = BlockBody._last_markup
+            markup = cursor.tag_markup
 
             if parse_context.line_number
-              newlines = BlockBody._last_newlines
+              newlines = cursor.tag_newlines
               parse_context.line_number += newlines if newlines > 0
             end
 
@@ -351,13 +352,7 @@ module Liquid
     def create_variable(token, parse_context)
       len = token.bytesize
       if len >= 4 && token.getbyte(len - 1) == CLOSE_CURLEY_BYTE && token.getbyte(len - 2) == CLOSE_CURLEY_BYTE
-        i = 2
+        markup = parse_context.cursor.parse_variable_token(token)
-        i = 3 if token.getbyte(i) == DASH_BYTE
-        parse_end = len - 3
-        parse_end -= 1 if token.getbyte(parse_end) == DASH_BYTE
-        markup_end = parse_end - i + 1
-        markup = markup_end <= 0 ? "" : token.byteslice(i, markup_end)
-
         return Variable.new(markup, parse_context)
       end
 

lib/liquid/cursor.rb

@@ -0,0 +1,304 @@
+# frozen_string_literal: true
+
+require "strscan"
+
+module Liquid
+  # Single-pass forward-only scanner for Liquid parsing.
+  # Wraps StringScanner with higher-level methods for common Liquid constructs.
+  # One Cursor per template parse — threaded through all parsing code.
+  class Cursor
+    # Byte constants
+    SPACE   = 32
+    TAB     = 9
+    NL      = 10
+    CR      = 13
+    FF      = 12
+    DASH    = 45  # '-'
+    DOT     = 46  # '.'
+    COLON   = 58  # ':'
+    PIPE    = 124 # '|'
+    QUOTE_S = 39  # "'"
+    QUOTE_D = 34  # '"'
+    LBRACK  = 91  # '['
+    RBRACK  = 93  # ']'
+    LPAREN  = 40  # '('
+    RPAREN  = 41  # ')'
+    QMARK   = 63  # '?'
+    HASH    = 35  # '#'
+    USCORE  = 95  # '_'
+    COMMA   = 44
+    ZERO    = 48
+    NINE    = 57
+    PCT     = 37  # '%'
+    LCURLY  = 123 # '{'
+    RCURLY  = 125 # '}'
+
+    attr_reader :ss
+
+    def initialize(source)
+      @source = source
+      @ss = StringScanner.new(source)
+    end
+
+    # ── Position ────────────────────────────────────────────────────
+    def pos;         @ss.pos;          end
+    def pos=(n);     @ss.pos = n;      end
+    def eos?;        @ss.eos?;         end
+    def peek_byte;   @ss.peek_byte;    end
+    def scan_byte;   @ss.scan_byte;    end
+
+    # Reset scanner to a new string (for reuse on sub-markup)
+    def reset(source)
+      @source = source
+      @ss.string = source
+    end
+
+    # ── Whitespace ──────────────────────────────────────────────────
+    # Skip spaces/tabs/newlines/cr, return count of newlines skipped
+    def skip_ws
+      nl = 0
+      while (b = @ss.peek_byte)
+        case b
+        when SPACE, TAB, CR, FF then @ss.scan_byte
+        when NL then @ss.scan_byte; nl += 1
+        else break
+        end
+      end
+      nl
+    end
+
+    # Check if remaining bytes are all whitespace
+    def rest_blank?
+      p = @ss.pos
+      len = @source.bytesize
+      while p < len
+        b = @source.getbyte(p)
+        return false unless b == SPACE || b == TAB || b == NL || b == CR || b == FF
+        p += 1
+      end
+      true
+    end
+
+    # ── Identifiers ─────────────────────────────────────────────────
+    # Scan a single identifier: [a-zA-Z_][\w-]*\??
+    # Returns the string or nil if not at an identifier
+    def scan_id
+      start = @ss.pos
+      b = @ss.peek_byte
+      return nil unless b && ((b >= 97 && b <= 122) || (b >= 65 && b <= 90) || b == USCORE)
+      @ss.scan_byte
+      while (b = @ss.peek_byte)
+        break unless (b >= 97 && b <= 122) || (b >= 65 && b <= 90) ||
+                     (b >= 48 && b <= 57) || b == USCORE || b == DASH
+        @ss.scan_byte
+      end
+      @ss.scan_byte if @ss.peek_byte == QMARK
+      @source.byteslice(start, @ss.pos - start)
+    end
+
+    # Scan a tag name: '#' or \w+
+    def scan_tag_name
+      if @ss.peek_byte == HASH
+        @ss.scan_byte
+        "#"
+      else
+        scan_id
+      end
+    end
+
+    # ── Numbers ─────────────────────────────────────────────────────
+    # Try to scan an integer or float. Returns the number or nil.
+    def scan_number
+      start = @ss.pos
+      b = @ss.peek_byte
+      return nil unless b
+
+      if b == DASH
+        @ss.scan_byte
+        b = @ss.peek_byte
+        unless b && b >= ZERO && b <= NINE
+          @ss.pos = start
+          return nil
+        end
+      elsif b >= ZERO && b <= NINE
+        # ok
+      else
+        return nil
+      end
+
+      # Scan digits
+      @ss.scan_byte
+      @ss.scan_byte while (b = @ss.peek_byte) && b >= ZERO && b <= NINE
+
+      if @ss.peek_byte == DOT
+        @ss.scan_byte
+        # Must have digit after dot for float
+        if (b = @ss.peek_byte) && b >= ZERO && b <= NINE
+          @ss.scan_byte
+          @ss.scan_byte while (b = @ss.peek_byte) && b >= ZERO && b <= NINE
+          return @source.byteslice(start, @ss.pos - start).to_f
+        else
+          # "123." — integer portion only, rewind past dot
+          @ss.pos -= 1
+        end
+      end
+
+      Integer(@source.byteslice(start, @ss.pos - start), 10)
+    end
+
+    # ── Strings ─────────────────────────────────────────────────────
+    # Scan a quoted string ('...' or "..."). Returns the content without quotes, or nil.
+    def scan_quoted_string
+      b = @ss.peek_byte
+      return nil unless b == QUOTE_S || b == QUOTE_D
+      quote = b
+      @ss.scan_byte
+      start = @ss.pos
+      @ss.scan_byte while (b = @ss.peek_byte) && b != quote
+      content = @source.byteslice(start, @ss.pos - start)
+      @ss.scan_byte if @ss.peek_byte == quote # consume closing quote
+      content
+    end
+
+    # Scan a quoted string including quotes. Returns the full "..." or '...' string, or nil.
+    def scan_quoted_string_raw
+      b = @ss.peek_byte
+      return nil unless b == QUOTE_S || b == QUOTE_D
+      quote = b
+      start = @ss.pos
+      @ss.scan_byte
+      @ss.scan_byte while (b = @ss.peek_byte) && b != quote
+      @ss.scan_byte if @ss.peek_byte == quote
+      @source.byteslice(start, @ss.pos - start)
+    end
+
+    # ── Expressions ─────────────────────────────────────────────────
+    # Scan a simple variable lookup: name(.name)* — no brackets, no filters
+    # Returns the string or nil
+    def scan_dotted_id
+      start = @ss.pos
+      return nil unless scan_id
+      while @ss.peek_byte == DOT
+        @ss.scan_byte
+        unless scan_id
+          @ss.pos -= 1 # rewind the dot
+          break
+        end
+      end
+      @source.byteslice(start, @ss.pos - start)
+    end
+
+    # Scan a "QuotedFragment" — a quoted string or non-whitespace/comma/pipe run
+    def scan_fragment
+      b = @ss.peek_byte
+      return nil unless b
+      if b == QUOTE_S || b == QUOTE_D
+        scan_quoted_string_raw
+      else
+        start = @ss.pos
+        while (b = @ss.peek_byte)
+          break if b == SPACE || b == TAB || b == NL || b == CR || b == COMMA || b == PIPE
+          @ss.scan_byte
+        end
+        len = @ss.pos - start
+        len > 0 ? @source.byteslice(start, len) : nil
+      end
+    end
+
+    # ── Comparison operators ────────────────────────────────────────
+    COMPARISON_OPS = {
+      '==' => '==', '!=' => '!=', '<>' => '<>',
+      '<=' => '<=', '>=' => '>=', '<' => '<', '>' => '>',
+      'contains' => 'contains',
+    }.freeze
+
+    # Scan a comparison operator. Returns frozen string or nil.
+    def scan_comparison_op
+      start = @ss.pos
+      b = @ss.peek_byte
+      case b
+      when 61, 33, 60, 62 # = ! < >
+        @ss.scan_byte
+        b2 = @ss.peek_byte
+        if b2 == 61 || b2 == 62 # second char of ==, !=, <=, >=, <>
+          @ss.scan_byte
+        end
+      when 99 # 'c' for contains
+        id = scan_id
+        return nil unless id == "contains"
+        return COMPARISON_OPS['contains']
+      else
+        return nil
+      end
+      op_str = @source.byteslice(start, @ss.pos - start)
+      COMPARISON_OPS[op_str] || (@ss.pos = start; nil)
+    end
+
+    # ── Tag parsing helpers ─────────────────────────────────────────
+    # Results from last parse_tag_token call (avoids array allocation)
+    attr_reader :tag_markup, :tag_newlines
+
+    # Parse the interior of a tag token: "{%[-] tag_name markup [-]%}"
+    # Caller provides the full token string. Sets cursor to the token.
+    # Returns tag_name string or nil. Sets tag_markup and tag_newlines.
+    def parse_tag_token(token)
+      reset(token)
+      @ss.pos = 2 # skip "{%"
+      @ss.scan_byte if peek_byte == DASH # skip whitespace control '-'
+      nl = skip_ws
+      tag_name = scan_tag_name
+      return nil unless tag_name
+      nl += skip_ws
+
+      # markup is everything up to optional '-' before '%}'
+      markup_end = token.bytesize - 2
+      markup_end -= 1 if markup_end > @ss.pos && token.getbyte(markup_end - 1) == DASH
+      @tag_markup = @ss.pos >= markup_end ? "" : token.byteslice(@ss.pos, markup_end - @ss.pos)
+      @tag_newlines = nl
+
+      tag_name
+    end
+
+    # Parse variable token interior: extract markup from "{{[-] ... [-]}}"
+    def parse_variable_token(token)
+      len = token.bytesize
+      return nil if len < 4
+      i = 2
+      i = 3 if token.getbyte(i) == DASH
+      parse_end = len - 3
+      parse_end -= 1 if token.getbyte(parse_end) == DASH
+      markup_len = parse_end - i + 1
+      markup_len <= 0 ? "" : token.byteslice(i, markup_len)
+    end
+
+    # ── Simple condition parser ─────────────────────────────────────
+    # Results from last parse_simple_condition call
+    attr_reader :cond_left, :cond_op, :cond_right
+
+    # Parse "expr [op expr]" from current position to end.
+    # Returns true on success, nil on failure. Sets cond_left, cond_op, cond_right.
+    def parse_simple_condition
+      skip_ws
+      @cond_left = scan_fragment
+      return nil unless @cond_left
+
+      skip_ws
+      if eos?
+        @cond_op = nil
+        @cond_right = nil
+        return true
+      end
+
+      @cond_op = scan_comparison_op
+      return nil unless @cond_op
+
+      skip_ws
+      @cond_right = scan_fragment
+      return nil unless @cond_right
+
+      skip_ws
+      return nil unless eos? # trailing junk
+      true
+    end
+  end
+end

lib/liquid/parse_context.rb

@@ -3,7 +3,7 @@
 module Liquid
   class ParseContext
     attr_accessor :locale, :line_number, :trim_whitespace, :depth
-    attr_reader :partial, :warnings, :error_mode, :environment, :expression_cache, :string_scanner
+    attr_reader :partial, :warnings, :error_mode, :environment, :expression_cache, :string_scanner, :cursor
 
     def initialize(options = Const::EMPTY_HASH)
       @environment = options.fetch(:environment, Environment.default)
@@ -24,6 +24,8 @@ module Liquid
         {}
       end
 
+      @cursor = Cursor.new("")
+
       self.depth   = 0
       self.partial = false
     end

lib/liquid/tags/if.rb

@@ -88,119 +88,21 @@ module Liquid
     # Fast path regex for simple conditions: "expr", "expr op expr" (no and/or)
     SIMPLE_CONDITION = /\A\s*(#{QuotedFragment})\s*(?:([=!<>a-z_]+)\s*(#{QuotedFragment}))?\s*\z/o
 
-    # Operators indexed by first byte for fast lookup
-    COMPARISON_OPS = {
-      '==' => '==', '!=' => '!=', '<>' => '<>',
-      '<=' => '<=', '>=' => '>=', '<' => '<', '>' => '>',
-      'contains' => 'contains',
-    }.freeze
-
-    # Parse a simple condition "expr [op expr]" without regex.
-    # Returns [left, op, right] or nil if not parseable.
-    def self.parse_simple_condition(markup)
-      len = markup.bytesize
-      pos = 0
-
-      # Skip leading whitespace
-      pos += 1 while pos < len && (b = markup.getbyte(pos)) && (b == 32 || b == 9 || b == 10 || b == 13)
-      return nil if pos >= len
-
-      # Scan left expression (QuotedFragment): quoted string or non-whitespace/comma/pipe sequence
-      left_start = pos
-      b = markup.getbyte(pos)
-      if b == 34 || b == 39 # quoted string
-        quote = b
-        pos += 1
-        pos += 1 while pos < len && markup.getbyte(pos) != quote
-        pos += 1 if pos < len # closing quote
-      else
-        # Non-whitespace, non-comma, non-pipe chars (QuotedFragment without quotes)
-        while pos < len
-          b = markup.getbyte(pos)
-          break if b == 32 || b == 9 || b == 10 || b == 13 || b == 44 || b == 124 # space, tab, \n, \r, comma, pipe
-          pos += 1
-        end
-      end
-      left_end = pos
-
-      return nil if left_start == left_end
-
-      # Skip whitespace
-      pos += 1 while pos < len && (b = markup.getbyte(pos)) && (b == 32 || b == 9 || b == 10 || b == 13)
-
-      # End of markup? Simple truthiness
-      if pos >= len
-        left = markup.byteslice(left_start, left_end - left_start)
-        return [left, nil, nil]
-      end
-
-      # Scan operator
-      op_start = pos
-      b = markup.getbyte(pos)
-      if b == 61 || b == 33 || b == 60 || b == 62 # =, !, <, >
-        pos += 1
-        b2 = markup.getbyte(pos)
-        pos += 1 if b2 && (b2 == 61 || b2 == 62) # second char of ==, !=, <=, >=, <>
-      elsif b == 99 # 'c' for 'contains'
-        while pos < len
-          b = markup.getbyte(pos)
-          break unless (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || b == 95
-          pos += 1
-        end
-      else
-        return nil # unknown operator start
-      end
-      op = markup.byteslice(op_start, pos - op_start)
-      return nil unless COMPARISON_OPS.key?(op)
-      op = COMPARISON_OPS[op] # use frozen string
-
-      # Skip whitespace
-      pos += 1 while pos < len && (b = markup.getbyte(pos)) && (b == 32 || b == 9 || b == 10 || b == 13)
-      return nil if pos >= len # op without right operand
-
-      # Scan right expression
-      right_start = pos
-      b = markup.getbyte(pos)
-      if b == 34 || b == 39
-        quote = b
-        pos += 1
-        pos += 1 while pos < len && markup.getbyte(pos) != quote
-        pos += 1 if pos < len
-      else
-        while pos < len
-          b = markup.getbyte(pos)
-          break if b == 32 || b == 9 || b == 10 || b == 13 || b == 44 || b == 124
-          pos += 1
-        end
-      end
-      right_end = pos
-
-      return nil if right_start == right_end
-
-      # Skip trailing whitespace
-      pos += 1 while pos < len && (b = markup.getbyte(pos)) && (b == 32 || b == 9 || b == 10 || b == 13)
-      return nil unless pos >= len # extra stuff after right expr
-
-      left = markup.byteslice(left_start, left_end - left_start)
-      right = markup.byteslice(right_start, right_end - right_start)
-      [left, op, right]
-    end
-
     def lax_parse(markup)
       # Fastest path: simple identifier truthiness like "product.available" or "forloop.first"
       if (simple = Variable.simple_variable_markup(markup))
         return Condition.new(parse_expression(simple))
       end
 
-      # Fast path: simple condition without and/or — manual byte parser
+      # Fast path: simple condition without and/or — use Cursor
       if !markup.include?(' and ') && !markup.include?(' or ')
-        parsed = If.parse_simple_condition(markup)
+        cursor = @parse_context.cursor
-        if parsed
+        cursor.reset(markup)
-          left, op, right = parsed
+        if cursor.parse_simple_condition
           return Condition.new(
-            parse_expression(left),
+            parse_expression(cursor.cond_left),
-            op,
+            cursor.cond_op,
-            right ? parse_expression(right) : nil,
+            cursor.cond_right ? parse_expression(cursor.cond_right) : nil,
           )
         end
       end