Replace StringScanner tokenizer with String#byteindex — 12% faster parse, no regex overhead for delimiter finding\n\nResult: {"status":"keep","combined_µs":3556,"parse_µs":2388,"render_µs":1168,"allocations":24882}

auto/autoresearch.ideas.md

@@ -0,0 +1,30 @@
+# Autoresearch Ideas
+
+## Dead Ends (tried and failed)
+
+- **Tag name interning** (skip+byte dispatch): saves 878 allocs but verification loop overhead kills speed
+- **String dedup (-@)** for filter names: no alloc savings, creates temp strings anyway
+- **Split-based tokenizer**: 2.5x faster C-level split but can't handle {{ followed by %} nesting
+- **Streaming tokenizer**: needs own StringScanner (+alloc), per-shift overhead worse than eager array
+- **Merge simple_lookup? into initialize**: logic overhead offsets saved index call
+- **Cursor for filter scanning**: cursor.reset overhead worse than inline byte loops
+- **Direct strainer call**: YJIT already inlines context.invoke_single well
+- **TruthyCondition subclass**: YJIT polymorphism at evaluate call site hurts more than 115 saved allocs
+- **Index loop for filters**: YJIT optimizes each+destructure MUCH better than manual filter[0]/filter[1]
+
+## Key Insights
+
+- YJIT monomorphism > allocation reduction at this scale
+- C-level StringScanner.scan/skip > Ruby-level byte loops (already applied)
+- String#split is 2.5x faster than manual tokenization, but Liquid's grammar is too complex for regex
+- 74% of total CPU time is GC — alloc reduction is the highest-leverage optimization
+- But YJIT-deoptimization from polymorphism costs more than the GC savings
+
+## Remaining Ideas
+
+- **Tokenizer: use String#index + byteslice instead of StringScanner**: avoid the StringScanner overhead entirely for the simple case of finding {%/{{ delimiters
+- **Pre-freeze all Condition operator lambdas**: reduce alloc in Condition initialization
+- **Avoid `@blocks = []` in If with single-element optimization**: use `@block` ivar for single condition, only create array for elsif
+- **Reduce ForloopDrop allocation**: reuse ForloopDrop objects across iterations or use a lighter-weight object
+- **VariableLookup: single-segment optimization**: for "product.title" (1 lookup), use an ivar instead of 1-element Array
+

autoresearch.jsonl

@@ -1 +1,13 @@
 {"type":"config","name":"Liquid parse+render performance (tenderlove-inspired)","metricName":"combined_µs","metricUnit":"µs","bestDirection":"lower"}
+{"run":1,"commit":"c09e722","metric":3818,"metrics":{"parse_µs":2722,"render_µs":1096,"allocations":24881},"status":"keep","description":"Baseline: 3,818µs combined, 24,881 allocs","timestamp":1773348490227}
+{"run":2,"commit":"c09e722","metric":4063,"metrics":{"parse_µs":2901,"render_µs":1162,"allocations":24003},"status":"discard","description":"Tag name interning via skip+byte dispatch: saves 878 allocs but verification loop slower than scan","timestamp":1773348738557,"segment":0}
+{"run":3,"commit":"c09e722","metric":3881,"metrics":{"parse_µs":2720,"render_µs":1161,"allocations":24881},"status":"discard","description":"String dedup (-@) for filter names: no alloc savings, no speed benefit","timestamp":1773348781481,"segment":0}
+{"run":4,"commit":"c09e722","metric":3970,"metrics":{"parse_µs":2829,"render_µs":1141,"allocations":24881},"status":"discard","description":"Streaming tokenizer: needs own StringScanner (+1 alloc), per-shift overhead worse than saved array","timestamp":1773348883093,"segment":0}
+{"run":5,"commit":"c09e722","metric":0,"metrics":{"parse_µs":0,"render_µs":0,"allocations":0},"status":"crash","description":"REVERTED: split-based tokenizer — regex can't handle unclosed tags inside raw blocks","timestamp":1773349089230,"segment":0}
+{"run":6,"commit":"c09e722","metric":0,"metrics":{"parse_µs":0,"render_µs":0,"allocations":0},"status":"crash","description":"REVERTED: split regex tokenizer v2 — can't handle {{ followed by %} (variable-becomes-tag nesting)","timestamp":1773349248313,"segment":0}
+{"run":7,"commit":"c09e722","metric":3861,"metrics":{"parse_µs":2744,"render_µs":1117,"allocations":24881},"status":"discard","description":"Merge simple_lookup? dot position into initialize — logic overhead offsets saved index call","timestamp":1773349376707,"segment":0}
+{"run":8,"commit":"c09e722","metric":4048,"metrics":{"parse_µs":2929,"render_µs":1119,"allocations":24881},"status":"discard","description":"Use Cursor regex for filter name scanning — cursor.reset + method dispatch overhead worse than inline bytes","timestamp":1773349447172,"segment":0}
+{"run":9,"commit":"c09e722","metric":3872,"metrics":{"parse_µs":2744,"render_µs":1128,"allocations":24881},"status":"discard","description":"Direct strainer call in Variable#render — YJIT already inlines context.invoke_single well","timestamp":1773349497593,"segment":0}
+{"run":10,"commit":"c09e722","metric":3839,"metrics":{"parse_µs":2732,"render_µs":1107,"allocations":24879},"status":"discard","description":"Array#[] fast path for slice_collection with limit/offset — only 2 alloc savings, not meaningful","timestamp":1773349555348,"segment":0}
+{"run":11,"commit":"c09e722","metric":3889,"metrics":{"parse_µs":2770,"render_µs":1119,"allocations":24766},"status":"discard","description":"TruthyCondition for simple if checks: -115 allocs but YJIT polymorphism at evaluate call site hurts speed","timestamp":1773349649377,"segment":0}
+{"run":12,"commit":"c09e722","metric":4150,"metrics":{"parse_µs":2769,"render_µs":1381,"allocations":24881},"status":"discard","description":"Index loop for filters: YJIT optimizes each+destructure better than manual indexing","timestamp":1773349699285,"segment":0}

lib/liquid/tokenizer.rb

@@ -54,108 +54,127 @@ module Liquid
       if @for_liquid_tag
         @tokens = @source.split("\n")
       else
-        @tokens << shift_normal until @ss.eos?
+        tokenize_fast
       end
 
       @source = nil
       @ss = nil
     end
 
-    def shift_normal
+    # Fast tokenizer using String#index instead of StringScanner regex.
-      token = next_token
+    # String#index is ~40% faster for finding { delimiters.
+    def tokenize_fast
+      src = @source
+      unless src.valid_encoding?
+        raise SyntaxError, "Invalid byte sequence in #{src.encoding}"
+      end
 
-      return unless token
+      len = src.bytesize
+      pos = 0
 
-      token
+      while pos < len
-    end
+        # Find next { which could start a tag or variable
+        idx = src.byteindex('{', pos)
 
-    def next_token
+        unless idx
-      # possible states: :text, :tag, :variable
+          # No more tags/variables — rest is text
-      byte_a = @ss.peek_byte
+          @tokens << src.byteslice(pos, len - pos) if pos < len
-
+          break
-      if byte_a == OPEN_CURLEY
-        @ss.scan_byte
-
-        byte_b = @ss.peek_byte
-
-        if byte_b == PERCENTAGE
-          @ss.scan_byte
-          return next_tag_token
-        elsif byte_b == OPEN_CURLEY
-          @ss.scan_byte
-          return next_variable_token
         end
 
-        @ss.pos -= 1
+        next_byte = idx + 1 < len ? src.getbyte(idx + 1) : nil
-      end
 
-      next_text_token
+        if next_byte == PERCENTAGE # {%
-    end
+          # Emit text before tag
+          @tokens << src.byteslice(pos, idx - pos) if idx > pos
 
-    def next_text_token
+          # Find %} to close the tag
-      start = @ss.pos
+          close = src.byteindex('%}', idx + 2)
-
+          if close
-      unless @ss.skip_until(TAG_OR_VARIABLE_START)
+            @tokens << src.byteslice(idx, close + 2 - idx)
-        token = @ss.rest
+            pos = close + 2
-        @ss.terminate
+          else
-        return token
+            @tokens << "{%"
-      end
+            pos = idx + 2
-
+          end
-      pos = @ss.pos -= 2
+        elsif next_byte == OPEN_CURLEY # {{
-      @source.byteslice(start, pos - start)
+          # Emit text before variable
-    rescue ::ArgumentError => e
+          @tokens << src.byteslice(pos, idx - pos) if idx > pos
-      if e.message == "invalid byte sequence in #{@ss.string.encoding}"
+
-        raise SyntaxError, "Invalid byte sequence in #{@ss.string.encoding}"
+          # Scan variable token — matches original tokenizer's byte-by-byte logic:
-      else
+          # Find } or {, then check next byte for }}/{% nesting
-        raise
+          scan_pos = idx + 2
-      end
+          found = false
-    end
+          while scan_pos < len
-
+            b = src.getbyte(scan_pos)
-    def next_variable_token
+            if b == CLOSE_CURLEY # }
-      start = @ss.pos - 2
+              if scan_pos + 1 >= len
-
+                # } at end of string — emit token up to here
-      byte_a = byte_b = @ss.scan_byte
+                @tokens << src.byteslice(idx, scan_pos + 1 - idx)
-
+                pos = scan_pos + 1
-      while byte_b
+                found = true
-        byte_a = @ss.scan_byte while byte_a && byte_a != CLOSE_CURLEY && byte_a != OPEN_CURLEY
+                break
-
+              end
-        break unless byte_a
+              b2 = src.getbyte(scan_pos + 1)
-
+              if b2 == CLOSE_CURLEY
-        if @ss.eos?
+                # Found }} — close variable
-          return byte_a == CLOSE_CURLEY ? @source.byteslice(start, @ss.pos - start) : "{{"
+                @tokens << src.byteslice(idx, scan_pos + 2 - idx)
-        end
+                pos = scan_pos + 2
-
+                found = true
-        byte_b = @ss.scan_byte
+                break
-
+              else
-        if byte_a == CLOSE_CURLEY
+                # } followed by non-} — emit token up to here (matches original: @ss.pos -= 1)
-          if byte_b == CLOSE_CURLEY
+                @tokens << src.byteslice(idx, scan_pos + 1 - idx)
-            return @source.byteslice(start, @ss.pos - start)
+                pos = scan_pos + 1
-          elsif byte_b != CLOSE_CURLEY
+                found = true
-            @ss.pos -= 1
+                break
-            return @source.byteslice(start, @ss.pos - start)
+              end
+            elsif b == OPEN_CURLEY
+              if scan_pos + 1 < len && src.getbyte(scan_pos + 1) == PERCENTAGE
+                # Found {% inside {{ — scan to %} and emit as one token
+                close = src.byteindex('%}', scan_pos + 2)
+                if close
+                  @tokens << src.byteslice(idx, close + 2 - idx)
+                  pos = close + 2
+                else
+                  @tokens << src.byteslice(idx, len - idx)
+                  pos = len
+                end
+                found = true
+                break
+              end
+              scan_pos += 1
+            else
+              scan_pos += 1
+            end
+          end
+
+          unless found
+            @tokens << "{{"
+            pos = idx + 2
+          end
+        else
+          # { followed by something else — it's text
+          # Keep scanning from after this {
+          # Find next { that could be {%  or {{
+          next_open = idx + 1
+          while next_open < len
+            ni = src.byteindex('{', next_open)
+            unless ni
+              @tokens << src.byteslice(pos, len - pos)
+              pos = len
+              break
+            end
+            nb = ni + 1 < len ? src.getbyte(ni + 1) : nil
+            if nb == PERCENTAGE || nb == OPEN_CURLEY
+              @tokens << src.byteslice(pos, ni - pos)
+              pos = ni
+              break
+            end
+            next_open = ni + 1
           end
-        elsif byte_a == OPEN_CURLEY && byte_b == PERCENTAGE
-          return next_tag_token_with_start(start)
         end
-
-        byte_a = byte_b
       end
-
-      "{{"
-    end
-
-    def next_tag_token
-      start = @ss.pos - 2
-      if (len = @ss.skip_until(TAG_END))
-        @source.byteslice(start, len + 2)
-      else
-        "{%"
-      end
-    end
-
-    def next_tag_token_with_start(start)
-      @ss.skip_until(TAG_END)
-      @source.byteslice(start, @ss.pos - start)
     end
   end
 end