From b37fa98c9121d84c9d68f4b000a269de6c71f339 Mon Sep 17 00:00:00 2001
From: Tobi Lutke <tobi@shopify.com>
Date: Thu, 12 Mar 2026 17:12:08 -0400
Subject: [PATCH] =?UTF-8?q?Clean=20up=20tokenizer:=20remove=20unused=20Str?=
 =?UTF-8?q?ingScanner=20setup=20and=20regex=20constants\n\nResult:=20{"sta?=
 =?UTF-8?q?tus":"keep","combined=5F=C2=B5s":3490,"parse=5F=C2=B5s":2331,"r?=
 =?UTF-8?q?ender=5F=C2=B5s":1159,"allocations":24882}?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 autoresearch.jsonl      |  1 +
 lib/liquid/tokenizer.rb | 10 +---------
 2 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/autoresearch.jsonl b/autoresearch.jsonl
index 8287d4b5..a7194b13 100644
--- a/autoresearch.jsonl
+++ b/autoresearch.jsonl
@@ -12,3 +12,4 @@
 {"run":11,"commit":"c09e722","metric":3889,"metrics":{"parse_µs":2770,"render_µs":1119,"allocations":24766},"status":"discard","description":"TruthyCondition for simple if checks: -115 allocs but YJIT polymorphism at evaluate call site hurts speed","timestamp":1773349649377,"segment":0}
 {"run":12,"commit":"c09e722","metric":4150,"metrics":{"parse_µs":2769,"render_µs":1381,"allocations":24881},"status":"discard","description":"Index loop for filters: YJIT optimizes each+destructure better than manual indexing","timestamp":1773349699285,"segment":0}
 {"run":13,"commit":"b7ae55f","metric":3556,"metrics":{"parse_µs":2388,"render_µs":1168,"allocations":24882},"status":"keep","description":"Replace StringScanner tokenizer with String#byteindex — 12% faster parse, no regex overhead for delimiter finding","timestamp":1773349875890,"segment":0}
+{"run":14,"commit":"e25f2f1","metric":3464,"metrics":{"parse_µs":2335,"render_µs":1129,"allocations":24882},"status":"keep","description":"Confirmation run: byteindex tokenizer consistently 3,400-3,600µs","timestamp":1773349889465,"segment":0}
diff --git a/lib/liquid/tokenizer.rb b/lib/liquid/tokenizer.rb
index 54b56a9a..59b4c47e 100644
--- a/lib/liquid/tokenizer.rb
+++ b/lib/liquid/tokenizer.rb
@@ -6,10 +6,6 @@ module Liquid
   class Tokenizer
     attr_reader :line_number, :for_liquid_tag
 
-    TAG_END = /%\}/
-    TAG_OR_VARIABLE_START = /\{[\{\%]/
-    NEWLINE = /\n/
-
     OPEN_CURLEY = "{".ord
     CLOSE_CURLEY = "}".ord
     PERCENTAGE = "%".ord
@@ -27,11 +23,7 @@ module Liquid
       @offset = 0
       @tokens = []
 
-      if @source
-        @ss = string_scanner
-        @ss.string = @source
-        tokenize
-      end
+      tokenize if @source
     end
 
     def shift