From d2b68b81a76e242e06f54521d136bf6f712303cc Mon Sep 17 00:00:00 2001
From: Michael Go <michael.go@shopify.com>
Date: Fri, 1 Nov 2024 17:02:09 -0300
Subject: [PATCH] undo mini opt

---
 lib/liquid/tokenizer.rb | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/liquid/tokenizer.rb b/lib/liquid/tokenizer.rb
index b7f85abd..6f2358ca 100644
--- a/lib/liquid/tokenizer.rb
+++ b/lib/liquid/tokenizer.rb
@@ -68,7 +68,6 @@ module Liquid
 
     def shift
       token = @tokens[@offset]
-      @tokens[@offset] = nil
 
       return unless token
 
@@ -205,6 +204,5 @@ module Liquid
   end
 
   # Remove this once we can depend on strscan >= 3.1.1
-  # Tokenizer = StringScanner.instance_methods.include?(:scan_byte) ? Tokenizer2 : Tokenizer1
-  Tokenizer = Tokenizer2
+  Tokenizer = StringScanner.instance_methods.include?(:scan_byte) ? Tokenizer2 : Tokenizer1
 end