From 78ef37287830657b399b2cf47094254b24409ee7 Mon Sep 17 00:00:00 2001
From: Michael Go <michael.go@shopify.com>
Date: Wed, 30 Oct 2024 14:31:55 -0300
Subject: [PATCH] backward compatible tokenizer

---
 lib/liquid/lexer.rb     |  1 +
 lib/liquid/tokenizer.rb | 47 ++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/lib/liquid/lexer.rb b/lib/liquid/lexer.rb
index 271cd8bf..f8f3dc7b 100644
--- a/lib/liquid/lexer.rb
+++ b/lib/liquid/lexer.rb
@@ -233,5 +233,6 @@ module Liquid
     end
   end
 
+  # TODO: Remove this once Ruby 3.4.0 release is out
   Lexer = StringScanner.instance_methods.include?(:scan_byte) ? Lexer2 : Lexer1
 end
diff --git a/lib/liquid/tokenizer.rb b/lib/liquid/tokenizer.rb
index bc26814b..a7cffdde 100644
--- a/lib/liquid/tokenizer.rb
+++ b/lib/liquid/tokenizer.rb
@@ -3,7 +3,49 @@
 require "strscan"
 
 module Liquid
-  class Tokenizer
+  class Tokenizer1
+    attr_reader :line_number, :for_liquid_tag
+
+    def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
+      @source         = source
+      @line_number    = line_number || (line_numbers ? 1 : nil)
+      @for_liquid_tag = for_liquid_tag
+      @offset         = 0
+      @tokens         = tokenize
+    end
+
+    def shift
+      token = @tokens[@offset]
+      return nil unless token
+
+      @offset += 1
+
+      if @line_number
+        @line_number += @for_liquid_tag ? 1 : token.count("\n")
+      end
+
+      token
+    end
+
+    private
+
+    def tokenize
+      return [] if @source.empty?
+
+      return @source.split("\n") if @for_liquid_tag
+
+      tokens = @source.split(TemplateParser)
+
+      # removes the rogue empty element at the beginning of the array
+      if tokens[0]&.empty?
+        @offset += 1
+      end
+
+      tokens
+    end
+  end
+
+  class Tokenizer2
     attr_reader :line_number, :for_liquid_tag
 
     TAG_END = /%\}/
@@ -153,4 +195,7 @@ module Liquid
       @source.byteslice(start, @ss.pos - start)
     end
   end
+
+  # TODO: Remove this once Ruby 3.4.0 release is out
+  Tokenizer = StringScanner.instance_methods.include?(:scan_byte) ? Tokenizer2 : Tokenizer1
 end