backward compatible tokenizer

lib/liquid/lexer.rb

@@ -233,5 +233,6 @@ module Liquid
     end
   end
 
+  # TODO: Remove this once Ruby 3.4.0 release is out
   Lexer = StringScanner.instance_methods.include?(:scan_byte) ? Lexer2 : Lexer1
 end

lib/liquid/tokenizer.rb

@@ -3,7 +3,49 @@
 require "strscan"
 
 module Liquid
-  class Tokenizer
+  class Tokenizer1
+    attr_reader :line_number, :for_liquid_tag
+
+    def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
+      @source         = source
+      @line_number    = line_number || (line_numbers ? 1 : nil)
+      @for_liquid_tag = for_liquid_tag
+      @offset         = 0
+      @tokens         = tokenize
+    end
+
+    def shift
+      token = @tokens[@offset]
+      return nil unless token
+
+      @offset += 1
+
+      if @line_number
+        @line_number += @for_liquid_tag ? 1 : token.count("\n")
+      end
+
+      token
+    end
+
+    private
+
+    def tokenize
+      return [] if @source.empty?
+
+      return @source.split("\n") if @for_liquid_tag
+
+      tokens = @source.split(TemplateParser)
+
+      # removes the rogue empty element at the beginning of the array
+      if tokens[0]&.empty?
+        @offset += 1
+      end
+
+      tokens
+    end
+  end
+
+  class Tokenizer2
     attr_reader :line_number, :for_liquid_tag
 
     TAG_END = /%\}/
@@ -153,4 +195,7 @@ module Liquid
       @source.byteslice(start, @ss.pos - start)
     end
   end
+
+  # TODO: Remove this once Ruby 3.4.0 release is out
+  Tokenizer = StringScanner.instance_methods.include?(:scan_byte) ? Tokenizer2 : Tokenizer1
 end