passing tokenizer unit test

lib/liquid/tokenizer.rb

@@ -8,18 +8,22 @@ module Liquid
       @source         = source
       @line_number    = line_number || (line_numbers ? 1 : nil)
       @for_liquid_tag = for_liquid_tag
-      @offset         = 0
-      @tokens         = tokenize
+      @offset = 0
+      @state = :outside
+      @ss = StringScanner.new(source)
     end
 
     def shift
-      token = @tokens[@offset]
-      return nil unless token
+      return nil if @ss.eos?
 
-      @offset += 1
+      token = take_new_next
 
-      if @line_number
-        @line_number += @for_liquid_tag ? 1 : token.count("\n")
+      until @state == :outside || @ss.eos?
+        token << if @state == :inside_variable || @state == :inside_tag
+          take_inside
+        else
+          take_rest
+        end
       end
 
       token
@@ -27,19 +31,65 @@ module Liquid
 
     private
 
-    def tokenize
-      return [] if @source.empty?
+    def take_rest
+      token = @ss.rest
+      @ss.terminate
+      @state = :outside
+      token
+    end
 
-      return @source.split("\n") if @for_liquid_tag
+    def take_new_next
+      start_pos = @ss.pos
 
-      tokens = @source.split(TemplateParser)
+      if (n = @ss.skip_until(VariableStart))
+        if n == 2
+          @state = :inside_variable
+          end_pos = @ss.pos
+        token = @source.byteslice(start_pos, end_pos - start_pos)
+        else
+          @ss.pos -= 2
+          token = @source.byteslice(start_pos, n - 2)
+        end
 
-      # removes the rogue empty element at the beginning of the array
-      if tokens[0]&.empty?
-        @offset += 1
+        @line_number += @for_liquid_tag ? 1 : token.count("\n") if @line_number
+        token
+      elsif (n = @ss.skip_until(TagStart))
+        if n == 2
+          @state = :inside_tag
+          end_pos = @ss.pos
+          token = @source.byteslice(start_pos, end_pos - start_pos)
+        else
+          @ss.pos -= 2
+          token = @source.byteslice(start_pos, n - 2)
+        end
+
+        @line_number += @for_liquid_tag ? 1 : token.count("\n") if @line_number
+        token
+      else
+        take_rest
       end
+    end
 
-      tokens
+    def take_inside
+      start_pos = @ss.pos
+
+      if @state == :inside_variable
+        if (n = @ss.skip_until(VariableEnd))
+          @state = :outside
+          token = @source.byteslice(start_pos, n)
+          @line_number += @for_liquid_tag ? 1 : token.count("\n") if @line_number
+          token
+        end
+      elsif @state == :inside_tag
+        if (n = @ss.skip_until(TagEnd))
+          @state = :outside
+          token = @source.byteslice(start_pos, n)
+          @line_number += @for_liquid_tag ? 1 : token.count("\n") if @line_number
+          token
+        end
+      else
+        raise SyntaxError, "Unknown state: #{@state}"
+      end
     end
   end
 end