Extract tokenize logic from Template to a RubyTokenizer

lib/liquid.rb

@@ -68,6 +68,7 @@ require 'liquid/template'
 require 'liquid/standardfilters'
 require 'liquid/condition'
 require 'liquid/utils'
+require 'liquid/tokenizer'
 require 'liquid/token'
 
 # Load all the tags of the standard library

lib/liquid/template.rb

@@ -228,28 +228,8 @@ module Liquid
 
     private
 
-    # Uses the <tt>Liquid::TemplateParser</tt> regexp to tokenize the passed source
     def tokenize(source)
-      source = source.source if source.respond_to?(:source)
+      Tokenizer.new(source, @line_numbers)
-      return [] if source.to_s.empty?
-
-      tokens = calculate_line_numbers(source.split(TemplateParser))
-
-      # removes the rogue empty element at the beginning of the array
-      tokens.shift if tokens[0] && tokens[0].empty?
-
-      tokens
-    end
-
-    def calculate_line_numbers(raw_tokens)
-      return raw_tokens unless @line_numbers
-
-      current_line = 1
-      raw_tokens.map do |token|
-        Token.new(token, current_line).tap do
-          current_line += token.count("\n")
-        end
-      end
     end
 
     def with_profiling

lib/liquid/tokenizer.rb

@@ -0,0 +1,38 @@
+module Liquid
+  class Tokenizer
+    attr_reader :tokens
+
+    def initialize(source, line_numbers = false)
+      @source, @line_numbers = source, line_numbers
+      @tokens = tokenize
+    end
+
+    def shift
+      @tokens.shift
+    end
+
+    private
+
+    def tokenize
+      @source = @source.source if @source.respond_to?(:source)
+      return [] if @source.to_s.empty?
+
+      tokens = @source.split(TemplateParser)
+      tokens = @line_numbers ? calculate_line_numbers(tokens) : tokens
+
+      # removes the rogue empty element at the beginning of the array
+      tokens.shift if tokens[0] && tokens[0].empty?
+
+      tokens
+    end
+
+    def calculate_line_numbers(tokens)
+      current_line = 1
+      tokens.map do |token|
+        Token.new(token, current_line).tap do
+          current_line += token.count("\n")
+        end
+      end
+    end
+  end
+end

test/unit/tokenizer_unit_test.rb

@@ -24,15 +24,16 @@ class TokenizerTest < Minitest::Test
   def test_calculate_line_numbers_per_token_with_profiling
     template = Liquid::Template.parse("", :profile => true)
 
-    assert_equal [1],       template.send(:tokenize, "{{funk}}").map(&:line_number)
+    assert_equal [1],       template.send(:tokenize, "{{funk}}").tokens.map(&:line_number)
-    assert_equal [1, 1, 1], template.send(:tokenize, " {{funk}} ").map(&:line_number)
+    assert_equal [1, 1, 1], template.send(:tokenize, " {{funk}} ").tokens.map(&:line_number)
-    assert_equal [1, 2, 2], template.send(:tokenize, "\n{{funk}}\n").map(&:line_number)
+    assert_equal [1, 2, 2], template.send(:tokenize, "\n{{funk}}\n").tokens.map(&:line_number)
-    assert_equal [1, 1, 3], template.send(:tokenize, " {{\n funk \n}} ").map(&:line_number)
+    assert_equal [1, 1, 3], template.send(:tokenize, " {{\n funk \n}} ").tokens.map(&:line_number)
   end
 
   private
 
   def tokenize(source)
-    Liquid::Template.new.send(:tokenize, source)
+    tokenizer = Liquid::Tokenizer.new(source)
+    tokenizer.tokens
   end
 end