From 8b98f92c7f1883989f9e54739c551872a503dde3 Mon Sep 17 00:00:00 2001
From: Gaurav Chande <gmail@gauravchande.com>
Date: Wed, 3 Jun 2015 19:40:49 +0000
Subject: [PATCH] Extract tokenize logic from Template to a RubyTokenizer

---
 lib/liquid.rb                    |  1 +
 lib/liquid/template.rb           | 22 +-----------------
 lib/liquid/tokenizer.rb          | 38 ++++++++++++++++++++++++++++++++
 test/unit/tokenizer_unit_test.rb | 11 ++++-----
 4 files changed, 46 insertions(+), 26 deletions(-)
 create mode 100644 lib/liquid/tokenizer.rb

diff --git a/lib/liquid.rb b/lib/liquid.rb
index 9724aac9..14afa757 100644
--- a/lib/liquid.rb
+++ b/lib/liquid.rb
@@ -68,6 +68,7 @@ require 'liquid/template'
 require 'liquid/standardfilters'
 require 'liquid/condition'
 require 'liquid/utils'
+require 'liquid/tokenizer'
 require 'liquid/token'
 
 # Load all the tags of the standard library
diff --git a/lib/liquid/template.rb b/lib/liquid/template.rb
index a6e36b1b..05b4114d 100644
--- a/lib/liquid/template.rb
+++ b/lib/liquid/template.rb
@@ -228,28 +228,8 @@ module Liquid
 
     private
 
-    # Uses the <tt>Liquid::TemplateParser</tt> regexp to tokenize the passed source
     def tokenize(source)
-      source = source.source if source.respond_to?(:source)
-      return [] if source.to_s.empty?
-
-      tokens = calculate_line_numbers(source.split(TemplateParser))
-
-      # removes the rogue empty element at the beginning of the array
-      tokens.shift if tokens[0] && tokens[0].empty?
-
-      tokens
-    end
-
-    def calculate_line_numbers(raw_tokens)
-      return raw_tokens unless @line_numbers
-
-      current_line = 1
-      raw_tokens.map do |token|
-        Token.new(token, current_line).tap do
-          current_line += token.count("\n")
-        end
-      end
+      Tokenizer.new(source, @line_numbers)
     end
 
     def with_profiling
diff --git a/lib/liquid/tokenizer.rb b/lib/liquid/tokenizer.rb
new file mode 100644
index 00000000..9196d2ae
--- /dev/null
+++ b/lib/liquid/tokenizer.rb
@@ -0,0 +1,38 @@
+module Liquid
+  class Tokenizer
+    attr_reader :tokens
+
+    def initialize(source, line_numbers = false)
+      @source, @line_numbers = source, line_numbers
+      @tokens = tokenize
+    end
+
+    def shift
+      @tokens.shift
+    end
+
+    private
+
+    def tokenize
+      @source = @source.source if @source.respond_to?(:source)
+      return [] if @source.to_s.empty?
+
+      tokens = @source.split(TemplateParser)
+      tokens = @line_numbers ? calculate_line_numbers(tokens) : tokens
+
+      # removes the rogue empty element at the beginning of the array
+      tokens.shift if tokens[0] && tokens[0].empty?
+
+      tokens
+    end
+
+    def calculate_line_numbers(tokens)
+      current_line = 1
+      tokens.map do |token|
+        Token.new(token, current_line).tap do
+          current_line += token.count("\n")
+        end
+      end
+    end
+  end
+end
diff --git a/test/unit/tokenizer_unit_test.rb b/test/unit/tokenizer_unit_test.rb
index f0ae7c99..71b6205e 100644
--- a/test/unit/tokenizer_unit_test.rb
+++ b/test/unit/tokenizer_unit_test.rb
@@ -24,15 +24,16 @@ class TokenizerTest < Minitest::Test
   def test_calculate_line_numbers_per_token_with_profiling
     template = Liquid::Template.parse("", :profile => true)
 
-    assert_equal [1],       template.send(:tokenize, "{{funk}}").map(&:line_number)
-    assert_equal [1, 1, 1], template.send(:tokenize, " {{funk}} ").map(&:line_number)
-    assert_equal [1, 2, 2], template.send(:tokenize, "\n{{funk}}\n").map(&:line_number)
-    assert_equal [1, 1, 3], template.send(:tokenize, " {{\n funk \n}} ").map(&:line_number)
+    assert_equal [1],       template.send(:tokenize, "{{funk}}").tokens.map(&:line_number)
+    assert_equal [1, 1, 1], template.send(:tokenize, " {{funk}} ").tokens.map(&:line_number)
+    assert_equal [1, 2, 2], template.send(:tokenize, "\n{{funk}}\n").tokens.map(&:line_number)
+    assert_equal [1, 1, 3], template.send(:tokenize, " {{\n funk \n}} ").tokens.map(&:line_number)
   end
 
   private
 
   def tokenize(source)
-    Liquid::Template.new.send(:tokenize, source)
+    tokenizer = Liquid::Tokenizer.new(source)
+    tokenizer.tokens
   end
 end