Create a Liquid::Tokenizer class in the C extension.

test/liquid/template_test.rb

@@ -25,26 +25,6 @@ end
 class TemplateTest < Test::Unit::TestCase
   include Liquid
 
-  def test_tokenize_strings
-    assert_equal [' '], Template.new.send(:tokenize, ' ')
-    assert_equal ['hello world'], Template.new.send(:tokenize, 'hello world')
-  end
-
-  def test_tokenize_variables
-    assert_equal ['{{funk}}'], Template.new.send(:tokenize, '{{funk}}')
-    assert_equal [' ', '{{funk}}', ' '], Template.new.send(:tokenize, ' {{funk}} ')
-    assert_equal [' ', '{{funk}}', ' ', '{{so}}', ' ', '{{brother}}', ' '], Template.new.send(:tokenize, ' {{funk}} {{so}} {{brother}} ')
-    assert_equal [' ', '{{  funk  }}', ' '], Template.new.send(:tokenize, ' {{  funk  }} ')
-  end
-
-  def test_tokenize_blocks
-    assert_equal ['{%comment%}'], Template.new.send(:tokenize, '{%comment%}')
-    assert_equal [' ', '{%comment%}', ' '], Template.new.send(:tokenize, ' {%comment%} ')
-
-    assert_equal [' ', '{%comment%}', ' ', '{%endcomment%}', ' '], Template.new.send(:tokenize, ' {%comment%} {%endcomment%} ')
-    assert_equal ['  ', '{% comment %}', ' ', '{% endcomment %}', ' '], Template.new.send(:tokenize, "  {% comment %} {% endcomment %} ")
-  end
-
   def test_instance_assigns_persist_on_same_template_object_between_parses
     t = Template.new
     assert_equal 'from instance assigns', t.parse("{% assign foo = 'from instance assigns' %}{{ foo }}").render

test/liquid/tokenizer_test.rb

@@ -0,0 +1,64 @@
+require 'test_helper'
+
+class TokenizerTest < Test::Unit::TestCase
+  def test_tokenize_strings
+    assert_equal [' '], tokenize(' ')
+    assert_equal ['hello world'], tokenize('hello world')
+  end
+
+  def test_tokenize_variables
+    assert_equal ['{{funk}}'], tokenize('{{funk}}')
+    assert_equal [' ', '{{funk}}', ' '], tokenize(' {{funk}} ')
+    assert_equal [' ', '{{funk}}', ' ', '{{so}}', ' ', '{{brother}}', ' '], tokenize(' {{funk}} {{so}} {{brother}} ')
+    assert_equal [' ', '{{  funk  }}', ' '], tokenize(' {{  funk  }} ')
+  end
+
+  def test_tokenize_blocks
+    assert_equal ['{%comment%}'], tokenize('{%comment%}')
+    assert_equal [' ', '{%comment%}', ' '], tokenize(' {%comment%} ')
+
+    assert_equal [' ', '{%comment%}', ' ', '{%endcomment%}', ' '], tokenize(' {%comment%} {%endcomment%} ')
+    assert_equal ['  ', '{% comment %}', ' ', '{% endcomment %}', ' '], tokenize("  {% comment %} {% endcomment %} ")
+  end
+
+  def test_tokenize_incomplete_end
+    assert_tokens 'before{{ incomplete }after', ['before', '{{ incomplete }', 'after']
+    assert_tokens 'before{% incomplete %after', ['before', '{%', ' incomplete %after']
+  end
+
+  def test_tokenize_no_end
+    assert_tokens 'before{{ unterminated ', ['before', '{{', ' unterminated ']
+    assert_tokens 'before{% unterminated ', ['before', '{%', ' unterminated ']
+  end
+
+  private
+
+  def assert_tokens(source, expected)
+    assert_equal expected, tokenize(source)
+    assert_equal expected, old_tokenize(source)
+  end
+
+  def tokenize(source)
+    tokenizer = Liquid::Tokenizer.new(source)
+    tokens = []
+    while token = tokenizer.next
+      tokens << token
+    end
+    tokens
+  end
+
+  AnyStartingTag        = /\{\{|\{\%/
+  VariableIncompleteEnd = /\}\}?/
+  PartialTemplateParser = /#{Liquid::TagStart}.*?#{Liquid::TagEnd}|#{Liquid::VariableStart}.*?#{VariableIncompleteEnd}/o
+  TemplateParser        = /(#{PartialTemplateParser}|#{AnyStartingTag})/o
+
+  def old_tokenize(source)
+    return [] if source.to_s.empty?
+    tokens = source.split(TemplateParser)
+
+    # removes the rogue empty element at the beginning of the array
+    tokens.shift if tokens[0] and tokens[0].empty?
+
+    tokens
+  end
+end