Implement tokenization in a C extension.

lib/liquid.rb

@@ -30,13 +30,9 @@ module Liquid
   VariableSegment             = /[\w\-]/
   VariableStart               = /\{\{/
   VariableEnd                 = /\}\}/
-  VariableIncompleteEnd       = /\}\}?/
   QuotedString                = /"[^"]*"|'[^']*'/
   QuotedFragment              = /#{QuotedString}|(?:[^\s,\|'"]|#{QuotedString})+/o
   TagAttributes               = /(\w+)\s*\:\s*(#{QuotedFragment})/o
-  AnyStartingTag              = /\{\{|\{\%/
-  PartialTemplateParser       = /#{TagStart}.*?#{TagEnd}|#{VariableStart}.*?#{VariableIncompleteEnd}/om
-  TemplateParser              = /(#{PartialTemplateParser}|#{AnyStartingTag})/om
   VariableParser              = /\[[^\]]+\]|#{VariableSegment}+\??/o
 end
 
@@ -64,3 +60,9 @@ require 'liquid/utils'
 # Load all the tags of the standard library
 #
 Dir[File.dirname(__FILE__) + '/liquid/tags/*.rb'].each { |f| require f }
+
+if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'ruby'
+  require 'liquid/liquid'
+else
+  require 'liquid/tokenizer'
+end

lib/liquid/template.rb

@@ -162,16 +162,9 @@ module Liquid
 
     private
 
-    # Uses the <tt>Liquid::TemplateParser</tt> regexp to tokenize the passed source
     def tokenize(source)
       source = source.source if source.respond_to?(:source)
-      return [] if source.to_s.empty?
-      tokens = source.split(TemplateParser)
-
-      # removes the rogue empty element at the beginning of the array
-      tokens.shift if tokens[0] and tokens[0].empty?
-
-      tokens
+      Tokenizer.new(source.to_s)
     end
 
   end

lib/liquid/tokenizer.rb

@@ -0,0 +1,20 @@
+module Liquid
+  class Tokenizer
+    VariableIncompleteEnd = /\}\}?/
+    AnyStartingTag        = /\{\{|\{\%/
+    PartialTemplateParser = /#{TagStart}.*?#{TagEnd}|#{VariableStart}.*?#{VariableIncompleteEnd}/om
+    TemplateParser        = /(#{PartialTemplateParser}|#{AnyStartingTag})/om
+
+    def initialize(source)
+      @tokens = source.split(TemplateParser)
+
+      # removes the rogue empty element at the beginning of the array
+      @tokens.shift if @tokens[0] && @tokens[0].empty?
+    end
+
+    def next
+      @tokens.shift
+    end
+    alias_method :shift, :next
+  end
+end