Bring back the lexer

lib/liquid/parser.rb

@@ -3,13 +3,24 @@ module Liquid
   # it provides helpers and encapsulates state
   class Parser
     def initialize(input)
-      @tokens = tokenize(input)
+      l = Lexer.new(input)
+      @tokens = l.tokenize
       @p = 0 # pointer to current location
     end
 
-    def tokenize(input)
-      # "foo.bar | filter: baz, qux" becomes ["foo", ".", "bar", "|", "filter", ":", "baz", ",", "qux"]
-      input.split(/\b/).map {|tok| tok.strip}
+    def consume(type)
+      token = @tokens[@p]
+      if match && token.type != type
+        raise SyntaxError, "Expected #{match} but found #{@tokens[@p]}"
+      end
+      @p += 1
+      token
     end
+
+    def look(type)
+      @tokens[@p].type == type
+    end
+
+    # === General Liquid parsing functions ===
   end
 end