Better lexer

lib/liquid/lexer.rb

@@ -25,7 +25,7 @@ module Liquid
     }
 
     def initialize(input)
-      @input = input.chars.to_a
+      @input = input
     end
 
     def tokenize
@@ -33,18 +33,26 @@ module Liquid
       @output = []
 
       loop do
+        tok = next_token
+        return @output unless tok
+        @output << tok
+      end
+    end
+
+    def next_token
       consume_whitespace
       c = @input[@p]
-
+      return nil unless c
-        # are we out of input?
-        return @output unless c
 
       if identifier?(c)
-          @output << consume_identifier
+        identifier
+      elsif c == '"' || c == '\''
+        string_literal
       elsif s = SPECIALS[c]
-          @output << Token[s]
         @p += 1
-        end
+        Token[s]
+      else
+        raise SyntaxError, "Unexpected character #{c}."
       end
     end
 
@@ -56,19 +64,39 @@ module Liquid
       c =~ /^\s$/
     end
 
+    def consume
+      c = @input[@p]
+      @p += 1
+      c
+    end
+
     def consume_whitespace
       while whitespace?(@input[@p])
         @p += 1
       end
     end
 
-    def consume_identifier
+    def identifier
       str = ""
       while identifier?(@input[@p])
         str << @input[@p]
         @p += 1
       end
-      Token[:identifier, str]
+      Token[:id, str]
     end
+
+    def string_literal
+      quote = consume()
+
+      start = @p
+      while @input[@p] != quote
+        @p += 1
+      end
+      @p += 1 # closing quote
+
+      Token[:string, @input[start..(@p-2)]]
+    end
+
+    def number_literal
   end
 end