Test a different lexer architechture

lib/liquid/lexer.rb

@@ -29,11 +29,17 @@ module Liquid
       '[' => :open_square,
       ']' => :close_square
     }
-    IDENTIFIER = /[\w\-]+/
-    SINGLE_STRING_LITERAL = /'[^\']*'/
-    DOUBLE_STRING_LITERAL = /"[^\"]*"/
-    INTEGER_LITERAL = /-?\d+/
-    FLOAT_LITERAL = /-?\d+(?:\.\d+)?/
+
+    MATCHERS = {
+      /([\w\-]+)/ => :id,
+      /('[^\']*')/ => :string,
+      /("[^\"]*")/ => :string,
+      /(-?\d+)/ => :integer,
+      /(-?\d+(?:\.\d+)?)/ => :float
+    }
+    MATCHER_REGEX = Regexp.union(MATCHERS.keys)
+    MATCHER_TOKENS = MATCHERS.values
+
 
     def initialize(input)
       @ss = StringScanner.new(input)
@@ -55,21 +61,24 @@ module Liquid
     def next_token
       consume_whitespace
       return if @ss.eos?
-      
-      case
-      when t = @ss.scan(IDENTIFIER) then Token[:id, t]
-      when t = @ss.scan(SINGLE_STRING_LITERAL) then Token[:string, t]
-      when t = @ss.scan(DOUBLE_STRING_LITERAL) then Token[:string, t]
-      when t = @ss.scan(INTEGER_LITERAL) then Token[:integer, t]
-      when t = @ss.scan(FLOAT_LITERAL) then Token[:float, t]
-      else
-        c = @ss.getch
-        if s = SPECIALS[c]
-          return Token[s,c]
-        end
 
-        raise SyntaxError, "Unexpected character #{c}."
+      tok_contents = @ss.scan(MATCHER_REGEX)
+      tok_type = nil
+      MATCHER_TOKENS.each_with_index do |type, i|
+        if @ss[i+1]
+          tok_type = type
+          break
+        end
       end
+
+      return Token[tok_type, tok_contents] if tok_type
+      
+      c = @ss.getch
+      if s = SPECIALS[c]
+        return Token[s,c]
+      end
+
+      raise SyntaxError, "Unexpected character #{c}."
     end
 
     def consume_whitespace