Test a different lexer architechture

lib/liquid/lexer.rb

@@ -29,11 +29,17 @@ module Liquid
       '[' => :open_square,
       ']' => :close_square
     }
-    IDENTIFIER = /[\w\-]+/
+
-    SINGLE_STRING_LITERAL = /'[^\']*'/
+    MATCHERS = {
-    DOUBLE_STRING_LITERAL = /"[^\"]*"/
+      /([\w\-]+)/ => :id,
-    INTEGER_LITERAL = /-?\d+/
+      /('[^\']*')/ => :string,
-    FLOAT_LITERAL = /-?\d+(?:\.\d+)?/
+      /("[^\"]*")/ => :string,
+      /(-?\d+)/ => :integer,
+      /(-?\d+(?:\.\d+)?)/ => :float
+    }
+    MATCHER_REGEX = Regexp.union(MATCHERS.keys)
+    MATCHER_TOKENS = MATCHERS.values
+
 
     def initialize(input)
       @ss = StringScanner.new(input)
@@ -56,13 +62,17 @@ module Liquid
       consume_whitespace
       return if @ss.eos?
 
-      case
+      tok_contents = @ss.scan(MATCHER_REGEX)
-      when t = @ss.scan(IDENTIFIER) then Token[:id, t]
+      tok_type = nil
-      when t = @ss.scan(SINGLE_STRING_LITERAL) then Token[:string, t]
+      MATCHER_TOKENS.each_with_index do |type, i|
-      when t = @ss.scan(DOUBLE_STRING_LITERAL) then Token[:string, t]
+        if @ss[i+1]
-      when t = @ss.scan(INTEGER_LITERAL) then Token[:integer, t]
+          tok_type = type
-      when t = @ss.scan(FLOAT_LITERAL) then Token[:float, t]
+          break
-      else
+        end
+      end
+
+      return Token[tok_type, tok_contents] if tok_type
+      
       c = @ss.getch
       if s = SPECIALS[c]
         return Token[s,c]
@@ -70,7 +80,6 @@ module Liquid
 
       raise SyntaxError, "Unexpected character #{c}."
     end
-    end
 
     def consume_whitespace
       @ss.skip(/\s*/)