From 24ddaf1a9ce4d8c6ee5e8302d682c1f62e61954d Mon Sep 17 00:00:00 2001
From: Tristan Hume <tristan.hume@jadedpixel.com>
Date: Fri, 26 Jul 2013 09:51:58 -0400
Subject: [PATCH] Test a different lexer architechture

---
 lib/liquid/lexer.rb | 45 +++++++++++++++++++++++++++------------------
 1 file changed, 27 insertions(+), 18 deletions(-)

diff --git a/lib/liquid/lexer.rb b/lib/liquid/lexer.rb
index 0d011b18..882ef572 100644
--- a/lib/liquid/lexer.rb
+++ b/lib/liquid/lexer.rb
@@ -29,11 +29,17 @@ module Liquid
       '[' => :open_square,
       ']' => :close_square
     }
-    IDENTIFIER = /[\w\-]+/
-    SINGLE_STRING_LITERAL = /'[^\']*'/
-    DOUBLE_STRING_LITERAL = /"[^\"]*"/
-    INTEGER_LITERAL = /-?\d+/
-    FLOAT_LITERAL = /-?\d+(?:\.\d+)?/
+
+    MATCHERS = {
+      /([\w\-]+)/ => :id,
+      /('[^\']*')/ => :string,
+      /("[^\"]*")/ => :string,
+      /(-?\d+)/ => :integer,
+      /(-?\d+(?:\.\d+)?)/ => :float
+    }
+    MATCHER_REGEX = Regexp.union(MATCHERS.keys)
+    MATCHER_TOKENS = MATCHERS.values
+
 
     def initialize(input)
       @ss = StringScanner.new(input)
@@ -55,21 +61,24 @@ module Liquid
     def next_token
       consume_whitespace
       return if @ss.eos?
-      
-      case
-      when t = @ss.scan(IDENTIFIER) then Token[:id, t]
-      when t = @ss.scan(SINGLE_STRING_LITERAL) then Token[:string, t]
-      when t = @ss.scan(DOUBLE_STRING_LITERAL) then Token[:string, t]
-      when t = @ss.scan(INTEGER_LITERAL) then Token[:integer, t]
-      when t = @ss.scan(FLOAT_LITERAL) then Token[:float, t]
-      else
-        c = @ss.getch
-        if s = SPECIALS[c]
-          return Token[s,c]
-        end
 
-        raise SyntaxError, "Unexpected character #{c}."
+      tok_contents = @ss.scan(MATCHER_REGEX)
+      tok_type = nil
+      MATCHER_TOKENS.each_with_index do |type, i|
+        if @ss[i+1]
+          tok_type = type
+          break
+        end
       end
+
+      return Token[tok_type, tok_contents] if tok_type
+      
+      c = @ss.getch
+      if s = SPECIALS[c]
+        return Token[s,c]
+      end
+
+      raise SyntaxError, "Unexpected character #{c}."
     end
 
     def consume_whitespace