Remove the Token class from the lexer in favour of less smart but faster arrays

lib/liquid/lexer.rb

@@ -1,30 +1,4 @@
 module Liquid
-  class Token
-    attr_accessor :type, :contents
-    def initialize(*args)
-      @type, @contents = args
-    end
-
-    def self.[](*args)
-      Token.new(*args)
-    end
-
-    def inspect
-      out = "<#{@type}"
-      out << ": \'#{@contents}\'" if contents
-      out << '>'
-    end
-
-    def to_s
-      self.inspect
-    end
-
-    def ==(other)
-      return unless other && other.respond_to?(:type) && other.respond_to?(:contents)
-      @type == other.type && @contents == other.contents
-    end
-  end
-
   class Lexer
     SPECIALS = {
       '|' => :pipe,
@@ -51,7 +25,7 @@ module Liquid
       loop do
         tok = next_token
         unless tok
-          @output << Token[:end_of_string]
+          @output << [:end_of_string]
           return @output
         end
         @output << tok
@@ -59,16 +33,16 @@ module Liquid
     end
 
     def next_token
-      consume_whitespace
+      @ss.skip(/\s*/)
       return if @ss.eos?
-      
+
       case
-      when t = @ss.scan(COMPARISON_OPERATOR) then Token[:comparison, t]
-      when t = @ss.scan(SINGLE_STRING_LITERAL) then Token[:string, t]
-      when t = @ss.scan(DOUBLE_STRING_LITERAL) then Token[:string, t]
-      when t = @ss.scan(FLOAT_LITERAL) then Token[:float, t]
-      when t = @ss.scan(INTEGER_LITERAL) then Token[:integer, t]
-      when t = @ss.scan(IDENTIFIER) then Token[:id, t]
+      when t = @ss.scan(COMPARISON_OPERATOR) then [:comparison, t]
+      when t = @ss.scan(SINGLE_STRING_LITERAL) then [:string, t]
+      when t = @ss.scan(DOUBLE_STRING_LITERAL) then [:string, t]
+      when t = @ss.scan(FLOAT_LITERAL) then [:float, t]
+      when t = @ss.scan(INTEGER_LITERAL) then [:integer, t]
+      when t = @ss.scan(IDENTIFIER) then [:id, t]
       else
         lex_specials
       end
@@ -78,14 +52,10 @@ module Liquid
     def lex_specials
       c = @ss.getch
       if s = SPECIALS[c]
-        return Token[s,c]
+        return [s,c]
       end
 
       raise SyntaxError, "Unexpected character #{c}."
     end
-
-    def consume_whitespace
-      @ss.skip(/\s*/)
-    end
   end
 end