Merged array_tokens into recursive-parsing

lib/liquid/lexer.rb

@@ -1,28 +1,5 @@
 require "strscan"
 module Liquid
-  class Token
-    attr_accessor :type, :contents
-    def initialize(type, contents = nil)
-      @type = type
-      @contents = contents
-    end
-
-    def inspect
-      out = "<#{@type}"
-      out << ": \'#{@contents}\'" if contents
-      out << '>'
-    end
-
-    def to_s
-      self.inspect
-    end
-
-    def ==(other)
-      return unless other && other.respond_to?(:type) && other.respond_to?(:contents)
-      @type == other.type && @contents == other.contents
-    end
-  end
-
   class Lexer
     SPECIALS = {
       '|' => :pipe,
@@ -47,31 +24,33 @@ module Liquid
       @output = []
 
       loop do
-        tok = next_token
+        @ss.skip(/\s*/)
+
+        tok = case
+        when @ss.eos? then nil
+        when t = @ss.scan(COMPARISON_OPERATOR) then [:comparison, t]
+        when t = @ss.scan(SINGLE_STRING_LITERAL) then [:string, t]
+        when t = @ss.scan(DOUBLE_STRING_LITERAL) then [:string, t]
+        when t = @ss.scan(FLOAT_LITERAL) then [:float, t]
+        when t = @ss.scan(INTEGER_LITERAL) then [:integer, t]
+        when t = @ss.scan(IDENTIFIER) then [:id, t]
+        else
+          c = @ss.getch
+          if s = SPECIALS[c]
+            [s,c]
+          else
+            raise SyntaxError, "Unexpected character #{c}."
+          end
+        end
+
         unless tok
-          @output << Token.new(:end_of_string)
+          @output << [:end_of_string]
           return @output
         end
         @output << tok
       end
     end
 
-    def next_token
-      consume_whitespace
-      return if @ss.eos?
-      
-      case
-      when t = @ss.scan(COMPARISON_OPERATOR) then Token.new(:comparison, t)
-      when t = @ss.scan(SINGLE_STRING_LITERAL) then Token.new(:string, t)
-      when t = @ss.scan(DOUBLE_STRING_LITERAL) then Token.new(:string, t)
-      when t = @ss.scan(FLOAT_LITERAL) then Token.new(:float, t)
-      when t = @ss.scan(INTEGER_LITERAL) then Token.new(:integer, t)
-      when t = @ss.scan(IDENTIFIER) then Token.new(:id, t)
-      else
-        lex_specials
-      end
-    end
-
     protected
     def lex_specials
       c = @ss.getch
@@ -81,9 +60,5 @@ module Liquid
 
       raise SyntaxError, "Unexpected character #{c}."
     end
-
-    def consume_whitespace
-      @ss.skip(/\s*/)
-    end
   end
 end

lib/liquid/parser.rb

@@ -14,11 +14,11 @@ module Liquid
 
     def consume(type = nil)
       token = @tokens[@p]
-      if type && token.type != type
+      if type && token[0] != type
         raise SyntaxError, "Expected #{type} but found #{@tokens[@p]}"
       end
       @p += 1
-      token.contents
+      token[1]
     end
 
     # Only consumes the token if it matches the type
@@ -26,35 +26,35 @@ module Liquid
     # or false otherwise.
     def consume?(type)
       token = @tokens[@p]
-      return false unless token && token.type == type
+      return false unless token && token[0] == type
       @p += 1
-      token.contents
+      token[1]
     end
 
     # Like consume? Except for an :id token of a certain name
     def id?(str)
       token = @tokens[@p]
-      return false unless token && token.type == :id
-      return false unless token.contents == str
+      return false unless token && token[0] == :id
+      return false unless token[1] == str
       @p += 1
-      token.contents
+      token[1]
     end
 
     def look(type, ahead = 0)
       tok = @tokens[@p + ahead]
       return false unless tok
-      tok.type == type
+      tok[0] == type
     end
 
     # === General Liquid parsing functions ===
 
     def expression
       token = @tokens[@p]
-      if token.type == :id
+      if token[0] == :id
         variable_signature
-      elsif [:string, :integer, :float].include? token.type
+      elsif [:string, :integer, :float].include? token[0]
         consume
-        token.contents
+        token[1]
       else
         raise SyntaxError, "#{token} is not a valid expression."
       end

performance/profile.rb

@@ -11,7 +11,7 @@ results  = profiler.run_profile
 puts 'Success'
 puts
 
-[RubyProf::FlatPrinter, RubyProf::GraphPrinter, RubyProf::GraphHtmlPrinter, RubyProf::CallTreePrinter].each do |klass|
+[RubyProf::FlatPrinter, RubyProf::GraphHtmlPrinter, RubyProf::CallTreePrinter, RubyProf::DotPrinter].each do |klass|
   filename = (ENV['TMP'] || '/tmp') + (klass.name.include?('Html') ? "/liquid.#{klass.name.downcase}.html" : "/callgrind.liquid.#{klass.name.downcase}.txt")
   filename.gsub!(/:+/, '_')
   File.open(filename, "w+") { |fp| klass.new(results).print(fp, :print_file => true) }

test/liquid/lexer_test.rb

@@ -5,39 +5,39 @@ class LexerTest < Test::Unit::TestCase
 
   def test_strings
     tokens = Lexer.new(%! 'this is a test""' "wat 'lol'"!).tokenize
-    assert_equal [Token.new(:string,%!'this is a test""'!), Token.new(:string, %!"wat 'lol'"!), Token.new(:end_of_string)], tokens
+    assert_equal [[:string,%!'this is a test""'!], [:string, %!"wat 'lol'"!], [:end_of_string]], tokens
   end
 
   def test_integer
     tokens = Lexer.new('hi 50').tokenize
-    assert_equal [Token.new(:id,'hi'), Token.new(:integer, '50'), Token.new(:end_of_string)], tokens
+    assert_equal [[:id,'hi'], [:integer, '50'], [:end_of_string]], tokens
   end
 
   def test_float
     tokens = Lexer.new('hi 5.0').tokenize
-    assert_equal [Token.new(:id,'hi'), Token.new(:float, '5.0'), Token.new(:end_of_string)], tokens
+    assert_equal [[:id,'hi'], [:float, '5.0'], [:end_of_string]], tokens
   end
 
   def test_comparison
     tokens = Lexer.new('== <> contains').tokenize
-    assert_equal [Token.new(:comparison,'=='), Token.new(:comparison, '<>'), Token.new(:comparison, 'contains'), Token.new(:end_of_string)], tokens
+    assert_equal [[:comparison,'=='], [:comparison, '<>'], [:comparison, 'contains'], [:end_of_string]], tokens
   end
 
   def test_specials
     tokens = Lexer.new('| .:').tokenize
-    assert_equal [Token.new(:pipe, '|'), Token.new(:dot, '.'), Token.new(:colon, ':'), Token.new(:end_of_string)], tokens
+    assert_equal [[:pipe, '|'], [:dot, '.'], [:colon, ':'], [:end_of_string]], tokens
     tokens = Lexer.new('[,]').tokenize
-    assert_equal [Token.new(:open_square, '['), Token.new(:comma, ','), Token.new(:close_square, ']'), Token.new(:end_of_string)], tokens
+    assert_equal [[:open_square, '['], [:comma, ','], [:close_square, ']'], [:end_of_string]], tokens
   end
 
   def test_fancy_identifiers
     tokens = Lexer.new('hi! five?').tokenize
-    assert_equal [Token.new(:id,'hi!'), Token.new(:id, 'five?'), Token.new(:end_of_string)], tokens
+    assert_equal [[:id,'hi!'], [:id, 'five?'], [:end_of_string]], tokens
   end
 
   def test_whitespace
     tokens = Lexer.new("five|\n\t ==").tokenize
-    assert_equal [Token.new(:id,'five'), Token.new(:pipe, '|'), Token.new(:comparison, '=='), Token.new(:end_of_string)], tokens
+    assert_equal [[:id,'five'], [:pipe, '|'], [:comparison, '=='], [:end_of_string]], tokens
   end
 
   def test_unexpected_character
@@ -45,11 +45,4 @@ class LexerTest < Test::Unit::TestCase
       Lexer.new("%").tokenize
     end
   end
-
-  def test_next_token
-    l = Lexer.new('hi 5.0')
-    assert_equal Token.new(:id, 'hi'), l.next_token
-    assert_equal Token.new(:float, '5.0'), l.next_token
-    assert_nil l.next_token
-  end
 end