New variable parser!

lib/liquid/lexer.rb

@@ -14,6 +14,10 @@ module Liquid
       out << ": \'#{@contents}\'" if contents
       out << '>'
     end
+
+    def to_s
+      self.inspect
+    end
   end
 
   class Lexer
@@ -40,7 +44,10 @@ module Liquid
 
       loop do
         tok = next_token
-        return @output unless tok
+        unless tok
+          @output << Token[:end_of_string]
+          return @output
+        end
         @output << tok
       end
     end

lib/liquid/parser.rb

@@ -8,6 +8,10 @@ module Liquid
       @p = 0 # pointer to current location
     end
 
+    def jump(point)
+      @p = point
+    end
+
     def consume(type = nil)
       token = @tokens[@p]
       if type && token.type != type
@@ -17,14 +21,24 @@ module Liquid
       token.contents
     end
 
+    # Only consumes the token if it matches the type
+    # Returns the token's contents if it was consumed
+    # or false otherwise.
+    def consume?(type)
+      token = @tokens[@p]
+      return false unless token && token.type == type
+      @p += 1
+      token.contents
+    end
+
     def cur_token()
       tok = @tokens[@p]
       raise SyntaxError, 'Expected more input.' unless tok
       tok
     end
 
-    def look(type)
+    def look(type, ahead = 0)
-      tok = @tokens[@p]
+      tok = @tokens[@p + ahead]
       return false unless tok
       tok.type == type
     end
@@ -36,22 +50,34 @@ module Liquid
       if token.type == :id
         variable_signature
       elsif [:string, :integer, :float].include? token.type
+        consume
         token.contents
       else
         raise SyntaxError, "#{token} is not a valid expression."
       end
     end
 
+    def argument
+      str = ""
+      # might be a keyword argument (identifier: expression)
+      if look(:id) && look(:colon, 1)
+        str << consume << consume << ' '
+      end
+
+      str << expression
+    end
+
     def variable_signature
       str = consume(:id)
-      if look(:dot)
+      if look(:open_square)
-        str << consume
-        str << variable_signature
-      elsif look(:open_square)
         str << consume
         str << expression
         str << consume(:close_square)
       end
+      if look(:dot)
+        str << consume
+        str << variable_signature
+      end
       str
     end
   end

lib/liquid/variable.rb

@@ -18,6 +18,10 @@ module Liquid
       @markup  = markup
       @name    = nil
       @filters = []
+      parse(markup)
+    end
+
+    def old_parse(markup)
       if match = markup.match(/\s*(#{QuotedFragment})(.*)/o)
         @name = match[1]
         if match[2].match(/#{FilterSeparator}\s*(.*)/o)
@@ -33,6 +37,28 @@ module Liquid
       end
     end
 
+    def parse(markup)
+      p = Parser.new(markup)
+      # Could be just filters with no input
+      @name = p.look(:pipe) ? '' : p.expression
+      while p.consume?(:pipe)
+        filtername = p.consume(:id)
+        filterargs = p.consume?(:colon) ? parse_filterargs(p) : []
+        @filters << [filtername, filterargs]
+      end
+      p.consume(:end_of_string)
+    end
+
+    def parse_filterargs(p)
+      # first argument
+      filterargs = [p.argument]
+      # followed by comma separated others
+      while p.consume?(:comma)
+        filterargs << p.argument
+      end
+      filterargs
+    end
+
     def render(context)
       return '' if @name.nil?
       @filters.inject(context[@name]) do |output, filter|

test/liquid/parsing_quirks_test.rb

@@ -31,9 +31,11 @@ class ParsingQuirksTest < Test::Unit::TestCase
 
   def test_error_on_empty_filter
     assert_nothing_raised do
-      Template.parse("{{test |a|b|}}")
       Template.parse("{{test}}")
-      Template.parse("{{|test|}}")
+      Template.parse("{{|test}}")
+    end
+    assert_raise(SyntaxError) do
+      Template.parse("{{test |a|b|}}")
     end
   end
 

test/liquid/variable_test.rb

@@ -73,8 +73,8 @@ class VariableTest < Test::Unit::TestCase
   end
 
   def test_symbol
-    var = Variable.new("http://disney.com/logo.gif | image: 'med' ")
+    var = Variable.new("'http://disney.com/logo.gif' | image: 'med' ")
-    assert_equal 'http://disney.com/logo.gif', var.name
+    assert_equal "'http://disney.com/logo.gif'", var.name
     assert_equal [["image",["'med'"]]], var.filters
   end
 
@@ -114,10 +114,10 @@ class VariableTest < Test::Unit::TestCase
     assert_equal [['things',["greeting: \"world\"","farewell: 'goodbye'"]]], var.filters
   end
 
-  def test_lax_filter_argument_parsing
+  def test_strict_filter_argument_parsing
-    var = Variable.new(%! number_of_comments | pluralize: 'comment': 'comments' !)
+    assert_raises(SyntaxError) do
-    assert_equal 'number_of_comments', var.name
+      Variable.new(%! number_of_comments | pluralize: 'comment': 'comments' !)
-    assert_equal [['pluralize',["'comment'","'comments'"]]], var.filters
+    end
   end
 end