Implement line numbers without the Liquid::Token class.

test/integration/filter_test.rb

@@ -39,13 +39,13 @@ class FiltersTest < Minitest::Test
     @context['var'] = 1000
     @context.add_filters(MoneyFilter)
 
-    assert_equal ' 1000$ ', Variable.new("var | money").render(@context)
+    assert_equal ' 1000$ ', Template.parse("{{var | money}}").render(@context)
   end
 
   def test_underscore_in_filter_name
     @context['var'] = 1000
     @context.add_filters(MoneyFilter)
-    assert_equal ' 1000$ ', Variable.new("var | money_with_underscore").render(@context)
+    assert_equal ' 1000$ ', Template.parse("{{var | money_with_underscore}}").render(@context)
   end
 
   def test_second_filter_overwrites_first
@@ -53,20 +53,20 @@ class FiltersTest < Minitest::Test
     @context.add_filters(MoneyFilter)
     @context.add_filters(CanadianMoneyFilter)
 
-    assert_equal ' 1000$ CAD ', Variable.new("var | money").render(@context)
+    assert_equal ' 1000$ CAD ', Template.parse("{{var | money}}").render(@context)
   end
 
   def test_size
     @context['var'] = 'abcd'
     @context.add_filters(MoneyFilter)
 
-    assert_equal 4, Variable.new("var | size").render(@context)
+    assert_equal '4', Template.parse("{{var | size}}").render(@context)
   end
 
   def test_join
     @context['var'] = [1, 2, 3, 4]
 
-    assert_equal "1 2 3 4", Variable.new("var | join").render(@context)
+    assert_equal "1 2 3 4", Template.parse("{{var | join}}").render(@context)
   end
 
   def test_sort
@@ -76,11 +76,11 @@ class FiltersTest < Minitest::Test
     @context['arrays'] = ['flower', 'are']
     @context['case_sensitive'] = ['sensitive', 'Expected', 'case']
 
-    assert_equal [1, 2, 3, 4], Variable.new("numbers | sort").render(@context)
-    assert_equal ['alphabetic', 'as', 'expected'], Variable.new("words | sort").render(@context)
-    assert_equal [3], Variable.new("value | sort").render(@context)
-    assert_equal ['are', 'flower'], Variable.new("arrays | sort").render(@context)
-    assert_equal ['Expected', 'case', 'sensitive'], Variable.new("case_sensitive | sort").render(@context)
+    assert_equal '1 2 3 4', Template.parse("{{numbers | sort | join}}").render(@context)
+    assert_equal 'alphabetic as expected', Template.parse("{{words | sort | join}}").render(@context)
+    assert_equal '3', Template.parse("{{value | sort}}").render(@context)
+    assert_equal 'are flower', Template.parse("{{arrays | sort | join}}").render(@context)
+    assert_equal 'Expected case sensitive', Template.parse("{{case_sensitive | sort | join}}").render(@context)
   end
 
   def test_sort_natural
@@ -89,19 +89,13 @@ class FiltersTest < Minitest::Test
     @context['objects'] = [TestObject.new('A'), TestObject.new('b'), TestObject.new('C')]
 
     # Test strings
-    assert_equal ['Assert', 'case', 'Insensitive'], Variable.new("words | sort_natural").render(@context)
+    assert_equal 'Assert case Insensitive', Template.parse("{{words | sort_natural | join}}").render(@context)
 
     # Test hashes
-    sorted = Variable.new("hashes | sort_natural: 'a'").render(@context)
-    assert_equal sorted[0]['a'], 'A'
-    assert_equal sorted[1]['a'], 'b'
-    assert_equal sorted[2]['a'], 'C'
+    assert_equal 'A b C', Template.parse("{{hashes | sort_natural: 'a' | map: 'a' | join}}").render(@context)
 
     # Test objects
-    sorted = Variable.new("objects | sort_natural: 'a'").render(@context)
-    assert_equal sorted[0].a, 'A'
-    assert_equal sorted[1].a, 'b'
-    assert_equal sorted[2].a, 'C'
+    assert_equal 'A b C', Template.parse("{{objects | sort_natural: 'a' | map: 'a' | join}}").render(@context)
   end
 
   def test_compact
@@ -110,49 +104,44 @@ class FiltersTest < Minitest::Test
     @context['objects'] = [TestObject.new('A'), TestObject.new(nil), TestObject.new('C')]
 
     # Test strings
-    assert_equal ['a', 'b', 'c'], Variable.new("words | compact").render(@context)
+    assert_equal 'a b c', Template.parse("{{words | compact | join}}").render(@context)
 
     # Test hashes
-    sorted = Variable.new("hashes | compact: 'a'").render(@context)
-    assert_equal sorted[0]['a'], 'A'
-    assert_equal sorted[1]['a'], 'C'
-    assert_nil sorted[2]
+    assert_equal 'A C', Template.parse("{{hashes | compact: 'a' | map: 'a' | join}}").render(@context)
 
     # Test objects
-    sorted = Variable.new("objects | compact: 'a'").render(@context)
-    assert_equal sorted[0].a, 'A'
-    assert_equal sorted[1].a, 'C'
-    assert_nil sorted[2]
+    assert_equal 'A C', Template.parse("{{objects | compact: 'a' | map: 'a' | join}}").render(@context)
   end
 
   def test_strip_html
     @context['var'] = "<b>bla blub</a>"
 
-    assert_equal "bla blub", Variable.new("var | strip_html").render(@context)
+    assert_equal "bla blub", Template.parse("{{ var | strip_html }}").render(@context)
   end
 
   def test_strip_html_ignore_comments_with_html
     @context['var'] = "<!-- split and some <ul> tag --><b>bla blub</a>"
 
-    assert_equal "bla blub", Variable.new("var | strip_html").render(@context)
+    assert_equal "bla blub", Template.parse("{{ var | strip_html }}").render(@context)
   end
 
   def test_capitalize
     @context['var'] = "blub"
 
-    assert_equal "Blub", Variable.new("var | capitalize").render(@context)
+    assert_equal "Blub", Template.parse("{{ var | capitalize }}").render(@context)
   end
 
   def test_nonexistent_filter_is_ignored
     @context['var'] = 1000
 
-    assert_equal 1000, Variable.new("var | xyzzy").render(@context)
+    assert_equal '1000', Template.parse("{{ var | xyzzy }}").render(@context)
   end
 
   def test_filter_with_keyword_arguments
     @context['surname'] = 'john'
+    @context['input'] = 'hello %{first_name}, %{last_name}'
     @context.add_filters(SubstituteFilter)
-    output = Variable.new(%( 'hello %{first_name}, %{last_name}' | substitute: first_name: surname, last_name: 'doe' )).render(@context)
+    output = Template.parse(%({{ input | substitute: first_name: surname, last_name: 'doe' }})).render(@context)
     assert_equal 'hello john, doe', output
   end
 
@@ -181,7 +170,7 @@ class FiltersInTemplate < Minitest::Test
   end
 end # FiltersTest
 
-class TestObject
+class TestObject < Liquid::Drop
   attr_accessor :a
   def initialize(a)
     @a = a

test/unit/tag_unit_test.rb

@@ -4,18 +4,18 @@ class TagUnitTest < Minitest::Test
   include Liquid
 
   def test_tag
-    tag = Tag.parse('tag', [], [], {})
+    tag = Tag.parse('tag', "", Tokenizer.new(""), ParseContext.new)
     assert_equal 'liquid::tag', tag.name
     assert_equal '', tag.render(Context.new)
   end
 
   def test_return_raw_text_of_tag
-    tag = Tag.parse("long_tag", "param1, param2, param3", [], {})
+    tag = Tag.parse("long_tag", "param1, param2, param3", Tokenizer.new(""), ParseContext.new)
     assert_equal("long_tag param1, param2, param3", tag.raw)
   end
 
   def test_tag_name_should_return_name_of_the_tag
-    tag = Tag.parse("some_tag", [], [], {})
+    tag = Tag.parse("some_tag", "", Tokenizer.new(""), ParseContext.new)
     assert_equal 'some_tag', tag.tag_name
   end
 end

test/unit/tokenizer_unit_test.rb

@@ -22,20 +22,34 @@ class TokenizerTest < Minitest::Test
   end
 
   def test_calculate_line_numbers_per_token_with_profiling
-    assert_equal [1],       tokenize("{{funk}}", true).map(&:line_number)
-    assert_equal [1, 1, 1], tokenize(" {{funk}} ", true).map(&:line_number)
-    assert_equal [1, 2, 2], tokenize("\n{{funk}}\n", true).map(&:line_number)
-    assert_equal [1, 1, 3], tokenize(" {{\n funk \n}} ", true).map(&:line_number)
+    assert_equal [1],       tokenize_line_numbers("{{funk}}")
+    assert_equal [1, 1, 1], tokenize_line_numbers(" {{funk}} ")
+    assert_equal [1, 2, 2], tokenize_line_numbers("\n{{funk}}\n")
+    assert_equal [1, 1, 3], tokenize_line_numbers(" {{\n funk \n}} ")
   end
 
   private
 
-  def tokenize(source, line_numbers = false)
-    tokenizer = Liquid::Tokenizer.new(source, line_numbers)
+  def tokenize(source)
+    tokenizer = Liquid::Tokenizer.new(source)
     tokens = []
     while t = tokenizer.shift
       tokens << t
     end
     tokens
   end
+
+  def tokenize_line_numbers(source)
+    tokenizer = Liquid::Tokenizer.new(source, true)
+    line_numbers = []
+    loop do
+      line_number = tokenizer.line_number
+      if tokenizer.shift
+        line_numbers << line_number
+      else
+        break
+      end
+    end
+    line_numbers
+  end
 end

test/unit/variable_unit_test.rb

@@ -4,133 +4,133 @@ class VariableUnitTest < Minitest::Test
   include Liquid
 
   def test_variable
-    var = Variable.new('hello')
+    var = create_variable('hello')
     assert_equal VariableLookup.new('hello'), var.name
   end
 
   def test_filters
-    var = Variable.new('hello | textileze')
+    var = create_variable('hello | textileze')
     assert_equal VariableLookup.new('hello'), var.name
     assert_equal [['textileze', []]], var.filters
 
-    var = Variable.new('hello | textileze | paragraph')
+    var = create_variable('hello | textileze | paragraph')
     assert_equal VariableLookup.new('hello'), var.name
     assert_equal [['textileze', []], ['paragraph', []]], var.filters
 
-    var = Variable.new(%( hello | strftime: '%Y'))
+    var = create_variable(%( hello | strftime: '%Y'))
     assert_equal VariableLookup.new('hello'), var.name
     assert_equal [['strftime', ['%Y']]], var.filters
 
-    var = Variable.new(%( 'typo' | link_to: 'Typo', true ))
+    var = create_variable(%( 'typo' | link_to: 'Typo', true ))
     assert_equal 'typo', var.name
     assert_equal [['link_to', ['Typo', true]]], var.filters
 
-    var = Variable.new(%( 'typo' | link_to: 'Typo', false ))
+    var = create_variable(%( 'typo' | link_to: 'Typo', false ))
     assert_equal 'typo', var.name
     assert_equal [['link_to', ['Typo', false]]], var.filters
 
-    var = Variable.new(%( 'foo' | repeat: 3 ))
+    var = create_variable(%( 'foo' | repeat: 3 ))
     assert_equal 'foo', var.name
     assert_equal [['repeat', [3]]], var.filters
 
-    var = Variable.new(%( 'foo' | repeat: 3, 3 ))
+    var = create_variable(%( 'foo' | repeat: 3, 3 ))
     assert_equal 'foo', var.name
     assert_equal [['repeat', [3, 3]]], var.filters
 
-    var = Variable.new(%( 'foo' | repeat: 3, 3, 3 ))
+    var = create_variable(%( 'foo' | repeat: 3, 3, 3 ))
     assert_equal 'foo', var.name
     assert_equal [['repeat', [3, 3, 3]]], var.filters
 
-    var = Variable.new(%( hello | strftime: '%Y, okay?'))
+    var = create_variable(%( hello | strftime: '%Y, okay?'))
     assert_equal VariableLookup.new('hello'), var.name
     assert_equal [['strftime', ['%Y, okay?']]], var.filters
 
-    var = Variable.new(%( hello | things: "%Y, okay?", 'the other one'))
+    var = create_variable(%( hello | things: "%Y, okay?", 'the other one'))
     assert_equal VariableLookup.new('hello'), var.name
     assert_equal [['things', ['%Y, okay?', 'the other one']]], var.filters
   end
 
   def test_filter_with_date_parameter
-    var = Variable.new(%( '2006-06-06' | date: "%m/%d/%Y"))
+    var = create_variable(%( '2006-06-06' | date: "%m/%d/%Y"))
     assert_equal '2006-06-06', var.name
     assert_equal [['date', ['%m/%d/%Y']]], var.filters
   end
 
   def test_filters_without_whitespace
-    var = Variable.new('hello | textileze | paragraph')
+    var = create_variable('hello | textileze | paragraph')
     assert_equal VariableLookup.new('hello'), var.name
     assert_equal [['textileze', []], ['paragraph', []]], var.filters
 
-    var = Variable.new('hello|textileze|paragraph')
+    var = create_variable('hello|textileze|paragraph')
     assert_equal VariableLookup.new('hello'), var.name
     assert_equal [['textileze', []], ['paragraph', []]], var.filters
 
-    var = Variable.new("hello|replace:'foo','bar'|textileze")
+    var = create_variable("hello|replace:'foo','bar'|textileze")
     assert_equal VariableLookup.new('hello'), var.name
     assert_equal [['replace', ['foo', 'bar']], ['textileze', []]], var.filters
   end
 
   def test_symbol
-    var = Variable.new("http://disney.com/logo.gif | image: 'med' ", error_mode: :lax)
+    var = create_variable("http://disney.com/logo.gif | image: 'med' ", error_mode: :lax)
     assert_equal VariableLookup.new('http://disney.com/logo.gif'), var.name
     assert_equal [['image', ['med']]], var.filters
   end
 
   def test_string_to_filter
-    var = Variable.new("'http://disney.com/logo.gif' | image: 'med' ")
+    var = create_variable("'http://disney.com/logo.gif' | image: 'med' ")
     assert_equal 'http://disney.com/logo.gif', var.name
     assert_equal [['image', ['med']]], var.filters
   end
 
   def test_string_single_quoted
-    var = Variable.new(%( "hello" ))
+    var = create_variable(%( "hello" ))
     assert_equal 'hello', var.name
   end
 
   def test_string_double_quoted
-    var = Variable.new(%( 'hello' ))
+    var = create_variable(%( 'hello' ))
     assert_equal 'hello', var.name
   end
 
   def test_integer
-    var = Variable.new(%( 1000 ))
+    var = create_variable(%( 1000 ))
     assert_equal 1000, var.name
   end
 
   def test_float
-    var = Variable.new(%( 1000.01 ))
+    var = create_variable(%( 1000.01 ))
     assert_equal 1000.01, var.name
   end
 
   def test_dashes
-    assert_equal VariableLookup.new('foo-bar'), Variable.new('foo-bar').name
-    assert_equal VariableLookup.new('foo-bar-2'), Variable.new('foo-bar-2').name
+    assert_equal VariableLookup.new('foo-bar'), create_variable('foo-bar').name
+    assert_equal VariableLookup.new('foo-bar-2'), create_variable('foo-bar-2').name
 
     with_error_mode :strict do
-      assert_raises(Liquid::SyntaxError) { Variable.new('foo - bar') }
-      assert_raises(Liquid::SyntaxError) { Variable.new('-foo') }
-      assert_raises(Liquid::SyntaxError) { Variable.new('2foo') }
+      assert_raises(Liquid::SyntaxError) { create_variable('foo - bar') }
+      assert_raises(Liquid::SyntaxError) { create_variable('-foo') }
+      assert_raises(Liquid::SyntaxError) { create_variable('2foo') }
     end
   end
 
   def test_string_with_special_chars
-    var = Variable.new(%( 'hello! $!@.;"ddasd" ' ))
+    var = create_variable(%( 'hello! $!@.;"ddasd" ' ))
     assert_equal 'hello! $!@.;"ddasd" ', var.name
   end
 
   def test_string_dot
-    var = Variable.new(%( test.test ))
+    var = create_variable(%( test.test ))
     assert_equal VariableLookup.new('test.test'), var.name
   end
 
   def test_filter_with_keyword_arguments
-    var = Variable.new(%( hello | things: greeting: "world", farewell: 'goodbye'))
+    var = create_variable(%( hello | things: greeting: "world", farewell: 'goodbye'))
     assert_equal VariableLookup.new('hello'), var.name
     assert_equal [['things', [], { 'greeting' => 'world', 'farewell' => 'goodbye' }]], var.filters
   end
 
   def test_lax_filter_argument_parsing
-    var = Variable.new(%( number_of_comments | pluralize: 'comment': 'comments' ), error_mode: :lax)
+    var = create_variable(%( number_of_comments | pluralize: 'comment': 'comments' ), error_mode: :lax)
     assert_equal VariableLookup.new('number_of_comments'), var.name
     assert_equal [['pluralize', ['comment', 'comments']]], var.filters
   end
@@ -138,13 +138,13 @@ class VariableUnitTest < Minitest::Test
   def test_strict_filter_argument_parsing
     with_error_mode(:strict) do
       assert_raises(SyntaxError) do
-        Variable.new(%( number_of_comments | pluralize: 'comment': 'comments' ))
+        create_variable(%( number_of_comments | pluralize: 'comment': 'comments' ))
       end
     end
   end
 
   def test_output_raw_source_of_variable
-    var = Variable.new(%( name_of_variable | upcase ))
+    var = create_variable(%( name_of_variable | upcase ))
     assert_equal " name_of_variable | upcase ", var.raw
   end
 
@@ -153,4 +153,10 @@ class VariableUnitTest < Minitest::Test
     assert_equal 'a', lookup.name
     assert_equal ['b', 'c'], lookup.lookups
   end
+
+  private
+
+  def create_variable(markup, options = {})
+    Variable.new(markup, ParseContext.new(options))
+  end
 end