Merge pull request #614 from Shopify/remove-token-class

Implement line numbers without the Liquid::Token class.

lib/liquid.rb

@@ -69,7 +69,7 @@ require 'liquid/standardfilters'
 require 'liquid/condition'
 require 'liquid/utils'
 require 'liquid/tokenizer'
-require 'liquid/token'
+require 'liquid/parse_context'
 
 # Load all the tags of the standard library
 #

lib/liquid/block.rb

@@ -23,29 +23,17 @@ module Liquid
       @body.nodelist
     end
 
-    # warnings of this block and all sub-tags
-    def warnings
-      all_warnings = []
-      all_warnings.concat(@warnings) if @warnings
-
-      (nodelist || []).each do |node|
-        all_warnings.concat(node.warnings || []) if node.respond_to?(:warnings)
-      end
-
-      all_warnings
-    end
-
     def unknown_tag(tag, _params, _tokens)
       case tag
       when 'else'.freeze
-        raise SyntaxError.new(options[:locale].t("errors.syntax.unexpected_else".freeze,
+        raise SyntaxError.new(parse_context.locale.t("errors.syntax.unexpected_else".freeze,
           block_name: block_name))
       when 'end'.freeze
-        raise SyntaxError.new(options[:locale].t("errors.syntax.invalid_delimiter".freeze,
+        raise SyntaxError.new(parse_context.locale.t("errors.syntax.invalid_delimiter".freeze,
           block_name: block_name,
           block_delimiter: block_delimiter))
       else
-        raise SyntaxError.new(options[:locale].t("errors.syntax.unknown_tag".freeze, tag: tag))
+        raise SyntaxError.new(parse_context.locale.t("errors.syntax.unknown_tag".freeze, tag: tag))
       end
     end
 
@@ -60,12 +48,12 @@ module Liquid
     protected
 
     def parse_body(body, tokens)
-      body.parse(tokens, options) do |end_tag_name, end_tag_params|
+      body.parse(tokens, parse_context) do |end_tag_name, end_tag_params|
         @blank &&= body.blank?
 
         return false if end_tag_name == block_delimiter
         unless end_tag_name
-          raise SyntaxError.new(@options[:locale].t("errors.syntax.tag_never_closed".freeze, block_name: block_name))
+          raise SyntaxError.new(parse_context.locale.t("errors.syntax.tag_never_closed".freeze, block_name: block_name))
         end
 
         # this tag is not registered with the system

lib/liquid/block_body.rb

@@ -12,44 +12,37 @@ module Liquid
       @blank = true
     end
 
-    def parse(tokens, options)
+    def parse(tokenizer, parse_context)
-      while token = tokens.shift
+      parse_context.line_number = tokenizer.line_number
-        begin
+      while token = tokenizer.shift
-          unless token.empty?
+        unless token.empty?
-            case
+          case
-            when token.start_with?(TAGSTART)
+          when token.start_with?(TAGSTART)
-              if token =~ FullToken
+            if token =~ FullToken
-                tag_name = $1
+              tag_name = $1
-                markup = $2
+              markup = $2
-                # fetch the tag from registered blocks
+              # fetch the tag from registered blocks
-                if tag = registered_tags[tag_name]
+              if tag = registered_tags[tag_name]
-                  markup = token.child(markup) if token.is_a?(Token)
+                new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
-                  new_tag = tag.parse(tag_name, markup, tokens, options)
+                @blank &&= new_tag.blank?
-                  new_tag.line_number = token.line_number if token.is_a?(Token)
+                @nodelist << new_tag
-                  @blank &&= new_tag.blank?
-                  @nodelist << new_tag
-                else
-                  # end parsing if we reach an unknown tag and let the caller decide
-                  # determine how to proceed
-                  return yield tag_name, markup
-                end
               else
-                raise_missing_tag_terminator(token, options)
+                # end parsing if we reach an unknown tag and let the caller decide
+                # determine how to proceed
+                return yield tag_name, markup
               end
-            when token.start_with?(VARSTART)
-              new_var = create_variable(token, options)
-              new_var.line_number = token.line_number if token.is_a?(Token)
-              @nodelist << new_var
-              @blank = false
             else
-              @nodelist << token
+              raise_missing_tag_terminator(token, parse_context)
-              @blank &&= !!(token =~ /\A\s*\z/)
             end
+          when token.start_with?(VARSTART)
+            @nodelist << create_variable(token, parse_context)
+            @blank = false
+          else
+            @nodelist << token
+            @blank &&= !!(token =~ /\A\s*\z/)
           end
-        rescue SyntaxError => e
-          e.set_line_number_from_token(token)
-          raise
         end
+        parse_context.line_number = tokenizer.line_number
       end
 
       yield nil, nil
@@ -59,14 +52,6 @@ module Liquid
       @blank
     end
 
-    def warnings
-      all_warnings = []
-      nodelist.each do |node|
-        all_warnings.concat(node.warnings || []) if node.respond_to?(:warnings)
-      end
-      all_warnings
-    end
-
     def render(context)
       output = []
       context.resource_limits.render_score += @nodelist.length
@@ -92,7 +77,7 @@ module Liquid
         rescue MemoryError => e
           raise e
         rescue ::StandardError => e
-          output << context.handle_error(e, token)
+          output << context.handle_error(e, token.line_number)
         end
       end
 
@@ -112,20 +97,20 @@ module Liquid
       node_output
     end
 
-    def create_variable(token, options)
+    def create_variable(token, parse_context)
       token.scan(ContentOfVariable) do |content|
-        markup = token.is_a?(Token) ? token.child(content.first) : content.first
+        markup = content.first
-        return Variable.new(markup, options)
+        return Variable.new(markup, parse_context)
       end
-      raise_missing_variable_terminator(token, options)
+      raise_missing_variable_terminator(token, parse_context)
     end
 
-    def raise_missing_tag_terminator(token, options)
+    def raise_missing_tag_terminator(token, parse_context)
-      raise SyntaxError.new(options[:locale].t("errors.syntax.tag_termination".freeze, token: token, tag_end: TagEnd.inspect))
+      raise SyntaxError.new(parse_context.locale.t("errors.syntax.tag_termination".freeze, token: token, tag_end: TagEnd.inspect))
     end
 
-    def raise_missing_variable_terminator(token, options)
+    def raise_missing_variable_terminator(token, parse_context)
-      raise SyntaxError.new(options[:locale].t("errors.syntax.variable_termination".freeze, token: token, tag_end: VariableEnd.inspect))
+      raise SyntaxError.new(parse_context.locale.t("errors.syntax.variable_termination".freeze, token: token, tag_end: VariableEnd.inspect))
     end
 
     def registered_tags

lib/liquid/context.rb

@@ -13,13 +13,14 @@ module Liquid
   #   context['bob']  #=> nil  class Context
   class Context
     attr_reader :scopes, :errors, :registers, :environments, :resource_limits
-    attr_accessor :exception_handler, :template_name
+    attr_accessor :exception_handler, :template_name, :partial
 
     def initialize(environments = {}, outer_scope = {}, registers = {}, rethrow_errors = false, resource_limits = nil)
       @environments     = [environments].flatten
       @scopes           = [(outer_scope || {})]
       @registers        = registers
       @errors           = []
+      @partial          = false
       @resource_limits  = resource_limits || ResourceLimits.new(Template.default_resource_limits)
       squash_instance_assigns_with_environments
 
@@ -66,10 +67,10 @@ module Liquid
       @interrupts.pop
     end
 
-    def handle_error(e, token = nil)
+    def handle_error(e, line_number = nil)
       if e.is_a?(Liquid::Error)
-        e.template_name = template_name
+        e.template_name ||= template_name
-        e.set_line_number_from_token(token)
+        e.line_number ||= line_number
       end
 
       output = nil
@@ -79,7 +80,10 @@ module Liquid
         case result
         when Exception
           e = result
-          e.set_line_number_from_token(token) if e.is_a?(Liquid::Error)
+          if e.is_a?(Liquid::Error)
+            e.template_name ||= template_name
+            e.line_number ||= line_number
+          end
         when String
           output = result
         else

lib/liquid/document.rb

@@ -1,12 +1,8 @@
 module Liquid
   class Document < BlockBody
-    DEFAULT_OPTIONS = {
-      locale: I18n.new
-    }
-
     def self.parse(tokens, options)
       doc = new
-      doc.parse(tokens, DEFAULT_OPTIONS.merge(options))
+      doc.parse(tokens, options)
       doc
     end
 

lib/liquid/errors.rb

@@ -17,12 +17,6 @@ module Liquid
       str
     end
 
-    def set_line_number_from_token(token)
-      return unless token.respond_to?(:line_number)
-      return if line_number
-      self.line_number = token.line_number
-    end
-
     def self.render(e)
       if e.is_a?(Liquid::Error)
         e.to_s

lib/liquid/parse_context.rb

@@ -0,0 +1,37 @@
+module Liquid
+  class ParseContext
+    attr_accessor :partial, :locale, :line_number
+    attr_reader :warnings, :error_mode
+
+    def initialize(options = {})
+      @template_options = options ? options.dup : {}
+      @locale = @template_options[:locale] ||= I18n.new
+      @warnings = []
+      self.partial = false
+    end
+
+    def [](option_key)
+      @options[option_key]
+    end
+
+    def partial=(value)
+      @partial = value
+      @options = value ? partial_options : @template_options
+      @error_mode = @options[:error_mode] || Template.error_mode
+      value
+    end
+
+    def partial_options
+      @partial_options ||= begin
+        dont_pass = @template_options[:include_options_blacklist]
+        if dont_pass == true
+          { locale: locale }
+        elsif dont_pass.is_a?(Array)
+          @template_options.reject { |k, v| dont_pass.include?(k) }
+        else
+          @template_options
+        end
+      end
+    end
+  end
+end

lib/liquid/parser_switching.rb

@@ -1,16 +1,14 @@
 module Liquid
   module ParserSwitching
     def parse_with_selected_parser(markup)
-      case @options[:error_mode] || Template.error_mode
+      case parse_context.error_mode
       when :strict then strict_parse_with_error_context(markup)
       when :lax    then lax_parse(markup)
       when :warn
         begin
           return strict_parse_with_error_context(markup)
         rescue SyntaxError => e
-          e.set_line_number_from_token(markup)
+          parse_context.warnings << e
-          @warnings ||= []
-          @warnings << e
           return lax_parse(markup)
         end
       end
@@ -21,6 +19,7 @@ module Liquid
     def strict_parse_with_error_context(markup)
       strict_parse(markup)
     rescue SyntaxError => e
+      e.line_number = line_number
       e.markup_context = markup_context(markup)
       raise e
     end

lib/liquid/tag.rb

@@ -1,23 +1,24 @@
 module Liquid
   class Tag
-    attr_accessor :options, :line_number
+    attr_reader :nodelist, :tag_name, :line_number, :options
-    attr_reader :nodelist, :warnings, :tag_name
+    alias_method :parse_context, :options
     include ParserSwitching
 
     class << self
-      def parse(tag_name, markup, tokens, options)
+      def parse(tag_name, markup, tokenizer, options)
         tag = new(tag_name, markup, options)
-        tag.parse(tokens)
+        tag.parse(tokenizer)
         tag
       end
 
       private :new
     end
 
-    def initialize(tag_name, markup, options)
+    def initialize(tag_name, markup, parse_context)
       @tag_name   = tag_name
       @markup     = markup
-      @options    = options
+      @options    = parse_context
+      @line_number = parse_context.line_number
     end
 
     def parse(_tokens)

lib/liquid/tags/assign.rb

@@ -15,7 +15,6 @@ module Liquid
       if markup =~ Syntax
         @to = $1
         @from = Variable.new($2, options)
-        @from.line_number = line_number
       else
         raise SyntaxError.new options[:locale].t("errors.syntax.assign".freeze)
       end

lib/liquid/tags/include.rb

@@ -53,8 +53,10 @@ module Liquid
       end
 
       old_template_name = context.template_name
+      old_partial = context.partial
       begin
         context.template_name = template_name
+        context.partial = true
         context.stack do
           @attributes.each do |key, value|
             context[key] = context.evaluate(value)
@@ -72,11 +74,15 @@ module Liquid
         end
       ensure
         context.template_name = old_template_name
+        context.partial = old_partial
       end
     end
 
     private
 
+    alias_method :parse_context, :options
+    private :parse_context
+
     def load_cached_partial(template_name, context)
       cached_partials = context.registers[:cached_partials] || {}
 
@@ -84,7 +90,12 @@ module Liquid
         return cached
       end
       source = read_template_from_file_system(context)
-      partial = Liquid::Template.parse(source, pass_options)
+      begin
+        parse_context.partial = true
+        partial = Liquid::Template.parse(source, parse_context)
+      ensure
+        parse_context.partial = false
+      end
       cached_partials[template_name] = partial
       context.registers[:cached_partials] = cached_partials
       partial
@@ -95,16 +106,6 @@ module Liquid
 
       file_system.read_template_file(context.evaluate(@template_name_expr))
     end
-
-    def pass_options
-      dont_pass = @options[:include_options_blacklist]
-      return { locale: @options[:locale] } if dont_pass == true
-      opts = @options.merge(included: true, include_options_blacklist: false)
-      if dont_pass.is_a?(Array)
-        dont_pass.each { |o| opts.delete(o) }
-      end
-      opts
-    end
   end
 
   Template.register_tag('include'.freeze, Include)

lib/liquid/tags/raw.rb

@@ -3,11 +3,11 @@ module Liquid
     Syntax = /\A\s*\z/
     FullTokenPossiblyInvalid = /\A(.*)#{TagStart}\s*(\w+)\s*(.*)?#{TagEnd}\z/om
 
-    def initialize(tag_name, markup, options)
+    def initialize(tag_name, markup, parse_context)
       super
 
       unless markup =~ Syntax
-        raise SyntaxError.new(@options[:locale].t("errors.syntax.tag_unexpected_args".freeze, tag: tag_name))
+        raise SyntaxError.new(parse_context.locale.t("errors.syntax.tag_unexpected_args".freeze, tag: tag_name))
       end
     end
 
@@ -21,7 +21,7 @@ module Liquid
         @body << token unless token.empty?
       end
 
-      raise SyntaxError.new(@options[:locale].t("errors.syntax.tag_never_closed".freeze, block_name: block_name))
+      raise SyntaxError.new(parse_context.locale.t("errors.syntax.tag_never_closed".freeze, block_name: block_name))
     end
 
     def render(_context)

lib/liquid/template.rb

@@ -14,7 +14,7 @@ module Liquid
   #
   class Template
     attr_accessor :root
-    attr_reader :resource_limits
+    attr_reader :resource_limits, :warnings
 
     @@file_system = BlankFileSystem.new
 
@@ -116,14 +116,13 @@ module Liquid
       @options = options
       @profiling = options[:profile]
       @line_numbers = options[:line_numbers] || @profiling
-      @root = Document.parse(tokenize(source), options)
+      parse_context = options.is_a?(ParseContext) ? options : ParseContext.new(options)
-      @warnings = nil
+      @root = Document.parse(tokenize(source), parse_context)
+      @warnings = parse_context.warnings
       self
-    end
+    rescue SyntaxError => e
-
+      e.line_number ||= parse_context.line_number
-    def warnings
+      raise
-      return [] unless @root
-      @warnings ||= @root.warnings
     end
 
     def registers
@@ -206,7 +205,7 @@ module Liquid
       begin
         # render the nodelist.
         # for performance reasons we get an array back here. join will make a string out of it.
-        result = with_profiling do
+        result = with_profiling(context) do
           @root.render(context)
         end
         result.respond_to?(:join) ? result.join : result
@@ -228,8 +227,8 @@ module Liquid
       Tokenizer.new(source, @line_numbers)
     end
 
-    def with_profiling
+    def with_profiling(context)
-      if @profiling && !@options[:included]
+      if @profiling && !context.partial
         raise "Profiler not loaded, require 'liquid/profiler' first" unless defined?(Liquid::Profiler)
 
         @profiler = Profiler.new

lib/liquid/token.rb

@@ -1,18 +0,0 @@
-module Liquid
-  class Token < String
-    attr_reader :line_number
-
-    def initialize(content, line_number)
-      super(content)
-      @line_number = line_number
-    end
-
-    def raw
-      "<raw>"
-    end
-
-    def child(string)
-      Token.new(string, @line_number)
-    end
-  end
-end

lib/liquid/tokenizer.rb

@@ -1,13 +1,17 @@
 module Liquid
   class Tokenizer
+    attr_reader :line_number
+
     def initialize(source, line_numbers = false)
       @source = source
-      @line_numbers = line_numbers
+      @line_number = 1 if line_numbers
       @tokens = tokenize
     end
 
     def shift
-      @tokens.shift
+      token = @tokens.shift
+      @line_number += token.count("\n") if @line_number && token
+      token
     end
 
     private
@@ -17,21 +21,11 @@ module Liquid
       return [] if @source.to_s.empty?
 
       tokens = @source.split(TemplateParser)
-      tokens = @line_numbers ? calculate_line_numbers(tokens) : tokens
 
       # removes the rogue empty element at the beginning of the array
       tokens.shift if tokens[0] && tokens[0].empty?
 
       tokens
     end
-
-    def calculate_line_numbers(tokens)
-      current_line = 1
-      tokens.map do |token|
-        Token.new(token, current_line).tap do
-          current_line += token.count("\n")
-        end
-      end
-    end
   end
 end

lib/liquid/variable.rb

@@ -11,14 +11,16 @@ module Liquid
   #
   class Variable
     FilterParser = /(?:\s+|#{QuotedFragment}|#{ArgumentSeparator})+/o
-    attr_accessor :filters, :name, :warnings
+    attr_accessor :filters, :name, :line_number
-    attr_accessor :line_number
+    attr_reader :options
+    alias_method :parse_context, :options
     include ParserSwitching
 
-    def initialize(markup, options = {})
+    def initialize(markup, parse_context)
       @markup  = markup
       @name    = nil
-      @options = options || {}
+      @options = parse_context
+      @line_number = parse_context.line_number
 
       parse_with_selected_parser(markup)
     end

test/integration/filter_test.rb

@@ -39,13 +39,13 @@ class FiltersTest < Minitest::Test
     @context['var'] = 1000
     @context.add_filters(MoneyFilter)
 
-    assert_equal ' 1000$ ', Variable.new("var | money").render(@context)
+    assert_equal ' 1000$ ', Template.parse("{{var | money}}").render(@context)
   end
 
   def test_underscore_in_filter_name
     @context['var'] = 1000
     @context.add_filters(MoneyFilter)
-    assert_equal ' 1000$ ', Variable.new("var | money_with_underscore").render(@context)
+    assert_equal ' 1000$ ', Template.parse("{{var | money_with_underscore}}").render(@context)
   end
 
   def test_second_filter_overwrites_first
@@ -53,20 +53,20 @@ class FiltersTest < Minitest::Test
     @context.add_filters(MoneyFilter)
     @context.add_filters(CanadianMoneyFilter)
 
-    assert_equal ' 1000$ CAD ', Variable.new("var | money").render(@context)
+    assert_equal ' 1000$ CAD ', Template.parse("{{var | money}}").render(@context)
   end
 
   def test_size
     @context['var'] = 'abcd'
     @context.add_filters(MoneyFilter)
 
-    assert_equal 4, Variable.new("var | size").render(@context)
+    assert_equal '4', Template.parse("{{var | size}}").render(@context)
   end
 
   def test_join
     @context['var'] = [1, 2, 3, 4]
 
-    assert_equal "1 2 3 4", Variable.new("var | join").render(@context)
+    assert_equal "1 2 3 4", Template.parse("{{var | join}}").render(@context)
   end
 
   def test_sort
@@ -76,11 +76,11 @@ class FiltersTest < Minitest::Test
     @context['arrays'] = ['flower', 'are']
     @context['case_sensitive'] = ['sensitive', 'Expected', 'case']
 
-    assert_equal [1, 2, 3, 4], Variable.new("numbers | sort").render(@context)
+    assert_equal '1 2 3 4', Template.parse("{{numbers | sort | join}}").render(@context)
-    assert_equal ['alphabetic', 'as', 'expected'], Variable.new("words | sort").render(@context)
+    assert_equal 'alphabetic as expected', Template.parse("{{words | sort | join}}").render(@context)
-    assert_equal [3], Variable.new("value | sort").render(@context)
+    assert_equal '3', Template.parse("{{value | sort}}").render(@context)
-    assert_equal ['are', 'flower'], Variable.new("arrays | sort").render(@context)
+    assert_equal 'are flower', Template.parse("{{arrays | sort | join}}").render(@context)
-    assert_equal ['Expected', 'case', 'sensitive'], Variable.new("case_sensitive | sort").render(@context)
+    assert_equal 'Expected case sensitive', Template.parse("{{case_sensitive | sort | join}}").render(@context)
   end
 
   def test_sort_natural
@@ -89,19 +89,13 @@ class FiltersTest < Minitest::Test
     @context['objects'] = [TestObject.new('A'), TestObject.new('b'), TestObject.new('C')]
 
     # Test strings
-    assert_equal ['Assert', 'case', 'Insensitive'], Variable.new("words | sort_natural").render(@context)
+    assert_equal 'Assert case Insensitive', Template.parse("{{words | sort_natural | join}}").render(@context)
 
     # Test hashes
-    sorted = Variable.new("hashes | sort_natural: 'a'").render(@context)
+    assert_equal 'A b C', Template.parse("{{hashes | sort_natural: 'a' | map: 'a' | join}}").render(@context)
-    assert_equal sorted[0]['a'], 'A'
-    assert_equal sorted[1]['a'], 'b'
-    assert_equal sorted[2]['a'], 'C'
 
     # Test objects
-    sorted = Variable.new("objects | sort_natural: 'a'").render(@context)
+    assert_equal 'A b C', Template.parse("{{objects | sort_natural: 'a' | map: 'a' | join}}").render(@context)
-    assert_equal sorted[0].a, 'A'
-    assert_equal sorted[1].a, 'b'
-    assert_equal sorted[2].a, 'C'
   end
 
   def test_compact
@@ -110,49 +104,44 @@ class FiltersTest < Minitest::Test
     @context['objects'] = [TestObject.new('A'), TestObject.new(nil), TestObject.new('C')]
 
     # Test strings
-    assert_equal ['a', 'b', 'c'], Variable.new("words | compact").render(@context)
+    assert_equal 'a b c', Template.parse("{{words | compact | join}}").render(@context)
 
     # Test hashes
-    sorted = Variable.new("hashes | compact: 'a'").render(@context)
+    assert_equal 'A C', Template.parse("{{hashes | compact: 'a' | map: 'a' | join}}").render(@context)
-    assert_equal sorted[0]['a'], 'A'
-    assert_equal sorted[1]['a'], 'C'
-    assert_nil sorted[2]
 
     # Test objects
-    sorted = Variable.new("objects | compact: 'a'").render(@context)
+    assert_equal 'A C', Template.parse("{{objects | compact: 'a' | map: 'a' | join}}").render(@context)
-    assert_equal sorted[0].a, 'A'
-    assert_equal sorted[1].a, 'C'
-    assert_nil sorted[2]
   end
 
   def test_strip_html
     @context['var'] = "<b>bla blub</a>"
 
-    assert_equal "bla blub", Variable.new("var | strip_html").render(@context)
+    assert_equal "bla blub", Template.parse("{{ var | strip_html }}").render(@context)
   end
 
   def test_strip_html_ignore_comments_with_html
     @context['var'] = "<!-- split and some <ul> tag --><b>bla blub</a>"
 
-    assert_equal "bla blub", Variable.new("var | strip_html").render(@context)
+    assert_equal "bla blub", Template.parse("{{ var | strip_html }}").render(@context)
   end
 
   def test_capitalize
     @context['var'] = "blub"
 
-    assert_equal "Blub", Variable.new("var | capitalize").render(@context)
+    assert_equal "Blub", Template.parse("{{ var | capitalize }}").render(@context)
   end
 
   def test_nonexistent_filter_is_ignored
     @context['var'] = 1000
 
-    assert_equal 1000, Variable.new("var | xyzzy").render(@context)
+    assert_equal '1000', Template.parse("{{ var | xyzzy }}").render(@context)
   end
 
   def test_filter_with_keyword_arguments
     @context['surname'] = 'john'
+    @context['input'] = 'hello %{first_name}, %{last_name}'
     @context.add_filters(SubstituteFilter)
-    output = Variable.new(%( 'hello %{first_name}, %{last_name}' | substitute: first_name: surname, last_name: 'doe' )).render(@context)
+    output = Template.parse(%({{ input | substitute: first_name: surname, last_name: 'doe' }})).render(@context)
     assert_equal 'hello john, doe', output
   end
 
@@ -181,7 +170,7 @@ class FiltersInTemplate < Minitest::Test
   end
 end # FiltersTest
 
-class TestObject
+class TestObject < Liquid::Drop
   attr_accessor :a
   def initialize(a)
     @a = a

test/unit/tag_unit_test.rb

@@ -4,18 +4,18 @@ class TagUnitTest < Minitest::Test
   include Liquid
 
   def test_tag
-    tag = Tag.parse('tag', [], [], {})
+    tag = Tag.parse('tag', "", Tokenizer.new(""), ParseContext.new)
     assert_equal 'liquid::tag', tag.name
     assert_equal '', tag.render(Context.new)
   end
 
   def test_return_raw_text_of_tag
-    tag = Tag.parse("long_tag", "param1, param2, param3", [], {})
+    tag = Tag.parse("long_tag", "param1, param2, param3", Tokenizer.new(""), ParseContext.new)
     assert_equal("long_tag param1, param2, param3", tag.raw)
   end
 
   def test_tag_name_should_return_name_of_the_tag
-    tag = Tag.parse("some_tag", [], [], {})
+    tag = Tag.parse("some_tag", "", Tokenizer.new(""), ParseContext.new)
     assert_equal 'some_tag', tag.tag_name
   end
 end

test/unit/tokenizer_unit_test.rb

@@ -22,20 +22,34 @@ class TokenizerTest < Minitest::Test
   end
 
   def test_calculate_line_numbers_per_token_with_profiling
-    assert_equal [1],       tokenize("{{funk}}", true).map(&:line_number)
+    assert_equal [1],       tokenize_line_numbers("{{funk}}")
-    assert_equal [1, 1, 1], tokenize(" {{funk}} ", true).map(&:line_number)
+    assert_equal [1, 1, 1], tokenize_line_numbers(" {{funk}} ")
-    assert_equal [1, 2, 2], tokenize("\n{{funk}}\n", true).map(&:line_number)
+    assert_equal [1, 2, 2], tokenize_line_numbers("\n{{funk}}\n")
-    assert_equal [1, 1, 3], tokenize(" {{\n funk \n}} ", true).map(&:line_number)
+    assert_equal [1, 1, 3], tokenize_line_numbers(" {{\n funk \n}} ")
   end
 
   private
 
-  def tokenize(source, line_numbers = false)
+  def tokenize(source)
-    tokenizer = Liquid::Tokenizer.new(source, line_numbers)
+    tokenizer = Liquid::Tokenizer.new(source)
     tokens = []
     while t = tokenizer.shift
       tokens << t
     end
     tokens
   end
+
+  def tokenize_line_numbers(source)
+    tokenizer = Liquid::Tokenizer.new(source, true)
+    line_numbers = []
+    loop do
+      line_number = tokenizer.line_number
+      if tokenizer.shift
+        line_numbers << line_number
+      else
+        break
+      end
+    end
+    line_numbers
+  end
 end

test/unit/variable_unit_test.rb

@@ -4,133 +4,133 @@ class VariableUnitTest < Minitest::Test
   include Liquid
 
   def test_variable
-    var = Variable.new('hello')
+    var = create_variable('hello')
     assert_equal VariableLookup.new('hello'), var.name
   end
 
   def test_filters
-    var = Variable.new('hello | textileze')
+    var = create_variable('hello | textileze')
     assert_equal VariableLookup.new('hello'), var.name
     assert_equal [['textileze', []]], var.filters
 
-    var = Variable.new('hello | textileze | paragraph')
+    var = create_variable('hello | textileze | paragraph')
     assert_equal VariableLookup.new('hello'), var.name
     assert_equal [['textileze', []], ['paragraph', []]], var.filters
 
-    var = Variable.new(%( hello | strftime: '%Y'))
+    var = create_variable(%( hello | strftime: '%Y'))
     assert_equal VariableLookup.new('hello'), var.name
     assert_equal [['strftime', ['%Y']]], var.filters
 
-    var = Variable.new(%( 'typo' | link_to: 'Typo', true ))
+    var = create_variable(%( 'typo' | link_to: 'Typo', true ))
     assert_equal 'typo', var.name
     assert_equal [['link_to', ['Typo', true]]], var.filters
 
-    var = Variable.new(%( 'typo' | link_to: 'Typo', false ))
+    var = create_variable(%( 'typo' | link_to: 'Typo', false ))
     assert_equal 'typo', var.name
     assert_equal [['link_to', ['Typo', false]]], var.filters
 
-    var = Variable.new(%( 'foo' | repeat: 3 ))
+    var = create_variable(%( 'foo' | repeat: 3 ))
     assert_equal 'foo', var.name
     assert_equal [['repeat', [3]]], var.filters
 
-    var = Variable.new(%( 'foo' | repeat: 3, 3 ))
+    var = create_variable(%( 'foo' | repeat: 3, 3 ))
     assert_equal 'foo', var.name
     assert_equal [['repeat', [3, 3]]], var.filters
 
-    var = Variable.new(%( 'foo' | repeat: 3, 3, 3 ))
+    var = create_variable(%( 'foo' | repeat: 3, 3, 3 ))
     assert_equal 'foo', var.name
     assert_equal [['repeat', [3, 3, 3]]], var.filters
 
-    var = Variable.new(%( hello | strftime: '%Y, okay?'))
+    var = create_variable(%( hello | strftime: '%Y, okay?'))
     assert_equal VariableLookup.new('hello'), var.name
     assert_equal [['strftime', ['%Y, okay?']]], var.filters
 
-    var = Variable.new(%( hello | things: "%Y, okay?", 'the other one'))
+    var = create_variable(%( hello | things: "%Y, okay?", 'the other one'))
     assert_equal VariableLookup.new('hello'), var.name
     assert_equal [['things', ['%Y, okay?', 'the other one']]], var.filters
   end
 
   def test_filter_with_date_parameter
-    var = Variable.new(%( '2006-06-06' | date: "%m/%d/%Y"))
+    var = create_variable(%( '2006-06-06' | date: "%m/%d/%Y"))
     assert_equal '2006-06-06', var.name
     assert_equal [['date', ['%m/%d/%Y']]], var.filters
   end
 
   def test_filters_without_whitespace
-    var = Variable.new('hello | textileze | paragraph')
+    var = create_variable('hello | textileze | paragraph')
     assert_equal VariableLookup.new('hello'), var.name
     assert_equal [['textileze', []], ['paragraph', []]], var.filters
 
-    var = Variable.new('hello|textileze|paragraph')
+    var = create_variable('hello|textileze|paragraph')
     assert_equal VariableLookup.new('hello'), var.name
     assert_equal [['textileze', []], ['paragraph', []]], var.filters
 
-    var = Variable.new("hello|replace:'foo','bar'|textileze")
+    var = create_variable("hello|replace:'foo','bar'|textileze")
     assert_equal VariableLookup.new('hello'), var.name
     assert_equal [['replace', ['foo', 'bar']], ['textileze', []]], var.filters
   end
 
   def test_symbol
-    var = Variable.new("http://disney.com/logo.gif | image: 'med' ", error_mode: :lax)
+    var = create_variable("http://disney.com/logo.gif | image: 'med' ", error_mode: :lax)
     assert_equal VariableLookup.new('http://disney.com/logo.gif'), var.name
     assert_equal [['image', ['med']]], var.filters
   end
 
   def test_string_to_filter
-    var = Variable.new("'http://disney.com/logo.gif' | image: 'med' ")
+    var = create_variable("'http://disney.com/logo.gif' | image: 'med' ")
     assert_equal 'http://disney.com/logo.gif', var.name
     assert_equal [['image', ['med']]], var.filters
   end
 
   def test_string_single_quoted
-    var = Variable.new(%( "hello" ))
+    var = create_variable(%( "hello" ))
     assert_equal 'hello', var.name
   end
 
   def test_string_double_quoted
-    var = Variable.new(%( 'hello' ))
+    var = create_variable(%( 'hello' ))
     assert_equal 'hello', var.name
   end
 
   def test_integer
-    var = Variable.new(%( 1000 ))
+    var = create_variable(%( 1000 ))
     assert_equal 1000, var.name
   end
 
   def test_float
-    var = Variable.new(%( 1000.01 ))
+    var = create_variable(%( 1000.01 ))
     assert_equal 1000.01, var.name
   end
 
   def test_dashes
-    assert_equal VariableLookup.new('foo-bar'), Variable.new('foo-bar').name
+    assert_equal VariableLookup.new('foo-bar'), create_variable('foo-bar').name
-    assert_equal VariableLookup.new('foo-bar-2'), Variable.new('foo-bar-2').name
+    assert_equal VariableLookup.new('foo-bar-2'), create_variable('foo-bar-2').name
 
     with_error_mode :strict do
-      assert_raises(Liquid::SyntaxError) { Variable.new('foo - bar') }
+      assert_raises(Liquid::SyntaxError) { create_variable('foo - bar') }
-      assert_raises(Liquid::SyntaxError) { Variable.new('-foo') }
+      assert_raises(Liquid::SyntaxError) { create_variable('-foo') }
-      assert_raises(Liquid::SyntaxError) { Variable.new('2foo') }
+      assert_raises(Liquid::SyntaxError) { create_variable('2foo') }
     end
   end
 
   def test_string_with_special_chars
-    var = Variable.new(%( 'hello! $!@.;"ddasd" ' ))
+    var = create_variable(%( 'hello! $!@.;"ddasd" ' ))
     assert_equal 'hello! $!@.;"ddasd" ', var.name
   end
 
   def test_string_dot
-    var = Variable.new(%( test.test ))
+    var = create_variable(%( test.test ))
     assert_equal VariableLookup.new('test.test'), var.name
   end
 
   def test_filter_with_keyword_arguments
-    var = Variable.new(%( hello | things: greeting: "world", farewell: 'goodbye'))
+    var = create_variable(%( hello | things: greeting: "world", farewell: 'goodbye'))
     assert_equal VariableLookup.new('hello'), var.name
     assert_equal [['things', [], { 'greeting' => 'world', 'farewell' => 'goodbye' }]], var.filters
   end
 
   def test_lax_filter_argument_parsing
-    var = Variable.new(%( number_of_comments | pluralize: 'comment': 'comments' ), error_mode: :lax)
+    var = create_variable(%( number_of_comments | pluralize: 'comment': 'comments' ), error_mode: :lax)
     assert_equal VariableLookup.new('number_of_comments'), var.name
     assert_equal [['pluralize', ['comment', 'comments']]], var.filters
   end
@@ -138,13 +138,13 @@ class VariableUnitTest < Minitest::Test
   def test_strict_filter_argument_parsing
     with_error_mode(:strict) do
       assert_raises(SyntaxError) do
-        Variable.new(%( number_of_comments | pluralize: 'comment': 'comments' ))
+        create_variable(%( number_of_comments | pluralize: 'comment': 'comments' ))
       end
     end
   end
 
   def test_output_raw_source_of_variable
-    var = Variable.new(%( name_of_variable | upcase ))
+    var = create_variable(%( name_of_variable | upcase ))
     assert_equal " name_of_variable | upcase ", var.raw
   end
 
@@ -153,4 +153,10 @@ class VariableUnitTest < Minitest::Test
     assert_equal 'a', lookup.name
     assert_equal ['b', 'c'], lookup.lookups
   end
+
+  private
+
+  def create_variable(markup, options = {})
+    Variable.new(markup, ParseContext.new(options))
+  end
 end