diff --git a/lib/liquid.rb b/lib/liquid.rb index 9724aac9..14afa757 100644 --- a/lib/liquid.rb +++ b/lib/liquid.rb @@ -68,6 +68,7 @@ require 'liquid/template' require 'liquid/standardfilters' require 'liquid/condition' require 'liquid/utils' +require 'liquid/tokenizer' require 'liquid/token' # Load all the tags of the standard library diff --git a/lib/liquid/block_body.rb b/lib/liquid/block_body.rb index 66eaffa1..be5b65e3 100644 --- a/lib/liquid/block_body.rb +++ b/lib/liquid/block_body.rb @@ -22,7 +22,7 @@ module Liquid tag_name = $1 markup = $2 # fetch the tag from registered blocks - if tag = Template.tags[tag_name] + if tag = registered_tags[tag_name] markup = token.child(markup) if token.is_a?(Token) new_tag = tag.parse(tag_name, markup, tokens, options) new_tag.line_number = token.line_number if token.is_a?(Token) @@ -127,5 +127,9 @@ module Liquid def raise_missing_variable_terminator(token, options) raise SyntaxError.new(options[:locale].t("errors.syntax.variable_termination".freeze, token: token, tag_end: VariableEnd.inspect)) end + + def registered_tags + Template.tags + end end end diff --git a/lib/liquid/document.rb b/lib/liquid/document.rb index 7ecced6e..c964ae84 100644 --- a/lib/liquid/document.rb +++ b/lib/liquid/document.rb @@ -1,8 +1,12 @@ module Liquid class Document < BlockBody + DEFAULT_OPTIONS = { + locale: I18n.new + } + def self.parse(tokens, options) doc = new - doc.parse(tokens, options) + doc.parse(tokens, DEFAULT_OPTIONS.merge(options)) doc end diff --git a/lib/liquid/template.rb b/lib/liquid/template.rb index a6e36b1b..7b2897d9 100644 --- a/lib/liquid/template.rb +++ b/lib/liquid/template.rb @@ -13,10 +13,6 @@ module Liquid # template.render('user_name' => 'bob') # class Template - DEFAULT_OPTIONS = { - locale: I18n.new - } - attr_accessor :root attr_reader :resource_limits @@ -120,7 +116,7 @@ module Liquid @options = options @profiling = options[:profile] @line_numbers = options[:line_numbers] || @profiling - @root = Document.parse(tokenize(source), DEFAULT_OPTIONS.merge(options)) + @root = Document.parse(tokenize(source), options) @warnings = nil self end @@ -228,28 +224,8 @@ module Liquid private - # Uses the Liquid::TemplateParser regexp to tokenize the passed source def tokenize(source) - source = source.source if source.respond_to?(:source) - return [] if source.to_s.empty? - - tokens = calculate_line_numbers(source.split(TemplateParser)) - - # removes the rogue empty element at the beginning of the array - tokens.shift if tokens[0] && tokens[0].empty? - - tokens - end - - def calculate_line_numbers(raw_tokens) - return raw_tokens unless @line_numbers - - current_line = 1 - raw_tokens.map do |token| - Token.new(token, current_line).tap do - current_line += token.count("\n") - end - end + Tokenizer.new(source, @line_numbers) end def with_profiling diff --git a/lib/liquid/tokenizer.rb b/lib/liquid/tokenizer.rb new file mode 100644 index 00000000..51347ed3 --- /dev/null +++ b/lib/liquid/tokenizer.rb @@ -0,0 +1,36 @@ +module Liquid + class Tokenizer + def initialize(source, line_numbers = false) + @source, @line_numbers = source, line_numbers + @tokens = tokenize + end + + def shift + @tokens.shift + end + + private + + def tokenize + @source = @source.source if @source.respond_to?(:source) + return [] if @source.to_s.empty? + + tokens = @source.split(TemplateParser) + tokens = @line_numbers ? calculate_line_numbers(tokens) : tokens + + # removes the rogue empty element at the beginning of the array + tokens.shift if tokens[0] && tokens[0].empty? + + tokens + end + + def calculate_line_numbers(tokens) + current_line = 1 + tokens.map do |token| + Token.new(token, current_line).tap do + current_line += token.count("\n") + end + end + end + end +end diff --git a/test/unit/tokenizer_unit_test.rb b/test/unit/tokenizer_unit_test.rb index f0ae7c99..e9cabfa0 100644 --- a/test/unit/tokenizer_unit_test.rb +++ b/test/unit/tokenizer_unit_test.rb @@ -22,17 +22,20 @@ class TokenizerTest < Minitest::Test end def test_calculate_line_numbers_per_token_with_profiling - template = Liquid::Template.parse("", :profile => true) - - assert_equal [1], template.send(:tokenize, "{{funk}}").map(&:line_number) - assert_equal [1, 1, 1], template.send(:tokenize, " {{funk}} ").map(&:line_number) - assert_equal [1, 2, 2], template.send(:tokenize, "\n{{funk}}\n").map(&:line_number) - assert_equal [1, 1, 3], template.send(:tokenize, " {{\n funk \n}} ").map(&:line_number) + assert_equal [1], tokenize("{{funk}}", true).map(&:line_number) + assert_equal [1, 1, 1], tokenize(" {{funk}} ", true).map(&:line_number) + assert_equal [1, 2, 2], tokenize("\n{{funk}}\n", true).map(&:line_number) + assert_equal [1, 1, 3], tokenize(" {{\n funk \n}} ", true).map(&:line_number) end private - def tokenize(source) - Liquid::Template.new.send(:tokenize, source) + def tokenize(source, line_numbers = false) + tokenizer = Liquid::Tokenizer.new(source, line_numbers) + tokens = [] + while t = tokenizer.shift + tokens << t + end + tokens end end