Add source migration support for all builtin tags

lib/liquid/block_body.rb

@@ -4,8 +4,8 @@ require 'English'
 
 module Liquid
   class BlockBody
-    LiquidTagToken      = /\A\s*(#{TagName})\s*(.*?)\s*\z/o
-    FullToken           = /\A#{TagStart}#{WhitespaceControl}?(\s*)(#{TagName})(\s*)(.*?)\s*?#{WhitespaceControl}?#{TagEnd}\z/om
+    LiquidTagToken      = /\A\s*(#{TagName})\s*(.*?)\z/o
+    FullToken           = /\A#{TagStart}#{WhitespaceControl}?(\s*)(#{TagName})(\s*)(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
     ContentOfVariable   = /\A#{VariableStart}#{WhitespaceControl}?(.*?)#{WhitespaceControl}?#{VariableEnd}\z/om
     WhitespaceOrNothing = /\A\s*\z/
     TAGSTART            = "{%"
@@ -80,6 +80,10 @@ module Liquid
         @markup_capture_number = markup_capture_number
       end
 
+      def original_tag_string
+        @match[0]
+      end
+
       def replaced_markup(new_markup)
         Utils.match_capture_replace(@match, @markup_capture_number, new_markup)
       end
@@ -88,16 +92,15 @@ module Liquid
     private_class_method def self.migrate_for_liquid_tag(tokenizer, parse_context)
       result = +""
       while (token = tokenizer.shift)
+        token += "\n" if tokenizer.more?
         if token.empty? || token.match?(WhitespaceOrNothing)
           result << token
-          result << "\n" if tokenizer.more?
         else
-          match = token.match(LiquidTagToken)
-          unless match
-            # Missing tag name, which was allowed in comment tags through its
-            # unknown tag handling
-            raise NotImplementedError, "TODO"
-          end
+          # modified version of LiquidTagToken with following changes:
+          # * TagName is optional, to continue supporting its absence in the comment tag
+          # * trailing spaces is allowed to support the newline appended above and so the tag
+          #   migrate method doesn't have to handle trailing whitespace
+          match = token.match(/\A\s*(#{TagName})?\s*(.*?)\s*\z/o)
           tag_name = match[1]
           markup   = match[2]
           unless (tag = Template.tags[tag_name])
@@ -108,10 +111,8 @@ module Liquid
             )
             return [result, unknown_tag]
           end
-          has_more_tokens = tokenizer.more?
           new_markup, new_tag_body = tag.migrate(tag_name, markup, tokenizer, parse_context)
           result << Utils.match_capture_replace(match, 2, new_markup)
-          result << "\n" if has_more_tokens
           result << new_tag_body.to_s
         end
         parse_context.line_number = tokenizer.line_number
@@ -243,12 +244,8 @@ module Liquid
         case
         when token.start_with?(TAGSTART)
           raise SyntaxError unless token.end_with?('%}')
-          match = token.match(FullToken)
-          unless match
-            # Missing tag name, which was allowed in comment tags through its
-            # unknown tag handling
-            raise NotImplementedError, "TODO"
-          end
+          # modified FullToken regex with optional tag name, to allow its absence in a comment tag
+          match = token.match(/\A#{TagStart}#{WhitespaceControl}?(\s*)(#{TagName})?(\s*)(.*?)#{WhitespaceControl}?#{TagEnd}\z/om)
           tag_name = match[2]
           markup   = match[4]
 
@@ -259,7 +256,9 @@ module Liquid
           end
 
           if tag_name == 'liquid'
-            new_markup = migrate_liquid_tag(markup, parse_context)
+            new_markup = Utils.migrate_stripped(markup) do |stripped_markup|
+              migrate_liquid_tag(stripped_markup, parse_context)
+            end
             result << Utils.match_capture_replace(match, 4, new_markup)
             next
           end
@@ -272,7 +271,11 @@ module Liquid
             )
             return [result, unknown_tag]
           end
-          new_markup, new_tag_body = tag.migrate(tag_name, markup, tokenizer, parse_context)
+          new_tag_body = nil
+          new_markup = Utils.migrate_stripped(markup) do |stripped_markup|
+            new_stripped_markup, new_tag_body = tag.migrate(tag_name, stripped_markup, tokenizer, parse_context)
+            new_stripped_markup
+          end
           result << Utils.match_capture_replace(match, 4, new_markup) << new_tag_body.to_s
         when token.start_with?(VARSTART)
           result << migrate_variable(token, parse_context)

lib/liquid/parse_context.rb

@@ -5,6 +5,9 @@ module Liquid
     attr_accessor :locale, :line_number, :trim_whitespace, :depth
     attr_reader :partial, :warnings, :error_mode
 
+    # @api private
+    attr_writer :error_mode
+
     def initialize(options = {})
       @template_options = options ? options.dup : {}
 

lib/liquid/parser_switching.rb

@@ -20,7 +20,7 @@ module Liquid
             strict_migrate(tag_name, markup, tokenizer, parse_context)
           rescue SyntaxError => e
             parse_context.warnings << e
-            lax_migrate(markup)
+            lax_migrate(tag_name, markup, tokenizer, parse_context)
           end
         end
       end

lib/liquid/tags/assign.rb

@@ -22,7 +22,7 @@ module Liquid
     end
 
     def self.migrate(tag_name, markup, tokenizer, parse_context)
-      match = markup.match(/\s*#{Syntax}/)
+      match = markup.match(Syntax)
       new_variable_markup = Variable.migrate(match[2], parse_context)
       new_markup = Utils.match_captures_replace(match, 2 => new_variable_markup)
 

lib/liquid/tags/break.rb

@@ -21,6 +21,10 @@ module Liquid
   class Break < Tag
     INTERRUPT = BreakInterrupt.new.freeze
 
+    def self.migrate(_tag_name, _markup, _tokenizer, _parse_context)
+      "" # markup was ignored
+    end
+
     def render_to_output_buffer(context, output)
       context.push_interrupt(INTERRUPT)
       output

lib/liquid/tags/capture.rb

@@ -18,6 +18,21 @@ module Liquid
   class Capture < Block
     Syntax = /(#{VariableSignature}+)/o
 
+    def self.migrate(tag_name, markup, tokenizer, parse_context)
+      match = markup.match(Syntax)
+
+      new_markup = match[1]
+
+      # replace scanned over characters with a space to ensure there is a space
+      # to separate the tag name and the variable name
+      new_markup.prepend(" ") if match.begin(0) > 0
+
+      new_body, unknown_tag = migrate_body(tag_name, tokenizer, parse_context)
+      raise SyntaxError if unknown_tag
+
+      [new_markup, new_body]
+    end
+
     def initialize(tag_name, markup, options)
       super
       if markup =~ Syntax

lib/liquid/tags/case.rb

@@ -28,6 +28,20 @@ module Liquid
 
     attr_reader :blocks, :left
 
+    def self.migrate(tag_name, markup, tokenizer, parse_context)
+      match = markup.match(/\s*#{Syntax}\s*/o) || raise(SyntaxError)
+
+      new_expression = Expression.lax_migrate(match[1])
+      new_markup = Utils.match_captures_replace(match, { 1 => new_expression })
+
+      # replace scanned over characters with a space to ensure there is a space
+      # to separate the tag name and the variable name
+      new_markup.prepend(" ") if match.begin(0) > 0
+
+      new_body = migrate_body(tag_name, tokenizer, parse_context)
+      [new_markup, new_body]
+    end
+
     def initialize(tag_name, markup, options)
       super
       @blocks = []
@@ -39,6 +53,37 @@ module Liquid
       end
     end
 
+    def self.migrate_body(start_tag_name, tokenizer, parse_context)
+      result = +""
+
+      # body before first `when` delimiter tag is ignored
+      unused_body, delimiter_tag = super(start_tag_name, tokenizer, parse_context)
+      unless delimiter_tag
+        raise NotImplementedError, "TODO: migrate `case` tag with no `when` or `else` tags"
+      end
+
+      result << Utils.migrate_stripped(unused_body) { "" } # just keep whitespace (e.g. newline and indent)
+
+      while delimiter_tag
+        break unless delimiter_tag
+
+        case delimiter_tag.tag_name
+        when "when"
+          new_markup = migrate_when_markup(delimiter_tag.markup)
+          result << delimiter_tag.replaced_markup(new_markup)
+        when "else"
+          result << delimiter_tag.original_tag_string
+        else
+          raise SyntaxError
+        end
+
+        new_body, delimiter_tag = super(start_tag_name, tokenizer, parse_context)
+        result << new_body
+      end
+
+      result
+    end
+
     def parse(tokens)
       body = case_body = new_body
       body = @blocks.last.attachment while parse_body(body, tokens)
@@ -91,6 +136,25 @@ module Liquid
 
     private
 
+    private_class_method def self.migrate_when_markup(unstripped_markup)
+      Utils.migrate_stripped(unstripped_markup) do |markup|
+        match = markup.match(WhenSyntax) || raise(SyntaxError)
+
+        replacements = { 1 => Expression.lax_migrate(match[1]) }
+        if (right = match[2])
+          replacements[2] = migrate_when_markup(right)
+        end
+
+        new_markup = Utils.match_captures_replace(match, replacements)
+
+        # replace scanned over characters with a space to ensure there is a space
+        # to separate the tag name and the variable name
+        new_markup.prepend(" ") if match.begin(0) > 0
+
+        new_markup
+      end
+    end
+
     def record_when_condition(markup)
       body = new_body
 

lib/liquid/tags/comment.rb

@@ -15,6 +15,24 @@ module Liquid
   #   {% endcomment %}
   # @liquid_syntax_keyword content The content of the comment.
   class Comment < Block
+    def self.migrate(tag_name, _markup, tokenizer, parse_context)
+      new_markup = "" # markup was ignored
+      new_body = migrate_body(tag_name, tokenizer, parse_context)
+      [new_markup, new_body]
+    end
+
+    def self.migrate_body(start_tag_name, tokenizer, parse_context)
+      result = +""
+      loop do
+        new_body, delimiter_tag = super(start_tag_name, tokenizer, parse_context)
+        result << new_body
+        break unless delimiter_tag
+
+        result << delimiter_tag.original_tag_string # unknown tags allowed
+      end
+      result
+    end
+
     def render_to_output_buffer(_context, output)
       output
     end

lib/liquid/tags/continue.rb

@@ -12,6 +12,10 @@ module Liquid
   class Continue < Tag
     INTERRUPT = ContinueInterrupt.new.freeze
 
+    def self.migrate(_tag_name, _markup, _tokenizer, _parse_context)
+      "" # markup was ignored
+    end
+
     def render_to_output_buffer(context, output)
       context.push_interrupt(INTERRUPT)
       output

lib/liquid/tags/cycle.rb

@@ -18,8 +18,43 @@ module Liquid
     SimpleSyntax = /\A#{QuotedFragment}+/o
     NamedSyntax  = /\A(#{QuotedFragment})\s*\:\s*(.*)/om
 
+    def self.migrate(tag_name, markup, tokenizer, parse_context)
+      new_markup = case markup
+      when NamedSyntax
+        match = Regexp.last_match
+
+        new_name_syntax = Expression.lax_migrate(match[1])
+        new_variables_markup = migrate_variables_from_string(match[2])
+
+        Utils.match_captures_replace(match, 1 => new_name_syntax, 2 => new_variables_markup)
+      when SimpleSyntax
+        match = Regexp.last_match
+        migrate_variables_from_string(markup)
+      else
+        raise SyntaxError
+      end
+
+      # replace scanned over characters with a space to ensure there is a space
+      # to separate the tag name and the variable name
+      new_markup.prepend(" ") if match.begin(0) > 0
+
+      new_markup
+    end
+
+    def self.migrate_variables_from_string(markup)
+      markup.split(',').collect do |var|
+        match = var.match(/\s*(#{QuotedFragment})\s*/o)
+        if match
+          Utils.match_captures_replace(match, 1 => Expression.lax_migrate(match[1]))
+        end
+      end.compact.join(",")
+    end
+
     attr_reader :variables
 
+    # @api private
+    attr_reader :name
+
     def initialize(tag_name, markup, options)
       super
       case markup

lib/liquid/tags/decrement.rb

@@ -21,6 +21,10 @@ module Liquid
   class Decrement < Tag
     attr_reader :variable_name
 
+    def self.migrate(_tag_name, markup, _tokenizer, _parse_context)
+      markup # no characters ignored, it just uses anything for the variable name
+    end
+
     def initialize(tag_name, markup, options)
       super
       @variable_name = markup.strip

lib/liquid/tags/echo.rb

@@ -21,6 +21,10 @@ module Liquid
   class Echo < Tag
     attr_reader :variable
 
+    def self.migrate(tag_name, markup, tokenizer, parse_context)
+      Variable.migrate(markup, parse_context)
+    end
+
     def initialize(tag_name, markup, parse_context)
       super
       @variable = Variable.new(markup, parse_context)

lib/liquid/tags/for.rb

@@ -25,10 +25,16 @@ module Liquid
   # @liquid_optional_param range [untyped] A custom numeric range to iterate over.
   # @liquid_optional_param reversed [untyped] Iterate in reverse order.
   class For < Block
-    Syntax = /\A(#{VariableSegment}+)\s+in\s+(#{QuotedFragment}+)\s*(reversed)?/o
+    Syntax = /\A(#{VariableSegment}+)\s+in\s+(#{QuotedFragment}+)(\s*reversed)?/o
 
     attr_reader :collection_name, :variable_name, :limit, :from
 
+    def self.migrate(tag_name, markup, tokenizer, parse_context)
+      new_markup = migrate_with_selected_parser(tag_name, markup, tokenizer, parse_context)
+      new_body = migrate_body(tag_name, tokenizer, parse_context)
+      [new_markup, new_body]
+    end
+
     def initialize(tag_name, markup, options)
       super
       @from = @limit = nil
@@ -53,6 +59,27 @@ module Liquid
       @else_block ? [@for_block, @else_block] : [@for_block]
     end
 
+    def self.migrate_body(start_tag_name, tokenizer, parse_context)
+      result = +""
+
+      new_body, delimiter_tag = super(start_tag_name, tokenizer, parse_context)
+      result << new_body
+
+      else_tag = delimiter_tag
+      else_body = nil
+      while delimiter_tag
+        raise SyntaxError unless delimiter_tag.tag_name == 'else'
+        else_tag = delimiter_tag
+        else_body, delimiter_tag = super(start_tag_name, tokenizer, parse_context)
+      end
+      if else_tag
+        result << else_tag.replaced_markup("") # markup was ignored in else tags
+        result << else_body
+      end
+
+      result
+    end
+
     def unknown_tag(tag, markup, tokens)
       return super unless tag == 'else'
       @else_block = new_body
@@ -72,6 +99,13 @@ module Liquid
 
     protected
 
+    private_class_method def self.lax_migrate(tag_name, markup, tokenizer, parse_context)
+      match = markup.match(Syntax) || raise(SyntaxError)
+      new_collection_name = Expression.lax_migrate(match[2])
+      new_markup = Utils.match_captures_replace(match, { 2 => new_collection_name }.compact)
+      new_markup << Utils.migrate_tag_attributes(markup)
+    end
+
     def lax_parse(markup)
       if markup =~ Syntax
         @variable_name   = Regexp.last_match(1)
@@ -87,6 +121,10 @@ module Liquid
       end
     end
 
+    private_class_method def self.strict_migrate(tag_name, markup, tokenizer, parse_context)
+      markup
+    end
+
     def strict_parse(markup)
       p = Parser.new(markup)
       @variable_name = p.consume(:id)
@@ -98,11 +136,12 @@ module Liquid
       @name     = "#{@variable_name}-#{collection_name}"
       @reversed = p.id?('reversed')
 
-      while p.look(:id) && p.look(:colon, 1)
+      while p.look(:comma) || p.look(:id)
+        p.consume?(:comma)
         unless (attribute = p.id?('limit') || p.id?('offset'))
           raise SyntaxError, options[:locale].t("errors.syntax.for_invalid_attribute")
         end
-        p.consume
+        p.consume(:colon)
         set_attribute(attribute, p.expression)
       end
       p.consume(:end_of_string)

lib/liquid/tags/if.rb

@@ -37,7 +37,7 @@ module Liquid
 
         case delimiter_tag.tag_name
         when "else"
-          result << delimiter_tag.replaced_markup("") # markup was ignored on end tags
+          result << delimiter_tag.replaced_markup("") # markup was ignored on else tags
         when "elsif"
           new_markup = migrate_with_selected_parser(delimiter_tag.tag_name, delimiter_tag.markup, tokenizer, parse_context)
           result << delimiter_tag.replaced_markup(new_markup)

lib/liquid/tags/ifchanged.rb

@@ -2,6 +2,15 @@
 
 module Liquid
   class Ifchanged < Block
+    def self.migrate(tag_name, _markup, tokenizer, parse_context)
+      new_markup = "" # markup was ignored
+
+      new_body, unknown_tag = migrate_body(tag_name, tokenizer, parse_context)
+      raise SyntaxError if unknown_tag
+
+      [new_markup, new_body]
+    end
+
     def render_to_output_buffer(context, output)
       block_output = +''
       super(context, block_output)

lib/liquid/tags/include.rb

@@ -25,6 +25,22 @@ module Liquid
 
     attr_reader :template_name_expr, :variable_name_expr, :attributes
 
+    def self.migrate(_tag_name, markup, _tokenizer, parse_context)
+      match = markup.match(SYNTAX) || raise(SyntaxError)
+
+      template_name = Expression.lax_migrate(match[1])
+      variable_name = Expression.lax_migrate(match[3]) if match[3]
+
+      new_markup = Utils.match_captures_replace(match, { 1 => template_name, 3 => variable_name }.compact)
+      new_markup << Utils.migrate_tag_attributes(markup)
+
+      # replace scanned over characters with a space to ensure there is a space
+      # to separate the tag name and the variable name
+      new_markup.prepend(" ") if match.begin(0) > 0
+
+      new_markup
+    end
+
     def initialize(tag_name, markup, options)
       super
 

lib/liquid/tags/increment.rb

@@ -21,6 +21,10 @@ module Liquid
   class Increment < Tag
     attr_reader :variable_name
 
+    def self.migrate(_tag_name, markup, _tokenizer, _parse_context)
+      markup # no characters ignored, it just uses anything for the variable name
+    end
+
     def initialize(tag_name, markup, options)
       super
       @variable_name = markup.strip

lib/liquid/tags/inline_comment.rb

@@ -2,6 +2,10 @@
 
 module Liquid
   class InlineComment < Tag
+    def self.migrate(_tag_name, markup, _tokenizer, _parse_context)
+      markup
+    end
+
     def initialize(tag_name, markup, options)
       super
 

lib/liquid/tags/raw.rb

@@ -16,6 +16,31 @@ module Liquid
     Syntax = /\A\s*\z/
     FullTokenPossiblyInvalid = /\A(.*)#{TagStart}\s*(\w+)\s*(.*)?#{TagEnd}\z/om
 
+    def self.migrate(tag_name, markup, tokenizer, parse_context)
+      raise SyntaxError unless Syntax.match?(markup)
+
+      new_body = migrate_body(tag_name, tokenizer, parse_context)
+      [markup, new_body]
+    end
+
+    def self.migrate_body(start_tag_name, tokenizer, parse_context)
+      block_delimiter = "end#{start_tag_name}"
+
+      body = +''
+      while (token = tokenizer.shift)
+        match = token.match(FullTokenPossiblyInvalid)
+        if match && block_delimiter == match[2]
+          body << Utils.match_captures_replace(match, { 3 => "" })
+          return body
+        end
+        body << token unless token.empty?
+      end
+
+      raise SyntaxError
+    end
+
+    attr_reader :body
+
     def initialize(tag_name, markup, parse_context)
       super
 

lib/liquid/tags/render.rb

@@ -29,6 +29,22 @@ module Liquid
     FOR = 'for'
     SYNTAX = /(#{QuotedString}+)(\s+(with|#{FOR})\s+(#{QuotedFragment}+))?(\s+(?:as)\s+(#{VariableSegment}+))?/o
 
+    def self.migrate(_tag_name, markup, _tokenizer, parse_context)
+      match = markup.match(SYNTAX)
+
+      template_name = Expression.lax_migrate(match[1])
+      variable_name = Expression.lax_migrate(match[4]) if match[4]
+
+      new_markup = Utils.match_captures_replace(match, { 1 => template_name, 4 => variable_name }.compact)
+      new_markup << Utils.migrate_tag_attributes(markup)
+
+      # replace scanned over characters with a space to ensure there is a space
+      # to separate the tag name and the variable name
+      new_markup.prepend(" ") if match.begin(0) > 0
+
+      new_markup
+    end
+
     disable_tags "include"
 
     attr_reader :template_name_expr, :variable_name_expr, :attributes, :alias_name

lib/liquid/tags/table_row.rb

@@ -26,6 +26,20 @@ module Liquid
   class TableRow < Block
     Syntax = /(\w+)\s+in\s+(#{QuotedFragment}+)/o
 
+    def self.migrate(tag_name, markup, tokenizer, parse_context)
+      match = markup.match(Syntax) || raise(SyntaxError)
+
+      new_collection_name = Expression.lax_migrate(match[2])
+
+      new_markup = Utils.match_captures_replace(match, { 2 => new_collection_name }.compact)
+      new_markup << Utils.migrate_tag_attributes(markup)
+
+      new_body, unknown_tag = migrate_body(tag_name, tokenizer, parse_context)
+      raise SyntaxError if unknown_tag
+
+      [new_markup, new_body]
+    end
+
     attr_reader :variable_name, :collection_name, :attributes
 
     def initialize(tag_name, markup, options)

lib/liquid/template.rb

@@ -118,7 +118,7 @@ module Liquid
     end
 
     def migrate(source, parse_options = {})
-      parse_context = configure_options(parse_options)
+      parse_context = configure_options(parse_options.merge(disable_liquid_c_nodes: true))
       tokenizer     = parse_context.new_tokenizer(source, start_line_number: @line_numbers && 1)
       Document.migrate(tokenizer, parse_context)
     end

lib/liquid/utils.rb

@@ -96,6 +96,21 @@ module Liquid
       Utils.match_captures_replace(match, 1 => new_markup)
     end
 
+    def self.migrate_tag_attributes(markup)
+      attributes = []
+      markup.scan(/\s*,?\s*#{TagAttributes}/) do
+        tag_match = Regexp.last_match
+        new_value_markup = Expression.lax_migrate(tag_match[2])
+        attribute_markup = Utils.match_captures_replace(tag_match, { 2 => new_value_markup })
+        unless attribute_markup.match?(/\A[,\s]/)
+          attribute_markup.prepend(", ")
+        end
+        attributes << attribute_markup
+      end
+      return "" if attributes.empty?
+      attributes.join
+    end
+
     # @api private
     def self.match_capture_replace(match, capture_number, replacement_string)
       match_captures_replace(match, { capture_number => replacement_string })

lib/liquid/variable.rb

@@ -41,7 +41,7 @@ module Liquid
       "in \"{{#{markup}}}\""
     end
 
-    STRICT_PARSE_CONTEXT = ParseContext.new(error_mode: :strict).freeze
+    STRICT_PARSE_CONTEXT = ParseContext.new(error_mode: :strict)
     private_constant :STRICT_PARSE_CONTEXT
 
     def self.migrate(markup, parse_context)

test/unit/migrate_unit_test.rb

@@ -97,7 +97,7 @@ class MigrateUnitTest < Minitest::Test
       LIQUID
       expect = <<~LIQUID
         {% liquid
-          assign   a = 1
+          assign  a = 1
           assign a =  b
         %}
       LIQUID