Revert "Add end tag support (#1823)" (#1824)

This reverts commit 27ead517ee.

README.md

@@ -1,4 +1,4 @@
-[![Build status](https://github.com/Shopify/liquid/actions/workflows/liquid.yml/badge.svg)](https://github.com/Shopify/liquid/actions/workflows/liquid.yml)
+[![Build Status](https://api.travis-ci.org/Shopify/liquid.svg?branch=master)](http://travis-ci.org/Shopify/liquid)
 [![Inline docs](http://inch-ci.org/github/Shopify/liquid.svg?branch=master)](http://inch-ci.org/github/Shopify/liquid)
 
 # Liquid template engine

lib/liquid/block_body.rb

@@ -6,7 +6,6 @@ module Liquid
   class BlockBody
     LiquidTagToken      = /\A\s*(#{TagName})\s*(.*?)\z/o
     FullToken           = /\A#{TagStart}#{WhitespaceControl}?(\s*)(#{TagName})(\s*)(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
-    FullTokenPossiblyInvalid = /\A(.*)#{TagStart}#{WhitespaceControl}?\s*(\w+)\s*(.*)?#{WhitespaceControl}?#{TagEnd}\z/om
     ContentOfVariable   = /\A#{VariableStart}#{WhitespaceControl}?(.*?)#{WhitespaceControl}?#{VariableEnd}\z/om
     WhitespaceOrNothing = /\A\s*\z/
     TAGSTART            = "{%"

lib/liquid/errors.rb

@@ -40,20 +40,19 @@ module Liquid
     end
   end
 
-  ArgumentError         = Class.new(Error)
-  ContextError          = Class.new(Error)
-  FileSystemError       = Class.new(Error)
-  StandardError         = Class.new(Error)
-  SyntaxError           = Class.new(Error)
-  StackLevelError       = Class.new(Error)
-  MemoryError           = Class.new(Error)
-  ZeroDivisionError     = Class.new(Error)
-  FloatDomainError      = Class.new(Error)
-  UndefinedVariable     = Class.new(Error)
-  UndefinedDropMethod   = Class.new(Error)
-  UndefinedFilter       = Class.new(Error)
-  MethodOverrideError   = Class.new(Error)
-  DisabledError         = Class.new(Error)
-  InternalError         = Class.new(Error)
-  TemplateEncodingError = Class.new(Error)
+  ArgumentError       = Class.new(Error)
+  ContextError        = Class.new(Error)
+  FileSystemError     = Class.new(Error)
+  StandardError       = Class.new(Error)
+  SyntaxError         = Class.new(Error)
+  StackLevelError     = Class.new(Error)
+  MemoryError         = Class.new(Error)
+  ZeroDivisionError   = Class.new(Error)
+  FloatDomainError    = Class.new(Error)
+  UndefinedVariable   = Class.new(Error)
+  UndefinedDropMethod = Class.new(Error)
+  UndefinedFilter     = Class.new(Error)
+  MethodOverrideError = Class.new(Error)
+  DisabledError       = Class.new(Error)
+  InternalError       = Class.new(Error)
 end

lib/liquid/locales/en.yml

@@ -15,7 +15,6 @@
       include: "Error in tag 'include' - Valid syntax: include '[template]' (with|for) [object|collection]"
       inline_comment_invalid: "Syntax error in tag '#' - Each line of comments must be prefixed by the '#' character"
       invalid_delimiter: "'%{tag}' is not a valid delimiter for %{block_name} tags. use %{block_delimiter}"
-      invalid_template_encoding: "Invalid template encoding"
       render: "Syntax error in tag 'render' - Template name must be a quoted string"
       table_row: "Syntax Error in 'table_row loop' - Valid syntax: table_row [item] in [collection] cols=3"
       tag_never_closed: "'%{block_name}' tag was never closed"

lib/liquid/standardfilters.rb

@@ -29,19 +29,6 @@ module Liquid
     )
     STRIP_HTML_TAGS = /<.*?>/m
 
-    class << self
-      def try_coerce_encoding(input, encoding:)
-        original_encoding = input.encoding
-        if input.encoding != encoding
-          input.force_encoding(encoding)
-          unless input.valid_encoding?
-            input.force_encoding(original_encoding)
-          end
-        end
-        input
-      end
-    end
-
     # @liquid_public_docs
     # @liquid_type filter
     # @liquid_category array
@@ -163,8 +150,7 @@ module Liquid
     # @liquid_syntax string | base64_decode
     # @liquid_return [string]
     def base64_decode(input)
-      input = input.to_s
-      StandardFilters.try_coerce_encoding(Base64.strict_decode64(input), encoding: input.encoding)
+      Base64.strict_decode64(input.to_s)
     rescue ::ArgumentError
       raise Liquid::ArgumentError, "invalid base64 provided to base64_decode"
     end
@@ -188,8 +174,7 @@ module Liquid
     # @liquid_syntax string | base64_url_safe_decode
     # @liquid_return [string]
     def base64_url_safe_decode(input)
-      input = input.to_s
-      StandardFilters.try_coerce_encoding(Base64.urlsafe_decode64(input), encoding: input.encoding)
+      Base64.urlsafe_decode64(input.to_s)
     rescue ::ArgumentError
       raise Liquid::ArgumentError, "invalid base64 provided to base64_url_safe_decode"
     end
@@ -907,11 +892,9 @@ module Liquid
         raise_property_error(property)
       end
 
-      result = InputIterator.new(values_for_sum, context).sum do |item|
+      InputIterator.new(values_for_sum, context).sum do |item|
         Utils.to_number(item)
       end
-
-      result.is_a?(BigDecimal) ? result.to_f : result
     end
 
     private

lib/liquid/tag.rb

@@ -54,8 +54,7 @@ module Liquid
     # of the `render_to_output_buffer` method will become the default and the `render`
     # method will be removed.
     def render_to_output_buffer(context, output)
-      render_result = render(context)
-      output << render_result if render_result
+      output << render(context)
       output
     end
 

lib/liquid/tags/comment.rb

@@ -25,65 +25,6 @@ module Liquid
     def blank?
       true
     end
-
-    private
-
-    def parse_body(body, tokenizer)
-      if parse_context.depth >= MAX_DEPTH
-        raise StackLevelError, "Nesting too deep"
-      end
-
-      parse_context.depth += 1
-      comment_tag_depth = 1
-
-      begin
-        # Consume tokens without creating child nodes.
-        # The children tag doesn't require to be a valid Liquid except the comment and raw tag.
-        # The child comment and raw tag must be closed.
-        while (token = tokenizer.send(:shift))
-          tag_name = if tokenizer.for_liquid_tag
-            next if token.empty? || token.match?(BlockBody::WhitespaceOrNothing)
-
-            tag_name_match = BlockBody::LiquidTagToken.match(token)
-
-            next if tag_name_match.nil?
-
-            tag_name_match[1]
-          else
-            token =~ BlockBody::FullToken
-            Regexp.last_match(2)
-          end
-
-          case tag_name
-          when "raw"
-            parse_raw_tag_body(tokenizer)
-          when "comment"
-            comment_tag_depth += 1
-          when "endcomment"
-            comment_tag_depth -= 1
-          end
-
-          if comment_tag_depth.zero?
-            parse_context.trim_whitespace = (token[-3] == WhitespaceControl) unless tokenizer.for_liquid_tag
-            return false
-          end
-        end
-
-        raise_tag_never_closed(block_name)
-      ensure
-        parse_context.depth -= 1
-      end
-
-      false
-    end
-
-    def parse_raw_tag_body(tokenizer)
-      while (token = tokenizer.send(:shift))
-        return if token =~ BlockBody::FullTokenPossiblyInvalid && "endraw" == Regexp.last_match(2)
-      end
-
-      raise_tag_never_closed("raw")
-    end
   end
 
   Template.register_tag('comment', Comment)

lib/liquid/tags/raw.rb

@@ -14,6 +14,7 @@ module Liquid
   # @liquid_syntax_keyword expression The expression to be output without being rendered.
   class Raw < Block
     Syntax = /\A\s*\z/
+    FullTokenPossiblyInvalid = /\A(.*)#{TagStart}#{WhitespaceControl}?\s*(\w+)\s*(.*)?#{WhitespaceControl}?#{TagEnd}\z/om
 
     def initialize(tag_name, markup, parse_context)
       super
@@ -24,7 +25,7 @@ module Liquid
     def parse(tokens)
       @body = +''
       while (token = tokens.shift)
-        if token =~ BlockBody::FullTokenPossiblyInvalid && block_delimiter == Regexp.last_match(2)
+        if token =~ FullTokenPossiblyInvalid && block_delimiter == Regexp.last_match(2)
           parse_context.trim_whitespace = (token[-3] == WhitespaceControl)
           @body << Regexp.last_match(1) if Regexp.last_match(1) != ""
           return

lib/liquid/template.rb

@@ -107,11 +107,6 @@ module Liquid
     # Returns self for easy chaining
     def parse(source, options = {})
       parse_context = configure_options(options)
-
-      unless source.valid_encoding?
-        raise TemplateEncodingError, parse_context.locale.t("errors.syntax.invalid_template_encoding")
-      end
-
       tokenizer     = parse_context.new_tokenizer(source, start_line_number: @line_numbers && 1)
       @root         = Document.parse(tokenizer, parse_context)
       self

test/integration/standard_filter_test.rb

@@ -176,17 +176,7 @@ class StandardFiltersTest < Minitest::Test
   end
 
   def test_base64_decode
-    decoded = @filters.base64_decode('b25lIHR3byB0aHJlZQ==')
-    assert_equal('one two three', decoded)
-    assert_equal(Encoding::UTF_8, decoded.encoding)
-
-    decoded = @filters.base64_decode('4pyF')
-    assert_equal('✅', decoded)
-    assert_equal(Encoding::UTF_8, decoded.encoding)
-
-    decoded = @filters.base64_decode("/w==")
-    assert_equal(Encoding::ASCII_8BIT, decoded.encoding)
-    assert_equal((+"\xFF").force_encoding(Encoding::ASCII_8BIT), decoded)
+    assert_equal('one two three', @filters.base64_decode('b25lIHR3byB0aHJlZQ=='))
 
     exception = assert_raises(Liquid::ArgumentError) do
       @filters.base64_decode("invalidbase64")
@@ -204,21 +194,10 @@ class StandardFiltersTest < Minitest::Test
   end
 
   def test_base64_url_safe_decode
-    decoded = @filters.base64_url_safe_decode('YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXogQUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVogMTIzNDU2Nzg5MCAhQCMkJV4mKigpLT1fKy8_Ljo7W117fVx8')
     assert_equal(
       'abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ 1234567890 !@#$%^&*()-=_+/?.:;[]{}\|',
-      decoded,
+      @filters.base64_url_safe_decode('YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXogQUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVogMTIzNDU2Nzg5MCAhQCMkJV4mKigpLT1fKy8_Ljo7W117fVx8'),
     )
-    assert_equal(Encoding::UTF_8, decoded.encoding)
-
-    decoded = @filters.base64_url_safe_decode('4pyF')
-    assert_equal('✅', decoded)
-    assert_equal(Encoding::UTF_8, decoded.encoding)
-
-    decoded = @filters.base64_url_safe_decode("_w==")
-    assert_equal(Encoding::ASCII_8BIT, decoded.encoding)
-    assert_equal((+"\xFF").force_encoding(Encoding::ASCII_8BIT), decoded)
-
     exception = assert_raises(Liquid::ArgumentError) do
       @filters.base64_url_safe_decode("invalidbase64")
     end
@@ -1015,42 +994,6 @@ class StandardFiltersTest < Minitest::Test
     assert(t.foo > 0)
   end
 
-  def test_sum_of_floats
-    input = [0.1, 0.2, 0.3]
-    assert_equal(0.6, @filters.sum(input))
-    assert_template_result("0.6", "{{ input | sum }}", { "input" => input })
-  end
-
-  def test_sum_of_negative_floats
-    input = [0.1, 0.2, -0.3]
-    assert_equal(0.0, @filters.sum(input))
-    assert_template_result("0.0", "{{ input | sum }}", { "input" => input })
-  end
-
-  def test_sum_with_float_strings
-    input = [0.1, "0.2", "0.3"]
-    assert_equal(0.6, @filters.sum(input))
-    assert_template_result("0.6", "{{ input | sum }}", { "input" => input })
-  end
-
-  def test_sum_resulting_in_negative_float
-    input = [0.1, -0.2, -0.3]
-    assert_equal(-0.4, @filters.sum(input))
-    assert_template_result("-0.4", "{{ input | sum }}", { "input" => input })
-  end
-
-  def test_sum_with_floats_and_indexable_map_values
-    input = [{ "quantity" => 1 }, { "quantity" => 0.2, "weight" => -0.3 }, { "weight" => 0.4 }]
-    assert_equal(0.0, @filters.sum(input))
-    assert_equal(1.2, @filters.sum(input, "quantity"))
-    assert_equal(0.1, @filters.sum(input, "weight"))
-    assert_equal(0.0, @filters.sum(input, "subtotal"))
-    assert_template_result("0", "{{ input | sum }}", { "input" => input })
-    assert_template_result("1.2", "{{ input | sum: 'quantity' }}", { "input" => input })
-    assert_template_result("0.1", "{{ input | sum: 'weight' }}", { "input" => input })
-    assert_template_result("0", "{{ input | sum: 'subtotal' }}", { "input" => input })
-  end
-
   private
 
   def with_timezone(tz)

test/integration/template_test.rb

@@ -337,16 +337,4 @@ class TemplateTest < Minitest::Test
     assert_equal("x=2", output)
     assert_instance_of(String, output)
   end
-
-  def test_raises_error_with_invalid_utf8
-    e = assert_raises(TemplateEncodingError) do
-      Template.parse(<<~LIQUID)
-        {% comment %}
-          \xC0
-        {% endcomment %}
-      LIQUID
-    end
-
-    assert_equal('Liquid error: Invalid template encoding', e.message)
-  end
 end

test/unit/tag_unit_test.rb

@@ -20,13 +20,4 @@ class TagUnitTest < Minitest::Test
     tag = Tag.parse("some_tag", "", Tokenizer.new(""), ParseContext.new)
     assert_equal('some_tag', tag.tag_name)
   end
-
-  class CustomTag < Liquid::Tag
-    def render(_context); end
-  end
-
-  def test_tag_render_to_output_buffer_nil_value
-    custom_tag = CustomTag.parse("some_tag", "", Tokenizer.new(""), ParseContext.new)
-    assert_equal('some string', custom_tag.render_to_output_buffer(Context.new, "some string"))
-  end
 end

test/unit/tags/comment_tag_unit_test.rb

@@ -1,202 +0,0 @@
-# frozen_string_literal: true
-
-require 'test_helper'
-
-class CommentTagUnitTest < Minitest::Test
-  def test_comment_inside_liquid_tag
-    assert_template_result("", <<~LIQUID.chomp)
-      {% liquid
-        if 1 != 1
-        comment
-        else
-          echo 123
-        endcomment
-        endif
-      %}
-    LIQUID
-  end
-
-  def test_does_not_parse_nodes_inside_a_comment
-    assert_template_result("", <<~LIQUID.chomp)
-      {% comment %}
-        {% if true %}
-        {% if ... %}
-        {%- for ? -%}
-        {% while true %}
-        {%
-          unless if
-        %}
-        {% endcase %}
-      {% endcomment %}
-    LIQUID
-  end
-
-  def test_allows_unclosed_tags
-    assert_template_result('', <<~LIQUID.chomp)
-      {% comment %}
-        {% if true %}
-      {% endcomment %}
-    LIQUID
-  end
-
-  def test_open_tags_in_comment
-    assert_template_result('', <<~LIQUID.chomp)
-      {% comment %}
-        {% assign a = 123 {% comment %}
-      {% endcomment %}
-    LIQUID
-
-    assert_raises(Liquid::SyntaxError) do
-      assert_template_result("", <<~LIQUID.chomp)
-        {% comment %}
-          {% assign foo = "1"
-        {% endcomment %}
-      LIQUID
-    end
-
-    assert_raises(Liquid::SyntaxError) do
-      assert_template_result("", <<~LIQUID.chomp)
-        {% comment %}
-          {% comment %}
-            {% invalid
-          {% endcomment %}
-        {% endcomment %}
-      LIQUID
-    end
-
-    assert_raises(Liquid::SyntaxError) do
-      assert_template_result("", <<~LIQUID.chomp)
-        {% comment %}
-        {% {{ {%- endcomment %}
-      LIQUID
-    end
-  end
-
-  def test_child_comment_tags_need_to_be_closed
-    assert_template_result("", <<~LIQUID.chomp)
-      {% comment %}
-        {% comment %}
-          {% comment %}{%    endcomment     %}
-        {% endcomment %}
-      {% endcomment %}
-    LIQUID
-
-    assert_raises(Liquid::SyntaxError) do
-      assert_template_result("", <<~LIQUID.chomp)
-        {% comment %}
-          {% comment %}
-            {% comment %}
-          {% endcomment %}
-        {% endcomment %}
-      LIQUID
-    end
-  end
-
-  def test_child_raw_tags_need_to_be_closed
-    assert_template_result("", <<~LIQUID.chomp)
-      {% comment %}
-        {% raw %}
-          {% endcomment %}
-        {% endraw %}
-      {% endcomment %}
-    LIQUID
-
-    assert_raises(Liquid::SyntaxError) do
-      Liquid::Template.parse(<<~LIQUID.chomp)
-        {% comment %}
-          {% raw %}
-          {% endcomment %}
-        {% endcomment %}
-      LIQUID
-    end
-  end
-
-  def test_error_line_number_is_correct
-    template = Liquid::Template.parse(<<~LIQUID.chomp, line_numbers: true)
-      {% comment %}
-        {% if true %}
-      {% endcomment %}
-      {{ errors.standard_error }}
-    LIQUID
-
-    output = template.render('errors' => ErrorDrop.new)
-    expected = <<~TEXT.chomp
-
-      Liquid error (line 4): standard error
-    TEXT
-
-    assert_equal(expected, output)
-  end
-
-  def test_comment_tag_delimiter_with_extra_strings
-    assert_template_result(
-      '',
-      <<~LIQUID.chomp,
-        {% comment %}
-          {% comment %}
-          {% endcomment
-          {% if true %}
-          {% endif %}
-        {% endcomment %}
-      LIQUID
-    )
-  end
-
-  def test_nested_comment_tag_with_extra_strings
-    assert_template_result(
-      '',
-      <<~LIQUID.chomp,
-        {% comment %}
-          {% comment
-            {% assign foo = 1 %}
-          {% endcomment
-          {% assign foo = 1 %}
-        {% endcomment %}
-      LIQUID
-    )
-  end
-
-  def test_ignores_delimiter_with_extra_strings
-    assert_template_result(
-      '',
-      <<~LIQUID.chomp,
-        {% if true %}
-          {% comment %}
-            {% commentXXXXX %}wut{% endcommentXXXXX %}
-          {% endcomment %}
-        {% endif %}
-      LIQUID
-    )
-  end
-
-  def test_delimiter_can_have_extra_strings
-    assert_template_result('', "{% comment %}123{% endcomment xyz %}")
-    assert_template_result('', "{% comment %}123{% endcomment\txyz %}")
-    assert_template_result('', "{% comment %}123{% endcomment\nxyz %}")
-    assert_template_result('', "{% comment %}123{% endcomment\n   xyz  endcomment %}")
-    assert_template_result('', "{%comment}{% assign a = 1 %}{%endcomment}{% endif %}")
-  end
-
-  def test_with_whitespace_control
-    assert_template_result("Hello!", "      {%- comment -%}123{%- endcomment -%}Hello!")
-    assert_template_result("Hello!", "{%- comment -%}123{%- endcomment -%}     Hello!")
-    assert_template_result("Hello!", "      {%- comment -%}123{%- endcomment -%}     Hello!")
-
-    assert_template_result("Hello!", <<~LIQUID.chomp)
-      {%- comment %}Whitespace control!{% endcomment -%}
-      Hello!
-    LIQUID
-  end
-
-  def test_dont_override_liquid_tag_whitespace_control
-    assert_template_result("Hello!World!", <<~LIQUID.chomp)
-      Hello!
-      {%- liquid
-        comment
-         this is inside a liquid tag
-        endcomment
-      -%}
-      World!
-    LIQUID
-  end
-end