add quirky raw tag parsing

lib/liquid/block_body.rb

@@ -6,7 +6,6 @@ module Liquid
   class BlockBody
     LiquidTagToken      = /\A\s*(#{TagName})\s*(.*?)\z/o
     FullToken           = /\A#{TagStart}#{WhitespaceControl}?(\s*)(#{TagName})(\s*)(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
-    FullTokenPossiblyInvalid = /\A(.*)#{TagStart}#{WhitespaceControl}?\s*(\w+)\s*(.*)?#{WhitespaceControl}?#{TagEnd}\z/om
     ContentOfVariable   = /\A#{VariableStart}#{WhitespaceControl}?(.*?)#{WhitespaceControl}?#{VariableEnd}\z/om
     WhitespaceOrNothing = /\A\s*\z/
     TAGSTART            = "{%"
@@ -46,12 +45,6 @@ module Liquid
           end
           tag_name = Regexp.last_match(1)
           markup   = Regexp.last_match(2)
-
-          if tag_name == 'liquid'
-            parse_context.line_number -= 1
-            next parse_liquid_tag(markup, parse_context)
-          end
-
           unless (tag = registered_tags[tag_name])
             # end parsing if we reach an unknown tag and let the caller decide
             # determine how to proceed

lib/liquid/condition.rb

@@ -69,9 +69,9 @@ module Liquid
 
         case condition.child_relation
         when :or
-          break if Liquid::Utils.to_liquid_value(result)
+          break if result
         when :and
-          break unless Liquid::Utils.to_liquid_value(result)
+          break unless result
         else
           break
         end

lib/liquid/tags/comment.rb

@@ -15,8 +15,6 @@ module Liquid
   #   {% endcomment %}
   # @liquid_syntax_keyword content The content of the comment.
   class Comment < Block
-    TAG_DELIMITER = /\A(.*)#{TagStart}#{WhitespaceControl}?\s*(endcomment)\s*(.*)?#{WhitespaceControl}?#{TagEnd}\z/om
-
     def render_to_output_buffer(_context, output)
       output
     end
@@ -27,71 +25,6 @@ module Liquid
     def blank?
       true
     end
-
-    private
-
-    def parse_body(body, tokenizer)
-      if parse_context.depth >= MAX_DEPTH
-        raise StackLevelError, "Nesting too deep"
-      end
-
-      parse_context.depth += 1
-      comment_tag_depth = 1
-
-      begin
-        # Consume tokens without creating child nodes.
-        # The children tag doesn't require to be a valid Liquid except the comment and raw tag.
-        # The child comment and raw tag must be closed.
-        while (token = tokenizer.send(:shift))
-          tag_name = if tokenizer.for_liquid_tag
-            next if token.empty? || token.match?(BlockBody::WhitespaceOrNothing)
-
-            tag_name_match = BlockBody::LiquidTagToken.match(token)
-
-            next if tag_name_match.nil?
-
-            tag_name_match[1]
-          elsif TAG_DELIMITER.match?(token)
-            # aggressively match comment delimiter
-            "endcomment"
-          elsif token =~ BlockBody::FullToken && Regexp.last_match(2) == "comment"
-            # aggressively match comment tag
-            "comment"
-          else
-            tag_name_match = BlockBody::FullTokenPossiblyInvalid.match(token)
-
-            next if tag_name_match.nil?
-
-            tag_name_match[2]
-          end
-
-          case tag_name
-          when "raw"
-            parse_raw_tag_body(tokenizer)
-          when "comment"
-            comment_tag_depth += 1
-          when "endcomment"
-            comment_tag_depth -= 1
-
-            return false if comment_tag_depth.zero?
-          end
-        end
-
-        raise_tag_never_closed(block_name)
-      ensure
-        parse_context.depth -= 1
-      end
-
-      false
-    end
-
-    def parse_raw_tag_body(tokenizer)
-      while (token = tokenizer.send(:shift))
-        return if token =~ BlockBody::FullTokenPossiblyInvalid && "endraw" == Regexp.last_match(2)
-      end
-
-      raise_tag_never_closed("raw")
-    end
   end
 
   Template.register_tag('comment', Comment)

lib/liquid/tags/raw.rb

@@ -14,6 +14,7 @@ module Liquid
   # @liquid_syntax_keyword expression The expression to be output without being rendered.
   class Raw < Block
     Syntax = /\A\s*\z/
+    FullTokenPossiblyInvalid = /\A(.*)#{TagStart}#{WhitespaceControl}?[^\w\d]*(\w+)[^\w\d]*(.*)?#{WhitespaceControl}?#{TagEnd}\z/om
 
     def initialize(tag_name, markup, parse_context)
       super
@@ -24,7 +25,7 @@ module Liquid
     def parse(tokens)
       @body = +''
       while (token = tokens.shift)
-        if token =~ BlockBody::FullTokenPossiblyInvalid && block_delimiter == Regexp.last_match(2)
+        if token =~ FullTokenPossiblyInvalid && block_delimiter == Regexp.last_match(2)
           parse_context.trim_whitespace = (token[-3] == WhitespaceControl)
           @body << Regexp.last_match(1) if Regexp.last_match(1) != ""
           return

test/integration/parsing_quirks_test.rb

@@ -131,4 +131,16 @@ class ParsingQuirksTest < Minitest::Test
   def test_contains_in_id
     assert_template_result(' YES ', '{% if containsallshipments == true %} YES {% endif %}', { 'containsallshipments' => true })
   end
+
+  def test_delimiter_can_have_characters_after
+    assert_template_result('123', "{% if true %}123{% endif this is a valid syntax %}")
+  end
+
+  def test_none_breaking_whitespaces_in_tag
+    assert_template_result('123', "{% raw %}123{%\u{00A0}endraw\u{00A0}%}")
+
+    assert_template_result('123', "{% raw %}123{%\u{2000}endraw%}")
+
+    assert_template_result('123', "{% raw %}123{%endraw\u{200B}%}")
+  end
 end # ParsingQuirksTest

test/integration/tags/liquid_tag_test.rb

@@ -113,37 +113,4 @@ class LiquidTagTest < Minitest::Test
       {% raw %}{% liquid echo 'test' %}{% endraw %}
     LIQUID
   end
-
-  def test_nested_liquid_tags
-    assert_template_result('good', <<~LIQUID)
-      {%- liquid
-        liquid
-          if true
-            echo "good"
-          endif
-      -%}
-    LIQUID
-  end
-
-  def test_nested_liquid_tags_on_same_line
-    assert_template_result('good', <<~LIQUID)
-      {%- liquid liquid liquid echo "good" -%}
-    LIQUID
-  end
-
-  def test_nested_liquid_liquid_is_not_skipped_if_used_in_non_tag_position
-    assert_template_result('liquid', <<~LIQUID, { 'liquid' => 'liquid' })
-      {%- liquid liquid liquid echo liquid -%}
-    LIQUID
-  end
-
-  def test_next_liquid_with_unclosed_if_tag
-    assert_match_syntax_error("Liquid syntax error (line 2): 'if' tag was never closed", <<~LIQUID)
-      {%- liquid
-        liquid if true
-          echo "good"
-        endif
-      -%}
-    LIQUID
-  end
 end

test/integration/variable_test.rb

@@ -34,7 +34,6 @@ class VariableTest < Minitest::Test
 
     assert_template_result('', '{% if foo %}true{% endif %}', { 'foo' => BooleanDrop.new(false) })
     assert_template_result('', '{% if foo == true %}True{% endif %}', { 'foo' => BooleanDrop.new(false) })
-    assert_template_result('', '{% if foo and true %}SHOULD NOT HAPPEN{% endif %}', { 'foo' => BooleanDrop.new(false) })
 
     assert_template_result('one', '{% if a contains x %}one{% endif %}', { 'a' => [1], 'x' => IntegerDrop.new(1) })
   end

test/unit/tags/comment_tag_unit_test.rb

@@ -1,138 +0,0 @@
-# frozen_string_literal: true
-
-require 'test_helper'
-
-class CommentTagUnitTest < Minitest::Test
-  def test_comment_inside_liquid_tag
-    assert_template_result("", <<~LIQUID.chomp)
-      {% liquid
-        if 1 != 1
-        comment
-        else
-          echo 123
-        endcomment
-        endif
-      %}
-    LIQUID
-  end
-
-  def test_does_not_parse_nodes_inside_a_comment
-    assert_template_result("", <<~LIQUID.chomp)
-      {% comment %}
-        {% if true %}
-        {% if ... %}
-        {%- for ? -%}
-        {% while true %}
-        {%
-          unless if
-        %}
-        {% endcase %}
-      {% endcomment %}
-    LIQUID
-  end
-
-  def test_allows_incomplete_tags_inside_a_comment
-    assert_template_result("", <<~LIQUID.chomp)
-      {% comment %}
-        {% assign foo = "1"
-      {% endcomment %}
-    LIQUID
-
-    assert_template_result("", <<~LIQUID.chomp)
-      {% comment %}
-        {% comment %}
-          {% invalid
-        {% endcomment %}
-      {% endcomment %}
-    LIQUID
-
-    assert_template_result("", <<~LIQUID.chomp)
-      {% comment %}
-      {% {{ {%- endcomment %}
-    LIQUID
-  end
-
-  def test_child_comment_tags_need_to_be_closed
-    assert_template_result("", <<~LIQUID.chomp)
-      {% comment %}
-        {% comment %}
-          {% comment %}{%    endcomment     %}
-        {% endcomment %}
-      {% endcomment %}
-    LIQUID
-
-    assert_raises(Liquid::SyntaxError) do
-      assert_template_result("", <<~LIQUID.chomp)
-        {% comment %}
-          {% comment %}
-            {% comment %}
-          {% endcomment %}
-        {% endcomment %}
-      LIQUID
-    end
-  end
-
-  def test_child_raw_tags_need_to_be_closed
-    assert_template_result("", <<~LIQUID.chomp)
-      {% comment %}
-        {% raw %}
-          {% endcomment %}
-        {% endraw %}
-      {% endcomment %}
-    LIQUID
-
-    assert_raises(Liquid::SyntaxError) do
-      Liquid::Template.parse(<<~LIQUID.chomp)
-        {% comment %}
-          {% raw %}
-          {% endcomment %}
-        {% endcomment %}
-      LIQUID
-    end
-  end
-
-  def test_error_line_number_is_correct
-    template = Liquid::Template.parse(<<~LIQUID.chomp, line_numbers: true)
-      {% comment %}
-        {% if true %}
-      {% endcomment %}
-      {{ errors.standard_error }}
-    LIQUID
-
-    output = template.render('errors' => ErrorDrop.new)
-    expected = <<~TEXT.chomp
-
-      Liquid error (line 4): standard error
-    TEXT
-
-    assert_equal(expected, output)
-  end
-
-  def test_comment_tag_delimiter_with_extra_strings
-    assert_template_result(
-      '',
-      <<~LIQUID.chomp,
-        {% comment %}
-          {% comment %}
-          {% endcomment
-          {% if true %}
-          {% endif %}
-        {% endcomment %}
-      LIQUID
-    )
-  end
-
-  def test_nested_comment_tag_with_extra_strings
-    assert_template_result(
-      '',
-      <<~LIQUID.chomp,
-        {% comment %}
-          {% comment
-            {% assign foo = 1 %}
-          {% endcomment
-          {% assign foo = 1 %}
-        {% endcomment %}
-      LIQUID
-    )
-  end
-end