Fix History.md

Raise SyntaxError on invalid UTF8 strings in lexer/tokenizer

History.md

@@ -1,18 +1,30 @@
 # Liquid Change Log
 
-## 5.8.0 (unreleased)
+## 5.8.1 (unreleased)
+
+## 5.8.0
+
+* Introduce the new `{% doc %}` tag [Guilherme Carreiro]
+
+## 5.7.3
+
+* Raise Liquid::SyntaxError when parsing invalidly encoded strings [Chris AtLee]
+
+## 5.7.2 2025-01-31
+
+* Fix array filters to not support nested properties [Guilherme Carreiro]
 
 ## 5.7.1 2025-01-24
 
-* Fix the `find` and `find_index`filters to return `nil` when filtering empty arrays
-* Fix the `has` filter to return `false` when filtering empty arrays
+* Fix the `find` and `find_index`filters to return `nil` when filtering empty arrays [Guilherme Carreiro]
+* Fix the `has` filter to return `false` when filtering empty arrays [Guilherme Carreiro]
 
 ## 5.7.0 2025-01-16
 
 ### Features
 
-* Add `find`, `find_index`, `has`, and `reject` filters to arrays
-* Compatibility with Ruby 3.4
+* Add `find`, `find_index`, `has`, and `reject` filters to arrays [Guilherme Carreiro]
+* Compatibility with Ruby 3.4 [Ian Ker-Seymer]
 
 ## 5.6.4 2025-01-14
 

lib/liquid/lexer.rb

@@ -161,6 +161,12 @@ module Liquid
         end
         # rubocop:enable Metrics/BlockNesting
         output << EOS
+      rescue ::ArgumentError => e
+        if e.message == "invalid byte sequence in #{ss.string.encoding}"
+          raise SyntaxError, "Invalid byte sequence in #{ss.string.encoding}"
+        else
+          raise
+        end
       end
 
       def raise_syntax_error(start_pos, ss)

lib/liquid/locales/en.yml

@@ -2,12 +2,14 @@
   errors:
     syntax:
       tag_unexpected_args: "Syntax Error in '%{tag}' - Valid syntax: %{tag}"
+      block_tag_unexpected_args: "Syntax Error in '%{tag}' - Valid syntax: {% %{tag} %}{% end%{tag} %}"
       assign: "Syntax Error in 'assign' - Valid syntax: assign [var] = [source]"
       capture: "Syntax Error in 'capture' - Valid syntax: capture [var]"
       case: "Syntax Error in 'case' - Valid syntax: case [condition]"
       case_invalid_when: "Syntax Error in tag 'case' - Valid when condition: {% when [condition] [or condition2...] %}"
       case_invalid_else: "Syntax Error in tag 'case' - Valid else condition: {% else %} (no parameters) "
       cycle: "Syntax Error in 'cycle' - Valid syntax: cycle [name :] var [, var2, var3 ...]"
+      doc_invalid_nested: "Syntax Error in 'doc' - Nested doc tags are not allowed"
       for: "Syntax Error in 'for loop' - Valid syntax: for [item] in [collection]"
       for_invalid_in: "For loops require an 'in' clause"
       for_invalid_attribute: "Invalid attribute in for loop. Valid attributes are limit and offset"

lib/liquid/standardfilters.rb

@@ -387,7 +387,7 @@ module Liquid
         end
       elsif ary.all? { |el| el.respond_to?(:[]) }
         begin
-          ary.sort { |a, b| nil_safe_compare(fetch_property(a, property), fetch_property(b, property)) }
+          ary.sort { |a, b| nil_safe_compare(a[property], b[property]) }
         rescue TypeError
           raise_property_error(property)
         end
@@ -416,7 +416,7 @@ module Liquid
         end
       elsif ary.all? { |el| el.respond_to?(:[]) }
         begin
-          ary.sort { |a, b| nil_safe_casecmp(fetch_property(a, property), fetch_property(b, property)) }
+          ary.sort { |a, b| nil_safe_casecmp(a[property], b[property]) }
         rescue TypeError
           raise_property_error(property)
         end
@@ -456,7 +456,7 @@ module Liquid
     #   Tests if any item in an array has a specific property value.
     # @liquid_description
     #   This requires you to provide both the property name and the associated value.
-    # @liquid_syntax array | some: string, string
+    # @liquid_syntax array | has: string, string
     # @liquid_return [boolean]
     def has(input, property, target_value = nil)
       filter_array(input, property, target_value, false) { |ary, &block| ary.any?(&block) }
@@ -504,7 +504,7 @@ module Liquid
         []
       else
         ary.uniq do |item|
-          fetch_property(item, property)
+          item[property]
         rescue TypeError
           raise_property_error(property)
         rescue NoMethodError
@@ -540,7 +540,7 @@ module Liquid
         if property == "to_liquid"
           e
         elsif e.respond_to?(:[])
-          r = fetch_property(e, property)
+          r = e[property]
           r.is_a?(Proc) ? r.call : r
         end
       end
@@ -564,7 +564,7 @@ module Liquid
         []
       else
         ary.reject do |item|
-          fetch_property(item, property).nil?
+          item[property].nil?
         rescue TypeError
           raise_property_error(property)
         rescue NoMethodError
@@ -950,7 +950,7 @@ module Liquid
         if property.nil?
           item
         elsif item.respond_to?(:[])
-          fetch_property(item, property)
+          item[property]
         else
           0
         end
@@ -976,9 +976,9 @@ module Liquid
 
       block.call(ary) do |item|
         if target_value.nil?
-          fetch_property(item, property)
+          item[property]
         else
-          fetch_property(item, property) == target_value
+          item[property] == target_value
         end
       rescue TypeError
         raise_property_error(property)
@@ -988,31 +988,6 @@ module Liquid
       end
     end
 
-    def fetch_property(drop, property_or_keys)
-      ##
-      # This keeps backward compatibility by supporting properties containing
-      # dots. This is valid in Liquid syntax and used in some runtimes, such as
-      # Shopify with metafields.
-      #
-      # Using this approach, properties like 'price.value' can be accessed in
-      # both of the following examples:
-      #
-      # ```
-      # [
-      #   { 'name' => 'Item 1', 'price.price' => 40000 },
-      #   { 'name' => 'Item 2', 'price' => { 'value' => 39900 } }
-      # ]
-      # ```
-      value = drop[property_or_keys]
-
-      return value if !value.nil? || !property_or_keys.is_a?(String)
-
-      keys = property_or_keys.split('.')
-      keys.reduce(drop) do |drop, key|
-        drop.respond_to?(:[]) ? drop[key] : drop
-      end
-    end
-
     def raise_property_error(property)
       raise Liquid::ArgumentError, "cannot select the property '#{property}'"
     end

lib/liquid/tags.rb

@@ -19,6 +19,7 @@ require_relative "tags/comment"
 require_relative "tags/raw"
 require_relative "tags/render"
 require_relative "tags/cycle"
+require_relative "tags/doc"
 
 module Liquid
   module Tags
@@ -42,6 +43,7 @@ module Liquid
       'if' => If,
       'echo' => Echo,
       'tablerow' => TableRow,
+      'doc' => Doc,
     }.freeze
   end
 end

lib/liquid/tags/doc.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Liquid
+  # @liquid_public_docs
+  # @liquid_type tag
+  # @liquid_category syntax
+  # @liquid_name doc
+  # @liquid_summary
+  #   Documents template elements with annotations.
+  # @liquid_description
+  #   The `doc` tag allows developers to include documentation within Liquid
+  #   templates. Any content inside `doc` tags is not rendered or outputted.
+  #   Liquid code inside will be parsed but not executed. This facilitates
+  #   tooling support for features like code completion, linting, and inline
+  #   documentation.
+  # @liquid_syntax
+  #   {% doc %}
+  #     Renders a message.
+  #
+  #     @param {string} foo - A foo value.
+  #     @param {string} [bar] - An optional bar value.
+  #
+  #     @example
+  #     {% render 'message', foo: 'Hello', bar: 'World' %}
+  #   {% enddoc %}
+  #   {{ foo }}, {{ bar }}!
+  class Doc < Block
+    NO_UNEXPECTED_ARGS = /\A\s*\z/
+
+    def initialize(tag_name, markup, parse_context)
+      super
+      ensure_valid_markup(tag_name, markup, parse_context)
+    end
+
+    def parse(tokens)
+      while (token = tokens.shift)
+        tag_name = token =~ BlockBody::FullTokenPossiblyInvalid && Regexp.last_match(2)
+
+        raise_nested_doc_error if tag_name == @tag_name
+
+        if tag_name == block_delimiter
+          parse_context.trim_whitespace = (token[-3] == WhitespaceControl)
+          return
+        end
+      end
+
+      raise_tag_never_closed(block_name)
+    end
+
+    def render_to_output_buffer(_context, output)
+      output
+    end
+
+    def blank?
+      true
+    end
+
+    private
+
+    def ensure_valid_markup(tag_name, markup, parse_context)
+      unless NO_UNEXPECTED_ARGS.match?(markup)
+        raise SyntaxError, parse_context.locale.t("errors.syntax.block_tag_unexpected_args", tag: tag_name)
+      end
+    end
+
+    def raise_nested_doc_error
+      raise SyntaxError, parse_context.locale.t("errors.syntax.doc_invalid_nested")
+    end
+  end
+end

lib/liquid/tokenizer.rb

@@ -103,6 +103,12 @@ module Liquid
 
       pos = @ss.pos -= 2
       @source.byteslice(start, pos - start)
+    rescue ::ArgumentError => e
+      if e.message == "invalid byte sequence in #{@ss.string.encoding}"
+        raise SyntaxError, "Invalid byte sequence in #{@ss.string.encoding}"
+      else
+        raise
+      end
     end
 
     def next_variable_token

lib/liquid/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module Liquid
-  VERSION = "5.7.1"
+  VERSION = "5.8.0"
 end

test/integration/standard_filter_test.rb

@@ -54,30 +54,6 @@ class TestEnumerable < Liquid::Drop
   end
 end
 
-class TestDeepEnumerable < Liquid::Drop
-  include Enumerable
-
-  class Product < Liquid::Drop
-    attr_reader :title, :price, :premium
-
-    def initialize(title:, price:, premium: nil)
-      @title = { "content" => title, "language" => "en" }
-      @price = { "value" => price, "unit" => "USD" }
-      @premium = { "category" => premium } if premium
-    end
-  end
-
-  def each(&block)
-    [
-      Product.new(title: "Pro goggles",    price: 1299),
-      Product.new(title: "Thermal gloves", price: 1299),
-      Product.new(title: "Alpine jacket",  price: 3999, premium: 'Basic'),
-      Product.new(title: "Mountain boots", price: 3899, premium: 'Pro'),
-      Product.new(title: "Safety helmet",  price: 1999)
-    ].each(&block)
-  end
-end
-
 class NumberLikeThing < Liquid::Drop
   def initialize(amount)
     @amount = amount
@@ -438,15 +414,6 @@ class StandardFiltersTest < Minitest::Test
     end
   end
 
-  def test_sort_natural_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | sort_natural: 'title.content' | map: 'title.content' | join: ', ' -}}
-    LIQUID
-    expected_output = "Alpine jacket, Mountain boots, Pro goggles, Safety helmet, Thermal gloves"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_legacy_sort_hash
     assert_equal([{ a: 1, b: 2 }], @filters.sort(a: 1, b: 2))
   end
@@ -483,15 +450,6 @@ class StandardFiltersTest < Minitest::Test
     end
   end
 
-  def test_uniq_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | uniq: 'price.value' | map: "title.content" | join: ', ' -}}
-    LIQUID
-    expected_output = "Pro goggles, Alpine jacket, Mountain boots, Safety helmet"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_compact_empty_array
     assert_equal([], @filters.compact([], "a"))
   end
@@ -508,15 +466,6 @@ class StandardFiltersTest < Minitest::Test
     end
   end
 
-  def test_compact_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | compact: 'premium.category' | map: 'title.content' | join: ', ' -}}
-    LIQUID
-    expected_output = "Alpine jacket, Mountain boots"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_reverse
     assert_equal([4, 3, 2, 1], @filters.reverse([1, 2, 3, 4]))
   end
@@ -626,15 +575,6 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result("213", '{{ foo | sort: "bar" | map: "foo" }}', { "foo" => TestEnumerable.new })
   end
 
-  def test_sort_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | sort: 'price.value' | map: 'title.content' | join: ', ' -}}
-    LIQUID
-    expected_output = "Pro goggles, Thermal gloves, Safety helmet, Mountain boots, Alpine jacket"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_first_and_last_call_to_liquid
     assert_template_result('foobar', '{{ foo | first }}', { 'foo' => [ThingWithToLiquid.new] })
     assert_template_result('foobar', '{{ foo | last }}', { 'foo' => [ThingWithToLiquid.new] })
@@ -951,15 +891,6 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result(expected_output, template, { "array" => array })
   end
 
-  def test_reject_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | reject: 'title.content', 'Pro goggles' | map: 'price.value' | join: ', ' -}}
-    LIQUID
-    expected_output = "1299, 3999, 3899, 1999"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_has
     array = [
       { "handle" => "alpha", "ok" => true },
@@ -1028,16 +959,6 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result(expected_output, template, { "array" => array })
   end
 
-  def test_has_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | has: 'title.content', 'Pro goggles' -}},
-      {{- products | has: 'title.content', 'foo' -}}
-    LIQUID
-    expected_output = "true,false"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_find_with_value
     products = [
       { "title" => "Pro goggles",    "price" => 1299 },
@@ -1056,16 +977,6 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result(expected_output, template, { "products" => products })
   end
 
-  def test_find_with_deep_enumerables
-    template = <<~LIQUID
-      {%- assign product = products | find: 'title.content', 'Pro goggles' -%}
-      {{- product.title.content -}}
-    LIQUID
-    expected_output = "Pro goggles"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_find_with_empty_arrays
     template = <<~LIQUID
       {%- assign product = products | find: 'title.content', 'Not found' -%}
@@ -1096,16 +1007,6 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result(expected_output, template, { "products" => products })
   end
 
-  def test_find_index_with_deep_enumerables
-    template = <<~LIQUID
-      {%- assign index = products | find_index: 'title.content', 'Alpine jacket' -%}
-      {{- index -}}
-    LIQUID
-    expected_output = "2"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_find_index_with_empty_arrays
     template = <<~LIQUID
       {%- assign index = products | find_index: 'title.content', 'Not found' -%}
@@ -1216,15 +1117,6 @@ class StandardFiltersTest < Minitest::Test
     assert_nil(@filters.where([nil], "ok"))
   end
 
-  def test_where_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | where: 'title.content', 'Pro goggles' | map: 'price.value' -}}
-    LIQUID
-    expected_output = "1299"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_all_filters_never_raise_non_liquid_exception
     test_drop = TestDrop.new(value: "test")
     test_drop.context = Context.new
@@ -1376,15 +1268,6 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result("0", "{{ input | sum: 'subtotal' }}", { "input" => input })
   end
 
-  def test_sum_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | sum: 'price.value' -}}
-    LIQUID
-    expected_output = "12495"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   private
 
   def with_timezone(tz)

test/unit/block_unit_test.rb

@@ -47,12 +47,18 @@ class BlockUnitTest < Minitest::Test
     )
   end
 
-  def test_with_block
+  def test_comment_tag_with_block
     template = Liquid::Template.parse("  {% comment %} {% endcomment %} ")
     assert_equal([String, Comment, String], block_types(template.root.nodelist))
     assert_equal(3, template.root.nodelist.size)
   end
 
+  def test_doc_tag_with_block
+    template = Liquid::Template.parse("  {% doc %} {% enddoc %} ")
+    assert_equal([String, Doc, String], block_types(template.root.nodelist))
+    assert_equal(3, template.root.nodelist.size)
+  end
+
   private
 
   def block_types(nodelist)

test/unit/lexer_unit_test.rb

@@ -131,6 +131,16 @@ class LexerUnitTest < Minitest::Test
     assert_equal([[:id, "false"], [:number, "1"], [:end_of_string]], tokenize("false 1"))
   end
 
+  def test_error_with_invalid_utf8
+    error = assert_raises(SyntaxError) do
+      tokenize("\x00\xff")
+    end
+    assert_equal(
+      'Liquid syntax error: Invalid byte sequence in UTF-8',
+      error.message,
+    )
+  end
+
   private
 
   def tokenize(input)

test/unit/tags/doc_tag_unit_test.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class DocTagUnitTest < Minitest::Test
+  def test_doc_tag
+    template = <<~LIQUID.chomp
+      {% doc %}
+        Renders loading-spinner.
+
+        @param {string} foo - some foo
+        @param {string} [bar] - optional bar
+
+        @example
+        {% render 'loading-spinner', foo: 'foo' %}
+        {% render 'loading-spinner', foo: 'foo', bar: 'bar' %}
+      {% enddoc %}
+    LIQUID
+
+    assert_template_result('', template)
+  end
+
+  def test_doc_tag_does_not_support_extra_arguments
+    error = assert_raises(Liquid::SyntaxError) do
+      template = <<~LIQUID.chomp
+        {% doc extra %}
+        {% enddoc %}
+      LIQUID
+
+      Liquid::Template.parse(template)
+    end
+
+    exp_error = "Liquid syntax error: Syntax Error in 'doc' - Valid syntax: {% doc %}{% enddoc %}"
+    act_error = error.message
+
+    assert_equal(exp_error, act_error)
+  end
+
+  def test_doc_tag_must_support_valid_tags
+    assert_match_syntax_error("Liquid syntax error (line 1): 'doc' tag was never closed", '{% doc %} foo')
+    assert_match_syntax_error("Liquid syntax error (line 1): Syntax Error in 'doc' - Valid syntax: {% doc %}{% enddoc %}", '{% doc } foo {% enddoc %}')
+    assert_match_syntax_error("Liquid syntax error (line 1): Syntax Error in 'doc' - Valid syntax: {% doc %}{% enddoc %}", '{% doc } foo %}{% enddoc %}')
+  end
+
+  def test_doc_tag_ignores_liquid_nodes
+    template = <<~LIQUID.chomp
+      {% doc %}
+        {% if true %}
+        {% if ... %}
+        {%- for ? -%}
+        {% while true %}
+        {%
+          unless if
+        %}
+        {% endcase %}
+      {% enddoc %}
+    LIQUID
+
+    assert_template_result('', template)
+  end
+
+  def test_doc_tag_ignores_unclosed_liquid_tags
+    template = <<~LIQUID.chomp
+      {% doc %}
+        {% if true %}
+      {% enddoc %}
+    LIQUID
+
+    assert_template_result('', template)
+  end
+
+  def test_doc_tag_does_not_allow_nested_docs
+    error = assert_raises(Liquid::SyntaxError) do
+      template = <<~LIQUID.chomp
+        {% doc %}
+          {% doc %}
+            {% doc %}
+        {% enddoc %}
+      LIQUID
+
+      Liquid::Template.parse(template)
+    end
+
+    exp_error = "Liquid syntax error: Syntax Error in 'doc' - Nested doc tags are not allowed"
+    act_error = error.message
+
+    assert_equal(exp_error, act_error)
+  end
+
+  def test_doc_tag_ignores_nested_raw_tags
+    template = <<~LIQUID.chomp
+      {% doc %}
+        {% raw %}
+      {% enddoc %}
+    LIQUID
+
+    assert_template_result('', template)
+  end
+
+  def test_doc_tag_ignores_unclosed_assign
+    template = <<~LIQUID.chomp
+      {% doc %}
+        {% assign foo = "1"
+      {% enddoc %}
+    LIQUID
+
+    assert_template_result('', template)
+  end
+
+  def test_doc_tag_ignores_malformed_syntax
+    template = <<~LIQUID.chomp
+      {% doc %}
+      {% {{ {%- enddoc %}
+    LIQUID
+
+    assert_template_result('', template)
+  end
+
+  def test_doc_tag_preserves_error_line_numbers
+    template = Liquid::Template.parse(<<~LIQUID.chomp, line_numbers: true)
+      {% doc %}
+        {% if true %}
+      {% enddoc %}
+      {{ errors.standard_error }}
+    LIQUID
+
+    expected = <<~TEXT.chomp
+
+      Liquid error (line 4): standard error
+    TEXT
+
+    assert_equal(expected, template.render('errors' => ErrorDrop.new))
+  end
+
+  def test_doc_tag_whitespace_control
+    # Basic whitespace control
+    assert_template_result("Hello!", "      {%- doc -%}123{%- enddoc -%}Hello!")
+    assert_template_result("Hello!", "{%- doc -%}123{%- enddoc -%}     Hello!")
+    assert_template_result("Hello!", "      {%- doc -%}123{%- enddoc -%}     Hello!")
+    assert_template_result("Hello!", <<~LIQUID.chomp)
+      {%- doc %}Whitespace control!{% enddoc -%}
+      Hello!
+    LIQUID
+  end
+
+  def test_doc_tag_delimiter_handling
+    assert_template_result('', <<~LIQUID.chomp)
+      {% if true %}
+        {% doc %}
+          {% docEXTRA %}wut{% enddocEXTRA %}xyz
+        {% enddoc %}
+      {% endif %}
+    LIQUID
+
+    assert_template_result('', "{% doc %}123{% enddoc xyz %}")
+    assert_template_result('', "{% doc %}123{% enddoc\txyz %}")
+    assert_template_result('', "{% doc %}123{% enddoc\nxyz %}")
+    assert_template_result('', "{% doc %}123{% enddoc\n   xyz  enddoc %}")
+  end
+end

test/unit/template_unit_test.rb

@@ -35,4 +35,15 @@ class TemplateUnitTest < Minitest::Test
   def test_template_inheritance
     assert_equal("foo", TemplateSubclass.parse("foo").render)
   end
+
+  def test_invalid_utf8
+    input = "\xff\x00"
+    error = assert_raises(SyntaxError) do
+      Liquid::Tokenizer.new(source: input, string_scanner: StringScanner.new(input))
+    end
+    assert_equal(
+      'Liquid syntax error: Invalid byte sequence in UTF-8',
+      error.message,
+    )
+  end
 end