Experiment: Hardcode parser to :strict everywhere

Add a link to the LiquidDoc tooling reference in `doc` tag documentation

.github/workflows/liquid.yml

@@ -14,8 +14,21 @@ jobs:
           - { ruby: 3.0, allowed-failure: false } # minimum supported
           - { ruby: 3.2, allowed-failure: false }
           - { ruby: 3.3, allowed-failure: false }
+          - { ruby: 3.3, allowed-failure: false }
           - { ruby: 3.4, allowed-failure: false } # latest
+          - {
+              ruby: 3.4,
+              allowed-failure: false,
+              rubyopt: "--enable-frozen-string-literal",
+            }
+          - { ruby: 3.4, allowed-failure: false, rubyopt: "--yjit" }
           - { ruby: ruby-head, allowed-failure: false }
+          - {
+              ruby: ruby-head,
+              allowed-failure: false,
+              rubyopt: "--enable-frozen-string-literal",
+            }
+          - { ruby: ruby-head, allowed-failure: false, rubyopt: "--yjit" }
     name: Test Ruby ${{ matrix.entry.ruby }}
     steps:
       - uses: actions/checkout@v3
@@ -26,6 +39,8 @@ jobs:
           bundler: latest
       - run: bundle exec rake
         continue-on-error: ${{ matrix.entry.allowed-failure }}
+        env:
+          RUBYOPT: ${{ matrix.entry.rubyopt }}
 
   memory_profile:
     runs-on: ubuntu-latest

History.md

@@ -1,22 +1,34 @@
 # Liquid Change Log
 
-## 5.8.0 (unreleased)
+## 5.8.1 (unreleased)
+
+## 5.8.1
+
+* Fix `{% doc %}` tag to be visitable [Guilherme Carreiro]
+
+## 5.8.0
+
+* Introduce the new `{% doc %}` tag [Guilherme Carreiro]
+
+## 5.7.3
+
+* Raise Liquid::SyntaxError when parsing invalidly encoded strings [Chris AtLee]
 
 ## 5.7.2 2025-01-31
 
-* Fix array filters to not support nested properties
+* Fix array filters to not support nested properties [Guilherme Carreiro]
 
 ## 5.7.1 2025-01-24
 
-* Fix the `find` and `find_index`filters to return `nil` when filtering empty arrays
-* Fix the `has` filter to return `false` when filtering empty arrays
+* Fix the `find` and `find_index`filters to return `nil` when filtering empty arrays [Guilherme Carreiro]
+* Fix the `has` filter to return `false` when filtering empty arrays [Guilherme Carreiro]
 
 ## 5.7.0 2025-01-16
 
 ### Features
 
-* Add `find`, `find_index`, `has`, and `reject` filters to arrays
-* Compatibility with Ruby 3.4
+* Add `find`, `find_index`, `has`, and `reject` filters to arrays [Guilherme Carreiro]
+* Compatibility with Ruby 3.4 [Ian Ker-Seymer]
 
 ## 5.6.4 2025-01-14
 

lib/liquid/environment.rb

@@ -4,10 +4,6 @@ module Liquid
   # The Environment is the container for all configuration options of Liquid, such as
   # the registered tags, filters, and the default error mode.
   class Environment
-    # The default error mode for all templates. This can be overridden on a
-    # per-template basis.
-    attr_accessor :error_mode
-
     # The tags that are available to use in the template.
     attr_accessor :tags
 
@@ -75,7 +71,7 @@ module Liquid
     # @api private
     def initialize
       @tags = Tags::STANDARD_TAGS.dup
-      @error_mode = :lax
+      @error_mode = :strict
       @strainer_template = Class.new(StrainerTemplate).tap do |klass|
         klass.add_filter(StandardFilters)
       end
@@ -85,6 +81,14 @@ module Liquid
       @strainer_template_class_cache = {}
     end
 
+    def error_mode
+      :strict
+    end
+
+    def error_mode=(mode)
+      :strict
+    end
+
     # Registers a new tag with the environment.
     #
     # @param name [String] The name of the tag.

lib/liquid/lexer.rb

@@ -161,6 +161,12 @@ module Liquid
         end
         # rubocop:enable Metrics/BlockNesting
         output << EOS
+      rescue ::ArgumentError => e
+        if e.message == "invalid byte sequence in #{ss.string.encoding}"
+          raise SyntaxError, "Invalid byte sequence in #{ss.string.encoding}"
+        else
+          raise
+        end
       end
 
       def raise_syntax_error(start_pos, ss)

lib/liquid/standardfilters.rb

@@ -456,7 +456,7 @@ module Liquid
     #   Tests if any item in an array has a specific property value.
     # @liquid_description
     #   This requires you to provide both the property name and the associated value.
-    # @liquid_syntax array | some: string, string
+    # @liquid_syntax array | has: string, string
     # @liquid_return [boolean]
     def has(input, property, target_value = nil)
       filter_array(input, property, target_value, false) { |ary, &block| ary.any?(&block) }
@@ -712,7 +712,16 @@ module Liquid
       input.gsub(/\r?\n/, "<br />\n")
     end
 
-    # Reformat a date using Ruby's core Time#strftime( string ) -> string
+    # @liquid_public_docs
+    # @liquid_type filter
+    # @liquid_category date
+    # @liquid_summary
+    #   Formats a date according to a specified format string.
+    # @liquid_description
+    #   This filter formats a date using various format specifiers. If the format string is empty,
+    #   the original input is returned. If the input cannot be converted to a date, the original input is returned.
+    #
+    #   The following format specifiers can be used:
     #
     #   %a - The abbreviated weekday name (``Sun'')
     #   %A - The  full  weekday  name (``Sunday'')
@@ -741,8 +750,8 @@ module Liquid
     #   %Y - Year with century
     #   %Z - Time zone name
     #   %% - Literal ``%'' character
-    #
-    #   See also: http://www.ruby-doc.org/core/Time.html#method-i-strftime
+    # @liquid_syntax date | date: string
+    # @liquid_return [string]
     def date(input, format)
       str_format = Utils.to_s(format)
       return input if str_format.empty?

lib/liquid/tags/doc.rb

@@ -13,12 +13,16 @@ module Liquid
   #   Liquid code inside will be parsed but not executed. This facilitates
   #   tooling support for features like code completion, linting, and inline
   #   documentation.
+  #
+  #   For detailed documentation syntax and examples, see the
+  #   [`LiquidDoc` reference](/docs/storefronts/themes/tools/liquid-doc).
+  #
   # @liquid_syntax
   #   {% doc %}
   #     Renders a message.
   #
-  #     @param {string} foo - A foo value.
-  #     @param {string} [bar] - An optional bar value.
+  #     @param {string} foo - A string value.
+  #     @param {string} [bar] - An optional string value.
   #
   #     @example
   #     {% render 'message', foo: 'Hello', bar: 'World' %}
@@ -55,6 +59,10 @@ module Liquid
       true
     end
 
+    def nodelist
+      []
+    end
+
     private
 
     def ensure_valid_markup(tag_name, markup, parse_context)

lib/liquid/template.rb

@@ -27,11 +27,11 @@ module Liquid
       # :strict will enforce correct syntax.
       def error_mode=(mode)
         Deprecations.warn("Template.error_mode=", "Environment#error_mode=")
-        Environment.default.error_mode = mode
+        Environment.default.error_mode = :strict
       end
 
       def error_mode
-        Environment.default.error_mode
+        :strict
       end
 
       def default_exception_renderer=(renderer)

lib/liquid/tokenizer.rb

@@ -103,6 +103,12 @@ module Liquid
 
       pos = @ss.pos -= 2
       @source.byteslice(start, pos - start)
+    rescue ::ArgumentError => e
+      if e.message == "invalid byte sequence in #{@ss.string.encoding}"
+        raise SyntaxError, "Invalid byte sequence in #{@ss.string.encoding}"
+      else
+        raise
+      end
     end
 
     def next_variable_token

lib/liquid/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module Liquid
-  VERSION = "5.7.2"
+  VERSION = "5.8.1"
 end

test/unit/lexer_unit_test.rb

@@ -131,6 +131,16 @@ class LexerUnitTest < Minitest::Test
     assert_equal([[:id, "false"], [:number, "1"], [:end_of_string]], tokenize("false 1"))
   end
 
+  def test_error_with_invalid_utf8
+    error = assert_raises(SyntaxError) do
+      tokenize("\x00\xff")
+    end
+    assert_equal(
+      'Liquid syntax error: Invalid byte sequence in UTF-8',
+      error.message,
+    )
+  end
+
   private
 
   def tokenize(input)

test/unit/tags/doc_tag_unit_test.rb

@@ -157,4 +157,28 @@ class DocTagUnitTest < Minitest::Test
     assert_template_result('', "{% doc %}123{% enddoc\nxyz %}")
     assert_template_result('', "{% doc %}123{% enddoc\n   xyz  enddoc %}")
   end
+
+  def test_doc_tag_visitor
+    template_source = '{% doc %}{% enddoc %}'
+
+    assert_equal(
+      [Liquid::Doc],
+      visit(template_source),
+    )
+  end
+
+  private
+
+  def traversal(template)
+    ParseTreeVisitor
+      .for(Template.parse(template).root)
+      .add_callback_for(Liquid::Doc) do |tag|
+        tag_class = tag.class
+        tag_class
+      end
+  end
+
+  def visit(template)
+    traversal(template).visit.flatten.compact
+  end
 end

test/unit/template_unit_test.rb

@@ -35,4 +35,15 @@ class TemplateUnitTest < Minitest::Test
   def test_template_inheritance
     assert_equal("foo", TemplateSubclass.parse("foo").render)
   end
+
+  def test_invalid_utf8
+    input = "\xff\x00"
+    error = assert_raises(SyntaxError) do
+      Liquid::Tokenizer.new(source: input, string_scanner: StringScanner.new(input))
+    end
+    assert_equal(
+      'Liquid syntax error: Invalid byte sequence in UTF-8',
+      error.message,
+    )
+  end
 end