more profile cleanups

Co-authored-by: Copilot <[email protected]>

.github/dependabot.yaml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: weekly

.github/workflows/liquid.yml

@@ -14,24 +14,39 @@ jobs:
           - { ruby: 3.0, allowed-failure: false } # minimum supported
           - { ruby: 3.2, allowed-failure: false }
           - { ruby: 3.3, allowed-failure: false }
+          - { ruby: 3.3, allowed-failure: false }
           - { ruby: 3.4, allowed-failure: false } # latest
+          - {
+              ruby: 3.4,
+              allowed-failure: false,
+              rubyopt: "--enable-frozen-string-literal",
+            }
+          - { ruby: 3.4, allowed-failure: false, rubyopt: "--yjit" }
           - { ruby: ruby-head, allowed-failure: false }
+          - {
+              ruby: ruby-head,
+              allowed-failure: false,
+              rubyopt: "--enable-frozen-string-literal",
+            }
+          - { ruby: ruby-head, allowed-failure: false, rubyopt: "--yjit" }
     name: Test Ruby ${{ matrix.entry.ruby }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: ruby/setup-ruby@v1
+      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      - uses: ruby/setup-ruby@dffc446db9ba5a0c4446edb5bca1c5c473a806c5 # v1.235.0
         with:
           ruby-version: ${{ matrix.entry.ruby }}
           bundler-cache: true
           bundler: latest
       - run: bundle exec rake
         continue-on-error: ${{ matrix.entry.allowed-failure }}
+        env:
+          RUBYOPT: ${{ matrix.entry.rubyopt }}
 
   memory_profile:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: ruby/setup-ruby@v1
+      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      - uses: ruby/setup-ruby@dffc446db9ba5a0c4446edb5bca1c5c473a806c5 # v1.235.0
         with:
           bundler-cache: true
       - run: bundle exec rake memory_profile:run

.ruby-version

@@ -1 +1 @@
-3.3.6
+3.4.1

CONTRIBUTING.md

@@ -29,6 +29,8 @@
 
 ## Releasing
 
-* Bump the version in `lib/liquid/version.rb` and merge it on `main`
+* Bump the version in `lib/liquid/version.rb`
+* Update the `History.md` file
+* Open a PR like [this one](https://github.com/Shopify/liquid/pull/1894) and merge it to `main`
 * Create a new release using the [GitHub UI](https://github.com/Shopify/liquid/releases/new)
 

Gemfile

@@ -20,6 +20,10 @@ group :benchmark, :test do
   end
 end
 
+group :development do
+  gem "webrick"
+end
+
 group :test do
   gem 'rubocop', '~> 1.61.0'
   gem 'rubocop-shopify', '~> 2.12.0', require: false

History.md

@@ -1,12 +1,77 @@
 # Liquid Change Log
 
-## 5.6.0 (unreleased)
+## 5.8.7
+* Expose body content in the `Doc` tag [James Meng]
+
+## 5.8.1
+
+* Fix `{% doc %}` tag to be visitable [Guilherme Carreiro]
+
+## 5.8.0
+
+* Introduce the new `{% doc %}` tag [Guilherme Carreiro]
+
+## 5.7.3
+
+* Raise Liquid::SyntaxError when parsing invalidly encoded strings [Chris AtLee]
+
+## 5.7.2 2025-01-31
+
+* Fix array filters to not support nested properties [Guilherme Carreiro]
+
+## 5.7.1 2025-01-24
+
+* Fix the `find` and `find_index`filters to return `nil` when filtering empty arrays [Guilherme Carreiro]
+* Fix the `has` filter to return `false` when filtering empty arrays [Guilherme Carreiro]
+
+## 5.7.0 2025-01-16
+
+### Features
+
+* Add `find`, `find_index`, `has`, and `reject` filters to arrays [Guilherme Carreiro]
+* Compatibility with Ruby 3.4 [Ian Ker-Seymer]
+
+## 5.6.4 2025-01-14
 
 ### Fixes
+* Add a default `string_scanner` to avoid errors with `Liquid::VariableLookup.parse("foo.bar")` [Ian Ker-Seymer]
 
+## 5.6.3 2025-01-13
+* Remove `lru_redux` dependency [Michael Go]
+
+## 5.6.2 2025-01-13
+
+### Fixes
+* Preserve the old behavior of requiring floats to start with a digit [Michael Go]
+
+## 5.6.1 2025-01-13
+
+### Performance improvements
+* Faster Expression parser / Tokenizer with StringScanner [Michael Go]
+
+## 5.6.0 2024-12-19
+
+### Architectural changes
+* Added new `Environment` class to manage configuration and state that was previously stored in `Template` [Ian Ker-Seymer]
+* Moved tag registration from `Template` to `Environment` [Ian Ker-Seymer]
+* Removed `StrainerFactory` in favor of `Environment`-based strainer creation [Ian Ker-Seymer]
+* Consolidated standard tags into a new `Tags` module with `STANDARD_TAGS` constant [Ian Ker-Seymer]
+
+### Performance improvements
+* Optimized `Lexer` with a new `Lexer2` implementation using jump tables for faster tokenization, requires Ruby 3.4 [Ian Ker-Seymer]
+* Improved variable rendering with specialized handling for different types [Michael Go]
+* Reduced array allocations by using frozen empty constants [Michael Go]
+
+### API changes
+* Deprecated several `Template` class methods in favor of `Environment` methods [Ian Ker-Seymer]
+* Added deprecation warnings system [Ian Ker-Seymer]
+* Changed how filters and tags are registered to use Environment [Ian Ker-Seymer]
+
+### Fixes
+* Fixed table row handling of break interrupts [Alex Coco]
+* Improved variable output handling for arrays [Ian Ker-Seymer]
 * Fix Tokenizer to handle null source value (#1873) [Bahar Pourazar]
 
-
 ## 5.5.0 2024-03-21
 
 Please reference the GitHub release for more information.

lib/liquid/lexer.rb

@@ -161,6 +161,12 @@ module Liquid
         end
         # rubocop:enable Metrics/BlockNesting
         output << EOS
+      rescue ::ArgumentError => e
+        if e.message == "invalid byte sequence in #{ss.string.encoding}"
+          raise SyntaxError, "Invalid byte sequence in #{ss.string.encoding}"
+        else
+          raise
+        end
       end
 
       def raise_syntax_error(start_pos, ss)

lib/liquid/locales/en.yml

@@ -2,12 +2,14 @@
   errors:
     syntax:
       tag_unexpected_args: "Syntax Error in '%{tag}' - Valid syntax: %{tag}"
+      block_tag_unexpected_args: "Syntax Error in '%{tag}' - Valid syntax: {% %{tag} %}{% end%{tag} %}"
       assign: "Syntax Error in 'assign' - Valid syntax: assign [var] = [source]"
       capture: "Syntax Error in 'capture' - Valid syntax: capture [var]"
       case: "Syntax Error in 'case' - Valid syntax: case [condition]"
       case_invalid_when: "Syntax Error in tag 'case' - Valid when condition: {% when [condition] [or condition2...] %}"
       case_invalid_else: "Syntax Error in tag 'case' - Valid else condition: {% else %} (no parameters) "
       cycle: "Syntax Error in 'cycle' - Valid syntax: cycle [name :] var [, var2, var3 ...]"
+      doc_invalid_nested: "Syntax Error in 'doc' - Nested doc tags are not allowed"
       for: "Syntax Error in 'for loop' - Valid syntax: for [item] in [collection]"
       for_invalid_in: "For loops require an 'in' clause"
       for_invalid_attribute: "Invalid attribute in for loop. Valid attributes are limit and offset"

lib/liquid/standardfilters.rb

@@ -3,7 +3,6 @@
 require 'cgi'
 require 'base64'
 require 'bigdecimal'
-
 module Liquid
   module StandardFilters
     MAX_I32 = (1 << 31) - 1
@@ -387,7 +386,7 @@ module Liquid
         end
       elsif ary.all? { |el| el.respond_to?(:[]) }
         begin
-          ary.sort { |a, b| nil_safe_compare(fetch_property(a, property), fetch_property(b, property)) }
+          ary.sort { |a, b| nil_safe_compare(a[property], b[property]) }
         rescue TypeError
           raise_property_error(property)
         end
@@ -416,7 +415,7 @@ module Liquid
         end
       elsif ary.all? { |el| el.respond_to?(:[]) }
         begin
-          ary.sort { |a, b| nil_safe_casecmp(fetch_property(a, property), fetch_property(b, property)) }
+          ary.sort { |a, b| nil_safe_casecmp(a[property], b[property]) }
         rescue TypeError
           raise_property_error(property)
         end
@@ -456,10 +455,10 @@ module Liquid
     #   Tests if any item in an array has a specific property value.
     # @liquid_description
     #   This requires you to provide both the property name and the associated value.
-    # @liquid_syntax array | some: string, string
+    # @liquid_syntax array | has: string, string
     # @liquid_return [boolean]
     def has(input, property, target_value = nil)
-      filter_array(input, property, target_value) { |ary, &block| ary.any?(&block) }
+      filter_array(input, property, target_value, false) { |ary, &block| ary.any?(&block) }
     end
 
     # @liquid_public_docs
@@ -472,7 +471,7 @@ module Liquid
     # @liquid_syntax array | find: string, string
     # @liquid_return [untyped]
     def find(input, property, target_value = nil)
-      filter_array(input, property, target_value) { |ary, &block| ary.find(&block) }
+      filter_array(input, property, target_value, nil) { |ary, &block| ary.find(&block) }
     end
 
     # @liquid_public_docs
@@ -485,7 +484,7 @@ module Liquid
     # @liquid_syntax array | find_index: string, string
     # @liquid_return [number]
     def find_index(input, property, target_value = nil)
-      filter_array(input, property, target_value) { |ary, &block| ary.find_index(&block) }
+      filter_array(input, property, target_value, nil) { |ary, &block| ary.find_index(&block) }
     end
 
     # @liquid_public_docs
@@ -504,7 +503,7 @@ module Liquid
         []
       else
         ary.uniq do |item|
-          fetch_property(item, property)
+          item[property]
         rescue TypeError
           raise_property_error(property)
         rescue NoMethodError
@@ -540,7 +539,7 @@ module Liquid
         if property == "to_liquid"
           e
         elsif e.respond_to?(:[])
-          r = fetch_property(e, property)
+          r = e[property]
           r.is_a?(Proc) ? r.call : r
         end
       end
@@ -564,7 +563,7 @@ module Liquid
         []
       else
         ary.reject do |item|
-          fetch_property(item, property).nil?
+          item[property].nil?
         rescue TypeError
           raise_property_error(property)
         rescue NoMethodError
@@ -712,7 +711,16 @@ module Liquid
       input.gsub(/\r?\n/, "<br />\n")
     end
 
-    # Reformat a date using Ruby's core Time#strftime( string ) -> string
+    # @liquid_public_docs
+    # @liquid_type filter
+    # @liquid_category date
+    # @liquid_summary
+    #   Formats a date according to a specified format string.
+    # @liquid_description
+    #   This filter formats a date using various format specifiers. If the format string is empty,
+    #   the original input is returned. If the input cannot be converted to a date, the original input is returned.
+    #
+    #   The following format specifiers can be used:
     #
     #   %a - The abbreviated weekday name (``Sun'')
     #   %A - The  full  weekday  name (``Sunday'')
@@ -741,8 +749,8 @@ module Liquid
     #   %Y - Year with century
     #   %Z - Time zone name
     #   %% - Literal ``%'' character
-    #
-    #   See also: http://www.ruby-doc.org/core/Time.html#method-i-strftime
+    # @liquid_syntax date | date: string
+    # @liquid_return [string]
     def date(input, format)
       str_format = Utils.to_s(format)
       return input if str_format.empty?
@@ -950,7 +958,7 @@ module Liquid
         if property.nil?
           item
         elsif item.respond_to?(:[])
-          fetch_property(item, property)
+          item[property]
         else
           0
         end
@@ -969,16 +977,16 @@ module Liquid
 
     attr_reader :context
 
-    def filter_array(input, property, target_value, &block)
+    def filter_array(input, property, target_value, default_value = [], &block)
       ary = InputIterator.new(input, context)
 
-      return [] if ary.empty?
+      return default_value if ary.empty?
 
       block.call(ary) do |item|
         if target_value.nil?
-          fetch_property(item, property)
+          item[property]
         else
-          fetch_property(item, property) == target_value
+          item[property] == target_value
         end
       rescue TypeError
         raise_property_error(property)
@@ -988,33 +996,8 @@ module Liquid
       end
     end
 
-    def fetch_property(drop, property_or_keys)
-      ##
-      # This keeps backward compatibility by supporting properties containing
-      # dots. This is valid in Liquid syntax and used in some runtimes, such as
-      # Shopify with metafields.
-      #
-      # Using this approach, properties like 'price.value' can be accessed in
-      # both of the following examples:
-      #
-      # ```
-      # [
-      #   { 'name' => 'Item 1', 'price.price' => 40000 },
-      #   { 'name' => 'Item 2', 'price' => { 'value' => 39900 } }
-      # ]
-      # ```
-      value = drop[property_or_keys]
-
-      return value if !value.nil? || !property_or_keys.is_a?(String)
-
-      keys = property_or_keys.split('.')
-      keys.reduce(drop) do |drop, key|
-        drop.respond_to?(:[]) ? drop[key] : drop
-      end
-    end
-
     def raise_property_error(property)
-      raise Liquid::ArgumentError, "cannot select the property '#{property}'"
+      raise Liquid::ArgumentError, "cannot select the property '#{Utils.to_s(property)}'"
     end
 
     def apply_operation(input, operand, operation)
@@ -1063,7 +1046,18 @@ module Liquid
       end
 
       def join(glue)
-        to_a.join(glue.to_s)
+        first = true
+        output = +""
+        each do |item|
+          if first
+            first = false
+          else
+            output << glue
+          end
+
+          output << Liquid::Utils.to_s(item)
+        end
+        output
       end
 
       def concat(args)
@@ -1075,7 +1069,10 @@ module Liquid
       end
 
       def uniq(&block)
-        to_a.uniq(&block)
+        to_a.uniq do |item|
+          item = Utils.to_liquid_value(item)
+          block ? yield(item) : item
+        end
       end
 
       def compact

lib/liquid/tags.rb

@@ -19,6 +19,7 @@ require_relative "tags/comment"
 require_relative "tags/raw"
 require_relative "tags/render"
 require_relative "tags/cycle"
+require_relative "tags/doc"
 
 module Liquid
   module Tags
@@ -42,6 +43,7 @@ module Liquid
       'if' => If,
       'echo' => Echo,
       'tablerow' => TableRow,
+      'doc' => Doc,
     }.freeze
   end
 end

lib/liquid/tags/assign.rb

@@ -9,6 +9,10 @@ module Liquid
   #   Creates a new variable.
   # @liquid_description
   #   You can create variables of any [basic type](/docs/api/liquid/basics#types), [object](/docs/api/liquid/objects), or object property.
+  #
+  #   > Caution:
+  #   > Predefined Liquid objects can be overridden by variables with the same name.
+  #   > To make sure that you can access all Liquid objects, make sure that your variable name doesn't match a predefined object's name.
   # @liquid_syntax
   #   {% assign variable_name = value %}
   # @liquid_syntax_keyword variable_name The name of the variable being created.

lib/liquid/tags/capture.rb

@@ -9,6 +9,10 @@ module Liquid
   #   Creates a new variable with a string value.
   # @liquid_description
   #   You can create complex strings with Liquid logic and variables.
+  #
+  #   > Caution:
+  #   > Predefined Liquid objects can be overridden by variables with the same name.
+  #   > To make sure that you can access all Liquid objects, make sure that your variable name doesn't match a predefined object's name.
   # @liquid_syntax
   #   {% capture variable %}
   #     value

lib/liquid/tags/decrement.rb

@@ -7,10 +7,14 @@ module Liquid
   # @liquid_name decrement
   # @liquid_summary
   #   Creates a new variable, with a default value of -1, that's decreased by 1 with each subsequent call.
+  #
+  #   > Caution:
+  #   > Predefined Liquid objects can be overridden by variables with the same name.
+  #   > To make sure that you can access all Liquid objects, make sure that your variable name doesn't match a predefined object's name.
   # @liquid_description
   #   Variables that are declared with `decrement` are unique to the [layout](/themes/architecture/layouts), [template](/themes/architecture/templates),
   #   or [section](/themes/architecture/sections) file that they're created in. However, the variable is shared across
-  #   [snippets](/themes/architecture#snippets) included in the file.
+  #   [snippets](/themes/architecture/snippets) included in the file.
   #
   #   Similarly, variables that are created with `decrement` are independent from those created with [`assign`](/docs/api/liquid/tags/assign)
   #   and [`capture`](/docs/api/liquid/tags/capture). However, `decrement` and [`increment`](/docs/api/liquid/tags/increment) share

lib/liquid/tags/doc.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Liquid
+  # @liquid_public_docs
+  # @liquid_type tag
+  # @liquid_category syntax
+  # @liquid_name doc
+  # @liquid_summary
+  #   Documents template elements with annotations.
+  # @liquid_description
+  #   The `doc` tag allows developers to include documentation within Liquid
+  #   templates. Any content inside `doc` tags is not rendered or outputted.
+  #   Liquid code inside will be parsed but not executed. This facilitates
+  #   tooling support for features like code completion, linting, and inline
+  #   documentation.
+  #
+  #   For detailed documentation syntax and examples, see the
+  #   [`LiquidDoc` reference](/docs/storefronts/themes/tools/liquid-doc).
+  #
+  # @liquid_syntax
+  #   {% doc %}
+  #     Renders a message.
+  #
+  #     @param {string} foo - A string value.
+  #     @param {string} [bar] - An optional string value.
+  #
+  #     @example
+  #     {% render 'message', foo: 'Hello', bar: 'World' %}
+  #   {% enddoc %}
+  class Doc < Block
+    NO_UNEXPECTED_ARGS = /\A\s*\z/
+
+    def initialize(tag_name, markup, parse_context)
+      super
+      ensure_valid_markup(tag_name, markup, parse_context)
+    end
+
+    def parse(tokens)
+      @body = +""
+
+      while (token = tokens.shift)
+        tag_name = token =~ BlockBody::FullTokenPossiblyInvalid && Regexp.last_match(2)
+
+        raise_nested_doc_error if tag_name == @tag_name
+
+        if tag_name == block_delimiter
+          parse_context.trim_whitespace = (token[-3] == WhitespaceControl)
+          @body << Regexp.last_match(1) if Regexp.last_match(1) != ""
+          return
+        end
+        @body << token unless token.empty?
+      end
+
+      raise_tag_never_closed(block_name)
+    end
+
+    def render_to_output_buffer(_context, output)
+      output
+    end
+
+    def blank?
+      @body.empty?
+    end
+
+    def nodelist
+      [@body]
+    end
+
+    private
+
+    def ensure_valid_markup(tag_name, markup, parse_context)
+      unless NO_UNEXPECTED_ARGS.match?(markup)
+        raise SyntaxError, parse_context.locale.t("errors.syntax.block_tag_unexpected_args", tag: tag_name)
+      end
+    end
+
+    def raise_nested_doc_error
+      raise SyntaxError, parse_context.locale.t("errors.syntax.doc_invalid_nested")
+    end
+  end
+end

lib/liquid/tags/include.rb

@@ -6,7 +6,7 @@ module Liquid
   # @liquid_category theme
   # @liquid_name include
   # @liquid_summary
-  #   Renders a [snippet](/themes/architecture#snippets).
+  #   Renders a [snippet](/themes/architecture/snippets).
   # @liquid_description
   #   Inside the snippet, you can access and alter variables that are [created](/docs/api/liquid/tags/variable-tags) outside of the
   #   snippet.

lib/liquid/tags/increment.rb

@@ -7,10 +7,14 @@ module Liquid
   # @liquid_name increment
   # @liquid_summary
   #   Creates a new variable, with a default value of 0, that's increased by 1 with each subsequent call.
+  #
+  #   > Caution:
+  #   > Predefined Liquid objects can be overridden by variables with the same name.
+  #   > To make sure that you can access all Liquid objects, make sure that your variable name doesn't match a predefined object's name.
   # @liquid_description
   #   Variables that are declared with `increment` are unique to the [layout](/themes/architecture/layouts), [template](/themes/architecture/templates),
   #   or [section](/themes/architecture/sections) file that they're created in. However, the variable is shared across
-  #   [snippets](/themes/architecture#snippets) included in the file.
+  #   [snippets](/themes/architecture/snippets) included in the file.
   #
   #   Similarly, variables that are created with `increment` are independent from those created with [`assign`](/docs/api/liquid/tags/assign)
   #   and [`capture`](/docs/api/liquid/tags/capture). However, `increment` and [`decrement`](/docs/api/liquid/tags/decrement) share

lib/liquid/tags/render.rb

@@ -6,7 +6,7 @@ module Liquid
   # @liquid_category theme
   # @liquid_name render
   # @liquid_summary
-  #   Renders a [snippet](/themes/architecture#snippets) or [app block](/themes/architecture/sections/section-schema#render-app-blocks).
+  #   Renders a [snippet](/themes/architecture/snippets) or [app block](/themes/architecture/sections/section-schema#render-app-blocks).
   # @liquid_description
   #   Inside snippets and app blocks, you can't directly access variables that are [created](/docs/api/liquid/tags/variable-tags) outside
   #   of the snippet or app block. However, you can [specify variables as parameters](/docs/api/liquid/tags/render#render-passing-variables-to-a-snippet)

lib/liquid/tokenizer.rb

@@ -103,6 +103,12 @@ module Liquid
 
       pos = @ss.pos -= 2
       @source.byteslice(start, pos - start)
+    rescue ::ArgumentError => e
+      if e.message == "invalid byte sequence in #{@ss.string.encoding}"
+        raise SyntaxError, "Invalid byte sequence in #{@ss.string.encoding}"
+      else
+        raise
+      end
     end
 
     def next_variable_token

lib/liquid/utils.rb

@@ -90,34 +90,38 @@ module Liquid
       obj
     end
 
-    if RUBY_VERSION >= '3.4'
-      def self.to_s(obj, seen = {})
-        case obj
-        when Hash
+    def self.to_s(obj, seen = {})
+      case obj
+      when Hash
+        # If the custom hash implementation overrides `#to_s`, use their
+        # custom implementation. Otherwise we use Liquid's default
+        # implementation.
+        if obj.class.instance_method(:to_s) == HASH_TO_S_METHOD
           hash_inspect(obj, seen)
-        when Array
-          array_inspect(obj, seen)
         else
           obj.to_s
         end
+      when Array
+        array_inspect(obj, seen)
+      else
+        obj.to_s
       end
+    end
 
-      def self.inspect(obj, seen = {})
-        case obj
-        when Hash
+    def self.inspect(obj, seen = {})
+      case obj
+      when Hash
+        # If the custom hash implementation overrides `#inspect`, use their
+        # custom implementation. Otherwise we use Liquid's default
+        # implementation.
+        if obj.class.instance_method(:inspect) == HASH_INSPECT_METHOD
           hash_inspect(obj, seen)
-        when Array
-          array_inspect(obj, seen)
         else
           obj.inspect
         end
-      end
-    else
-      def self.to_s(obj, seen = nil)
-        obj.to_s
-      end
-
-      def self.inspect(obj, seen = nil)
+      when Array
+        array_inspect(obj, seen)
+      else
         obj.inspect
       end
     end
@@ -175,5 +179,11 @@ module Liquid
     ensure
       seen.delete(hash.object_id)
     end
+
+    HASH_TO_S_METHOD = Hash.instance_method(:to_s)
+    private_constant :HASH_TO_S_METHOD
+
+    HASH_INSPECT_METHOD = Hash.instance_method(:inspect)
+    private_constant :HASH_INSPECT_METHOD
   end
 end

lib/liquid/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module Liquid
-  VERSION = "5.6.5"
+  VERSION = "5.8.7"
 end

performance/benchmark.rb

@@ -3,7 +3,13 @@
 require 'benchmark/ips'
 require_relative 'theme_runner'
 
-RubyVM::YJIT.enable if defined?(RubyVM::YJIT)
+if defined?(RubyVM::YJIT)
+  RubyVM::YJIT.enable
+  puts "* YJIT enabled"
+else
+  puts "* YJIT not enabled"
+end
+
 Liquid::Environment.default.error_mode = ARGV.first.to_sym if ARGV.first
 
 profiler = ThemeRunner.new
@@ -18,8 +24,8 @@ Benchmark.ips do |x|
 
   phase = ENV["PHASE"] || "all"
 
-  x.report("tokenize:") { profiler.tokenize } if phase == "all" || phase == "tokenize"
-  x.report("parse:") { profiler.compile } if phase == "all" || phase == "parse"
-  x.report("render:") { profiler.render } if phase == "all" || phase == "render"
-  x.report("parse & render:") { profiler.run } if phase == "all" || phase == "run"
+  x.report("tokenize:") { profiler.tokenize_all } if phase == "all" || phase == "tokenize"
+  x.report("parse:") { profiler.compile_all } if phase == "all" || phase == "parse"
+  x.report("render:") { profiler.render_all } if phase == "all" || phase == "render"
+  x.report("parse & render:") { profiler.run_all } if phase == "all" || phase == "run"
 end

performance/memory_profile.rb

@@ -57,7 +57,7 @@ Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
 
 runner = ThemeRunner.new
 Profiler.run do |x|
-  x.profile('parse') { runner.compile }
-  x.profile('render') { runner.render }
+  x.profile('parse') { runner.compile_all }
+  x.profile('render') { runner.render_all }
   x.tabulate
 end

performance/profile.rb

@@ -1,26 +1,42 @@
 # frozen_string_literal: true
 
 require 'stackprof'
+require 'fileutils'
 require_relative 'theme_runner'
 
+output_dir = ENV['OUTPUT_DIR'] || "/tmp/liquid-performance"
+FileUtils.mkdir_p(output_dir)
+
 Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
 profiler = ThemeRunner.new
-profiler.run
+profiler.run_all # warmup
 
-[:cpu, :object].each do |profile_type|
-  puts "Profiling in #{profile_type} mode..."
-  results = StackProf.run(mode: profile_type) do
+[:cpu, :object].each do |mode|
+  puts
+  puts "Profiling in #{mode} mode..."
+  puts "writing to #{output_dir}/#{mode}.profile:"
+  puts
+  StackProf.run(mode: mode, raw: true, out: "#{output_dir}/#{mode}.profile") do
     200.times do
-      profiler.run
+      profiler.run_all
     end
   end
 
-  if profile_type == :cpu && (graph_filename = ENV['GRAPH_FILENAME'])
-    File.open(graph_filename, 'w') do |f|
-      StackProf::Report.new(results).print_graphviz(nil, f)
+  result = StackProf.run(mode: mode) do
+    100.times do
+      profiler.run_all
     end
   end
 
-  StackProf::Report.new(results).print_text(false, 20)
-  File.write(ENV['FILENAME'] + "." + profile_type.to_s, Marshal.dump(results)) if ENV['FILENAME']
+  StackProf::Report.new(result).print_text(false, 30)
 end
+
+puts
+puts "files in #{output_dir}:"
+Dir.glob("#{output_dir}/*").each do |file|
+  puts "  #{file}"
+end
+puts "Recommended:"
+puts "  stackprof --d3-flamegraph #{output_dir}/cpu.profile > #{output_dir}/flame.html"
+puts "  stackprof --method #{output_dir}/cpu.profile"
+puts "  etc"

performance/shopify/database.rb

@@ -32,19 +32,36 @@ module Database
         end
       end
 
-      # Some standard direct accessors so that the specialized templates
-      # render correctly
-      db['collection'] = db['collections'].values.first
       db['product']    = db['products'].values.first
       db['blog']       = db['blogs'].values.first
       db['article']    = db['blog']['articles'].first
 
-      db['cart']       = {
+      # Some standard direct accessors so that the specialized templates
+      # render correctly
+      db['collection'] = db['collections'].values.first
+      db['collection']['tags'] = db['collection']['products'].map { |product| product['tags'] }.flatten.uniq.sort
+
+      db['tags'] = db['collection']['tags'][0..1]
+      db['all_tags'] = db['products'].values.map { |product| product['tags'] }.flatten.uniq.sort
+      db['current_tags'] = db['collection']['tags'][0..1]
+      db['handle'] = db['collection']['handle']
+
+      db['cart'] = {
         'total_price' => db['line_items'].values.inject(0) { |sum, item| sum + item['line_price'] * item['quantity'] },
         'item_count' => db['line_items'].values.inject(0) { |sum, item| sum + item['quantity'] },
         'items' => db['line_items'].values,
       }
 
+      db['linklists'] = db['link_lists']
+
+      db['shop'] = {
+        'name' => 'Snowdevil',
+        'currency' => 'USD',
+        'money_format' => '${{amount}}',
+        'money_with_currency_format' => '${{amount}} USD',
+        'money_format_with_currency' => 'USD ${{amount}}',
+      }
+
       db
     end
   end

performance/shopify/vision.database.yml

@@ -345,8 +345,7 @@ products:
     featured_image: products/arbor_draft.jpg
     images:
       - products/arbor_draft.jpg
-    description:
-      The Arbor Draft snowboard wouldn't exist if Polynesians hadn't figured out how to surf hundreds of years ago. But the Draft does exist, and it's here to bring your urban and park riding to a new level. The board's freaky Tiki design pays homage to culture that inspired snowboarding. It's designed to spin with ease, land smoothly, lock hook-free onto rails, and take the abuse of a pavement pounding or twelve. The Draft will pop off kickers with authority and carve solidly across the pipe. The Draft features targeted Koa wood die cuts inlayed into the deck that enhance the flex pattern. Now bow down to riding's ancestors.
+    description: The Arbor Draft snowboard wouldn't exist if Polynesians hadn't figured out how to surf hundreds of years ago. But the Draft does exist, and it's here to bring your urban and park riding to a new level. The board's freaky Tiki design pays homage to culture that inspired snowboarding. It's designed to spin with ease, land smoothly, lock hook-free onto rails, and take the abuse of a pavement pounding or twelve. The Draft will pop off kickers with authority and carve solidly across the pipe. The Draft features targeted Koa wood die cuts inlayed into the deck that enhance the flex pattern. Now bow down to riding's ancestors.
     variants:
       - *product-1-var-1
       - *product-1-var-2
@@ -377,8 +376,7 @@ products:
     featured_image: products/element58.jpg
     images:
       - products/element58.jpg
-    description:
-      The Element is a technically advanced all-mountain board for riders who readily transition from one terrain, snow condition, or riding style to another. Its balanced design provides the versatility needed for the true ride-it-all experience. The Element is exceedingly lively, freely initiates, and holds a tight edge at speed. Its structural real-wood topsheet is made with book-matched Koa.
+    description: The Element is a technically advanced all-mountain board for riders who readily transition from one terrain, snow condition, or riding style to another. Its balanced design provides the versatility needed for the true ride-it-all experience. The Element is exceedingly lively, freely initiates, and holds a tight edge at speed. Its structural real-wood topsheet is made with book-matched Koa.
     variants:
       - *product-2-var-1
 
@@ -411,8 +409,7 @@ products:
       - products/technine1.jpg
       - products/technine2.jpg
       - products/technine_detail.jpg
-    description:
-      2005 Technine Comic Series Description The Comic series was developed to be the ultimate progressive freestyle board in the Technine line. Dependable edge control and a perfect flex pattern for jumping in the park or out of bounds. Landins and progression will come easy with this board and it will help your riding progress to the next level. Street rails, park jibs, backcountry booters and park jumps, this board will do it all.
+    description: 2005 Technine Comic Series Description The Comic series was developed to be the ultimate progressive freestyle board in the Technine line. Dependable edge control and a perfect flex pattern for jumping in the park or out of bounds. Landins and progression will come easy with this board and it will help your riding progress to the next level. Street rails, park jibs, backcountry booters and park jumps, this board will do it all.
     variants:
       - *product-3-var-1
       - *product-3-var-2
@@ -446,8 +443,7 @@ products:
     images:
       - products/technine3.jpg
       - products/technine4.jpg
-    description:
-      2005 Technine Comic Series Description The Comic series was developed to be the ultimate progressive freestyle board in the Technine line. Dependable edge control and a perfect flex pattern for jumping in the park or out of bounds. Landins and progression will come easy with this board and it will help your riding progress to the next level. Street rails, park jibs, backcountry booters and park jumps, this board will do it all.
+    description: 2005 Technine Comic Series Description The Comic series was developed to be the ultimate progressive freestyle board in the Technine line. Dependable edge control and a perfect flex pattern for jumping in the park or out of bounds. Landins and progression will come easy with this board and it will help your riding progress to the next level. Street rails, park jibs, backcountry booters and park jumps, this board will do it all.
     variants:
       - *product-4-var-1
 
@@ -478,8 +474,7 @@ products:
     featured_image: products/burton.jpg
     images:
       - products/burton.jpg
-    description:
-      The Burton boots are particularly well on snowboards. The very best thing about them is that the according picture is cubic. This makes testing in a Vision testing environment very easy.
+    description: The Burton boots are particularly well on snowboards. The very best thing about them is that the according picture is cubic. This makes testing in a Vision testing environment very easy.
     variants:
       - *product-5-var-1
       - *product-5-var-2
@@ -516,8 +511,7 @@ products:
     featured_image: products/ducati.jpg
     images:
       - products/ducati.jpg
-    description:
-      <h3>‘S’ PERFORMANCE</h3>
+    description: <h3>‘S’ PERFORMANCE</h3>
       <p>Producing 170hp (125kW) and with a dry weight of just 169kg (372.6lb), the new 1198 S now incorporates more World Superbike technology than ever before by taking the 1198 motor and adding top-of-the-range suspension, lightweight chassis components and a true racing-style traction control system designed for road use.</p>
       <p>The high performance, fully adjustable 43mm Öhlins forks, which sport low friction titanium nitride-treated fork sliders, respond effortlessly to every imperfection in the tarmac. Beyond their advanced engineering solutions, one of the most important characteristics of Öhlins forks is their ability to communicate the condition and quality of the tyre-to-road contact patch, a feature that puts every rider in superior control. The suspension set-up at the rear is complemented with a fully adjustable Öhlins rear shock equipped with a ride enhancing top-out spring and mounted to a single-sided swingarm for outstanding drive and traction. The front-to-rear Öhlins package is completed with a control-enhancing adjustable steering damper.</p>
     variants:
@@ -636,7 +630,6 @@ products:
       - *product-9-var-2
       - *product-9-var-3
 
-
 # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
 #   Line Items
 # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
@@ -644,8 +637,8 @@ products:
 line_items:
   - &line_item-1
     id: 1
-    title: 'Arbor Draft'
-    subtitle: '151cm'
+    title: "Arbor Draft"
+    subtitle: "151cm"
     price: 29900
     line_price: 29900
     quantity: 1
@@ -654,8 +647,8 @@ line_items:
 
   - &line_item-2
     id: 2
-    title: 'Comic ~ Orange'
-    subtitle: '159cm'
+    title: "Comic ~ Orange"
+    subtitle: "159cm"
     price: 19900
     line_price: 39800
     quantity: 2
@@ -681,7 +674,7 @@ links:
   - &link-4
     id: 4
     title: Powered by Shopify
-    url: 'http://shopify.com'
+    url: "http://shopify.com"
   - &link-5
     id: 5
     title: About Us
@@ -715,8 +708,6 @@ links:
     title: Catalog
     url: /collections/all
 
-
-
 # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
 #   Link Lists
 # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
@@ -724,8 +715,8 @@ links:
 link_lists:
   - &link-list-1
     id: 1
-    title: 'Main Menu'
-    handle: 'main-menu'
+    title: "Main Menu"
+    handle: "main-menu"
     links:
       - *link-12
       - *link-5
@@ -733,8 +724,8 @@ link_lists:
       - *link-8
   - &link-list-2
     id: 1
-    title: 'Footer Menu'
-    handle: 'footer'
+    title: "Footer Menu"
+    handle: "footer"
     links:
       - *link-5
       - *link-6
@@ -768,8 +759,7 @@ collections:
     title: Snowboards
     handle: snowboards
     url: /collections/snowboards
-    description:
-      <p>This is a description for my <strong>Snowboards</strong> collection.</p>
+    description: <p>This is a description for my <strong>Snowboards</strong> collection.</p>
     products:
       - *product-1
       - *product-2
@@ -787,8 +777,8 @@ collections:
   - &collection-5
     id: 5
     title: Paginated Sale
-    handle: 'paginated-sale'
-    url: '/collections/paginated-sale'
+    handle: "paginated-sale"
+    url: "/collections/paginated-sale"
     products:
       - *product-1
       - *product-2
@@ -799,8 +789,8 @@ collections:
   - &collection-6
     id: 6
     title: All products
-    handle: 'all'
-    url: '/collections/all'
+    handle: "all"
+    url: "/collections/all"
     products:
       - *product-7
       - *product-8
@@ -812,7 +802,6 @@ collections:
       - *product-4
       - *product-5
 
-
 # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
 #   Pages and Blogs
 # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
@@ -823,8 +812,7 @@ pages:
     handle: contact
     url: /pages/contact
     author: Tobi
-    content:
-      "<p>You can contact us via phone under (555) 567-2222.</p>
+    content: "<p>You can contact us via phone under (555) 567-2222.</p>
       <p>Our retail store is located at <em>Rue d'Avignon 32, Avignon (Provence)</em>.</p>
       <p><strong>Opening Hours:</strong><br />Monday through Friday: 9am - 6pm<br />Saturday: 10am - 3pm<br />Sunday: closed</p>"
     created_at: 2005-04-04 12:00
@@ -874,12 +862,12 @@ blogs:
     url: /blogs/news
     articles:
       - id: 3
-        title: 'Welcome to the new Foo Shop'
+        title: "Welcome to the new Foo Shop"
         author: Daniel
         content: <p><strong>Welcome to your Shopify store! The jaded Pixel crew is really glad you decided to take Shopify for a spin.</strong></p><p>To help you get you started with Shopify, here are a couple of tips regarding what you see on this page.</p><p>The text you see here is an article. To edit this article, create new articles or create new pages you can go to the <a href="/admin/pages">Blogs &amp; Pages</a> tab of the administration menu.</p><p>The Shopify t-shirt above is a product and selling products is what Shopify is all about. To edit this product, or create new products you can go to the <a href="/admin/products">Products Tab</a> in of the administration menu.</p><p>While you're looking around be sure to check out the <a href="/admin/collections">Collections</a> and <a href="/admin/links">Navigations</a> tabs and soon you will be well on your way to populating your site.</p><p>And of course don't forget to browse the <a href="admin/design/appearance/themes">theme gallery</a> to pick a new look for your shop!</p><p><strong>Shopify is in beta</strong><br />If you would like to make comments or suggestions please visit us in the <a href="http://forums.shopify.com/community">Shopify Forums</a> or drop us an <a href="mailto:[email protected]">email</a>.</p>
         created_at: 2005-04-04 16:00
       - id: 4
-        title: 'Breaking News: Restock on all sales products'
+        title: "Breaking News: Restock on all sales products"
         author: Tobi
         content: Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
         created_at: 2005-04-04 12:00
@@ -891,13 +879,12 @@ blogs:
     url: /blogs/bigcheese-blog
     articles:
       - id: 1
-        title: 'One thing you probably did not know yet...'
+        title: "One thing you probably did not know yet..."
         author: Justin
         content: Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
         created_at: 2005-04-04 16:00
         comments:
-          -
-            id: 1
+          - id: 1
             author: John Smith
             email: [email protected]
             content: Wow...great article man.
@@ -905,8 +892,7 @@ blogs:
             created_at: 2009-01-01 12:00
             updated_at: 2009-02-01 12:00
             url: ""
-          -
-            id: 2
+          - id: 2
             author: John Jones
             email: [email protected]
             content: I really enjoyed this article. And I love your shop! It's awesome. Shopify rocks!
@@ -932,7 +918,7 @@ blogs:
     url: /blogs/paginated-blog
     articles:
       - id: 6
-        title: 'One thing you probably did not know yet...'
+        title: "One thing you probably did not know yet..."
         author: Justin
         content: Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
         created_at: 2005-04-04 16:00

performance/theme_runner.rb

@@ -25,23 +25,41 @@ class ThemeRunner
 
   # Initialize a new liquid ThemeRunner instance
   # Will load all templates into memory, do this now so that we don't profile IO.
-  def initialize
-    @tests = Dir[__dir__ + '/tests/**/*.liquid'].collect do |test|
+  def initialize(strictness: {})
+    @strictness = strictness
+    @tests = []
+    Dir[__dir__ + '/tests/**/*.liquid'].each do |test|
       next if File.basename(test) == 'theme.liquid'
 
-      theme_path = File.dirname(test) + '/theme.liquid'
-      {
-        liquid: File.read(test),
-        layout: (File.file?(theme_path) ? File.read(theme_path) : nil),
-        template_name: test,
-      }
-    end.compact
+      theme_path = File.realpath(File.dirname(test))
+      theme_name = File.basename(theme_path)
+      test_name = theme_name + "/" + File.basename(test)
+      template_name = File.basename(test, '.liquid')
+      layout_path = theme_path + '/theme.liquid'
 
-    compile_all_tests
+      test = {
+        test_name: test_name,
+        liquid: File.read(test),
+        layout: File.file?(layout_path) ? File.read(layout_path) : nil,
+        template_name: template_name,
+        theme_name: theme_name,
+        theme_path: theme_path,
+      }
+
+      @tests << test
+    end
   end
 
+  def find_test(test_name)
+    @tests.find do |test_hash|
+      test_hash[:test_name] == test_name
+    end
+  end
+
+  attr_reader :tests
+
   # `compile` will test just the compilation portion of liquid without any templates
-  def compile
+  def compile_all
     @tests.each do |test_hash|
       Liquid::Template.new.parse(test_hash[:liquid])
       Liquid::Template.new.parse(test_hash[:layout])
@@ -49,7 +67,7 @@ class ThemeRunner
   end
 
   # `tokenize` will just test the tokenizen portion of liquid without any templates
-  def tokenize
+  def tokenize_all
     ss = StringScanner.new("")
     @tests.each do |test_hash|
       tokenizer = Liquid::Tokenizer.new(
@@ -62,78 +80,72 @@ class ThemeRunner
   end
 
   # `run` is called to benchmark rendering and compiling at the same time
-  def run
-    each_test do |liquid, layout, assigns, page_template, template_name|
-      compile_and_render(liquid, layout, assigns, page_template, template_name)
+  def run_all
+    @tests.each do |test|
+      compile_and_render(test)
     end
   end
 
   # `render` is called to benchmark just the render portion of liquid
-  def render
+  def render_all
+    @compiled_tests ||= compile_all_tests
     @compiled_tests.each do |test|
-      tmpl    = test[:tmpl]
-      assigns = test[:assigns]
-      layout  = test[:layout]
-
-      if layout
-        assigns['content_for_layout'] = tmpl.render!(assigns)
-        layout.render!(assigns)
-      else
-        tmpl.render!(assigns)
-      end
+      render_template(test)
     end
   end
 
+  def run_one_test(test_name)
+    test = find_test(test_name)
+    compile_and_render(test)
+  end
+
   private
 
-  def render_layout(template, layout, assigns)
-    assigns['content_for_layout'] = template.render!(assigns)
-    layout&.render!(assigns)
+  def render_template(compiled_test)
+    tmpl, layout, assigns = compiled_test.values_at(:tmpl, :layout, :assigns)
+    if layout
+      assigns['content_for_layout'] = tmpl.render!(assigns, @strictness)
+      rendered_layout = layout.render!(assigns, @strictness)
+      rendered_layout
+    else
+      tmpl.render!(assigns, @strictness)
+    end
   end
 
-  def compile_and_render(template, layout, assigns, page_template, template_file)
-    compiled_test = compile_test(template, layout, assigns, page_template, template_file)
-    render_layout(compiled_test[:tmpl], compiled_test[:layout], compiled_test[:assigns])
+  def compile_and_render(test)
+    compiled_test = compile_test(test)
+    render_template(compiled_test)
   end
 
   def compile_all_tests
     @compiled_tests = []
-    each_test do |liquid, layout, assigns, page_template, template_name|
-      @compiled_tests << compile_test(liquid, layout, assigns, page_template, template_name)
+    @tests.each do |test_hash|
+      @compiled_tests << compile_test(test_hash)
     end
     @compiled_tests
   end
 
-  def compile_test(template, layout, assigns, page_template, template_file)
-    tmpl            = init_template(page_template, template_file)
-    parsed_template = tmpl.parse(template).dup
+  def compile_test(test_hash)
+    theme_path, template_name, layout, liquid = test_hash.values_at(:theme_path, :template_name, :layout, :liquid)
+
+    assigns = Database.tables.dup
+    assigns.merge!({
+      'title' => 'Page title',
+      'page_title' => 'Page title',
+      'content_for_header' => '',
+      'template' => template_name,
+    })
+
+    fs = ThemeRunner::FileSystem.new(theme_path)
+
+    result = {}
+    result[:assigns] = assigns
+    result[:tmpl] = Liquid::Template.parse(liquid, registers: { file_system: fs })
 
     if layout
-      parsed_layout = tmpl.parse(layout)
-      { tmpl: parsed_template, assigns: assigns, layout: parsed_layout }
-    else
-      { tmpl: parsed_template, assigns: assigns }
+      result[:layout] = Liquid::Template.parse(layout, registers: { file_system: fs })
     end
-  end
 
-  # utility method with similar functionality needed in `compile_all_tests` and `run`
-  def each_test
-    # Dup assigns because will make some changes to them
-    assigns = Database.tables.dup
-
-    @tests.each do |test_hash|
-      # Compute page_template outside of profiler run, uninteresting to profiler
-      page_template = File.basename(test_hash[:template_name], File.extname(test_hash[:template_name]))
-      yield(test_hash[:liquid], test_hash[:layout], assigns, page_template, test_hash[:template_name])
-    end
-  end
-
-  # set up a new Liquid::Template object for use in `compile_and_render` and `compile_test`
-  def init_template(page_template, template_file)
-    tmpl                         = Liquid::Template.new
-    tmpl.assigns['page_title']   = 'Page title'
-    tmpl.assigns['template']     = page_template
-    tmpl.registers[:file_system] = ThemeRunner::FileSystem.new(File.dirname(template_file))
-    tmpl
+    result
   end
 end

test/integration/hash_rendering_test.rb

@@ -77,7 +77,30 @@ class HashRenderingTest < Minitest::Test
     assert_template_result("{\"numbers\"=>[{:foo=>42}]}", "{{ my_hash }}", { "my_hash" => { "numbers" => [{ foo: 42 }] } })
   end
 
+  def test_join_filter_with_hash
+    array = [{ "key1" => "value1" }, { "key2" => "value2" }]
+    glue = { "lol" => "wut" }
+    assert_template_result("{\"key1\"=>\"value1\"}{\"lol\"=>\"wut\"}{\"key2\"=>\"value2\"}", "{{ my_array | join: glue }}", { "my_array" => array, "glue" => glue })
+  end
+
   def test_render_hash_with_hash_key
     assert_template_result("{{\"foo\"=>\"bar\"}=>42}", "{{ my_hash }}", { "my_hash" => { Hash["foo" => "bar"] => 42 } })
   end
+
+  def test_rendering_hash_with_custom_to_s_method_uses_custom_to_s
+    my_hash = Class.new(Hash) do
+      def to_s
+        "kewl"
+      end
+    end.new
+
+    assert_template_result("kewl", "{{ my_hash }}", { "my_hash" => my_hash })
+  end
+
+  def test_rendering_hash_without_custom_to_s_uses_default_inspect
+    my_hash = Class.new(Hash).new
+    my_hash[:foo] = :bar
+
+    assert_template_result("{:foo=>:bar}", "{{ my_hash }}", { "my_hash" => my_hash })
+  end
 end

test/integration/standard_filter_test.rb

@@ -54,30 +54,6 @@ class TestEnumerable < Liquid::Drop
   end
 end
 
-class TestDeepEnumerable < Liquid::Drop
-  include Enumerable
-
-  class Product < Liquid::Drop
-    attr_reader :title, :price, :premium
-
-    def initialize(title:, price:, premium: nil)
-      @title = { "content" => title, "language" => "en" }
-      @price = { "value" => price, "unit" => "USD" }
-      @premium = { "category" => premium } if premium
-    end
-  end
-
-  def each(&block)
-    [
-      Product.new(title: "Pro goggles",    price: 1299),
-      Product.new(title: "Thermal gloves", price: 1299),
-      Product.new(title: "Alpine jacket",  price: 3999, premium: 'Basic'),
-      Product.new(title: "Mountain boots", price: 3899, premium: 'Pro'),
-      Product.new(title: "Safety helmet",  price: 1999)
-    ].each(&block)
-  end
-end
-
 class NumberLikeThing < Liquid::Drop
   def initialize(amount)
     @amount = amount
@@ -157,6 +133,18 @@ class StandardFiltersTest < Minitest::Test
     assert_equal([], @filters.slice(input, -(1 << 63), 6))
   end
 
+  def test_find_on_empty_array
+    assert_nil(@filters.find([], 'foo', 'bar'))
+  end
+
+  def test_find_index_on_empty_array
+    assert_nil(@filters.find_index([], 'foo', 'bar'))
+  end
+
+  def test_has_on_empty_array
+    refute(@filters.has([], 'foo', 'bar'))
+  end
+
   def test_truncate
     assert_equal('1234...', @filters.truncate('1234567890', 7))
     assert_equal('1234567890', @filters.truncate('1234567890', 20))
@@ -305,6 +293,16 @@ class StandardFiltersTest < Minitest::Test
     assert_equal('1121314', @filters.join([1, 2, 3, 4], 1))
   end
 
+  def test_join_calls_to_liquid_on_each_element
+    drop = Class.new(Liquid::Drop) do
+      def to_liquid
+        'i did it'
+      end
+    end
+
+    assert_equal('i did it, i did it', @filters.join([drop.new, drop.new], ", "))
+  end
+
   def test_sort
     assert_equal([1, 2, 3, 4], @filters.sort([4, 3, 2, 1]))
     assert_equal([{ "a" => 1 }, { "a" => 2 }, { "a" => 3 }, { "a" => 4 }], @filters.sort([{ "a" => 4 }, { "a" => 3 }, { "a" => 1 }, { "a" => 2 }], "a"))
@@ -416,15 +414,6 @@ class StandardFiltersTest < Minitest::Test
     end
   end
 
-  def test_sort_natural_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | sort_natural: 'title.content' | map: 'title.content' | join: ', ' -}}
-    LIQUID
-    expected_output = "Alpine jacket, Mountain boots, Pro goggles, Safety helmet, Thermal gloves"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_legacy_sort_hash
     assert_equal([{ a: 1, b: 2 }], @filters.sort(a: 1, b: 2))
   end
@@ -461,15 +450,6 @@ class StandardFiltersTest < Minitest::Test
     end
   end
 
-  def test_uniq_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | uniq: 'price.value' | map: "title.content" | join: ', ' -}}
-    LIQUID
-    expected_output = "Pro goggles, Alpine jacket, Mountain boots, Safety helmet"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_compact_empty_array
     assert_equal([], @filters.compact([], "a"))
   end
@@ -486,15 +466,6 @@ class StandardFiltersTest < Minitest::Test
     end
   end
 
-  def test_compact_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | compact: 'premium.category' | map: 'title.content' | join: ', ' -}}
-    LIQUID
-    expected_output = "Alpine jacket, Mountain boots"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_reverse
     assert_equal([4, 3, 2, 1], @filters.reverse([1, 2, 3, 4]))
   end
@@ -589,12 +560,23 @@ class StandardFiltersTest < Minitest::Test
     end
   end
 
-  def test_map_returns_empty_with_no_property
+  def test_map_with_value_property
+    array = [
+      { "handle" => "alpha", "value" => "A" },
+      { "handle" => "beta", "value" => "B" },
+      { "handle" => "gamma", "value" => "C" }
+    ]
+
+    assert_template_result("A B C", "{{ array | map: 'value' | join: ' ' }}", { "array" => array })
+  end
+
+  def test_map_returns_input_with_no_property
     foo = [
       [1],
       [2],
       [3],
     ]
+
     assert_raises(Liquid::ArgumentError) do
       @filters.map(foo, nil)
     end
@@ -604,15 +586,6 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result("213", '{{ foo | sort: "bar" | map: "foo" }}', { "foo" => TestEnumerable.new })
   end
 
-  def test_sort_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | sort: 'price.value' | map: 'title.content' | join: ', ' -}}
-    LIQUID
-    expected_output = "Pro goggles, Thermal gloves, Safety helmet, Mountain boots, Alpine jacket"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_first_and_last_call_to_liquid
     assert_template_result('foobar', '{{ foo | first }}', { 'foo' => [ThingWithToLiquid.new] })
     assert_template_result('foobar', '{{ foo | last }}', { 'foo' => [ThingWithToLiquid.new] })
@@ -929,15 +902,6 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result(expected_output, template, { "array" => array })
   end
 
-  def test_reject_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | reject: 'title.content', 'Pro goggles' | map: 'price.value' | join: ', ' -}}
-    LIQUID
-    expected_output = "1299, 3999, 3899, 1999"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_has
     array = [
       { "handle" => "alpha", "ok" => true },
@@ -966,6 +930,18 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result(expected_output, "{{ array | has: 'ok', true }}", { "array" => array })
   end
 
+  def test_has_with_empty_arrays
+    template = <<~LIQUID
+      {%- assign has_product = products | has: 'title.content', 'Not found' -%}
+      {%- unless has_product -%}
+        Product not found.
+      {%- endunless -%}
+    LIQUID
+    expected_output = "Product not found."
+
+    assert_template_result(expected_output, template, { "products" => [] })
+  end
+
   def test_has_with_false_value
     array = [
       { "handle" => "alpha", "ok" => true },
@@ -994,16 +970,6 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result(expected_output, template, { "array" => array })
   end
 
-  def test_has_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | has: 'title.content', 'Pro goggles' -}},
-      {{- products | has: 'title.content', 'foo' -}}
-    LIQUID
-    expected_output = "true,false"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_find_with_value
     products = [
       { "title" => "Pro goggles",    "price" => 1299 },
@@ -1022,14 +988,16 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result(expected_output, template, { "products" => products })
   end
 
-  def test_find_with_deep_enumerables
+  def test_find_with_empty_arrays
     template = <<~LIQUID
-      {%- assign product = products | find: 'title.content', 'Pro goggles' -%}
-      {{- product.title.content -}}
+      {%- assign product = products | find: 'title.content', 'Not found' -%}
+      {%- unless product -%}
+        Product not found.
+      {%- endunless -%}
     LIQUID
-    expected_output = "Pro goggles"
+    expected_output = "Product not found."
 
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
+    assert_template_result(expected_output, template, { "products" => [] })
   end
 
   def test_find_index_with_value
@@ -1050,14 +1018,16 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result(expected_output, template, { "products" => products })
   end
 
-  def test_find_index_with_deep_enumerables
+  def test_find_index_with_empty_arrays
     template = <<~LIQUID
-      {%- assign index = products | find_index: 'title.content', 'Alpine jacket' -%}
-      {{- index -}}
+      {%- assign index = products | find_index: 'title.content', 'Not found' -%}
+      {%- unless index -%}
+        Index not found.
+      {%- endunless -%}
     LIQUID
-    expected_output = "2"
+    expected_output = "Index not found."
 
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
+    assert_template_result(expected_output, template, { "products" => [] })
   end
 
   def test_where
@@ -1074,6 +1044,23 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result(expected_output, template, { "array" => array })
   end
 
+  def test_where_with_empty_string_is_a_no_op
+    environment = { "array" => ["alpha", "beta", "gamma"] }
+    expected_output = "alpha beta gamma"
+    template = "{{ array | where: '' | join: ' ' }}"
+
+    assert_template_result(expected_output, template, environment)
+  end
+
+  def test_where_with_nil_is_a_no_op
+    environment = { "array" => ["alpha", "beta", "gamma"] }
+    template = "{{ array | where: nil | join: ' ' }}"
+
+    assert_raises(Liquid::ArgumentError) do
+      assert_template_result("alpha beta gamma", template, environment)
+    end
+  end
+
   def test_where_with_value
     array = [
       { "handle" => "alpha", "ok" => true },
@@ -1158,15 +1145,6 @@ class StandardFiltersTest < Minitest::Test
     assert_nil(@filters.where([nil], "ok"))
   end
 
-  def test_where_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | where: 'title.content', 'Pro goggles' | map: 'price.value' -}}
-    LIQUID
-    expected_output = "1299"
-
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
-  end
-
   def test_all_filters_never_raise_non_liquid_exception
     test_drop = TestDrop.new(value: "test")
     test_drop.context = Context.new
@@ -1318,13 +1296,31 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result("0", "{{ input | sum: 'subtotal' }}", { "input" => input })
   end
 
-  def test_sum_with_deep_enumerables
-    template = <<~LIQUID
-      {{- products | sum: 'price.value' -}}
-    LIQUID
-    expected_output = "12495"
+  def test_sum_with_non_string_property
+    input = [{ true => 1 }, { 1.0 => 0.2, 1 => -0.3 }, { 1..5 => 0.4 }]
 
-    assert_template_result(expected_output, template, { "products" => TestDeepEnumerable.new })
+    assert_equal(1, @filters.sum(input, true))
+    assert_equal(0.2, @filters.sum(input, 1.0))
+    assert_equal(-0.3, @filters.sum(input, 1))
+    assert_equal(0.4, @filters.sum(input, (1..5)))
+    assert_equal(0, @filters.sum(input, nil))
+    assert_equal(0, @filters.sum(input, ""))
+  end
+
+  def test_uniq_with_to_liquid_value
+    input = [StringDrop.new("foo"), StringDrop.new("bar"), "foo"]
+    expected = [StringDrop.new("foo"), StringDrop.new("bar")]
+    result = @filters.uniq(input)
+
+    assert_equal(expected, result)
+  end
+
+  def test_uniq_with_to_liquid_value_pick_correct_classes
+    input = ["foo", StringDrop.new("foo"), StringDrop.new("bar")]
+    expected = [String, StringDrop]
+    result = @filters.uniq(input).map(&:class)
+
+    assert_equal(expected, result)
   end
 
   private

test/test_helper.rb

@@ -146,6 +146,35 @@ class BooleanDrop < Liquid::Drop
   end
 end
 
+class StringDrop < Liquid::Drop
+  include Comparable
+
+  def initialize(value)
+    super()
+    @value = value
+  end
+
+  def to_liquid_value
+    @value
+  end
+
+  def to_s
+    @value
+  end
+
+  def to_str
+    @value
+  end
+
+  def inspect
+    "#<StringDrop @value=#{@value.inspect}>"
+  end
+
+  def <=>(other)
+    to_liquid_value <=> Liquid::Utils.to_liquid_value(other)
+  end
+end
+
 class ErrorDrop < Liquid::Drop
   def standard_error
     raise Liquid::StandardError, 'standard error'

test/unit/block_unit_test.rb

@@ -47,12 +47,18 @@ class BlockUnitTest < Minitest::Test
     )
   end
 
-  def test_with_block
+  def test_comment_tag_with_block
     template = Liquid::Template.parse("  {% comment %} {% endcomment %} ")
     assert_equal([String, Comment, String], block_types(template.root.nodelist))
     assert_equal(3, template.root.nodelist.size)
   end
 
+  def test_doc_tag_with_block
+    template = Liquid::Template.parse("  {% doc %} {% enddoc %} ")
+    assert_equal([String, Doc, String], block_types(template.root.nodelist))
+    assert_equal(3, template.root.nodelist.size)
+  end
+
   private
 
   def block_types(nodelist)

test/unit/environment_test.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class EnvironmentTest < Minitest::Test
+  include Liquid
+
+  class UnsubscribeFooter < Liquid::Tag
+    def render(_context)
+      'Unsubscribe Footer'
+    end
+  end
+
+  def test_custom_tag
+    email_environment = Liquid::Environment.build do |environment|
+      environment.register_tag("unsubscribe_footer", UnsubscribeFooter)
+    end
+
+    assert(email_environment.tags["unsubscribe_footer"])
+    assert(email_environment.tag_for_name("unsubscribe_footer"))
+    template = Liquid::Template.parse("{% unsubscribe_footer %}", environment: email_environment)
+
+    assert_equal('Unsubscribe Footer', template.render)
+  end
+end

test/unit/lexer_unit_test.rb

@@ -131,6 +131,16 @@ class LexerUnitTest < Minitest::Test
     assert_equal([[:id, "false"], [:number, "1"], [:end_of_string]], tokenize("false 1"))
   end
 
+  def test_error_with_invalid_utf8
+    error = assert_raises(SyntaxError) do
+      tokenize("\x00\xff")
+    end
+    assert_equal(
+      'Liquid syntax error: Invalid byte sequence in UTF-8',
+      error.message,
+    )
+  end
+
   private
 
   def tokenize(input)

test/unit/tags/doc_tag_unit_test.rb

@@ -0,0 +1,287 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class DocTagUnitTest < Minitest::Test
+  def test_doc_tag
+    template = <<~LIQUID.chomp
+      {% doc %}
+        Renders loading-spinner.
+
+        @param {string} foo - some foo
+        @param {string} [bar] - optional bar
+
+        @example
+        {% render 'loading-spinner', foo: 'foo' %}
+        {% render 'loading-spinner', foo: 'foo', bar: 'bar' %}
+      {% enddoc %}
+    LIQUID
+
+    assert_template_result('', template)
+  end
+
+  def test_doc_tag_body_content
+    doc_content = "  Documentation content\n  @param {string} foo - test\n"
+    template_source = "{% doc %}#{doc_content}{% enddoc %}"
+
+    doc_tag = nil
+    ParseTreeVisitor
+      .for(Template.parse(template_source).root)
+      .add_callback_for(Liquid::Doc) do |tag|
+        doc_tag = tag
+      end
+      .visit
+
+    assert_equal(doc_content, doc_tag.nodelist.first.to_s)
+  end
+
+  def test_doc_tag_does_not_support_extra_arguments
+    error = assert_raises(Liquid::SyntaxError) do
+      template = <<~LIQUID.chomp
+        {% doc extra %}
+        {% enddoc %}
+      LIQUID
+
+      Liquid::Template.parse(template)
+    end
+
+    exp_error = "Liquid syntax error: Syntax Error in 'doc' - Valid syntax: {% doc %}{% enddoc %}"
+    act_error = error.message
+
+    assert_equal(exp_error, act_error)
+  end
+
+  def test_doc_tag_must_support_valid_tags
+    assert_match_syntax_error("Liquid syntax error (line 1): 'doc' tag was never closed", '{% doc %} foo')
+    assert_match_syntax_error("Liquid syntax error (line 1): Syntax Error in 'doc' - Valid syntax: {% doc %}{% enddoc %}", '{% doc } foo {% enddoc %}')
+    assert_match_syntax_error("Liquid syntax error (line 1): Syntax Error in 'doc' - Valid syntax: {% doc %}{% enddoc %}", '{% doc } foo %}{% enddoc %}')
+  end
+
+  def test_doc_tag_ignores_liquid_nodes
+    template = <<~LIQUID.chomp
+      {% doc %}
+        {% if true %}
+        {% if ... %}
+        {%- for ? -%}
+        {% while true %}
+        {%
+          unless if
+        %}
+        {% endcase %}
+      {% enddoc %}
+    LIQUID
+
+    assert_template_result('', template)
+  end
+
+  def test_doc_tag_ignores_unclosed_liquid_tags
+    template = <<~LIQUID.chomp
+      {% doc %}
+        {% if true %}
+      {% enddoc %}
+    LIQUID
+
+    assert_template_result('', template)
+  end
+
+  def test_doc_tag_does_not_allow_nested_docs
+    error = assert_raises(Liquid::SyntaxError) do
+      template = <<~LIQUID.chomp
+        {% doc %}
+          {% doc %}
+            {% doc %}
+        {% enddoc %}
+      LIQUID
+
+      Liquid::Template.parse(template)
+    end
+
+    exp_error = "Liquid syntax error: Syntax Error in 'doc' - Nested doc tags are not allowed"
+    act_error = error.message
+
+    assert_equal(exp_error, act_error)
+  end
+
+  def test_doc_tag_ignores_nested_raw_tags
+    template = <<~LIQUID.chomp
+      {% doc %}
+        {% raw %}
+      {% enddoc %}
+    LIQUID
+
+    assert_template_result('', template)
+  end
+
+  def test_doc_tag_ignores_unclosed_assign
+    template = <<~LIQUID.chomp
+      {% doc %}
+        {% assign foo = "1"
+      {% enddoc %}
+    LIQUID
+
+    assert_template_result('', template)
+  end
+
+  def test_doc_tag_ignores_malformed_syntax
+    template = <<~LIQUID.chomp
+      {% doc %}
+      {% {{ {%- enddoc %}
+    LIQUID
+
+    assert_template_result('', template)
+  end
+
+  def test_doc_tag_captures_token_before_enddoc
+    template_source = "{% doc %}{{ incomplete{% enddoc %}"
+
+    doc_tag = nil
+    ParseTreeVisitor
+      .for(Template.parse(template_source).root)
+      .add_callback_for(Liquid::Doc) do |tag|
+        doc_tag = tag
+      end
+      .visit
+
+    assert_equal("{{ incomplete", doc_tag.nodelist.first.to_s)
+  end
+
+  def test_doc_tag_preserves_error_line_numbers
+    template = Liquid::Template.parse(<<~LIQUID.chomp, line_numbers: true)
+      {% doc %}
+        {% if true %}
+      {% enddoc %}
+      {{ errors.standard_error }}
+    LIQUID
+
+    expected = <<~TEXT.chomp
+
+      Liquid error (line 4): standard error
+    TEXT
+
+    assert_equal(expected, template.render('errors' => ErrorDrop.new))
+  end
+
+  def test_doc_tag_whitespace_control
+    # Basic whitespace control
+    assert_template_result("Hello!", "      {%- doc -%}123{%- enddoc -%}Hello!")
+    assert_template_result("Hello!", "{%- doc -%}123{%- enddoc -%}     Hello!")
+    assert_template_result("Hello!", "      {%- doc -%}123{%- enddoc -%}     Hello!")
+    assert_template_result("Hello!", <<~LIQUID.chomp)
+      {%- doc %}Whitespace control!{% enddoc -%}
+      Hello!
+    LIQUID
+  end
+
+  def test_doc_tag_delimiter_handling
+    assert_template_result('', <<~LIQUID.chomp)
+      {%- if true -%}
+        {%- doc -%}
+          {%- docEXTRA -%}wut{% enddocEXTRA -%}xyz
+        {%- enddoc -%}
+      {%- endif -%}
+    LIQUID
+
+    assert_template_result('', "{% doc %}123{% enddoc xyz %}")
+    assert_template_result('', "{% doc %}123{% enddoc\txyz %}")
+    assert_template_result('', "{% doc %}123{% enddoc\nxyz %}")
+    assert_template_result('', "{% doc %}123{% enddoc\n   xyz  enddoc %}")
+  end
+
+  def test_doc_tag_visitor
+    template_source = '{% doc %}{% enddoc %}'
+
+    assert_equal(
+      [Liquid::Doc],
+      visit(template_source),
+    )
+  end
+
+  def test_doc_tag_blank_with_empty_content
+    template_source = "{% doc %}{% enddoc %}"
+
+    doc_tag = nil
+    ParseTreeVisitor
+      .for(Template.parse(template_source).root)
+      .add_callback_for(Liquid::Doc) do |tag|
+        doc_tag = tag
+      end
+      .visit
+
+    assert_equal(true, doc_tag.blank?)
+  end
+
+  def test_doc_tag_blank_with_content
+    template_source = "{% doc %}Some documentation{% enddoc %}"
+
+    doc_tag = nil
+    ParseTreeVisitor
+      .for(Template.parse(template_source).root)
+      .add_callback_for(Liquid::Doc) do |tag|
+        doc_tag = tag
+      end
+      .visit
+
+    assert_equal(false, doc_tag.blank?)
+  end
+
+  def test_doc_tag_blank_with_whitespace_only
+    template_source = "{% doc %}    {% enddoc %}"
+
+    doc_tag = nil
+    ParseTreeVisitor
+      .for(Template.parse(template_source).root)
+      .add_callback_for(Liquid::Doc) do |tag|
+        doc_tag = tag
+      end
+      .visit
+
+    assert_equal(false, doc_tag.blank?)
+  end
+
+  def test_doc_tag_nodelist_returns_array_with_body
+    doc_content = "Documentation content\n@param {string} foo"
+    template_source = "{% doc %}#{doc_content}{% enddoc %}"
+
+    doc_tag = nil
+    ParseTreeVisitor
+      .for(Template.parse(template_source).root)
+      .add_callback_for(Liquid::Doc) do |tag|
+        doc_tag = tag
+      end
+      .visit
+
+    assert_equal([doc_content], doc_tag.nodelist)
+    assert_equal(1, doc_tag.nodelist.length)
+    assert_equal(doc_content, doc_tag.nodelist.first)
+  end
+
+  def test_doc_tag_nodelist_with_empty_content
+    template_source = "{% doc %}{% enddoc %}"
+
+    doc_tag = nil
+    ParseTreeVisitor
+      .for(Template.parse(template_source).root)
+      .add_callback_for(Liquid::Doc) do |tag|
+        doc_tag = tag
+      end
+      .visit
+
+    assert_equal([""], doc_tag.nodelist)
+    assert_equal(1, doc_tag.nodelist.length)
+  end
+
+  private
+
+  def traversal(template)
+    ParseTreeVisitor
+      .for(Template.parse(template).root)
+      .add_callback_for(Liquid::Doc) do |tag|
+        tag_class = tag.class
+        tag_class
+      end
+  end
+
+  def visit(template)
+    traversal(template).visit.flatten.compact
+  end
+end

test/unit/template_unit_test.rb

@@ -35,4 +35,15 @@ class TemplateUnitTest < Minitest::Test
   def test_template_inheritance
     assert_equal("foo", TemplateSubclass.parse("foo").render)
   end
+
+  def test_invalid_utf8
+    input = "\xff\x00"
+    error = assert_raises(SyntaxError) do
+      Liquid::Tokenizer.new(source: input, string_scanner: StringScanner.new(input))
+    end
+    assert_equal(
+      'Liquid syntax error: Invalid byte sequence in UTF-8',
+      error.message,
+    )
+  end
 end