Make whitespace filters Unicode-aware

Previously we were only leveraging Ruby's `String#strip` to handle the
logic in these filters but that only covers ASCII whitespace. When
rendering Liquid templates into HTML it would be confusing for these
filters to not strip *all* whitespace.

Additionally, it's helpful when trying to compare two values in, say, a
Liquid conditional.

.github/workflows/liquid.yml

@@ -22,6 +22,7 @@ jobs:
             }
           - { ruby: 4.0, allowed-failure: false, rubyopt: "--yjit" }
           - { ruby: 4.0, allowed-failure: false, rubyopt: "--zjit" }
+          - { ruby: truffleruby, allowed-failure: false }
 
           # Head can have failures due to being in development
           - { ruby: head, allowed-failure: true }

.ruby-version

@@ -1 +1 @@
-3.4.1
+4.0.2

Rakefile

@@ -151,6 +151,8 @@ end
 
 desc('run liquid-spec suite across all adapters')
 task :spec do
-  adapters = Dir['./spec/*.rb'].join(',')
-  sh "bundle exec liquid-spec matrix --adapters=#{adapters} --reference=ruby_liquid"
+  Dir['./spec/*.rb'].sort.each do |adapter|
+    puts "=== Running #{adapter} ==="
+    sh 'bundle', 'exec', 'liquid-spec', 'run', adapter, '--no-max-failures'
+  end
 end

lib/liquid/parser.rb

@@ -55,7 +55,7 @@ module Liquid
         str << variable_lookups
       when :open_square
         if @reject_bare_brackets
-          raise SyntaxError, "Bare bracket access is not allowed in strict2 mode. Use #{Expression::SELF}['...'] instead"
+          raise SyntaxError, "Bare bracket access is not allowed. Use #{Expression::SELF}['...'] instead"
         end
         str = consume.dup
         str << expression

lib/liquid/standardfilters.rb

@@ -8,10 +8,19 @@ module Liquid
     MAX_I32 = (1 << 31) - 1
     private_constant :MAX_I32
 
-    MIN_I64 = -(1 << 63)
-    MAX_I64 = (1 << 63) - 1
-    I64_RANGE = MIN_I64..MAX_I64
-    private_constant :MIN_I64, :MAX_I64, :I64_RANGE
+    supports_64bit_indices = begin
+      [][1 << 33, 1 << 33]
+      true
+    rescue RangeError
+      false
+    end
+
+    INDEX_RANGE = if supports_64bit_indices
+      (-(1 << 63))..((1 << 63) - 1)
+    else
+      (-(1 << 31))..((1 << 31) - 1)
+    end
+    private_constant :INDEX_RANGE
 
     HTML_ESCAPE = {
       '&' => '&amp;',
@@ -27,6 +36,19 @@ module Liquid
       %r{<style.*?</style>}m,
     )
     STRIP_HTML_TAGS = /<.*?>/m
+    # Use POSIX whitespace matching so filters handle whitespace beyond Ruby String#strip's ASCII set.
+    WHITESPACE_LEFT = /\A[[:space:]]+/
+    WHITESPACE_RIGHT = /[[:space:]]+\z/
+    WHITESPACE_EDGES = Regexp.union(WHITESPACE_LEFT, WHITESPACE_RIGHT)
+    # Optimized runs regex to find 2 or more [[:space:]] OR a single [[:space:]]
+    # that isn't already `" " `.
+    WHITESPACE_RUNS = /([[:space:]]{2,}|[[[:space:]]&&[^ ]])/
+    private_constant(
+      :WHITESPACE_EDGES,
+      :WHITESPACE_LEFT,
+      :WHITESPACE_RIGHT,
+      :WHITESPACE_RUNS,
+    )
 
     class << self
       def try_coerce_encoding(input, encoding:)
@@ -214,11 +236,11 @@ module Liquid
           Utils.to_s(input).slice(offset, length) || ''
         end
       rescue RangeError
-        if I64_RANGE.cover?(length) && I64_RANGE.cover?(offset)
+        if INDEX_RANGE.cover?(length) && INDEX_RANGE.cover?(offset)
           raise # unexpected error
         end
-        offset = offset.clamp(I64_RANGE)
-        length = length.clamp(I64_RANGE)
+        offset = offset.clamp(INDEX_RANGE)
+        length = length.clamp(INDEX_RANGE)
         retry
       end
     end
@@ -303,7 +325,7 @@ module Liquid
     def squish(input)
       return if input.nil?
 
-      Utils.to_s(input).strip.gsub(/\s+/, ' ')
+      Utils.to_s(input).gsub(WHITESPACE_RUNS, ' ').strip
     end
 
     # @liquid_public_docs
@@ -315,7 +337,7 @@ module Liquid
     # @liquid_return [string]
     def strip(input)
       input = Utils.to_s(input)
-      input.strip
+      input.gsub(WHITESPACE_EDGES, ' ').strip
     end
 
     # @liquid_public_docs
@@ -327,7 +349,7 @@ module Liquid
     # @liquid_return [string]
     def lstrip(input)
       input = Utils.to_s(input)
-      input.lstrip
+      input.gsub(WHITESPACE_LEFT, ' ').lstrip
     end
 
     # @liquid_public_docs
@@ -339,7 +361,7 @@ module Liquid
     # @liquid_return [string]
     def rstrip(input)
       input = Utils.to_s(input)
-      input.rstrip
+      input.gsub(WHITESPACE_RIGHT, ' ').rstrip
     end
 
     # @liquid_public_docs

lib/liquid/tags/decrement.rb

@@ -23,13 +23,29 @@ module Liquid
   #   {% decrement variable_name %}
   # @liquid_syntax_keyword variable_name The name of the variable being decremented.
   class Decrement < Tag
+    include ParserSwitching
+
     attr_reader :variable_name
 
     def initialize(tag_name, markup, options)
       super
+      parse_with_selected_parser(markup)
+    end
+
+    def lax_parse(markup)
       @variable_name = markup.strip
     end
 
+    def strict_parse(markup)
+      lax_parse(markup)
+    end
+
+    def strict2_parse(markup)
+      p = @parse_context.new_parser(markup.strip)
+      @variable_name = p.consume(:id)
+      p.consume(:end_of_string)
+    end
+
     def render_to_output_buffer(context, output)
       counter_environment = context.environments.first
       value = counter_environment[@variable_name] || 0

lib/liquid/tags/increment.rb

@@ -23,13 +23,29 @@ module Liquid
   #   {% increment variable_name %}
   # @liquid_syntax_keyword variable_name The name of the variable being incremented.
   class Increment < Tag
+    include ParserSwitching
+
     attr_reader :variable_name
 
     def initialize(tag_name, markup, options)
       super
+      parse_with_selected_parser(markup)
+    end
+
+    def lax_parse(markup)
       @variable_name = markup.strip
     end
 
+    def strict_parse(markup)
+      lax_parse(markup)
+    end
+
+    def strict2_parse(markup)
+      p = @parse_context.new_parser(markup.strip)
+      @variable_name = p.consume(:id)
+      p.consume(:end_of_string)
+    end
+
     def render_to_output_buffer(context, output)
       counter_environment = context.environments.first
       value = counter_environment[@variable_name] || 0

spec/ruby_liquid.rb

@@ -6,14 +6,24 @@
 
 $LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
 require 'liquid'
+require_relative 'support/liquid_spec_adapter_helper'
 
 LiquidSpec.configure do |config|
-  # Run core Liquid specs
-  config.features = [:core]
+  config.missing_features = [
+    :activesupport,
+    :lax_parsing,
+    :shopify_filters,
+    :shopify_includes,
+    :shopify_blank,
+    :shopify_error_handling,
+    :shopify_error_format,
+    :shopify_string_access,
+  ]
 end
 
 # Compile a template string into a Liquid::Template
 LiquidSpec.compile do |ctx, source, options|
+  options[:error_mode] ||= :strict
   ctx[:template] = Liquid::Template.parse(source, **options)
 end
 
@@ -28,9 +38,12 @@ LiquidSpec.render do |ctx, assigns, options|
     static_environments: assigns,
     registers: registers,
     rethrow_errors: options[:strict_errors],
+    resource_limits: LiquidSpecAdapterHelper.resource_limits(options),
   )
 
   context.exception_renderer = options[:exception_renderer] if options[:exception_renderer]
 
-  ctx[:template].render(context)
+  LiquidSpecAdapterHelper.with_frozen_time do
+    ctx[:template].render(context)
+  end
 end

spec/ruby_liquid_lax.rb

@@ -6,15 +6,24 @@
 
 $LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
 require 'liquid'
+require_relative 'support/liquid_spec_adapter_helper'
 
 LiquidSpec.configure do |config|
-  config.features = [:core, :lax_parsing]
+  config.missing_features = [
+    :activesupport,
+    :shopify_filters,
+    :shopify_includes,
+    :shopify_blank,
+    :shopify_error_handling,
+    :shopify_error_format,
+    :shopify_string_access,
+  ]
 end
 
 # Compile a template string into a Liquid::Template
 LiquidSpec.compile do |ctx, source, options|
-  # Force lax mode
-  options = options.merge(error_mode: :lax)
+  # Default to lax mode while still honoring specs that explicitly set error_mode.
+  options = { error_mode: :lax }.merge(options)
   ctx[:template] = Liquid::Template.parse(source, **options)
 end
 
@@ -26,9 +35,12 @@ LiquidSpec.render do |ctx, assigns, options|
     static_environments: assigns,
     registers: registers,
     rethrow_errors: options[:strict_errors],
+    resource_limits: LiquidSpecAdapterHelper.resource_limits(options),
   )
 
   context.exception_renderer = options[:exception_renderer] if options[:exception_renderer]
 
-  ctx[:template].render(context)
+  LiquidSpecAdapterHelper.with_frozen_time do
+    ctx[:template].render(context)
+  end
 end

spec/ruby_liquid_with_active_support.rb

@@ -7,14 +7,23 @@
 $LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
 require 'active_support/all'
 require 'liquid'
+require_relative 'support/liquid_spec_adapter_helper'
 
 LiquidSpec.configure do |config|
-  # Run core Liquid specs plus ActiveSupport SafeBuffer tests
-  config.features = [:core, :activesupport]
+  config.missing_features = [
+    :lax_parsing,
+    :shopify_filters,
+    :shopify_includes,
+    :shopify_blank,
+    :shopify_error_handling,
+    :shopify_error_format,
+    :shopify_string_access,
+  ]
 end
 
 # Compile a template string into a Liquid::Template
 LiquidSpec.compile do |ctx, source, options|
+  options[:error_mode] ||= :strict
   ctx[:template] = Liquid::Template.parse(source, **options)
 end
 
@@ -29,9 +38,12 @@ LiquidSpec.render do |ctx, assigns, options|
     static_environments: assigns,
     registers: registers,
     rethrow_errors: options[:strict_errors],
+    resource_limits: LiquidSpecAdapterHelper.resource_limits(options),
   )
 
   context.exception_renderer = options[:exception_renderer] if options[:exception_renderer]
 
-  ctx[:template].render(context)
+  LiquidSpecAdapterHelper.with_frozen_time do
+    ctx[:template].render(context)
+  end
 end

spec/ruby_liquid_yjit.rb

@@ -13,9 +13,18 @@ end
 
 require 'active_support/all'
 require 'liquid'
+require_relative 'support/liquid_spec_adapter_helper'
 
 LiquidSpec.configure do |config|
-  config.features = [:core, :activesupport]
+  config.missing_features = [
+    :lax_parsing,
+    :shopify_filters,
+    :shopify_includes,
+    :shopify_blank,
+    :shopify_error_handling,
+    :shopify_error_format,
+    :shopify_string_access,
+  ]
 end
 
 # Compile a template string into a Liquid::Template
@@ -33,9 +42,12 @@ LiquidSpec.render do |ctx, assigns, options|
     static_environments: assigns,
     registers: registers,
     rethrow_errors: options[:strict_errors],
+    resource_limits: LiquidSpecAdapterHelper.resource_limits(options),
   )
 
   context.exception_renderer = options[:exception_renderer] if options[:exception_renderer]
 
-  ctx[:template].render(context)
+  LiquidSpecAdapterHelper.with_frozen_time do
+    ctx[:template].render(context)
+  end
 end

spec/support/liquid_spec_adapter_helper.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module LiquidSpecAdapterHelper
+  extend self
+
+  def resource_limits(render_options)
+    return unless render_options[:resource_limits]
+
+    Liquid::ResourceLimits.new({}).tap do |limits|
+      render_options[:resource_limits].each do |key, value|
+        limits.public_send(:"#{key}=", value)
+      end
+    end
+  end
+
+  def with_frozen_time(&block)
+    original_tz = ENV['TZ']
+    ENV['TZ'] = 'UTC'
+
+    Liquid::Spec::TimeFreezer.freeze(Liquid::Spec::AdapterRunner::TEST_TIME, &block)
+  ensure
+    ENV['TZ'] = original_tz
+  end
+end

test/integration/security_test.rb

@@ -44,32 +44,39 @@ class SecurityTest < Minitest::Test
   end
 
   def test_does_not_permanently_add_filters_to_symbol_table
-    current_symbols = Symbol.all_symbols
+    assert_no_new_symbols do
+      # MRI imprecisely marks objects found on the C stack, which can result
+      # in uninitialized memory being marked. This can even result in the test failing
+      # deterministically for a given compilation of ruby. Using a separate thread will
+      # keep these writes of the symbol pointer on a separate stack that will be garbage
+      # collected after Thread#join.
+      Thread.new do
+        test = %( {{ "some_string" | a_bad_filter }} )
+        Template.parse(test).render!
+        nil
+      end.join
 
-    # MRI imprecisely marks objects found on the C stack, which can result
-    # in uninitialized memory being marked. This can even result in the test failing
-    # deterministically for a given compilation of ruby. Using a separate thread will
-    # keep these writes of the symbol pointer on a separate stack that will be garbage
-    # collected after Thread#join.
-    Thread.new do
-      test = %( {{ "some_string" | a_bad_filter }} )
-      Template.parse(test).render!
-      nil
-    end.join
-
-    GC.start
-
-    assert_equal([], Symbol.all_symbols - current_symbols)
+      GC.start
+    end
   end
 
   def test_does_not_add_drop_methods_to_symbol_table
+    assert_no_new_symbols do
+      assigns = { 'drop' => Drop.new }
+      assert_equal("", Template.parse("{{ drop.custom_method_1 }}", assigns).render!)
+      assert_equal("", Template.parse("{{ drop.custom_method_2 }}", assigns).render!)
+      assert_equal("", Template.parse("{{ drop.custom_method_3 }}", assigns).render!)
+    end
+  end
+
+  def assert_no_new_symbols
+    # Run once to trigger any first-time initialization which might create some symbols,
+    # for example autoload or lazy method parsing might create symbols on first execution.
+    yield
+
+    # Ensure no new symbols for further runs, i.e. the code does not leak symbols
     current_symbols = Symbol.all_symbols
-
-    assigns = { 'drop' => Drop.new }
-    assert_equal("", Template.parse("{{ drop.custom_method_1 }}", assigns).render!)
-    assert_equal("", Template.parse("{{ drop.custom_method_2 }}", assigns).render!)
-    assert_equal("", Template.parse("{{ drop.custom_method_3 }}", assigns).render!)
-
+    yield
     assert_equal([], Symbol.all_symbols - current_symbols)
   end
 

test/integration/standard_filter_test.rb

@@ -169,6 +169,15 @@ class StandardFiltersTest < Minitest::Test
 \t   boo   " | squish }})).render)
     assert_equal("", Liquid::Template.parse('{{ nil | squish }}').render)
     assert_equal("", Liquid::Template.parse('{{ " " | squish }}').render)
+
+    unicode_spaces = "\u00A0\u202F\u2009\u2007"
+
+    assert_template_result(
+      "foo bar boo",
+      "{{ source | squish }}",
+      { 'source' => "#{unicode_spaces}foo\u202F\u2009bar\t\n\u2007boo#{unicode_spaces}" },
+    )
+    assert_template_result("\u200Bfoo\u200B", "{{ source | squish }}", { 'source' => "\u200Bfoo\u200B" })
   end
 
   def test_escape
@@ -703,16 +712,42 @@ class StandardFiltersTest < Minitest::Test
   def test_strip
     assert_template_result('ab c', "{{ source | strip }}", { 'source' => " ab c  " })
     assert_template_result('ab c', "{{ source | strip }}", { 'source' => " \tab c  \n \t" })
+
+    unicode_spaces = "\u00A0\u202F\u2009\u2007"
+
+    assert_template_result(
+      'ab c',
+      "{{ source | strip }}",
+      { 'source' => "#{unicode_spaces}ab c#{unicode_spaces}" },
+    )
+    assert_template_result("a\u00A0b\u202Fc", "{{ source | strip }}", { 'source' => "a\u00A0b\u202Fc" })
+    assert_template_result("\u200Bfoo\u200B", "{{ source | strip }}", { 'source' => "\u200Bfoo\u200B" })
   end
 
   def test_lstrip
     assert_template_result('ab c  ', "{{ source | lstrip }}", { 'source' => " ab c  " })
     assert_template_result("ab c  \n \t", "{{ source | lstrip }}", { 'source' => " \tab c  \n \t" })
+
+    unicode_spaces = "\u00A0\u202F\u2009\u2007"
+
+    assert_template_result(
+      "ab c#{unicode_spaces}",
+      "{{ source | lstrip }}",
+      { 'source' => "#{unicode_spaces}ab c#{unicode_spaces}" },
+    )
   end
 
   def test_rstrip
     assert_template_result(" ab c", "{{ source | rstrip }}", { 'source' => " ab c  " })
     assert_template_result(" \tab c", "{{ source | rstrip }}", { 'source' => " \tab c  \n \t" })
+
+    unicode_spaces = "\u00A0\u202F\u2009\u2007"
+
+    assert_template_result(
+      "#{unicode_spaces}ab c",
+      "{{ source | rstrip }}",
+      { 'source' => "#{unicode_spaces}ab c#{unicode_spaces}" },
+    )
   end
 
   def test_strip_newlines
@@ -1181,6 +1216,8 @@ class StandardFiltersTest < Minitest::Test
   end
 
   def test_all_filters_never_raise_non_liquid_exception
+    skip("too slow on non-CRuby due to many exceptions") unless RUBY_ENGINE == 'ruby'
+
     test_drop = TestDrop.new(value: "test")
     test_drop.context = Context.new
     test_enum = TestEnumerable.new

test/integration/tags/increment_tag_test.rb

@@ -27,4 +27,50 @@ class IncrementTagTest < Minitest::Test
       '{%decrement starboard %}',
     )
   end
+
+  def test_increment_strict2_rejects_invalid_variable_name
+    assert_raises(Liquid::SyntaxError) do
+      Template.parse('{% increment foo bar %}', error_mode: :strict2)
+    end
+  end
+
+  def test_increment_strict2_rejects_variable_starting_with_number
+    assert_raises(Liquid::SyntaxError) do
+      Template.parse('{% increment 11aa %}', error_mode: :strict2)
+    end
+  end
+
+  def test_increment_strict2_accepts_valid_variable_name
+    template = Template.parse('{% increment my-var %}', error_mode: :strict2)
+    assert_equal('0', template.render)
+  end
+
+  def test_decrement_strict2_rejects_invalid_variable_name
+    assert_raises(Liquid::SyntaxError) do
+      Template.parse('{% decrement foo bar %}', error_mode: :strict2)
+    end
+  end
+
+  def test_decrement_strict2_rejects_variable_starting_with_number
+    assert_raises(Liquid::SyntaxError) do
+      Template.parse('{% decrement 11aa %}', error_mode: :strict2)
+    end
+  end
+
+  def test_decrement_strict2_accepts_valid_variable_name
+    template = Template.parse('{% decrement my-var %}', error_mode: :strict2)
+    assert_equal('-1', template.render)
+  end
+
+  def test_increment_strict2_rejects_empty_variable_name
+    assert_raises(Liquid::SyntaxError) do
+      Template.parse('{% increment %}', error_mode: :strict2)
+    end
+  end
+
+  def test_decrement_strict2_rejects_empty_variable_name
+    assert_raises(Liquid::SyntaxError) do
+      Template.parse('{% decrement %}', error_mode: :strict2)
+    end
+  end
 end

test/integration/tags/render_tag_test.rb

@@ -174,16 +174,16 @@ class RenderTagTest < Minitest::Test
   def test_increment_is_isolated_between_renders
     assert_template_result(
       '010',
-      '{% increment %}{% increment %}{% render "incr" %}',
-      partials: { 'incr' => '{% increment %}' },
+      '{% increment port %}{% increment port %}{% render "incr" %}',
+      partials: { 'incr' => '{% increment port %}' },
     )
   end
 
   def test_decrement_is_isolated_between_renders
     assert_template_result(
       '-1-2-1',
-      '{% decrement %}{% decrement %}{% render "decr" %}',
-      partials: { 'decr' => '{% decrement %}' },
+      '{% decrement port %}{% decrement port %}{% render "decr" %}',
+      partials: { 'decr' => '{% decrement port %}' },
     )
   end