add variable parser timeout unit tests

Allow raw tag to be used with whitespace control

.github/workflows/liquid.yml

@@ -1,5 +1,10 @@
 name: Liquid
 on: [push, pull_request]
+
+env:
+  BUNDLE_JOBS: 4
+  BUNDLE_RETRY: 3
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -7,22 +12,18 @@ jobs:
       matrix:
         entry:
           - { ruby: 2.7, allowed-failure: false } # minimum supported
-          - { ruby: 3.1, allowed-failure: false } # latest
+          - { ruby: 3.2, allowed-failure: false } # latest
           - { ruby: ruby-head, allowed-failure: true }
-    name: test (${{ matrix.entry.ruby }})
+    name: Test Ruby ${{ matrix.entry.ruby }}
     steps:
       - uses: actions/checkout@v3
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.entry.ruby }}
-      - uses: actions/cache@v1
-        with:
-          path: vendor/bundle
-          key: ${{ runner.os }}-gems-${{ hashFiles('Gemfile') }}
-          restore-keys: ${{ runner.os }}-gems-
-      - run: bundle install --jobs=3 --retry=3 --path=vendor/bundle
+          bundler-cache: true
       - run: bundle exec rake
         continue-on-error: ${{ matrix.entry.allowed-failure }}
+
   memory_profile:
     runs-on: ubuntu-latest
     steps:
@@ -30,10 +31,5 @@ jobs:
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: 2.7
-      - uses: actions/cache@v1
-        with:
-          path: vendor/bundle
-          key: ${{ runner.os }}-gems-${{ hashFiles('Gemfile') }}
-          restore-keys: ${{ runner.os }}-gems-
-      - run: bundle install --jobs=3 --retry=3 --path=vendor/bundle
+          bundler-cache: true
       - run: bundle exec rake memory_profile:run

lib/liquid.rb

@@ -41,7 +41,7 @@ module Liquid
   AnyStartingTag              = /#{TagStart}|#{VariableStart}/o
   PartialTemplateParser       = /#{TagStart}.*?#{TagEnd}|#{VariableStart}.*?#{VariableIncompleteEnd}/om
   TemplateParser              = /(#{PartialTemplateParser}|#{AnyStartingTag})/om
-  VariableParser              = /\[[^\]]+\]|#{VariableSegment}+\??/o
+  VariableParser              = /\[(?>[^\[\]]+|\g<0>)*\]|#{VariableSegment}+\??/o
 
   RAISE_EXCEPTION_LAMBDA = ->(_e) { raise }
 

lib/liquid/drop.rb

@@ -25,6 +25,10 @@ module Liquid
   class Drop
     attr_writer :context
 
+    def initialize
+      @context = nil
+    end
+
     # Catch all for the method
     def liquid_method_missing(method)
       return nil unless @context&.strict_variables

lib/liquid/standardfilters.rb

@@ -757,7 +757,7 @@ module Liquid
     # @liquid_type filter
     # @liquid_category math
     # @liquid_summary
-    #   Divides a number by a given number.
+    #   Divides a number by a given number. The `divided_by` filter produces a result of the same type as the divisor. This means if you divide by an integer, the result will be an integer, and if you divide by a float, the result will be a float.
     # @liquid_syntax number | divided_by: number
     # @liquid_return [number]
     def divided_by(input, operand)

lib/liquid/tags/for.rb

@@ -98,11 +98,12 @@ module Liquid
       @name     = "#{@variable_name}-#{collection_name}"
       @reversed = p.id?('reversed')
 
-      while p.look(:id) && p.look(:colon, 1)
+      while p.look(:comma) || p.look(:id)
+        p.consume?(:comma)
         unless (attribute = p.id?('limit') || p.id?('offset'))
           raise SyntaxError, options[:locale].t("errors.syntax.for_invalid_attribute")
         end
-        p.consume
+        p.consume(:colon)
         set_attribute(attribute, p.expression)
       end
       p.consume(:end_of_string)

lib/liquid/tags/raw.rb

@@ -14,7 +14,7 @@ module Liquid
   # @liquid_syntax_keyword expression The expression to be output without being rendered.
   class Raw < Block
     Syntax = /\A\s*\z/
-    FullTokenPossiblyInvalid = /\A(.*)#{TagStart}\s*(\w+)\s*(.*)?#{TagEnd}\z/om
+    FullTokenPossiblyInvalid = /\A(.*)#{TagStart}#{WhitespaceControl}?\s*(\w+)\s*(.*)?#{WhitespaceControl}?#{TagEnd}\z/om
 
     def initialize(tag_name, markup, parse_context)
       super

lib/liquid/tags/table_row.rb

@@ -45,13 +45,13 @@ module Liquid
     def render_to_output_buffer(context, output)
       (collection = context.evaluate(@collection_name)) || (return '')
 
-      from = @attributes.key?('offset') ? context.evaluate(@attributes['offset']).to_i : 0
-      to   = @attributes.key?('limit')  ? from + context.evaluate(@attributes['limit']).to_i : nil
+      from = @attributes.key?('offset') ? to_integer(context.evaluate(@attributes['offset'])) : 0
+      to = @attributes.key?('limit') ? from + to_integer(context.evaluate(@attributes['limit'])) : nil
 
       collection = Utils.slice_collection(collection, from, to)
       length     = collection.length
 
-      cols = @attributes.key?('cols') ? context.evaluate(@attributes['cols']).to_i : length
+      cols = @attributes.key?('cols') ? to_integer(context.evaluate(@attributes['cols'])) : length
 
       output << "<tr class=\"row1\">\n"
       context.stack do
@@ -82,6 +82,14 @@ module Liquid
         super + @node.attributes.values + [@node.collection_name]
       end
     end
+
+    private
+
+    def to_integer(value)
+      value.to_i
+    rescue NoMethodError
+      raise Liquid::ArgumentError, "invalid integer"
+    end
   end
 
   Template.register_tag('tablerow', TableRow)

lib/liquid/utils.rb

@@ -16,7 +16,11 @@ module Liquid
 
       # Maintains Ruby 1.8.7 String#each behaviour on 1.9
       if collection.is_a?(String)
-        return collection.empty? ? [] : [collection]
+        return [] if collection.empty?
+        if from > 0 || to == 0
+          Usage.increment("string_slice_bug")
+        end
+        return [collection]
       end
       return [] unless collection.respond_to?(:each)
 

test/integration/tags/for_tag_test.rb

@@ -103,6 +103,7 @@ HERE
     assert_template_result('1234', '{%for i in array limit:4 %}{{ i }}{%endfor%}', assigns)
     assert_template_result('3456', '{%for i in array limit:4 offset:2 %}{{ i }}{%endfor%}', assigns)
     assert_template_result('3456', '{%for i in array limit: 4 offset: 2 %}{{ i }}{%endfor%}', assigns)
+    assert_template_result('3456', '{%for i in array, limit: 4, offset: 2 %}{{ i }}{%endfor%}', assigns)
   end
 
   def test_limiting_with_invalid_limit

test/integration/tags/raw_tag_test.rb

@@ -11,7 +11,8 @@ class RawTagTest < Minitest::Test
   end
 
   def test_output_in_raw
-    assert_template_result('{{ test }}', '{% raw %}{{ test }}{% endraw %}')
+    # assert_template_result('{{ test }}', '{% raw %}{{ test }}{% endraw %}')
+    assert_template_result('>{{ test }}<', '> {%- raw -%}{{ test }}{%- endraw -%} <')
   end
 
   def test_open_tag_in_raw

test/integration/tags/table_row_test.rb

@@ -80,6 +80,32 @@ class TableRowTest < Minitest::Test
       { "var" => nil })
   end
 
+  def test_nil_limit_is_treated_as_zero
+    expect = "<tr class=\"row1\">\n" \
+      "</tr>\n"
+
+    assert_template_result(expect,
+      "{% tablerow i in (1..2) limit:nil %}{{ i }}{% endtablerow %}")
+
+    assert_template_result(expect,
+      "{% tablerow i in (1..2) limit:var %}{{ i }}{% endtablerow %}",
+      { "var" => nil })
+  end
+
+  def test_nil_offset_is_treated_as_zero
+    expect = "<tr class=\"row1\">\n" \
+      "<td class=\"col1\">1:false</td>" \
+      "<td class=\"col2\">2:true</td>" \
+      "</tr>\n"
+
+    assert_template_result(expect,
+      "{% tablerow i in (1..2) offset:nil %}{{ i }}:{{ tablerowloop.col_last }}{% endtablerow %}")
+
+    assert_template_result(expect,
+      "{% tablerow i in (1..2) offset:var %}{{ i }}:{{ tablerowloop.col_last }}{% endtablerow %}",
+      { "var" => nil })
+  end
+
   def test_tablerow_loop_drop_attributes
     template = <<~LIQUID.chomp
       {% tablerow i in (1...2) %}
@@ -131,4 +157,24 @@ class TableRowTest < Minitest::Test
 
     assert_template_result(expected_output, template)
   end
+
+  def test_table_row_renders_correct_error_message_for_invalid_parameters
+    assert_template_result(
+      "Liquid error (line 1): invalid integer",
+      '{% tablerow n in (1...10) limit:true %} {{n}} {% endtablerow %}',
+      render_errors: true,
+    )
+
+    assert_template_result(
+      "Liquid error (line 1): invalid integer",
+      '{% tablerow n in (1...10) offset:true %} {{n}} {% endtablerow %}',
+      render_errors: true,
+    )
+
+    assert_template_result(
+      "Liquid error (line 1): invalid integer",
+      '{% tablerow n in (1...10) cols:true %} {{n}} {% endtablerow %}',
+      render_errors: true,
+    )
+  end
 end

test/integration/variable_test.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'test_helper'
+require 'timeout'
 
 class VariableTest < Minitest::Test
   include Liquid
@@ -24,17 +25,22 @@ class VariableTest < Minitest::Test
 
   def test_if_tag_calls_to_liquid_value
     assert_template_result('one', '{% if foo == 1 %}one{% endif %}', { 'foo' => IntegerDrop.new('1') })
+    assert_template_result('one', '{% if foo == eqv %}one{% endif %}', { 'foo' => IntegerDrop.new(1), 'eqv' => IntegerDrop.new(1) })
     assert_template_result('one', '{% if 0 < foo %}one{% endif %}', { 'foo' => IntegerDrop.new('1') })
     assert_template_result('one', '{% if foo > 0 %}one{% endif %}', { 'foo' => IntegerDrop.new('1') })
+    assert_template_result('one', '{% if b > a %}one{% endif %}', { 'b' => IntegerDrop.new(1), 'a' => IntegerDrop.new(0) })
     assert_template_result('true', '{% if foo == true %}true{% endif %}', { 'foo' => BooleanDrop.new(true) })
     assert_template_result('true', '{% if foo %}true{% endif %}', { 'foo' => BooleanDrop.new(true) })
 
     assert_template_result('', '{% if foo %}true{% endif %}', { 'foo' => BooleanDrop.new(false) })
     assert_template_result('', '{% if foo == true %}True{% endif %}', { 'foo' => BooleanDrop.new(false) })
+
+    assert_template_result('one', '{% if a contains x %}one{% endif %}', { 'a' => [1], 'x' => IntegerDrop.new(1) })
   end
 
   def test_unless_tag_calls_to_liquid_value
     assert_template_result('', '{% unless foo %}true{% endunless %}', { 'foo' => BooleanDrop.new(true) })
+    assert_template_result('true', '{% unless foo %}true{% endunless %}', { 'foo' => BooleanDrop.new(false) })
   end
 
   def test_case_tag_calls_to_liquid_value
@@ -130,4 +136,72 @@ class VariableTest < Minitest::Test
   def test_raw_value_variable
     assert_template_result('bar', '{{ [key] }}', { 'key' => 'foo', 'foo' => 'bar' })
   end
+
+  def test_dynamic_find_var_with_drop
+    assert_template_result(
+      'bar',
+      '{{ [list[settings.zero]] }}',
+      {
+        'list' => ['foo'],
+        'settings' => SettingsDrop.new("zero" => 0),
+        'foo' => 'bar',
+      }
+    )
+
+    assert_template_result(
+      'foo',
+      '{{ [list[settings.zero]["foo"]] }}',
+      {
+        'list' => [{ 'foo' => 'bar' }],
+        'settings' => SettingsDrop.new("zero" => 0),
+        'bar' => 'foo',
+      }
+    )
+  end
+
+  def test_double_nested_variable_lookup
+    assert_template_result(
+      'bar',
+      '{{ list[list[settings.zero]]["foo"] }}',
+      {
+        'list' => [1, { 'foo' => 'bar' }],
+        'settings' => SettingsDrop.new("zero" => 0),
+        'bar' => 'foo',
+      }
+    )
+  end
+
+  def test_variable_lookup_should_not_hang_with_invalid_syntax
+    Timeout.timeout(1) do
+      assert_template_result(
+        'bar',
+        "{{['foo'}}",
+        {
+          'foo' => 'bar',
+        },
+        error_mode: :lax,
+      )
+    end
+
+    very_long_key = "1234567890" * 100
+
+    template_list = [
+      "{{['#{very_long_key}']}}", # valid
+      "{{['#{very_long_key}'}}", # missing closing bracket
+      "{{[['#{very_long_key}']}}", # extra open bracket
+    ]
+
+    template_list.each do |template|
+      Timeout.timeout(1) do
+        assert_template_result(
+          'bar',
+          template,
+          {
+            very_long_key => 'bar',
+          },
+          error_mode: :lax,
+        )
+      end
+    end
+  end
 end

test/test_helper.rb

@@ -125,16 +125,23 @@ class ThingWithToLiquid
   end
 end
 
+class SettingsDrop < Liquid::Drop
+  def initialize(settings)
+    super()
+    @settings = settings
+  end
+
+  def liquid_method_missing(key)
+    @settings[key]
+  end
+end
+
 class IntegerDrop < Liquid::Drop
   def initialize(value)
     super()
     @value = value.to_i
   end
 
-  def ==(other)
-    @value == other
-  end
-
   def to_s
     @value.to_s
   end
@@ -150,10 +157,6 @@ class BooleanDrop < Liquid::Drop
     @value = value
   end
 
-  def ==(other)
-    @value == other
-  end
-
   def to_liquid_value
     @value
   end

test/unit/regexp_unit_test.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'test_helper'
+require 'timeout'
 
 class RegexpUnitTest < Minitest::Test
   include Liquid
@@ -37,10 +38,22 @@ class RegexpUnitTest < Minitest::Test
 
   def test_variable_parser
     assert_equal(['var'],                               'var'.scan(VariableParser))
+    assert_equal(['[var]'],                             '[var]'.scan(VariableParser))
     assert_equal(['var', 'method'],                     'var.method'.scan(VariableParser))
     assert_equal(['var', '[method]'],                   'var[method]'.scan(VariableParser))
     assert_equal(['var', '[method]', '[0]'],            'var[method][0]'.scan(VariableParser))
     assert_equal(['var', '["method"]', '[0]'],          'var["method"][0]'.scan(VariableParser))
     assert_equal(['var', '[method]', '[0]', 'method'],  'var[method][0].method'.scan(VariableParser))
   end
+
+  def test_variable_parser_with_large_input
+    Timeout.timeout(1) { assert_equal(['[var]'], '[var]'.scan(VariableParser)) }
+
+    very_long_string = "foo" * 1000
+
+    # valid dynamic lookup
+    Timeout.timeout(1) { assert_equal(["[#{very_long_string}]"], "[#{very_long_string}]".scan(VariableParser)) }
+    # invalid dynamic lookup with missing closing bracket
+    Timeout.timeout(1) { assert_equal([very_long_string], "[#{very_long_string}".scan(VariableParser)) }
+  end
 end # RegexpTest

test/unit/tags/for_tag_unit_test.rb

@@ -12,4 +12,34 @@ class ForTagUnitTest < Minitest::Test
     template = Liquid::Template.parse('{% for item in items %}FOR{% else %}ELSE{% endfor %}')
     assert_equal(['FOR', 'ELSE'], template.root.nodelist[0].nodelist.map(&:nodelist).flatten)
   end
+
+  def test_for_string_slice_bug_usage
+    template = Liquid::Template.parse("{% for x in str, offset: 1 %}{{ x }},{% endfor %}")
+    assert_usage("string_slice_bug") do
+      assert_equal("abc,", template.render({ "str" => "abc" }))
+    end
+  end
+
+  def test_for_string_0_limit_usage
+    template = Liquid::Template.parse("{% for x in str, limit: 0 %}{{ x }},{% endfor %}")
+    assert_usage("string_slice_bug") do
+      assert_equal("abc,", template.render({ "str" => "abc" }))
+    end
+  end
+
+  def test_for_string_no_slice_usage
+    template = Liquid::Template.parse("{% for x in str, offset: 0, limit: 1 %}{{ x }},{% endfor %}")
+    assert_usage("string_slice_bug", times: 0) do
+      assert_equal("abc,", template.render({ "str" => "abc" }))
+    end
+  end
+
+  private
+
+  def assert_usage(name, times: 1, &block)
+    count = 0
+    result = Liquid::Usage.stub(:increment, ->(n) { count += 1 if n == name }, &block)
+    assert_equal(times, count)
+    result
+  end
 end