Fix kwarg parsing inconsistency with Liquid::C

Liquid::C parses liquid filter arguments with dashes in them, Liquid does not.

For tags that accept kwargs and dumps them on the HTML tag, this is an important feature.

e.g. {{ ... | image_tag: loading: 'lazy', data-something: 'value!' }}

Without this change, Liquid would incorrectly parse the
`data-something` kwarg as a single argument and would skip over the
invalid characters.

See https://github.com/Shopify/theme-check/issues/539 for more context

.rubocop.yml

@@ -10,7 +10,7 @@ Performance:
   Enabled: true
 
 AllCops:
-  TargetRubyVersion: 2.4
+  TargetRubyVersion: 2.5
   NewCops: disable
   Exclude:
     - 'vendor/bundle/**/*'
@@ -18,3 +18,7 @@ AllCops:
 Naming/MethodName:
   Exclude:
     - 'example/server/liquid_servlet.rb'
+
+# Backport https://github.com/Shopify/ruby-style-guide/pull/258
+Layout/BeginEndAlignment:
+  Enabled: true

History.md

@@ -1,9 +1,15 @@
 # Liquid Change Log
 
-## 5.0.2 (unreleased)
+## 5.1.0 / 2021-09-09
+
+### Features
+* Add `base64_encode`, `base64_decode`, `base64_url_safe_encode`, and `base64_url_safe_decode` filters (#1450) [Daniel Insley]
+* Introduce `to_liquid_value` in `Liquid::Drop` (#1441) [Michael Go]
 
 ### Fixes
 * Fix support for using a String subclass for the liquid source (#1421) [Dylan Thacker-Smith]
+* Add `ParseTreeVisitor` to `RangeLookup` (#1470) [CP Clermont]
+* Translate `RangeError` to `Liquid::Error` for `truncatewords` with large int (#1431) [Dylan Thacker-Smith]
 
 ## 5.0.1 / 2021-03-24
 

README.md

@@ -5,7 +5,7 @@
 
 * [Contributing guidelines](CONTRIBUTING.md)
 * [Version history](History.md)
-* [Liquid documentation from Shopify](http://docs.shopify.com/themes/liquid-basics)
+* [Liquid documentation from Shopify](https://shopify.dev/api/liquid)
 * [Liquid Wiki at GitHub](https://github.com/Shopify/liquid/wiki)
 * [Website](http://liquidmarkup.org/)
 
@@ -56,7 +56,7 @@ For standard use you can just pass it the content of a file and call render with
 
 Setting the error mode of Liquid lets you specify how strictly you want your templates to be interpreted.
 Normally the parser is very lax and will accept almost anything without error. Unfortunately this can make
-it very hard to debug and can lead to unexpected behaviour. 
+it very hard to debug and can lead to unexpected behaviour.
 
 Liquid also comes with a stricter parser that can be used when editing templates to give better error messages
 when templates are invalid. You can enable this new parser like this:

lib/liquid.rb

@@ -36,7 +36,7 @@ module Liquid
   VariableIncompleteEnd       = /\}\}?/
   QuotedString                = /"[^"]*"|'[^']*'/
   QuotedFragment              = /#{QuotedString}|(?:[^\s,\|'"]|#{QuotedString})+/o
-  TagAttributes               = /(\w+)\s*\:\s*(#{QuotedFragment})/o
+  TagAttributes               = /(\w[\w-]*)\s*\:\s*(#{QuotedFragment})/o
   AnyStartingTag              = /#{TagStart}|#{VariableStart}/o
   PartialTemplateParser       = /#{TagStart}.*?#{TagEnd}|#{VariableStart}.*?#{VariableIncompleteEnd}/om
   TemplateParser              = /(#{PartialTemplateParser}|#{AnyStartingTag})/om

lib/liquid/condition.rb

@@ -8,7 +8,7 @@ module Liquid
   #   c = Condition.new(1, '==', 1)
   #   c.evaluate #=> true
   #
-  class Condition #:nodoc:
+  class Condition # :nodoc:
     @@operators = {
       '==' => ->(cond, left, right) {  cond.send(:equal_variables, left, right) },
       '!=' => ->(cond, left, right) { !cond.send(:equal_variables, left, right) },
@@ -134,8 +134,8 @@ module Liquid
       # return this as the result.
       return context.evaluate(left) if op.nil?
 
-      left  = context.evaluate(left)
-      right = context.evaluate(right)
+      left  = Liquid::Utils.to_liquid_value(context.evaluate(left))
+      right = Liquid::Utils.to_liquid_value(context.evaluate(right))
 
       operation = self.class.operators[op] || raise(Liquid::ArgumentError, "Unknown operator #{op}")
 

lib/liquid/context.rb

@@ -124,7 +124,7 @@ module Liquid
     #      context['var'] = 'hi'
     #   end
     #
-    #   context['var]  #=> nil
+    #   context['var']  #=> nil
     def stack(new_scope = {})
       push(new_scope)
       yield

lib/liquid/range_lookup.rb

@@ -12,6 +12,8 @@ module Liquid
       end
     end
 
+    attr_reader :start_obj, :end_obj
+
     def initialize(start_obj, end_obj)
       @start_obj = start_obj
       @end_obj   = end_obj
@@ -35,5 +37,11 @@ module Liquid
         Utils.to_integer(input)
       end
     end
+
+    class ParseTreeVisitor < Liquid::ParseTreeVisitor
+      def children
+        [@node.start_obj, @node.end_obj]
+      end
+    end
   end
 end

lib/liquid/standardfilters.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
 require 'cgi'
+require 'base64'
 require 'bigdecimal'
 
 module Liquid
   module StandardFilters
+    MAX_INT = (1 << 31) - 1
     HTML_ESCAPE = {
       '&' => '&amp;',
       '>' => '&gt;',
@@ -62,6 +64,26 @@ module Liquid
       result
     end
 
+    def base64_encode(input)
+      Base64.strict_encode64(input.to_s)
+    end
+
+    def base64_decode(input)
+      Base64.strict_decode64(input.to_s)
+    rescue ::ArgumentError
+      raise Liquid::ArgumentError, "invalid base64 provided to base64_decode"
+    end
+
+    def base64_url_safe_encode(input)
+      Base64.urlsafe_encode64(input.to_s)
+    end
+
+    def base64_url_safe_decode(input)
+      Base64.urlsafe_decode64(input.to_s)
+    rescue ::ArgumentError
+      raise Liquid::ArgumentError, "invalid base64 provided to base64_url_safe_decode"
+    end
+
     def slice(input, offset, length = nil)
       offset = Utils.to_integer(offset)
       length = length ? Utils.to_integer(length) : 1
@@ -93,7 +115,13 @@ module Liquid
       words = Utils.to_integer(words)
       words = 1 if words <= 0
 
-      wordlist = input.split(" ", words + 1)
+      wordlist = begin
+        input.split(" ", words + 1)
+      rescue RangeError
+        raise if words + 1 < MAX_INT
+        # e.g. integer #{words} too big to convert to `int'
+        raise Liquid::ArgumentError, "integer #{words} too big for truncatewords"
+      end
       return input if wordlist.length <= words
 
       wordlist.pop
@@ -440,7 +468,7 @@ module Liquid
     #
     def default(input, default_value = '', options = {})
       options = {} unless options.is_a?(Hash)
-      false_check = options['allow_false'] ? input.nil? : !input
+      false_check = options['allow_false'] ? input.nil? : !Liquid::Utils.to_liquid_value(input)
       false_check || (input.respond_to?(:empty?) && input.empty?) ? default_value : input
     end
 

lib/liquid/tags/case.rb

@@ -52,7 +52,14 @@ module Liquid
       @blocks.each do |block|
         if block.else?
           block.attachment.render_to_output_buffer(context, output) if execute_else_block
-        elsif block.evaluate(context)
+          next
+        end
+
+        result = Liquid::Utils.to_liquid_value(
+          block.evaluate(context)
+        )
+
+        if result
           execute_else_block = false
           block.attachment.render_to_output_buffer(context, output)
         end

lib/liquid/tags/if.rb

@@ -50,7 +50,11 @@ module Liquid
 
     def render_to_output_buffer(context, output)
       @blocks.each do |block|
-        if block.evaluate(context)
+        result = Liquid::Utils.to_liquid_value(
+          block.evaluate(context)
+        )
+
+        if result
           return block.attachment.render_to_output_buffer(context, output)
         end
       end

lib/liquid/tags/unless.rb

@@ -11,13 +11,21 @@ module Liquid
     def render_to_output_buffer(context, output)
       # First condition is interpreted backwards ( if not )
       first_block = @blocks.first
-      unless first_block.evaluate(context)
+      result = Liquid::Utils.to_liquid_value(
+        first_block.evaluate(context)
+      )
+
+      unless result
         return first_block.attachment.render_to_output_buffer(context, output)
       end
 
       # After the first condition unless works just like if
       @blocks[1..-1].each do |block|
-        if block.evaluate(context)
+        result = Liquid::Utils.to_liquid_value(
+          block.evaluate(context)
+        )
+
+        if result
           return block.attachment.render_to_output_buffer(context, output)
         end
       end

lib/liquid/utils.rb

@@ -81,5 +81,13 @@ module Liquid
     rescue ::ArgumentError
       nil
     end
+
+    def self.to_liquid_value(obj)
+      # Enable "obj" to represent itself as a primitive value like integer, string, or boolean
+      return obj.to_liquid_value if obj.respond_to?(:to_liquid_value)
+
+      # Otherwise return the object itself
+      obj
+    end
   end
 end

lib/liquid/variable_lookup.rb

@@ -40,6 +40,9 @@ module Liquid
       @lookups.each_index do |i|
         key = context.evaluate(@lookups[i])
 
+        # Cast "key" to its liquid value to enable it to act as a primitive value
+        key = Liquid::Utils.to_liquid_value(key)
+
         # If object is a hash- or array-like object we look for the
         # presence of the key and if its available we return it
         if object.respond_to?(:[]) &&

lib/liquid/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module Liquid
-  VERSION = "5.0.2.alpha"
+  VERSION = "5.1.0"
 end

performance/shopify/database.rb

@@ -3,11 +3,19 @@
 require 'yaml'
 
 module Database
+  DATABASE_FILE_PATH = "#{__dir__}/vision.database.yml"
+
   # Load the standard vision toolkit database and re-arrage it to be simply exportable
   # to liquid as assigns. All this is based on Shopify
   def self.tables
     @tables ||= begin
-      db = YAML.load_file("#{__dir__}/vision.database.yml")
+      db =
+        if YAML.respond_to?(:unsafe_load_file) # Only Psych 4+ can use unsafe_load_file
+          # unsafe_load_file is needed for YAML references
+          YAML.unsafe_load_file(DATABASE_FILE_PATH)
+        else
+          YAML.load_file(DATABASE_FILE_PATH)
+        end
 
       # From vision source
       db['products'].each do |product|

test/integration/filter_kwarg_test.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class FilterKwargTest < Minitest::Test
+  module KwargFilter
+    def html_tag(_tag, attributes)
+      attributes
+        .map { |key, value| "#{key}='#{value}'" }
+        .join(' ')
+    end
+  end
+
+  include Liquid
+
+  def test_can_parse_data_kwargs
+    with_global_filter(KwargFilter) do
+      assert_equal(
+        "data-src='src' data-widths='100, 200'",
+        Template.parse("{{ 'img' | html_tag: data-src: 'src', data-widths: '100, 200' }}").render(nil, nil)
+      )
+    end
+  end
+end

test/integration/standard_filter_test.rb

@@ -145,6 +145,40 @@ class StandardFiltersTest < Minitest::Test
     assert_equal('&lt;strong&gt;Hulk&lt;/strong&gt;', @filters.escape_once('&lt;strong&gt;Hulk</strong>'))
   end
 
+  def test_base64_encode
+    assert_equal('b25lIHR3byB0aHJlZQ==', @filters.base64_encode('one two three'))
+    assert_equal('', @filters.base64_encode(nil))
+  end
+
+  def test_base64_decode
+    assert_equal('one two three', @filters.base64_decode('b25lIHR3byB0aHJlZQ=='))
+
+    exception = assert_raises(Liquid::ArgumentError) do
+      @filters.base64_decode("invalidbase64")
+    end
+
+    assert_equal('Liquid error: invalid base64 provided to base64_decode', exception.message)
+  end
+
+  def test_base64_url_safe_encode
+    assert_equal(
+      'YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXogQUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVogMTIzNDU2Nzg5MCAhQCMkJV4mKigpLT1fKy8_Ljo7W117fVx8',
+      @filters.base64_url_safe_encode('abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ 1234567890 !@#$%^&*()-=_+/?.:;[]{}\|')
+    )
+    assert_equal('', @filters.base64_url_safe_encode(nil))
+  end
+
+  def test_base64_url_safe_decode
+    assert_equal(
+      'abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ 1234567890 !@#$%^&*()-=_+/?.:;[]{}\|',
+      @filters.base64_url_safe_decode('YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXogQUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVogMTIzNDU2Nzg5MCAhQCMkJV4mKigpLT1fKy8_Ljo7W117fVx8')
+    )
+    exception = assert_raises(Liquid::ArgumentError) do
+      @filters.base64_url_safe_decode("invalidbase64")
+    end
+    assert_equal('Liquid error: invalid base64 provided to base64_url_safe_decode', exception.message)
+  end
+
   def test_url_encode
     assert_equal('foo%2B1%40example.com', @filters.url_encode('[email protected]'))
     assert_equal('1', @filters.url_encode(1))
@@ -178,6 +212,10 @@ class StandardFiltersTest < Minitest::Test
     assert_equal('one two three...', @filters.truncatewords("one  two\tthree\nfour", 3))
     assert_equal('one two...', @filters.truncatewords("one two three four", 2))
     assert_equal('one...', @filters.truncatewords("one two three four", 0))
+    exception = assert_raises(Liquid::ArgumentError) do
+      @filters.truncatewords("one two three four", 1 << 31)
+    end
+    assert_equal("Liquid error: integer #{1 << 31} too big for truncatewords", exception.message)
   end
 
   def test_strip_html
@@ -690,6 +728,8 @@ class StandardFiltersTest < Minitest::Test
     assert_equal("bar", @filters.default([], "bar"))
     assert_equal("bar", @filters.default({}, "bar"))
     assert_template_result('bar', "{{ false | default: 'bar' }}")
+    assert_template_result('bar', "{{ drop | default: 'bar' }}", 'drop' => BooleanDrop.new(false))
+    assert_template_result('Yay', "{{ drop | default: 'bar' }}", 'drop' => BooleanDrop.new(true))
   end
 
   def test_default_handle_false
@@ -700,6 +740,8 @@ class StandardFiltersTest < Minitest::Test
     assert_equal("bar", @filters.default([], "bar", "allow_false" => true))
     assert_equal("bar", @filters.default({}, "bar", "allow_false" => true))
     assert_template_result('false', "{{ false | default: 'bar', allow_false: true }}")
+    assert_template_result('Nay', "{{ drop | default: 'bar', allow_false: true }}", 'drop' => BooleanDrop.new(false))
+    assert_template_result('Yay', "{{ drop | default: 'bar', allow_false: true }}", 'drop' => BooleanDrop.new(true))
   end
 
   def test_cannot_access_private_methods
@@ -728,6 +770,18 @@ class StandardFiltersTest < Minitest::Test
     assert_equal(expectation, @filters.where(input, "ok"))
   end
 
+  def test_where_string_keys
+    input = [
+      "alpha", "beta", "gamma", "delta"
+    ]
+
+    expectation = [
+      "beta",
+    ]
+
+    assert_equal(expectation, @filters.where(input, "be"))
+  end
+
   def test_where_no_key_set
     input = [
       { "handle" => "alpha", "ok" => true },

test/integration/variable_test.rb

@@ -15,6 +15,33 @@ class VariableTest < Minitest::Test
     assert_template_result('foobar', '{{ foo }}', 'foo' => ThingWithToLiquid.new)
   end
 
+  def test_variable_lookup_calls_to_liquid_value
+    assert_template_result('1', '{{ foo }}', 'foo' => IntegerDrop.new('1'))
+    assert_template_result('2', '{{ list[foo] }}', 'foo' => IntegerDrop.new('1'), 'list' => [1, 2, 3])
+    assert_template_result('one', '{{ list[foo] }}', 'foo' => IntegerDrop.new('1'), 'list' => { 1 => 'one' })
+    assert_template_result('Yay', '{{ foo }}', 'foo' => BooleanDrop.new(true))
+    assert_template_result('YAY', '{{ foo | upcase }}', 'foo' => BooleanDrop.new(true))
+  end
+
+  def test_if_tag_calls_to_liquid_value
+    assert_template_result('one', '{% if foo == 1 %}one{% endif %}', 'foo' => IntegerDrop.new('1'))
+    assert_template_result('one', '{% if 0 < foo %}one{% endif %}', 'foo' => IntegerDrop.new('1'))
+    assert_template_result('one', '{% if foo > 0 %}one{% endif %}', 'foo' => IntegerDrop.new('1'))
+    assert_template_result('true', '{% if foo == true %}true{% endif %}', 'foo' => BooleanDrop.new(true))
+    assert_template_result('true', '{% if foo %}true{% endif %}', 'foo' => BooleanDrop.new(true))
+
+    assert_template_result('', '{% if foo %}true{% endif %}', 'foo' => BooleanDrop.new(false))
+    assert_template_result('', '{% if foo == true %}True{% endif %}', 'foo' => BooleanDrop.new(false))
+  end
+
+  def test_unless_tag_calls_to_liquid_value
+    assert_template_result('', '{% unless foo %}true{% endunless %}', 'foo' => BooleanDrop.new(true))
+  end
+
+  def test_case_tag_calls_to_liquid_value
+    assert_template_result('One', '{% case foo %}{% when 1 %}One{% endcase %}', 'foo' => IntegerDrop.new('1'))
+  end
+
   def test_simple_with_whitespaces
     template = Template.parse(%(  {{ test }}  ))
     assert_equal('  worked  ', template.render!('test' => 'worked'))
@@ -104,4 +131,8 @@ class VariableTest < Minitest::Test
   def test_dynamic_find_var
     assert_template_result('bar', '{{ [key] }}', 'key' => 'foo', 'foo' => 'bar')
   end
+
+  def test_raw_value_variable
+    assert_template_result('bar', '{{ [key] }}', 'key' => 'foo', 'foo' => 'bar')
+  end
 end

test/test_helper.rb

@@ -119,6 +119,44 @@ class ThingWithToLiquid
   end
 end
 
+class IntegerDrop < Liquid::Drop
+  def initialize(value)
+    super()
+    @value = value.to_i
+  end
+
+  def ==(other)
+    @value == other
+  end
+
+  def to_s
+    @value.to_s
+  end
+
+  def to_liquid_value
+    @value
+  end
+end
+
+class BooleanDrop < Liquid::Drop
+  def initialize(value)
+    super()
+    @value = value
+  end
+
+  def ==(other)
+    @value == other
+  end
+
+  def to_liquid_value
+    @value
+  end
+
+  def to_s
+    @value ? "Yay" : "Nay"
+  end
+end
+
 class ErrorDrop < Liquid::Drop
   def standard_error
     raise Liquid::StandardError, 'standard error'

test/unit/parse_tree_visitor_test.rb

@@ -159,6 +159,13 @@ class ParseTreeVisitorTest < Minitest::Test
     )
   end
 
+  def test_for_range
+    assert_equal(
+      ["test"],
+      visit(%({% for x in (1..test) %}{% endfor %}))
+    )
+  end
+
   def test_tablerow_in
     assert_equal(
       ["test"],