Prototype

fix parsing quirky incomplete expressions

lib/liquid/block_body.rb

@@ -246,10 +246,17 @@ module Liquid
     end
 
     def create_variable(token, parse_context)
-      if token =~ ContentOfVariable
-        markup = Regexp.last_match(1)
+      if token.end_with?("}}")
+        i = 2
+        i = 3 if token[i] == "-"
+        parse_end = token.length - 3
+        parse_end -= 1 if token[parse_end] == "-"
+        markup_end = parse_end - i + 1
+        markup = markup_end <= 0 ? "" : token.slice(i, markup_end)
+
         return Variable.new(markup, parse_context)
       end
+
       BlockBody.raise_missing_variable_terminator(token, parse_context)
     end
 

lib/liquid/lexer.rb

@@ -73,7 +73,6 @@ module Liquid
     COMPARISON_LESS_THAN = [:comparison, "<"].freeze
     COMPARISON_LESS_THAN_OR_EQUAL = [:comparison, "<="].freeze
     COMPARISON_NOT_EQUAL_ALT = [:comparison, "<>"].freeze
-    CONTAINS = /contains(?=\s)/
     DASH = [:dash, "-"].freeze
     DOT = [:dot, "."].freeze
     DOTDOT = [:dotdot, ".."].freeze
@@ -90,7 +89,12 @@ module Liquid
     SINGLE_STRING_LITERAL = /'[^\']*'/
     WHITESPACE_OR_NOTHING = /\s*/
 
-    COMPARISON_JUMP_TABLE = [].tap do |table|
+    SINGLE_COMPARISON_TOKENS = [].tap do |table|
+      table["<".ord] = COMPARISON_LESS_THAN
+      table[">".ord] = COMPARISON_GREATER_THAN
+    end
+
+    TWO_CHARS_COMPARISON_JUMP_TABLE = [].tap do |table|
       table["=".ord] = [].tap do |sub_table|
         sub_table["=".ord] = COMPARISON_EQUAL
         sub_table.freeze
@@ -99,6 +103,9 @@ module Liquid
         sub_table["=".ord] = COMPARISION_NOT_EQUAL
         sub_table.freeze
       end
+    end
+
+    COMPARISON_JUMP_TABLE = [].tap do |table|
       table["<".ord] = [].tap do |sub_table|
         sub_table["=".ord] = COMPARISON_LESS_THAN_OR_EQUAL
         sub_table[">".ord] = COMPARISON_NOT_EQUAL_ALT
@@ -163,6 +170,7 @@ module Liquid
 
         break if @ss.eos?
 
+        start_pos = @ss.pos
         peeked = @ss.peek_byte
 
         if (special = SPECIAL_TABLE[peeked])
@@ -173,7 +181,7 @@ module Liquid
             @output << DOTDOT
           elsif special == DASH
             # Special case for negative numbers
-            if NUMBER_TABLE[@ss.peek_byte]
+            if (peeked_byte = @ss.peek_byte) && NUMBER_TABLE[peeked_byte]
               @ss.pos -= 1
               @output << [:number, @ss.scan(NUMBER_LITERAL)]
             else
@@ -182,26 +190,34 @@ module Liquid
           else
             @output << special
           end
-        elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
+        elsif (sub_table = TWO_CHARS_COMPARISON_JUMP_TABLE[peeked])
           @ss.scan_byte
-          if (found = sub_table[@ss.peek_byte])
+          if (peeked_byte = @ss.peek_byte) && (found = sub_table[peeked_byte])
             @output << found
             @ss.scan_byte
           else
-            raise SyntaxError, "Unexpected character #{peeked.chr}"
+            raise_syntax_error(start_pos)
+          end
+        elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
+          @ss.scan_byte
+          if (peeked_byte = @ss.peek_byte) && (found = sub_table[peeked_byte])
+            @output << found
+            @ss.scan_byte
+          else
+            @output << SINGLE_COMPARISON_TOKENS[peeked]
           end
         else
           type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked]
 
           if type && (t = @ss.scan(pattern))
             # Special case for "contains"
-            @output << if type == :id && t == "contains"
+            @output << if type == :id && t == "contains" && @output.last&.first != :dot
               COMPARISON_CONTAINS
             else
               [type, t]
             end
           else
-            raise SyntaxError, "Unexpected character #{peeked.chr}"
+            raise_syntax_error(start_pos)
           end
         end
       end
@@ -209,6 +225,12 @@ module Liquid
 
       @output << EOS
     end
+
+    def raise_syntax_error(start_pos)
+      @ss.pos = start_pos
+      # the character could be a UTF-8 character, use getch to get all the bytes
+      raise SyntaxError, "Unexpected character #{@ss.getch}"
+    end
   end
 
   Lexer = StringScanner.instance_methods.include?(:scan_byte) ? Lexer2 : Lexer1

lib/liquid/standardfilters.rb

@@ -207,12 +207,18 @@ module Liquid
     def slice(input, offset, length = nil)
       offset = Utils.to_integer(offset)
       length = length ? Utils.to_integer(length) : 1
+      default_value = []
 
       begin
-        if input.is_a?(Array)
-          input.slice(offset, length) || []
+        unless input.is_a?(Array)
+          default_value = ''
+          input = input.to_s
+        end
+
+        if length.negative?
+          input[offset...length] || default_value
         else
-          input.to_s.slice(offset, length) || ''
+          input.slice(offset, length) || default_value
         end
       rescue RangeError
         if I64_RANGE.cover?(length) && I64_RANGE.cover?(offset)
@@ -424,29 +430,59 @@ module Liquid
     # @liquid_syntax array | where: string, string
     # @liquid_return [array[untyped]]
     def where(input, property, target_value = nil)
-      ary = InputIterator.new(input, context)
+      filter_array(input, property, target_value, :select)
+    end
 
-      if ary.empty?
-        []
-      elsif target_value.nil?
-        ary.select do |item|
-          item[property]
-        rescue TypeError
-          raise_property_error(property)
-        rescue NoMethodError
-          return nil unless item.respond_to?(:[])
-          raise
-        end
-      else
-        ary.select do |item|
-          item[property] == target_value
-        rescue TypeError
-          raise_property_error(property)
-        rescue NoMethodError
-          return nil unless item.respond_to?(:[])
-          raise
-        end
-      end
+    # @liquid_public_docs
+    # @liquid_type filter
+    # @liquid_category array
+    # @liquid_summary
+    #   Filters an array to exclude items with a specific property value.
+    # @liquid_description
+    #   This requires you to provide both the property name and the associated value.
+    # @liquid_syntax array | reject: string, string
+    # @liquid_return [array[untyped]]
+    def reject(input, property, target_value = nil)
+      filter_array(input, property, target_value, :reject)
+    end
+
+    # @liquid_public_docs
+    # @liquid_type filter
+    # @liquid_category array
+    # @liquid_summary
+    #   Tests if any item in an array has a specific property value.
+    # @liquid_description
+    #   This requires you to provide both the property name and the associated value.
+    # @liquid_syntax array | some: string, string
+    # @liquid_return [boolean]
+    def has?(input, property, target_value = nil)
+      filter_array(input, property, target_value, :any?)
+    end
+
+    # @liquid_public_docs
+    # @liquid_type filter
+    # @liquid_category array
+    # @liquid_summary
+    #   Returns the first item in an array with a specific property value.
+    # @liquid_description
+    #   This requires you to provide both the property name and the associated value.
+    # @liquid_syntax array | find: string, string
+    # @liquid_return [untyped]
+    def find(input, property, target_value = nil)
+      filter_array(input, property, target_value, :find)
+    end
+
+    # @liquid_public_docs
+    # @liquid_type filter
+    # @liquid_category array
+    # @liquid_summary
+    #   Returns the index of the first item in an array with a specific property value.
+    # @liquid_description
+    #   This requires you to provide both the property name and the associated value.
+    # @liquid_syntax array | find_index: string, string
+    # @liquid_return [number]
+    def find_index(input, property, target_value = nil)
+      filter_array(input, property, target_value, :find_index)
     end
 
     # @liquid_public_docs
@@ -877,7 +913,7 @@ module Liquid
     #   - [`nil`](/docs/api/liquid/basics#nil)
     # @liquid_syntax variable | default: variable
     # @liquid_return [untyped]
-    # @liquid_optional_param allow_false [boolean] Whether to use false values instead of the default.
+    # @liquid_optional_param allow_false: [boolean] Whether to use false values instead of the default.
     def default(input, default_value = '', options = {})
       options = {} unless options.is_a?(Hash)
       false_check = options['allow_false'] ? input.nil? : !Liquid::Utils.to_liquid_value(input)
@@ -918,6 +954,47 @@ module Liquid
 
     attr_reader :context
 
+    def filter_array(input, property, target_value, method)
+      ary = InputIterator.new(input, context)
+
+      return [] if ary.empty?
+
+      ary.public_send(method) do |item|
+        case target_value
+        when nil
+          item[property]
+        when Hash
+          compare_with_operator(item[property], target_value)
+        else
+          item[property] == target_value
+        end
+      rescue TypeError
+        raise_property_error(property)
+      rescue NoMethodError
+        return nil unless item.respond_to?(:[])
+        raise
+      end
+    end
+
+    def compare_with_operator(value, comparison)
+      operation = comparison.keys.first
+      compare_value = comparison.values.first
+      operators = {
+        'greater' => ->(a, b) { a > b },
+        'less' => ->(a, b) { a < b },
+        'greater_or_equal' => ->(a, b) { a >= b },
+        'less_or_equal' => ->(a, b) { a <= b },
+        'contains' => ->(a, b) { a.to_s.include?(b.to_s) },
+      }
+
+      operator = operators[operation]
+      operator && operator.call(value, compare_value)
+    rescue NoMethodError, TypeError, ArgumentError
+      false
+    rescue StandardError
+      false
+    end
+
     def raise_property_error(property)
       raise Liquid::ArgumentError, "cannot select the property '#{property}'"
     end

test/integration/parsing_quirks_test.rb

@@ -131,4 +131,16 @@ class ParsingQuirksTest < Minitest::Test
   def test_contains_in_id
     assert_template_result(' YES ', '{% if containsallshipments == true %} YES {% endif %}', { 'containsallshipments' => true })
   end
+
+  def test_incomplete_expression
+    with_error_mode(:lax) do
+      assert_template_result("false", "{% liquid assign foo = false -\n%}{{ foo }}")
+      assert_template_result("false", "{% liquid assign foo = false >\n%}{{ foo }}")
+      assert_template_result("false", "{% liquid assign foo = false <\n%}{{ foo }}")
+      assert_template_result("false", "{% liquid assign foo = false =\n%}{{ foo }}")
+      assert_template_result("false", "{% liquid assign foo = false !\n%}{{ foo }}")
+      assert_template_result("false", "{% liquid assign foo = false 1\n%}{{ foo }}")
+      assert_template_result("false", "{% liquid assign foo = false a\n%}{{ foo }}")
+    end
+  end
 end # ParsingQuirksTest

test/integration/standard_filter_test.rb

@@ -131,6 +131,9 @@ class StandardFiltersTest < Minitest::Test
     assert_equal(input, @filters.slice(input, 0, 1 << 63))
     assert_equal([], @filters.slice(input, 1 << 63, 6))
     assert_equal([], @filters.slice(input, -(1 << 63), 6))
+    assert_equal(%w(b a), @filters.slice(input, 3, -1))
+    assert_equal(%w(o b), @filters.slice(input, 2, -2))
+    assert_equal(%w(f o o), @filters.slice(input, 0, -3))
   end
 
   def test_truncate
@@ -778,14 +781,69 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result('bcd', "{{ a | append: b}}", assigns)
   end
 
-  def test_concat
-    assert_equal([1, 2, 3, 4], @filters.concat([1, 2], [3, 4]))
-    assert_equal([1, 2, 'a'],  @filters.concat([1, 2], ['a']))
-    assert_equal([1, 2, 10],   @filters.concat([1, 2], [10]))
+  def test_append_with_arrays
+    products = [
+      "Snowdevil pro goggles",
+      "Snowdevil alpine jacket",
+    ]
 
-    assert_raises(Liquid::ArgumentError, "concat filter requires an array argument") do
-      @filters.concat([1, 2], 10)
-    end
+    template = <<~LIQUID
+      {{
+        products
+        | append: 'Snowdevil bonus gift'
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "Snowdevil pro goggles, Snowdevil alpine jacket, Snowdevil bonus gift"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_concat
+    array1 = [1, 2]
+    array2 = [3, 4]
+
+    template = <<~LIQUID
+      {{
+        array1
+        | concat: array2
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "1, 2, 3, 4"
+
+    assert_template_result(expected_output, template, { "array1" => array1, "array2" => array2 })
+  end
+
+  def test_concat_with_string
+    array1 = [1, 2]
+    array2 = ['a']
+
+    template = <<~LIQUID
+      {{
+        array1
+        | concat: array2
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "1, 2, a"
+
+    assert_template_result(expected_output, template, { "array1" => array1, "array2" => array2 })
+  end
+
+  def test_concat_with_number
+    array = [1, 2]
+
+    template = <<~LIQUID
+      {{
+        array
+        | concat: 10
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "1, 2, 10"
+
+    assert_template_result(expected_output, template, { "array" => array })
   end
 
   def test_prepend
@@ -794,6 +852,24 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result('abc', "{{ a | prepend: b}}", assigns)
   end
 
+  def test_prepend_with_arrays
+    products = [
+      "Snowdevil pro goggles",
+      "Snowdevil alpine jacket",
+    ]
+
+    template = <<~LIQUID
+      {{
+        products
+        | prepend: 'Snowdevil bonus gift'
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "Snowdevil bonus gift, Snowdevil pro goggles, Snowdevil alpine jacket"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
   def test_default
     assert_equal("foo", @filters.default("foo", "bar"))
     assert_equal("bar", @filters.default(nil, "bar"))
@@ -827,21 +903,306 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result('abc', "{{ 'abc' | date: '%D' }}")
   end
 
-  def test_where
-    input = [
+  def test_where_with_value
+    array = [
       { "handle" => "alpha", "ok" => true },
       { "handle" => "beta", "ok" => false },
       { "handle" => "gamma", "ok" => false },
       { "handle" => "delta", "ok" => true },
     ]
 
-    expectation = [
+    template = "{{ array | where: 'ok', true | map: 'handle' | join: ' ' }}"
+    expected_output = "alpha delta"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_where_without_value
+    array = [
       { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
       { "handle" => "delta", "ok" => true },
     ]
 
-    assert_equal(expectation, @filters.where(input, "ok", true))
-    assert_equal(expectation, @filters.where(input, "ok"))
+    template = "{{ array | where: 'ok' | map: 'handle' | join: ' ' }}"
+    expected_output = "alpha delta"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_where_without_value
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | where: 'ok', false | map: 'handle' | join: ' ' }}"
+    expected_output = "beta gamma"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_where_with_greater_than
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = <<~LIQUID
+      {{
+        products
+        | where: 'price', greater: 300
+        | map: 'title'
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "Snowdevil alpine jacket, Snowdevil mountain boots"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_where_with_less_than
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = <<~LIQUID
+      {{
+        products
+        | where: 'price', less: 150
+        | map: 'title'
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "Snowdevil pro goggles, Snowdevil thermal gloves"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_where_with_greater_or_equal
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = <<~LIQUID
+      {{
+        products
+        | where: 'price', greater_or_equal: 389.99
+        | map: 'title'
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "Snowdevil alpine jacket, Snowdevil mountain boots"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_where_with_less_or_equal
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = <<~LIQUID
+      {{
+        products
+        | where: 'price', less_or_equal: 149.99
+        | map: 'title'
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "Snowdevil pro goggles, Snowdevil thermal gloves"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_where_with_contains
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = <<~LIQUID
+      {{
+        products
+        | where: 'title', contains: 'es'
+        | map: 'title'
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "Snowdevil pro goggles, Snowdevil thermal gloves"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_reject_with_value
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | reject: 'ok', true | map: 'handle' | join: ' ' }}"
+    expected_output = "beta gamma"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def xtest_reject_with_value2
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | add:array.first }}"
+    expected_output = "beta gamma"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_reject_with_greater_operator
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = <<~LIQUID
+      {{
+        products
+        | reject: 'price', greater: 190
+        | map: 'title'
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "Snowdevil pro goggles, Snowdevil thermal gloves"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_reject_without_value
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | reject: 'ok' | map: 'handle' | join: ' ' }}"
+    expected_output = "beta gamma"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_reject_with_false_value
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | reject: 'ok', false | map: 'handle' | join: ' ' }}"
+    expected_output = "alpha delta"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_some_with_value
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | some: 'ok', true }}"
+    expected_output = "true"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_some_without_value
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | some: 'ok' }}"
+    expected_output = "true"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_some_with_false_value
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | some: 'ok', false }}"
+    expected_output = "true"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_some_with_contains
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = "{{ products | some: 'title', contains: 'safety' }}"
+    expected_output = "true"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_some_with_contains
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = "{{ products | some: 'title', contains: 'game boy' }}"
+    expected_output = "false"
+
+    assert_template_result(expected_output, template, { "products" => products })
   end
 
   def test_where_string_keys
@@ -873,6 +1234,42 @@ class StandardFiltersTest < Minitest::Test
     assert_equal(expectation, @filters.where(input, "ok"))
   end
 
+  def test_find_with_value
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = <<~LIQUID
+      {%- assign product = products | find: 'price', greater: 150 -%}
+      {{- product.title -}}
+    LIQUID
+    expected_output = "Snowdevil alpine jacket"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_find_index_with_value
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = <<~LIQUID
+      {%- assign index = products | find_index: 'price', greater: 150 -%}
+      {{- index -}}
+    LIQUID
+    expected_output = "2"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
   def test_where_non_array_map_input
     assert_equal([{ "a" => "ok" }], @filters.where({ "a" => "ok" }, "a", "ok"))
     assert_equal([], @filters.where({ "a" => "not ok" }, "a", "ok"))

test/unit/block_unit_test.rb

@@ -32,6 +32,12 @@ class BlockUnitTest < Minitest::Test
     assert_equal(String, template.root.nodelist[2].class)
   end
 
+  def test_variable_with_multibyte_character
+    template = Liquid::Template.parse("{{ '❤️' }}")
+    assert_equal(1, template.root.nodelist.size)
+    assert_equal(Variable, template.root.nodelist[0].class)
+  end
+
   def test_variable_many_embedded_fragments
     template = Liquid::Template.parse("  {{funk}} {{so}} {{brother}} ")
     assert_equal(7, template.root.nodelist.size)

test/unit/lexer_unit_test.rb

@@ -6,58 +6,134 @@ class LexerUnitTest < Minitest::Test
   include Liquid
 
   def test_strings
-    tokens = Lexer.new(%( 'this is a test""' "wat 'lol'")).tokenize
-    assert_equal([[:string, %('this is a test""')], [:string, %("wat 'lol'")], [:end_of_string]], tokens)
+    assert_equal(
+      [[:string, %('this is a test""')], [:string, %("wat 'lol'")], [:end_of_string]],
+      tokenize(%( 'this is a test""' "wat 'lol'")),
+    )
   end
 
   def test_integer
-    tokens = Lexer.new('hi 50').tokenize
-    assert_equal([[:id, 'hi'], [:number, '50'], [:end_of_string]], tokens)
+    assert_equal(
+      [[:id, 'hi'], [:number, '50'], [:end_of_string]],
+      tokenize('hi 50'),
+    )
   end
 
   def test_float
-    tokens = Lexer.new('hi 5.0').tokenize
-    assert_equal([[:id, 'hi'], [:number, '5.0'], [:end_of_string]], tokens)
+    assert_equal(
+      [[:id, 'hi'], [:number, '5.0'], [:end_of_string]],
+      tokenize('hi 5.0'),
+    )
   end
 
   def test_comparison
-    tokens = Lexer.new('== <> contains ').tokenize
-    assert_equal([[:comparison, '=='], [:comparison, '<>'], [:comparison, 'contains'], [:end_of_string]], tokens)
+    assert_equal(
+      [[:comparison, '=='], [:comparison, '<>'], [:comparison, 'contains'], [:end_of_string]],
+      tokenize('== <> contains '),
+    )
+  end
+
+  def test_comparison_without_whitespace
+    assert_equal(
+      [[:number, '1'], [:comparison, '>'], [:number, '0'], [:end_of_string]],
+      tokenize('1>0'),
+    )
+  end
+
+  def test_comparison_with_negative_number
+    assert_equal(
+      [[:number, '1'], [:comparison, '>'], [:number, '-1'], [:end_of_string]],
+      tokenize('1>-1'),
+    )
+  end
+
+  def test_raise_for_invalid_comparison
+    assert_raises(SyntaxError) do
+      tokenize('1>!1')
+    end
+
+    assert_raises(SyntaxError) do
+      tokenize('1=<1')
+    end
+
+    assert_raises(SyntaxError) do
+      tokenize('1!!1')
+    end
   end
 
   def test_specials
-    tokens = Lexer.new('| .:').tokenize
-    assert_equal([[:pipe, '|'], [:dot, '.'], [:colon, ':'], [:end_of_string]], tokens)
-    tokens = Lexer.new('[,]').tokenize
-    assert_equal([[:open_square, '['], [:comma, ','], [:close_square, ']'], [:end_of_string]], tokens)
+    assert_equal(
+      [[:pipe, '|'], [:dot, '.'], [:colon, ':'], [:end_of_string]],
+      tokenize('| .:'),
+    )
+
+    assert_equal(
+      [[:open_square, '['], [:comma, ','], [:close_square, ']'], [:end_of_string]],
+      tokenize('[,]'),
+    )
   end
 
   def test_fancy_identifiers
-    tokens = Lexer.new('hi five?').tokenize
-    assert_equal([[:id, 'hi'], [:id, 'five?'], [:end_of_string]], tokens)
+    assert_equal([[:id, 'hi'], [:id, 'five?'], [:end_of_string]], tokenize('hi five?'))
 
-    tokens = Lexer.new('2foo').tokenize
-    assert_equal([[:number, '2'], [:id, 'foo'], [:end_of_string]], tokens)
+    assert_equal([[:number, '2'], [:id, 'foo'], [:end_of_string]], tokenize('2foo'))
   end
 
   def test_whitespace
-    tokens = Lexer.new("five|\n\t ==").tokenize
-    assert_equal([[:id, 'five'], [:pipe, '|'], [:comparison, '=='], [:end_of_string]], tokens)
+    assert_equal(
+      [[:id, 'five'], [:pipe, '|'], [:comparison, '=='], [:end_of_string]],
+      tokenize("five|\n\t =="),
+    )
   end
 
   def test_unexpected_character
     assert_raises(SyntaxError) do
-      Lexer.new("%").tokenize
+      tokenize("%")
     end
   end
 
   def test_negative_numbers
-    tokens = Lexer.new("foo | default: -1").tokenize
-    assert_equal([[:id, 'foo'], [:pipe, '|'], [:id, 'default'], [:colon, ":"], [:number, '-1'], [:end_of_string]], tokens)
+    assert_equal(
+      [[:id, 'foo'], [:pipe, '|'], [:id, 'default'], [:colon, ":"], [:number, '-1'], [:end_of_string]],
+      tokenize("foo | default: -1"),
+    )
   end
 
   def test_greater_than_two_digits
-    tokens = Lexer.new("foo > 12").tokenize
-    assert_equal([[:id, 'foo'], [:comparison, '>'], [:number, '12'], [:end_of_string]], tokens)
+    assert_equal(
+      [[:id, 'foo'], [:comparison, '>'], [:number, '12'], [:end_of_string]],
+      tokenize("foo > 12"),
+    )
+  end
+
+  def test_error_with_utf8_character
+    error = assert_raises(SyntaxError) do
+      tokenize("1 < 1Ø")
+    end
+
+    assert_equal(
+      'Liquid syntax error: Unexpected character Ø',
+      error.message,
+    )
+  end
+
+  def test_contains_as_attribute_name
+    assert_equal(
+      [[:id, "a"], [:dot, "."], [:id, "contains"], [:dot, "."], [:id, "b"], [:end_of_string]],
+      tokenize("a.contains.b"),
+    )
+  end
+
+  def test_tokenize_incomplete_expression
+    assert_equal([[:id, "false"], [:dash, "-"], [:end_of_string]], tokenize("false -"))
+    assert_equal([[:id, "false"], [:comparison, "<"], [:end_of_string]], tokenize("false <"))
+    assert_equal([[:id, "false"], [:comparison, ">"], [:end_of_string]], tokenize("false >"))
+    assert_equal([[:id, "false"], [:number, "1"], [:end_of_string]], tokenize("false 1"))
+  end
+
+  private
+
+  def tokenize(input)
+    Lexer.new(input).tokenize
   end
 end