Prototype

fix parsing quirky incomplete expressions

.github/workflows/liquid.yml

@@ -11,10 +11,10 @@ jobs:
     strategy:
       matrix:
         entry:
-          - { ruby: 2.7, allowed-failure: false } # minimum supported
+          - { ruby: 3.0, allowed-failure: false } # minimum supported
           - { ruby: 3.2, allowed-failure: false }
           - { ruby: 3.3, allowed-failure: false } # latest
-          - { ruby: ruby-head, allowed-failure: true }
+          - { ruby: ruby-head, allowed-failure: false }
     name: Test Ruby ${{ matrix.entry.ruby }}
     steps:
       - uses: actions/checkout@v3

.gitignore

@@ -6,3 +6,4 @@ pkg
 .rvmrc
 .bundle
 .byebug_history
+Gemfile.lock

.ruby-version

@@ -1 +1 @@
-3.3.0
+3.3.4

Gemfile.lock

@@ -1,75 +0,0 @@
-GIT
-  remote: https://github.com/Shopify/liquid-c.git
-  revision: 5a786af7284df55e013ea20551c4b688d02e8326
-  ref: main
-  specs:
-    liquid-c (4.2.0)
-      liquid (>= 5.0.1)
-
-PATH
-  remote: .
-  specs:
-    liquid (5.6.0.alpha)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    ast (2.4.2)
-    base64 (0.2.0)
-    benchmark-ips (2.13.0)
-    json (2.7.2)
-    language_server-protocol (3.17.0.3)
-    memory_profiler (1.0.1)
-    minitest (5.22.3)
-    parallel (1.24.0)
-    parser (3.3.0.5)
-      ast (~> 2.4.1)
-      racc
-    racc (1.7.3)
-    rainbow (3.1.1)
-    rake (13.2.1)
-    regexp_parser (2.9.0)
-    rexml (3.2.6)
-    rubocop (1.61.0)
-      json (~> 2.3)
-      language_server-protocol (>= 3.17.0)
-      parallel (~> 1.10)
-      parser (>= 3.3.0.2)
-      rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.30.0, < 2.0)
-      ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.31.2)
-      parser (>= 3.3.0.4)
-    rubocop-performance (1.19.1)
-      rubocop (>= 1.7.0, < 2.0)
-      rubocop-ast (>= 0.4.0)
-    rubocop-shopify (2.12.0)
-      rubocop (~> 1.44)
-    ruby-progressbar (1.13.0)
-    stackprof (0.2.26)
-    terminal-table (3.0.2)
-      unicode-display_width (>= 1.1.1, < 3)
-    unicode-display_width (2.5.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  base64
-  benchmark-ips
-  liquid!
-  liquid-c!
-  memory_profiler
-  minitest
-  rake (~> 13.0)
-  rubocop (~> 1.61.0)
-  rubocop-performance
-  rubocop-shopify (~> 2.12.0)
-  stackprof
-  terminal-table
-
-BUNDLED WITH
-   2.5.7

Rakefile

@@ -81,6 +81,14 @@ namespace :benchmark do
   task :strict do
     ruby "./performance/benchmark.rb strict"
   end
+
+  desc "Run unit benchmarks"
+  task :unit do
+    Dir["./performance/unit/*_benchmark.rb"].each do |file|
+      puts "🧪 Running #{file}"
+      ruby file
+    end
+  end
 end
 
 namespace :profile do

lib/liquid/block_body.rb

@@ -246,10 +246,17 @@ module Liquid
     end
 
     def create_variable(token, parse_context)
-      if token =~ ContentOfVariable
-        markup = Regexp.last_match(1)
+      if token.end_with?("}}")
+        i = 2
+        i = 3 if token[i] == "-"
+        parse_end = token.length - 3
+        parse_end -= 1 if token[parse_end] == "-"
+        markup_end = parse_end - i + 1
+        markup = markup_end <= 0 ? "" : token.slice(i, markup_end)
+
         return Variable.new(markup, parse_context)
       end
+
       BlockBody.raise_missing_variable_terminator(token, parse_context)
     end
 

lib/liquid/lexer.rb

@@ -1,8 +1,9 @@
 # frozen_string_literal: true
 
 require "strscan"
+
 module Liquid
-  class Lexer
+  class Lexer1
     SPECIALS = {
       '|' => :pipe,
       '.' => :dot,
@@ -58,4 +59,179 @@ module Liquid
       @output << [:end_of_string]
     end
   end
+
+  class Lexer2
+    CLOSE_ROUND = [:close_round, ")"].freeze
+    CLOSE_SQUARE = [:close_square, "]"].freeze
+    COLON = [:colon, ":"].freeze
+    COMMA = [:comma, ","].freeze
+    COMPARISION_NOT_EQUAL = [:comparison, "!="].freeze
+    COMPARISON_CONTAINS = [:comparison, "contains"].freeze
+    COMPARISON_EQUAL = [:comparison, "=="].freeze
+    COMPARISON_GREATER_THAN = [:comparison, ">"].freeze
+    COMPARISON_GREATER_THAN_OR_EQUAL = [:comparison, ">="].freeze
+    COMPARISON_LESS_THAN = [:comparison, "<"].freeze
+    COMPARISON_LESS_THAN_OR_EQUAL = [:comparison, "<="].freeze
+    COMPARISON_NOT_EQUAL_ALT = [:comparison, "<>"].freeze
+    DASH = [:dash, "-"].freeze
+    DOT = [:dot, "."].freeze
+    DOTDOT = [:dotdot, ".."].freeze
+    DOT_ORD = ".".ord
+    DOUBLE_STRING_LITERAL = /"[^\"]*"/
+    EOS = [:end_of_string].freeze
+    IDENTIFIER            = /[a-zA-Z_][\w-]*\??/
+    NUMBER_LITERAL        = /-?\d+(\.\d+)?/
+    OPEN_ROUND = [:open_round, "("].freeze
+    OPEN_SQUARE = [:open_square, "["].freeze
+    PIPE = [:pipe, "|"].freeze
+    QUESTION = [:question, "?"].freeze
+    RUBY_WHITESPACE = [" ", "\t", "\r", "\n", "\f"].freeze
+    SINGLE_STRING_LITERAL = /'[^\']*'/
+    WHITESPACE_OR_NOTHING = /\s*/
+
+    SINGLE_COMPARISON_TOKENS = [].tap do |table|
+      table["<".ord] = COMPARISON_LESS_THAN
+      table[">".ord] = COMPARISON_GREATER_THAN
+    end
+
+    TWO_CHARS_COMPARISON_JUMP_TABLE = [].tap do |table|
+      table["=".ord] = [].tap do |sub_table|
+        sub_table["=".ord] = COMPARISON_EQUAL
+        sub_table.freeze
+      end
+      table["!".ord] = [].tap do |sub_table|
+        sub_table["=".ord] = COMPARISION_NOT_EQUAL
+        sub_table.freeze
+      end
+    end
+
+    COMPARISON_JUMP_TABLE = [].tap do |table|
+      table["<".ord] = [].tap do |sub_table|
+        sub_table["=".ord] = COMPARISON_LESS_THAN_OR_EQUAL
+        sub_table[">".ord] = COMPARISON_NOT_EQUAL_ALT
+        RUBY_WHITESPACE.each { |c| sub_table[c.ord] = COMPARISON_LESS_THAN }
+        sub_table.freeze
+      end
+      table[">".ord] = [].tap do |sub_table|
+        sub_table["=".ord] = COMPARISON_GREATER_THAN_OR_EQUAL
+        RUBY_WHITESPACE.each { |c| sub_table[c.ord] = COMPARISON_GREATER_THAN }
+        sub_table.freeze
+      end
+      table.freeze
+    end
+
+    NEXT_MATCHER_JUMP_TABLE = [].tap do |table|
+      "a".upto("z") do |c|
+        table[c.ord] = [:id, IDENTIFIER].freeze
+        table[c.upcase.ord] = [:id, IDENTIFIER].freeze
+      end
+      table["_".ord] = [:id, IDENTIFIER].freeze
+
+      "0".upto("9") do |c|
+        table[c.ord] = [:number, NUMBER_LITERAL].freeze
+      end
+      table["-".ord] = [:number, NUMBER_LITERAL].freeze
+
+      table["'".ord] = [:string, SINGLE_STRING_LITERAL].freeze
+      table["\"".ord] = [:string, DOUBLE_STRING_LITERAL].freeze
+      table.freeze
+    end
+
+    SPECIAL_TABLE = [].tap do |table|
+      table["|".ord] = PIPE
+      table[".".ord] = DOT
+      table[":".ord] = COLON
+      table[",".ord] = COMMA
+      table["[".ord] = OPEN_SQUARE
+      table["]".ord] = CLOSE_SQUARE
+      table["(".ord] = OPEN_ROUND
+      table[")".ord] = CLOSE_ROUND
+      table["?".ord] = QUESTION
+      table["-".ord] = DASH
+    end
+
+    NUMBER_TABLE = [].tap do |table|
+      "0".upto("9") do |c|
+        table[c.ord] = true
+      end
+      table.freeze
+    end
+
+    def initialize(input)
+      @ss = StringScanner.new(input)
+    end
+
+    # rubocop:disable Metrics/BlockNesting
+    def tokenize
+      @output = []
+
+      until @ss.eos?
+        @ss.skip(WHITESPACE_OR_NOTHING)
+
+        break if @ss.eos?
+
+        start_pos = @ss.pos
+        peeked = @ss.peek_byte
+
+        if (special = SPECIAL_TABLE[peeked])
+          @ss.scan_byte
+          # Special case for ".."
+          if special == DOT && @ss.peek_byte == DOT_ORD
+            @ss.scan_byte
+            @output << DOTDOT
+          elsif special == DASH
+            # Special case for negative numbers
+            if (peeked_byte = @ss.peek_byte) && NUMBER_TABLE[peeked_byte]
+              @ss.pos -= 1
+              @output << [:number, @ss.scan(NUMBER_LITERAL)]
+            else
+              @output << special
+            end
+          else
+            @output << special
+          end
+        elsif (sub_table = TWO_CHARS_COMPARISON_JUMP_TABLE[peeked])
+          @ss.scan_byte
+          if (peeked_byte = @ss.peek_byte) && (found = sub_table[peeked_byte])
+            @output << found
+            @ss.scan_byte
+          else
+            raise_syntax_error(start_pos)
+          end
+        elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
+          @ss.scan_byte
+          if (peeked_byte = @ss.peek_byte) && (found = sub_table[peeked_byte])
+            @output << found
+            @ss.scan_byte
+          else
+            @output << SINGLE_COMPARISON_TOKENS[peeked]
+          end
+        else
+          type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked]
+
+          if type && (t = @ss.scan(pattern))
+            # Special case for "contains"
+            @output << if type == :id && t == "contains" && @output.last&.first != :dot
+              COMPARISON_CONTAINS
+            else
+              [type, t]
+            end
+          else
+            raise_syntax_error(start_pos)
+          end
+        end
+      end
+      # rubocop:enable Metrics/BlockNesting
+
+      @output << EOS
+    end
+
+    def raise_syntax_error(start_pos)
+      @ss.pos = start_pos
+      # the character could be a UTF-8 character, use getch to get all the bytes
+      raise SyntaxError, "Unexpected character #{@ss.getch}"
+    end
+  end
+
+  Lexer = StringScanner.instance_methods.include?(:scan_byte) ? Lexer2 : Lexer1
 end

lib/liquid/parser.rb

@@ -53,7 +53,7 @@ module Liquid
         str = consume
         str << variable_lookups
       when :open_square
-        str = consume
+        str = consume.dup
         str << expression
         str << consume(:close_square)
         str << variable_lookups

lib/liquid/standardfilters.rb

@@ -207,12 +207,18 @@ module Liquid
     def slice(input, offset, length = nil)
       offset = Utils.to_integer(offset)
       length = length ? Utils.to_integer(length) : 1
+      default_value = []
 
       begin
-        if input.is_a?(Array)
-          input.slice(offset, length) || []
+        unless input.is_a?(Array)
+          default_value = ''
+          input = input.to_s
+        end
+
+        if length.negative?
+          input[offset...length] || default_value
         else
-          input.to_s.slice(offset, length) || ''
+          input.slice(offset, length) || default_value
         end
       rescue RangeError
         if I64_RANGE.cover?(length) && I64_RANGE.cover?(offset)
@@ -424,29 +430,59 @@ module Liquid
     # @liquid_syntax array | where: string, string
     # @liquid_return [array[untyped]]
     def where(input, property, target_value = nil)
-      ary = InputIterator.new(input, context)
+      filter_array(input, property, target_value, :select)
+    end
 
-      if ary.empty?
-        []
-      elsif target_value.nil?
-        ary.select do |item|
-          item[property]
-        rescue TypeError
-          raise_property_error(property)
-        rescue NoMethodError
-          return nil unless item.respond_to?(:[])
-          raise
-        end
-      else
-        ary.select do |item|
-          item[property] == target_value
-        rescue TypeError
-          raise_property_error(property)
-        rescue NoMethodError
-          return nil unless item.respond_to?(:[])
-          raise
-        end
-      end
+    # @liquid_public_docs
+    # @liquid_type filter
+    # @liquid_category array
+    # @liquid_summary
+    #   Filters an array to exclude items with a specific property value.
+    # @liquid_description
+    #   This requires you to provide both the property name and the associated value.
+    # @liquid_syntax array | reject: string, string
+    # @liquid_return [array[untyped]]
+    def reject(input, property, target_value = nil)
+      filter_array(input, property, target_value, :reject)
+    end
+
+    # @liquid_public_docs
+    # @liquid_type filter
+    # @liquid_category array
+    # @liquid_summary
+    #   Tests if any item in an array has a specific property value.
+    # @liquid_description
+    #   This requires you to provide both the property name and the associated value.
+    # @liquid_syntax array | some: string, string
+    # @liquid_return [boolean]
+    def has?(input, property, target_value = nil)
+      filter_array(input, property, target_value, :any?)
+    end
+
+    # @liquid_public_docs
+    # @liquid_type filter
+    # @liquid_category array
+    # @liquid_summary
+    #   Returns the first item in an array with a specific property value.
+    # @liquid_description
+    #   This requires you to provide both the property name and the associated value.
+    # @liquid_syntax array | find: string, string
+    # @liquid_return [untyped]
+    def find(input, property, target_value = nil)
+      filter_array(input, property, target_value, :find)
+    end
+
+    # @liquid_public_docs
+    # @liquid_type filter
+    # @liquid_category array
+    # @liquid_summary
+    #   Returns the index of the first item in an array with a specific property value.
+    # @liquid_description
+    #   This requires you to provide both the property name and the associated value.
+    # @liquid_syntax array | find_index: string, string
+    # @liquid_return [number]
+    def find_index(input, property, target_value = nil)
+      filter_array(input, property, target_value, :find_index)
     end
 
     # @liquid_public_docs
@@ -877,7 +913,7 @@ module Liquid
     #   - [`nil`](/docs/api/liquid/basics#nil)
     # @liquid_syntax variable | default: variable
     # @liquid_return [untyped]
-    # @liquid_optional_param allow_false [boolean] Whether to use false values instead of the default.
+    # @liquid_optional_param allow_false: [boolean] Whether to use false values instead of the default.
     def default(input, default_value = '', options = {})
       options = {} unless options.is_a?(Hash)
       false_check = options['allow_false'] ? input.nil? : !Liquid::Utils.to_liquid_value(input)
@@ -918,6 +954,47 @@ module Liquid
 
     attr_reader :context
 
+    def filter_array(input, property, target_value, method)
+      ary = InputIterator.new(input, context)
+
+      return [] if ary.empty?
+
+      ary.public_send(method) do |item|
+        case target_value
+        when nil
+          item[property]
+        when Hash
+          compare_with_operator(item[property], target_value)
+        else
+          item[property] == target_value
+        end
+      rescue TypeError
+        raise_property_error(property)
+      rescue NoMethodError
+        return nil unless item.respond_to?(:[])
+        raise
+      end
+    end
+
+    def compare_with_operator(value, comparison)
+      operation = comparison.keys.first
+      compare_value = comparison.values.first
+      operators = {
+        'greater' => ->(a, b) { a > b },
+        'less' => ->(a, b) { a < b },
+        'greater_or_equal' => ->(a, b) { a >= b },
+        'less_or_equal' => ->(a, b) { a <= b },
+        'contains' => ->(a, b) { a.to_s.include?(b.to_s) },
+      }
+
+      operator = operators[operation]
+      operator && operator.call(value, compare_value)
+    rescue NoMethodError, TypeError, ArgumentError
+      false
+    rescue StandardError
+      false
+    end
+
     def raise_property_error(property)
       raise Liquid::ArgumentError, "cannot select the property '#{property}'"
     end

liquid.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |s|
   s.license     = "MIT"
   # s.description = "A secure, non-evaling end user template engine with aesthetic markup."
 
-  s.required_ruby_version     = ">= 2.7.0"
+  s.required_ruby_version     = ">= 3.0.0"
   s.required_rubygems_version = ">= 1.3.7"
 
   s.metadata['allowed_push_host'] = 'https://rubygems.org'
@@ -28,6 +28,9 @@ Gem::Specification.new do |s|
 
   s.require_path = "lib"
 
+  s.add_dependency("strscan")
+  s.add_dependency("bigdecimal")
+
   s.add_development_dependency('rake', '~> 13.0')
   s.add_development_dependency('minitest')
 end

performance/benchmark.rb

@@ -3,6 +3,7 @@
 require 'benchmark/ips'
 require_relative 'theme_runner'
 
+RubyVM::YJIT.enable if defined?(RubyVM::YJIT)
 Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
 profiler = ThemeRunner.new
 

performance/unit/lexer_benchmark.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "benchmark/ips"
+
+# benchmark liquid lexing
+
+require 'liquid'
+
+RubyVM::YJIT.enable
+
+EXPRESSIONS = [
+  "foo[1..2].baz",
+  "12.0",
+  "foo.bar.based",
+  "21 - 62",
+  "foo.bar.baz",
+  "foo > 12",
+  "foo < 12",
+  "foo <= 12",
+  "foo >= 12",
+  "foo <> 12",
+  "foo == 12",
+  "foo != 12",
+  "foo contains 12",
+  "foo contains 'bar'",
+  "foo != 'bar'",
+  "'foo' contains 'bar'",
+  '234089',
+  "foo | default: -1",
+]
+
+EXPRESSIONS.each do |expr|
+  lexer_1_result = Liquid::Lexer1.new(expr).tokenize
+  lexer_2_result = Liquid::Lexer2.new(expr).tokenize
+
+  next if lexer_1_result == lexer_2_result
+
+  warn "Lexer1 and Lexer2 results are different for expression: #{expr}"
+  warn "expected: #{lexer_1_result}"
+  warn "got: #{lexer_2_result}"
+  abort
+end
+
+Benchmark.ips do |x|
+  x.config(time: 10, warmup: 5)
+
+  x.report("Liquid::Lexer1#tokenize") do
+    EXPRESSIONS.each do |expr|
+      l = Liquid::Lexer1.new(expr)
+      l.tokenize
+    end
+  end
+
+  x.report("Liquid::Lexer2#tokenize") do
+    EXPRESSIONS.each do |expr|
+      l = Liquid::Lexer2.new(expr)
+      l.tokenize
+    end
+  end
+
+  x.compare!
+end

test/integration/parsing_quirks_test.rb

@@ -131,4 +131,16 @@ class ParsingQuirksTest < Minitest::Test
   def test_contains_in_id
     assert_template_result(' YES ', '{% if containsallshipments == true %} YES {% endif %}', { 'containsallshipments' => true })
   end
+
+  def test_incomplete_expression
+    with_error_mode(:lax) do
+      assert_template_result("false", "{% liquid assign foo = false -\n%}{{ foo }}")
+      assert_template_result("false", "{% liquid assign foo = false >\n%}{{ foo }}")
+      assert_template_result("false", "{% liquid assign foo = false <\n%}{{ foo }}")
+      assert_template_result("false", "{% liquid assign foo = false =\n%}{{ foo }}")
+      assert_template_result("false", "{% liquid assign foo = false !\n%}{{ foo }}")
+      assert_template_result("false", "{% liquid assign foo = false 1\n%}{{ foo }}")
+      assert_template_result("false", "{% liquid assign foo = false a\n%}{{ foo }}")
+    end
+  end
 end # ParsingQuirksTest

test/integration/standard_filter_test.rb

@@ -32,7 +32,7 @@ class TestDrop < Liquid::Drop
   attr_reader :value
 
   def registers
-    { @value => @context.registers[@value] }
+    "{#{@value.inspect}=>#{@context.registers[@value].inspect}}"
   end
 end
 
@@ -131,6 +131,9 @@ class StandardFiltersTest < Minitest::Test
     assert_equal(input, @filters.slice(input, 0, 1 << 63))
     assert_equal([], @filters.slice(input, 1 << 63, 6))
     assert_equal([], @filters.slice(input, -(1 << 63), 6))
+    assert_equal(%w(b a), @filters.slice(input, 3, -1))
+    assert_equal(%w(o b), @filters.slice(input, 2, -2))
+    assert_equal(%w(f o o), @filters.slice(input, 0, -3))
   end
 
   def test_truncate
@@ -778,14 +781,69 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result('bcd', "{{ a | append: b}}", assigns)
   end
 
-  def test_concat
-    assert_equal([1, 2, 3, 4], @filters.concat([1, 2], [3, 4]))
-    assert_equal([1, 2, 'a'],  @filters.concat([1, 2], ['a']))
-    assert_equal([1, 2, 10],   @filters.concat([1, 2], [10]))
+  def test_append_with_arrays
+    products = [
+      "Snowdevil pro goggles",
+      "Snowdevil alpine jacket",
+    ]
 
-    assert_raises(Liquid::ArgumentError, "concat filter requires an array argument") do
-      @filters.concat([1, 2], 10)
-    end
+    template = <<~LIQUID
+      {{
+        products
+        | append: 'Snowdevil bonus gift'
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "Snowdevil pro goggles, Snowdevil alpine jacket, Snowdevil bonus gift"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_concat
+    array1 = [1, 2]
+    array2 = [3, 4]
+
+    template = <<~LIQUID
+      {{
+        array1
+        | concat: array2
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "1, 2, 3, 4"
+
+    assert_template_result(expected_output, template, { "array1" => array1, "array2" => array2 })
+  end
+
+  def test_concat_with_string
+    array1 = [1, 2]
+    array2 = ['a']
+
+    template = <<~LIQUID
+      {{
+        array1
+        | concat: array2
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "1, 2, a"
+
+    assert_template_result(expected_output, template, { "array1" => array1, "array2" => array2 })
+  end
+
+  def test_concat_with_number
+    array = [1, 2]
+
+    template = <<~LIQUID
+      {{
+        array
+        | concat: 10
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "1, 2, 10"
+
+    assert_template_result(expected_output, template, { "array" => array })
   end
 
   def test_prepend
@@ -794,6 +852,24 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result('abc', "{{ a | prepend: b}}", assigns)
   end
 
+  def test_prepend_with_arrays
+    products = [
+      "Snowdevil pro goggles",
+      "Snowdevil alpine jacket",
+    ]
+
+    template = <<~LIQUID
+      {{
+        products
+        | prepend: 'Snowdevil bonus gift'
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "Snowdevil bonus gift, Snowdevil pro goggles, Snowdevil alpine jacket"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
   def test_default
     assert_equal("foo", @filters.default("foo", "bar"))
     assert_equal("bar", @filters.default(nil, "bar"))
@@ -827,21 +903,306 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result('abc', "{{ 'abc' | date: '%D' }}")
   end
 
-  def test_where
-    input = [
+  def test_where_with_value
+    array = [
       { "handle" => "alpha", "ok" => true },
       { "handle" => "beta", "ok" => false },
       { "handle" => "gamma", "ok" => false },
       { "handle" => "delta", "ok" => true },
     ]
 
-    expectation = [
+    template = "{{ array | where: 'ok', true | map: 'handle' | join: ' ' }}"
+    expected_output = "alpha delta"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_where_without_value
+    array = [
       { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
       { "handle" => "delta", "ok" => true },
     ]
 
-    assert_equal(expectation, @filters.where(input, "ok", true))
-    assert_equal(expectation, @filters.where(input, "ok"))
+    template = "{{ array | where: 'ok' | map: 'handle' | join: ' ' }}"
+    expected_output = "alpha delta"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_where_without_value
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | where: 'ok', false | map: 'handle' | join: ' ' }}"
+    expected_output = "beta gamma"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_where_with_greater_than
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = <<~LIQUID
+      {{
+        products
+        | where: 'price', greater: 300
+        | map: 'title'
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "Snowdevil alpine jacket, Snowdevil mountain boots"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_where_with_less_than
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = <<~LIQUID
+      {{
+        products
+        | where: 'price', less: 150
+        | map: 'title'
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "Snowdevil pro goggles, Snowdevil thermal gloves"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_where_with_greater_or_equal
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = <<~LIQUID
+      {{
+        products
+        | where: 'price', greater_or_equal: 389.99
+        | map: 'title'
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "Snowdevil alpine jacket, Snowdevil mountain boots"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_where_with_less_or_equal
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = <<~LIQUID
+      {{
+        products
+        | where: 'price', less_or_equal: 149.99
+        | map: 'title'
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "Snowdevil pro goggles, Snowdevil thermal gloves"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_where_with_contains
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = <<~LIQUID
+      {{
+        products
+        | where: 'title', contains: 'es'
+        | map: 'title'
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "Snowdevil pro goggles, Snowdevil thermal gloves"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_reject_with_value
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | reject: 'ok', true | map: 'handle' | join: ' ' }}"
+    expected_output = "beta gamma"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def xtest_reject_with_value2
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | add:array.first }}"
+    expected_output = "beta gamma"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_reject_with_greater_operator
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = <<~LIQUID
+      {{
+        products
+        | reject: 'price', greater: 190
+        | map: 'title'
+        | join: ', '
+      -}}
+    LIQUID
+    expected_output = "Snowdevil pro goggles, Snowdevil thermal gloves"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_reject_without_value
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | reject: 'ok' | map: 'handle' | join: ' ' }}"
+    expected_output = "beta gamma"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_reject_with_false_value
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | reject: 'ok', false | map: 'handle' | join: ' ' }}"
+    expected_output = "alpha delta"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_some_with_value
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | some: 'ok', true }}"
+    expected_output = "true"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_some_without_value
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | some: 'ok' }}"
+    expected_output = "true"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_some_with_false_value
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | some: 'ok', false }}"
+    expected_output = "true"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_some_with_contains
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = "{{ products | some: 'title', contains: 'safety' }}"
+    expected_output = "true"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_some_with_contains
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = "{{ products | some: 'title', contains: 'game boy' }}"
+    expected_output = "false"
+
+    assert_template_result(expected_output, template, { "products" => products })
   end
 
   def test_where_string_keys
@@ -873,6 +1234,42 @@ class StandardFiltersTest < Minitest::Test
     assert_equal(expectation, @filters.where(input, "ok"))
   end
 
+  def test_find_with_value
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = <<~LIQUID
+      {%- assign product = products | find: 'price', greater: 150 -%}
+      {{- product.title -}}
+    LIQUID
+    expected_output = "Snowdevil alpine jacket"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_find_index_with_value
+    products = [
+      { "title" => "Snowdevil pro goggles",    "price" => 129.99 },
+      { "title" => "Snowdevil thermal gloves", "price" => 149.99 },
+      { "title" => "Snowdevil alpine jacket",  "price" => 399.99 },
+      { "title" => "Snowdevil mountain boots", "price" => 389.99 },
+      { "title" => "Snowdevil safety helmet",  "price" => 199.99 }
+    ]
+
+    template = <<~LIQUID
+      {%- assign index = products | find_index: 'price', greater: 150 -%}
+      {{- index -}}
+    LIQUID
+    expected_output = "2"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
   def test_where_non_array_map_input
     assert_equal([{ "a" => "ok" }], @filters.where({ "a" => "ok" }, "a", "ok"))
     assert_equal([], @filters.where({ "a" => "not ok" }, "a", "ok"))

test/unit/block_unit_test.rb

@@ -32,6 +32,12 @@ class BlockUnitTest < Minitest::Test
     assert_equal(String, template.root.nodelist[2].class)
   end
 
+  def test_variable_with_multibyte_character
+    template = Liquid::Template.parse("{{ '❤️' }}")
+    assert_equal(1, template.root.nodelist.size)
+    assert_equal(Variable, template.root.nodelist[0].class)
+  end
+
   def test_variable_many_embedded_fragments
     template = Liquid::Template.parse("  {{funk}} {{so}} {{brother}} ")
     assert_equal(7, template.root.nodelist.size)

test/unit/lexer_unit_test.rb

@@ -6,48 +6,134 @@ class LexerUnitTest < Minitest::Test
   include Liquid
 
   def test_strings
-    tokens = Lexer.new(%( 'this is a test""' "wat 'lol'")).tokenize
-    assert_equal([[:string, %('this is a test""')], [:string, %("wat 'lol'")], [:end_of_string]], tokens)
+    assert_equal(
+      [[:string, %('this is a test""')], [:string, %("wat 'lol'")], [:end_of_string]],
+      tokenize(%( 'this is a test""' "wat 'lol'")),
+    )
   end
 
   def test_integer
-    tokens = Lexer.new('hi 50').tokenize
-    assert_equal([[:id, 'hi'], [:number, '50'], [:end_of_string]], tokens)
+    assert_equal(
+      [[:id, 'hi'], [:number, '50'], [:end_of_string]],
+      tokenize('hi 50'),
+    )
   end
 
   def test_float
-    tokens = Lexer.new('hi 5.0').tokenize
-    assert_equal([[:id, 'hi'], [:number, '5.0'], [:end_of_string]], tokens)
+    assert_equal(
+      [[:id, 'hi'], [:number, '5.0'], [:end_of_string]],
+      tokenize('hi 5.0'),
+    )
   end
 
   def test_comparison
-    tokens = Lexer.new('== <> contains ').tokenize
-    assert_equal([[:comparison, '=='], [:comparison, '<>'], [:comparison, 'contains'], [:end_of_string]], tokens)
+    assert_equal(
+      [[:comparison, '=='], [:comparison, '<>'], [:comparison, 'contains'], [:end_of_string]],
+      tokenize('== <> contains '),
+    )
+  end
+
+  def test_comparison_without_whitespace
+    assert_equal(
+      [[:number, '1'], [:comparison, '>'], [:number, '0'], [:end_of_string]],
+      tokenize('1>0'),
+    )
+  end
+
+  def test_comparison_with_negative_number
+    assert_equal(
+      [[:number, '1'], [:comparison, '>'], [:number, '-1'], [:end_of_string]],
+      tokenize('1>-1'),
+    )
+  end
+
+  def test_raise_for_invalid_comparison
+    assert_raises(SyntaxError) do
+      tokenize('1>!1')
+    end
+
+    assert_raises(SyntaxError) do
+      tokenize('1=<1')
+    end
+
+    assert_raises(SyntaxError) do
+      tokenize('1!!1')
+    end
   end
 
   def test_specials
-    tokens = Lexer.new('| .:').tokenize
-    assert_equal([[:pipe, '|'], [:dot, '.'], [:colon, ':'], [:end_of_string]], tokens)
-    tokens = Lexer.new('[,]').tokenize
-    assert_equal([[:open_square, '['], [:comma, ','], [:close_square, ']'], [:end_of_string]], tokens)
+    assert_equal(
+      [[:pipe, '|'], [:dot, '.'], [:colon, ':'], [:end_of_string]],
+      tokenize('| .:'),
+    )
+
+    assert_equal(
+      [[:open_square, '['], [:comma, ','], [:close_square, ']'], [:end_of_string]],
+      tokenize('[,]'),
+    )
   end
 
   def test_fancy_identifiers
-    tokens = Lexer.new('hi five?').tokenize
-    assert_equal([[:id, 'hi'], [:id, 'five?'], [:end_of_string]], tokens)
+    assert_equal([[:id, 'hi'], [:id, 'five?'], [:end_of_string]], tokenize('hi five?'))
 
-    tokens = Lexer.new('2foo').tokenize
-    assert_equal([[:number, '2'], [:id, 'foo'], [:end_of_string]], tokens)
+    assert_equal([[:number, '2'], [:id, 'foo'], [:end_of_string]], tokenize('2foo'))
   end
 
   def test_whitespace
-    tokens = Lexer.new("five|\n\t ==").tokenize
-    assert_equal([[:id, 'five'], [:pipe, '|'], [:comparison, '=='], [:end_of_string]], tokens)
+    assert_equal(
+      [[:id, 'five'], [:pipe, '|'], [:comparison, '=='], [:end_of_string]],
+      tokenize("five|\n\t =="),
+    )
   end
 
   def test_unexpected_character
     assert_raises(SyntaxError) do
-      Lexer.new("%").tokenize
+      tokenize("%")
     end
   end
+
+  def test_negative_numbers
+    assert_equal(
+      [[:id, 'foo'], [:pipe, '|'], [:id, 'default'], [:colon, ":"], [:number, '-1'], [:end_of_string]],
+      tokenize("foo | default: -1"),
+    )
+  end
+
+  def test_greater_than_two_digits
+    assert_equal(
+      [[:id, 'foo'], [:comparison, '>'], [:number, '12'], [:end_of_string]],
+      tokenize("foo > 12"),
+    )
+  end
+
+  def test_error_with_utf8_character
+    error = assert_raises(SyntaxError) do
+      tokenize("1 < 1Ø")
+    end
+
+    assert_equal(
+      'Liquid syntax error: Unexpected character Ø',
+      error.message,
+    )
+  end
+
+  def test_contains_as_attribute_name
+    assert_equal(
+      [[:id, "a"], [:dot, "."], [:id, "contains"], [:dot, "."], [:id, "b"], [:end_of_string]],
+      tokenize("a.contains.b"),
+    )
+  end
+
+  def test_tokenize_incomplete_expression
+    assert_equal([[:id, "false"], [:dash, "-"], [:end_of_string]], tokenize("false -"))
+    assert_equal([[:id, "false"], [:comparison, "<"], [:end_of_string]], tokenize("false <"))
+    assert_equal([[:id, "false"], [:comparison, ">"], [:end_of_string]], tokenize("false >"))
+    assert_equal([[:id, "false"], [:number, "1"], [:end_of_string]], tokenize("false 1"))
+  end
+
+  private
+
+  def tokenize(input)
+    Lexer.new(input).tokenize
+  end
 end