[WIP] Support for nested boolean expressions in parentheses

- Added unit tests for range syntax
- Added logical expression unit tests
- parser respects parentheses during expression traversal

.github/workflows/liquid.yml

@@ -13,8 +13,22 @@ jobs:
         entry:
           - { ruby: 3.0, allowed-failure: false } # minimum supported
           - { ruby: 3.2, allowed-failure: false }
-          - { ruby: 3.3, allowed-failure: false } # latest
+          - { ruby: 3.3, allowed-failure: false }
+          - { ruby: 3.3, allowed-failure: false }
+          - { ruby: 3.4, allowed-failure: false } # latest
+          - {
+              ruby: 3.4,
+              allowed-failure: false,
+              rubyopt: "--enable-frozen-string-literal",
+            }
+          - { ruby: 3.4, allowed-failure: false, rubyopt: "--yjit" }
           - { ruby: ruby-head, allowed-failure: false }
+          - {
+              ruby: ruby-head,
+              allowed-failure: false,
+              rubyopt: "--enable-frozen-string-literal",
+            }
+          - { ruby: ruby-head, allowed-failure: false, rubyopt: "--yjit" }
     name: Test Ruby ${{ matrix.entry.ruby }}
     steps:
       - uses: actions/checkout@v3
@@ -25,6 +39,8 @@ jobs:
           bundler: latest
       - run: bundle exec rake
         continue-on-error: ${{ matrix.entry.allowed-failure }}
+        env:
+          RUBYOPT: ${{ matrix.entry.rubyopt }}
 
   memory_profile:
     runs-on: ubuntu-latest

.ruby-version

@@ -1 +1 @@
-3.3.4
+3.4.1

CONTRIBUTING.md

@@ -26,3 +26,11 @@
 * If it makes sense, add tests for your code and/or run a performance benchmark
 * Make sure all tests pass (`bundle exec rake`)
 * Create a pull request
+
+## Releasing
+
+* Bump the version in `lib/liquid/version.rb`
+* Update the `History.md` file
+* Open a PR like [this one](https://github.com/Shopify/liquid/pull/1894) and merge it to `main`
+* Create a new release using the [GitHub UI](https://github.com/Shopify/liquid/releases/new)
+

Gemfile

@@ -13,18 +13,21 @@ group :benchmark, :test do
   gem 'benchmark-ips'
   gem 'memory_profiler'
   gem 'terminal-table'
+  gem "lru_redux"
 
   install_if -> { RUBY_PLATFORM !~ /mingw|mswin|java/ && RUBY_ENGINE != 'truffleruby' } do
     gem 'stackprof'
   end
 end
 
+group :development do
+  gem "webrick"
+end
+
 group :test do
+  gem 'ruby-lsp'
+  gem 'debug'
   gem 'rubocop', '~> 1.61.0'
   gem 'rubocop-shopify', '~> 2.12.0', require: false
   gem 'rubocop-performance', require: false
-
-  platform :mri, :truffleruby do
-    gem 'liquid-c', github: 'Shopify/liquid-c', ref: 'main'
-  end
 end

History.md

@@ -1,5 +1,76 @@
 # Liquid Change Log
 
+## 5.8.1 (unreleased)
+
+## 5.8.1
+
+* Fix `{% doc %}` tag to be visitable [Guilherme Carreiro]
+
+## 5.8.0
+
+* Introduce the new `{% doc %}` tag [Guilherme Carreiro]
+
+## 5.7.3
+
+* Raise Liquid::SyntaxError when parsing invalidly encoded strings [Chris AtLee]
+
+## 5.7.2 2025-01-31
+
+* Fix array filters to not support nested properties [Guilherme Carreiro]
+
+## 5.7.1 2025-01-24
+
+* Fix the `find` and `find_index`filters to return `nil` when filtering empty arrays [Guilherme Carreiro]
+* Fix the `has` filter to return `false` when filtering empty arrays [Guilherme Carreiro]
+
+## 5.7.0 2025-01-16
+
+### Features
+
+* Add `find`, `find_index`, `has`, and `reject` filters to arrays [Guilherme Carreiro]
+* Compatibility with Ruby 3.4 [Ian Ker-Seymer]
+
+## 5.6.4 2025-01-14
+
+### Fixes
+* Add a default `string_scanner` to avoid errors with `Liquid::VariableLookup.parse("foo.bar")` [Ian Ker-Seymer]
+
+## 5.6.3 2025-01-13
+* Remove `lru_redux` dependency [Michael Go]
+
+## 5.6.2 2025-01-13
+
+### Fixes
+* Preserve the old behavior of requiring floats to start with a digit [Michael Go]
+
+## 5.6.1 2025-01-13
+
+### Performance improvements
+* Faster Expression parser / Tokenizer with StringScanner [Michael Go]
+
+## 5.6.0 2024-12-19
+
+### Architectural changes
+* Added new `Environment` class to manage configuration and state that was previously stored in `Template` [Ian Ker-Seymer]
+* Moved tag registration from `Template` to `Environment` [Ian Ker-Seymer]
+* Removed `StrainerFactory` in favor of `Environment`-based strainer creation [Ian Ker-Seymer]
+* Consolidated standard tags into a new `Tags` module with `STANDARD_TAGS` constant [Ian Ker-Seymer]
+
+### Performance improvements
+* Optimized `Lexer` with a new `Lexer2` implementation using jump tables for faster tokenization, requires Ruby 3.4 [Ian Ker-Seymer]
+* Improved variable rendering with specialized handling for different types [Michael Go]
+* Reduced array allocations by using frozen empty constants [Michael Go]
+
+### API changes
+* Deprecated several `Template` class methods in favor of `Environment` methods [Ian Ker-Seymer]
+* Added deprecation warnings system [Ian Ker-Seymer]
+* Changed how filters and tags are registered to use Environment [Ian Ker-Seymer]
+
+### Fixes
+* Fixed table row handling of break interrupts [Alex Coco]
+* Improved variable output handling for arrays [Ian Ker-Seymer]
+* Fix Tokenizer to handle null source value (#1873) [Bahar Pourazar]
+
 ## 5.5.0 2024-03-21
 
 Please reference the GitHub release for more information.

README.md

@@ -91,7 +91,7 @@ Liquid::Template.parse(<<~LIQUID, environment: email_environment)
 LIQUID
 ```
 
-By using Environments, you ensure that custom tags and filters are only available in the contexts where they are needed, making your Liquid templates more robust and easier to manage.
+By using Environments, you ensure that custom tags and filters are only available in the contexts where they are needed, making your Liquid templates more robust and easier to manage. For smaller projects, a global environment is available via `Liquid::Environment.default`.
 
 ### Error Modes
 

Rakefile

@@ -43,8 +43,6 @@ task :test do
   Rake::Task['base_test'].invoke
 
   if RUBY_ENGINE == 'ruby' || RUBY_ENGINE == 'truffleruby'
-    ENV['LIQUID_C'] = '1'
-
     ENV['LIQUID_PARSER_MODE'] = 'lax'
     Rake::Task['integration_test'].reenable
     Rake::Task['integration_test'].invoke
@@ -73,7 +71,7 @@ end
 
 namespace :benchmark do
   desc "Run the liquid benchmark with lax parsing"
-  task :run do
+  task :lax do
     ruby "./performance/benchmark.rb lax"
   end
 
@@ -82,11 +80,30 @@ namespace :benchmark do
     ruby "./performance/benchmark.rb strict"
   end
 
+  desc "Run the liquid benchmark with both lax and strict parsing"
+  task run: [:lax, :strict]
+
   desc "Run unit benchmarks"
-  task :unit do
-    Dir["./performance/unit/*_benchmark.rb"].each do |file|
-      puts "🧪 Running #{file}"
-      ruby file
+  namespace :unit do
+    task :all do
+      Dir["./performance/unit/*_benchmark.rb"].each do |file|
+        puts "🧪 Running #{file}"
+        ruby file
+      end
+    end
+
+    task :lexer do
+      Dir["./performance/unit/lexer_benchmark.rb"].each do |file|
+        puts "🧪 Running #{file}"
+        ruby file
+      end
+    end
+
+    task :expression do
+      Dir["./performance/unit/expression_benchmark.rb"].each do |file|
+        puts "🧪 Running #{file}"
+        ruby file
+      end
     end
   end
 end

lib/liquid.rb

@@ -21,6 +21,8 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+require "strscan"
+
 module Liquid
   FilterSeparator             = /\|/
   ArgumentSeparator           = ','
@@ -44,6 +46,7 @@ module Liquid
   VariableParser              = /\[(?>[^\[\]]+|\g<0>)*\]|#{VariableSegment}+\??/o
 
   RAISE_EXCEPTION_LAMBDA = ->(_e) { raise }
+  HAS_STRING_SCANNER_SCAN_BYTE = StringScanner.instance_methods.include?(:scan_byte)
 end
 
 require "liquid/version"
@@ -68,7 +71,6 @@ require 'liquid/extensions'
 require 'liquid/errors'
 require 'liquid/interrupts'
 require 'liquid/strainer_template'
-require 'liquid/expression'
 require 'liquid/context'
 require 'liquid/tag'
 require 'liquid/block_body'
@@ -77,6 +79,9 @@ require 'liquid/variable'
 require 'liquid/variable_lookup'
 require 'liquid/range_lookup'
 require 'liquid/resource_limits'
+require 'liquid/expression'
+require 'liquid/expression/comparison_expression'
+require 'liquid/expression/logical_expression'
 require 'liquid/template'
 require 'liquid/condition'
 require 'liquid/utils'

lib/liquid/condition.rb

@@ -52,6 +52,10 @@ module Liquid
       @@method_literals[markup] || parse_context.parse_expression(markup)
     end
 
+    def self.parse(markup, ss, cache)
+      @@method_literals[markup] || Expression.parse(markup, ss, cache)
+    end
+
     attr_reader :attachment, :child_condition
     attr_accessor :left, :operator, :right
 
@@ -112,11 +116,15 @@ module Liquid
     private
 
     def equal_variables(left, right)
+      if left.is_a?(MethodLiteral) && right.is_a?(MethodLiteral)
+        return left.to_s == right.to_s
+      end
+
       if left.is_a?(MethodLiteral)
         if right.respond_to?(left.method_name)
           return right.send(left.method_name)
         else
-          return nil
+          return left.to_s == right
         end
       end
 
@@ -124,7 +132,7 @@ module Liquid
         if left.respond_to?(right.method_name)
           return left.send(right.method_name)
         else
-          return nil
+          return right.to_s == left
         end
       end
 

lib/liquid/context.rb

@@ -40,6 +40,10 @@ module Liquid
       @global_filter       = nil
       @disabled_tags       = {}
 
+      # Instead of constructing new StringScanner objects for each Expression parse,
+      # we recycle the same one.
+      @string_scanner = StringScanner.new("")
+
       @registers.static[:cached_partials] ||= {}
       @registers.static[:file_system] ||= environment.file_system
       @registers.static[:template_factory] ||= Liquid::TemplateFactory.new
@@ -176,7 +180,7 @@ module Liquid
     # Example:
     #   products == empty #=> products.empty?
     def [](expression)
-      evaluate(Expression.parse(expression))
+      evaluate(Expression.parse(expression, @string_scanner))
     end
 
     def key?(key)

lib/liquid/expression.rb

@@ -10,37 +10,114 @@ module Liquid
       'true' => true,
       'false' => false,
       'blank' => '',
-      'empty' => ''
+      'empty' => '',
+      # in lax mode, minus sign can be a VariableLookup
+      # For simplicity and performace, we treat it like a literal
+      '-' => VariableLookup.parse("-", nil).freeze,
     }.freeze
 
-    INTEGERS_REGEX       = /\A(-?\d+)\z/
-    FLOATS_REGEX         = /\A(-?\d[\d\.]+)\z/
+    DOT = ".".ord
+    ZERO = "0".ord
+    NINE = "9".ord
+    DASH = "-".ord
 
     # Use an atomic group (?>...) to avoid pathological backtracing from
     # malicious input as described in https://github.com/Shopify/liquid/issues/1357
-    RANGES_REGEX         = /\A\(\s*(?>(\S+)\s*\.\.)\s*(\S+)\s*\)\z/
+    RANGES_REGEX = /\A\(\s*(?>(\S+)\s*\.\.)\s*(\S+)\s*\)\z/
+    INTEGER_REGEX = /\A(-?\d+)\z/
+    FLOAT_REGEX = /\A(-?\d+)\.\d+\z/
+    QUOTED_STRING = /\A#{QuotedString}\z/
 
-    def self.parse(markup)
-      return nil unless markup
+    class << self
+      def parse(markup, ss = StringScanner.new(""), cache = nil)
+        return unless markup
 
-      markup = markup.strip
-      if (markup.start_with?('"') && markup.end_with?('"')) ||
-         (markup.start_with?("'") && markup.end_with?("'"))
-        return markup[1..-2]
+        markup = markup.strip # markup can be a frozen string
+
+        return markup[1..-2] if QUOTED_STRING.match?(markup)
+
+        return LITERALS[markup] if LITERALS.key?(markup)
+
+        # Cache only exists during parsing
+        if cache
+          return cache[markup] if cache.key?(markup)
+
+          cache[markup] = inner_parse(markup, ss, cache).freeze
+        else
+          inner_parse(markup, ss, nil).freeze
+        end
       end
 
-      case markup
-      when INTEGERS_REGEX
-        Regexp.last_match(1).to_i
-      when RANGES_REGEX
-        RangeLookup.parse(Regexp.last_match(1), Regexp.last_match(2))
-      when FLOATS_REGEX
-        Regexp.last_match(1).to_f
-      else
-        if LITERALS.key?(markup)
-          LITERALS[markup]
+      def inner_parse(markup, ss, cache)
+        return LogicalExpression.parse(markup, ss, cache)    if LogicalExpression.logical?(markup)
+        return ComparisonExpression.parse(markup, ss, cache) if ComparisonExpression.comparison?(markup)
+
+        if (markup.start_with?("(") && markup.end_with?(")")) && markup =~ RANGES_REGEX
+          return RangeLookup.parse(
+            Regexp.last_match(1),
+            Regexp.last_match(2),
+            ss,
+            cache,
+          )
+        end
+
+        if (num = parse_number(markup, ss))
+          num
         else
-          VariableLookup.parse(markup)
+          VariableLookup.parse(markup, ss, cache)
+        end
+      end
+
+      def parse_number(markup, ss)
+        # check if the markup is simple integer or float
+        case markup
+        when INTEGER_REGEX
+          return Integer(markup, 10)
+        when FLOAT_REGEX
+          return markup.to_f
+        end
+
+        ss.string = markup
+        # the first byte must be a digit or  a dash
+        byte = ss.scan_byte
+
+        return false if byte != DASH && (byte < ZERO || byte > NINE)
+
+        if byte == DASH
+          peek_byte = ss.peek_byte
+
+          # if it starts with a dash, the next byte must be a digit
+          return false if peek_byte.nil? || !(peek_byte >= ZERO && peek_byte <= NINE)
+        end
+
+        # The markup could be a float with multiple dots
+        first_dot_pos = nil
+        num_end_pos = nil
+
+        while (byte = ss.scan_byte)
+          return false if byte != DOT && (byte < ZERO || byte > NINE)
+
+          # we found our number and now we are just scanning the rest of the string
+          next if num_end_pos
+
+          if byte == DOT
+            if first_dot_pos.nil?
+              first_dot_pos = ss.pos
+            else
+              # we found another dot, so we know that the number ends here
+              num_end_pos = ss.pos - 1
+            end
+          end
+        end
+
+        num_end_pos = markup.length if ss.eos?
+
+        if num_end_pos
+          # number ends with a number "123.123"
+          markup.byteslice(0, num_end_pos).to_f
+        else
+          # number ends with a dot "123."
+          markup.byteslice(0, first_dot_pos).to_f
         end
       end
     end

lib/liquid/expression/comparison_expression.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Liquid
+  class Expression
+    class ComparisonExpression
+      # We can improve the resiliency of lax parsing by not expecting whitespace
+      # surrounding the operator (ie \s+ => \s*).
+      # However this is not in parity with existing lax parsing behavior.
+      COMPARISON_REGEX = /\A\s*(.+?)\s+(==|!=|<>|<=|>=|<|>|contains)\s+(.+)\s*\z/
+
+      class << self
+        def comparison?(markup)
+          markup.match(COMPARISON_REGEX)
+        end
+
+        def parse(markup, ss, cache)
+          match = comparison?(markup)
+
+          if match
+            left = Condition.parse(match[1].strip, ss, cache)
+            operator = match[2].strip
+            right = Condition.parse(match[3].strip, ss, cache)
+            return Condition.new(left, operator, right)
+          end
+
+          Condition.new(parse(markup, ss, cache), nil, nil)
+        end
+      end
+    end
+  end
+end

lib/liquid/expression/logical_expression.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Liquid
+  class Expression
+    class LogicalExpression
+      LOGICAL_REGEX = /\A\s*(.+?)\s+(and|or)\s+(.+)\s*\z/i
+      EXPRESSIONS_AND_OPERATORS = /(?:\b(?:\s?and\s?|\s?or\s?)\b|(?:\s*(?!\b(?:\s?and\s?|\s?or\s?)\b)(?:#{QuotedFragment}|\S+)\s*)+)/o
+      BOOLEAN_OPERATORS = ['and', 'or'].freeze
+
+      class << self
+        def logical?(markup)
+          markup.match(LOGICAL_REGEX)
+        end
+
+        def boolean_operator?(markup)
+          BOOLEAN_OPERATORS.include?(markup)
+        end
+
+        def parse(markup, ss, cache)
+          expressions = markup.scan(EXPRESSIONS_AND_OPERATORS)
+
+          expression = expressions.pop
+          condition  = parse_condition(expression, ss, cache)
+
+          until expressions.empty?
+            operator = expressions.pop.to_s.strip
+
+            next unless boolean_operator?(operator)
+
+            expression    = expressions.pop.to_s.strip
+            new_condition = parse_condition(expression, ss, cache)
+
+            case operator
+            when 'and' then new_condition.and(condition)
+            when 'or'  then new_condition.or(condition)
+            end
+
+            condition = new_condition
+          end
+
+          condition
+        end
+
+        private
+
+        def parse_condition(expr, ss, cache)
+          return ComparisonExpression.parse(expr, ss, cache) if comparison?(expr)
+          return LogicalExpression.parse(expr, ss, cache)    if logical?(expr)
+
+          Condition.new(Expression.parse(expr, ss, cache), nil, nil)
+        end
+
+        def comparison?(...)
+          ComparisonExpression.comparison?(...)
+        end
+      end
+    end
+  end
+end

lib/liquid/lexer.rb

@@ -1,66 +1,7 @@
 # frozen_string_literal: true
 
-require "strscan"
-
 module Liquid
-  class Lexer1
-    SPECIALS = {
-      '|' => :pipe,
-      '.' => :dot,
-      ':' => :colon,
-      ',' => :comma,
-      '[' => :open_square,
-      ']' => :close_square,
-      '(' => :open_round,
-      ')' => :close_round,
-      '?' => :question,
-      '-' => :dash,
-    }.freeze
-    IDENTIFIER            = /[a-zA-Z_][\w-]*\??/
-    SINGLE_STRING_LITERAL = /'[^\']*'/
-    DOUBLE_STRING_LITERAL = /"[^\"]*"/
-    STRING_LITERAL        = Regexp.union(SINGLE_STRING_LITERAL, DOUBLE_STRING_LITERAL)
-    NUMBER_LITERAL        = /-?\d+(\.\d+)?/
-    DOTDOT                = /\.\./
-    COMPARISON_OPERATOR   = /==|!=|<>|<=?|>=?|contains(?=\s)/
-    WHITESPACE_OR_NOTHING = /\s*/
-
-    def initialize(input)
-      @ss = StringScanner.new(input)
-    end
-
-    def tokenize
-      @output = []
-
-      until @ss.eos?
-        @ss.skip(WHITESPACE_OR_NOTHING)
-        break if @ss.eos?
-        tok      = if (t = @ss.scan(COMPARISON_OPERATOR))
-          [:comparison, t]
-        elsif (t = @ss.scan(STRING_LITERAL))
-          [:string, t]
-        elsif (t = @ss.scan(NUMBER_LITERAL))
-          [:number, t]
-        elsif (t = @ss.scan(IDENTIFIER))
-          [:id, t]
-        elsif (t = @ss.scan(DOTDOT))
-          [:dotdot, t]
-        else
-          c     = @ss.getch
-          if (s = SPECIALS[c])
-            [s, c]
-          else
-            raise SyntaxError, "Unexpected character #{c}"
-          end
-        end
-        @output << tok
-      end
-
-      @output << [:end_of_string]
-    end
-  end
-
-  class Lexer2
+  class Lexer
     CLOSE_ROUND = [:close_round, ")"].freeze
     CLOSE_SQUARE = [:close_square, "]"].freeze
     COLON = [:colon, ":"].freeze
@@ -73,6 +14,8 @@ module Liquid
     COMPARISON_LESS_THAN = [:comparison, "<"].freeze
     COMPARISON_LESS_THAN_OR_EQUAL = [:comparison, "<="].freeze
     COMPARISON_NOT_EQUAL_ALT = [:comparison, "<>"].freeze
+    BOOLEAN_AND = [:boolean_operator, "and"].freeze
+    BOOLEAN_OR = [:boolean_operator, "or"].freeze
     DASH = [:dash, "-"].freeze
     DOT = [:dot, "."].freeze
     DOTDOT = [:dotdot, ".."].freeze
@@ -92,6 +35,7 @@ module Liquid
     SINGLE_COMPARISON_TOKENS = [].tap do |table|
       table["<".ord] = COMPARISON_LESS_THAN
       table[">".ord] = COMPARISON_GREATER_THAN
+      table.freeze
     end
 
     TWO_CHARS_COMPARISON_JUMP_TABLE = [].tap do |table|
@@ -103,18 +47,17 @@ module Liquid
         sub_table["=".ord] = COMPARISION_NOT_EQUAL
         sub_table.freeze
       end
+      table.freeze
     end
 
     COMPARISON_JUMP_TABLE = [].tap do |table|
       table["<".ord] = [].tap do |sub_table|
         sub_table["=".ord] = COMPARISON_LESS_THAN_OR_EQUAL
         sub_table[">".ord] = COMPARISON_NOT_EQUAL_ALT
-        RUBY_WHITESPACE.each { |c| sub_table[c.ord] = COMPARISON_LESS_THAN }
         sub_table.freeze
       end
       table[">".ord] = [].tap do |sub_table|
         sub_table["=".ord] = COMPARISON_GREATER_THAN_OR_EQUAL
-        RUBY_WHITESPACE.each { |c| sub_table[c.ord] = COMPARISON_GREATER_THAN }
         sub_table.freeze
       end
       table.freeze
@@ -157,81 +100,86 @@ module Liquid
       table.freeze
     end
 
-    def initialize(input)
-      @ss = StringScanner.new(input)
-    end
-
     # rubocop:disable Metrics/BlockNesting
-    def tokenize
-      @output = []
+    class << self
+      def tokenize(ss)
+        output = []
 
-      until @ss.eos?
-        @ss.skip(WHITESPACE_OR_NOTHING)
+        until ss.eos?
+          ss.skip(WHITESPACE_OR_NOTHING)
 
-        break if @ss.eos?
+          break if ss.eos?
 
-        start_pos = @ss.pos
-        peeked = @ss.peek_byte
+          start_pos = ss.pos
+          peeked = ss.peek_byte
 
-        if (special = SPECIAL_TABLE[peeked])
-          @ss.scan_byte
-          # Special case for ".."
-          if special == DOT && @ss.peek_byte == DOT_ORD
-            @ss.scan_byte
-            @output << DOTDOT
-          elsif special == DASH
-            # Special case for negative numbers
-            if (peeked_byte = @ss.peek_byte) && NUMBER_TABLE[peeked_byte]
-              @ss.pos -= 1
-              @output << [:number, @ss.scan(NUMBER_LITERAL)]
+          if (special = SPECIAL_TABLE[peeked])
+            ss.scan_byte
+            # Special case for ".."
+            if special == DOT && ss.peek_byte == DOT_ORD
+              ss.scan_byte
+              output << DOTDOT
+            elsif special == DASH
+              # Special case for negative numbers
+              if (peeked_byte = ss.peek_byte) && NUMBER_TABLE[peeked_byte]
+                ss.pos -= 1
+                output << [:number, ss.scan(NUMBER_LITERAL)]
+              else
+                output << special
+              end
             else
-              @output << special
+              output << special
+            end
+          elsif (sub_table = TWO_CHARS_COMPARISON_JUMP_TABLE[peeked])
+            ss.scan_byte
+            if (peeked_byte = ss.peek_byte) && (found = sub_table[peeked_byte])
+              output << found
+              ss.scan_byte
+            else
+              raise_syntax_error(start_pos, ss)
+            end
+          elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
+            ss.scan_byte
+            if (peeked_byte = ss.peek_byte) && (found = sub_table[peeked_byte])
+              output << found
+              ss.scan_byte
+            else
+              output << SINGLE_COMPARISON_TOKENS[peeked]
             end
           else
-            @output << special
-          end
-        elsif (sub_table = TWO_CHARS_COMPARISON_JUMP_TABLE[peeked])
-          @ss.scan_byte
-          if (peeked_byte = @ss.peek_byte) && (found = sub_table[peeked_byte])
-            @output << found
-            @ss.scan_byte
-          else
-            raise_syntax_error(start_pos)
-          end
-        elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
-          @ss.scan_byte
-          if (peeked_byte = @ss.peek_byte) && (found = sub_table[peeked_byte])
-            @output << found
-            @ss.scan_byte
-          else
-            @output << SINGLE_COMPARISON_TOKENS[peeked]
-          end
-        else
-          type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked]
+            type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked]
 
-          if type && (t = @ss.scan(pattern))
-            # Special case for "contains"
-            @output << if type == :id && t == "contains" && @output.last&.first != :dot
-              COMPARISON_CONTAINS
+            if type && (t = ss.scan(pattern))
+              # Special case for "contains"
+              output << if type == :id && t == "contains" && output.last&.first != :dot
+                COMPARISON_CONTAINS
+              elsif type == :id && t == "and" && output.last&.first != :dot
+                BOOLEAN_AND
+              elsif type == :id && t == "or" && output.last&.first != :dot
+                BOOLEAN_OR
+              else
+                [type, t]
+              end
             else
-              [type, t]
+              raise_syntax_error(start_pos, ss)
             end
-          else
-            raise_syntax_error(start_pos)
           end
         end
+        # rubocop:enable Metrics/BlockNesting
+        output << EOS
+      rescue ::ArgumentError => e
+        if e.message == "invalid byte sequence in #{ss.string.encoding}"
+          raise SyntaxError, "Invalid byte sequence in #{ss.string.encoding}"
+        else
+          raise
+        end
       end
-      # rubocop:enable Metrics/BlockNesting
 
-      @output << EOS
-    end
-
-    def raise_syntax_error(start_pos)
-      @ss.pos = start_pos
-      # the character could be a UTF-8 character, use getch to get all the bytes
-      raise SyntaxError, "Unexpected character #{@ss.getch}"
+      def raise_syntax_error(start_pos, ss)
+        ss.pos = start_pos
+        # the character could be a UTF-8 character, use getch to get all the bytes
+        raise SyntaxError, "Unexpected character #{ss.getch}"
+      end
     end
   end
-
-  Lexer = StringScanner.instance_methods.include?(:scan_byte) ? Lexer2 : Lexer1
 end

lib/liquid/locales/en.yml

@@ -2,12 +2,14 @@
   errors:
     syntax:
       tag_unexpected_args: "Syntax Error in '%{tag}' - Valid syntax: %{tag}"
+      block_tag_unexpected_args: "Syntax Error in '%{tag}' - Valid syntax: {% %{tag} %}{% end%{tag} %}"
       assign: "Syntax Error in 'assign' - Valid syntax: assign [var] = [source]"
       capture: "Syntax Error in 'capture' - Valid syntax: capture [var]"
       case: "Syntax Error in 'case' - Valid syntax: case [condition]"
       case_invalid_when: "Syntax Error in tag 'case' - Valid when condition: {% when [condition] [or condition2...] %}"
       case_invalid_else: "Syntax Error in tag 'case' - Valid else condition: {% else %} (no parameters) "
       cycle: "Syntax Error in 'cycle' - Valid syntax: cycle [name :] var [, var2, var3 ...]"
+      doc_invalid_nested: "Syntax Error in 'doc' - Nested doc tags are not allowed"
       for: "Syntax Error in 'for loop' - Valid syntax: for [item] in [collection]"
       for_invalid_in: "For loops require an 'in' clause"
       for_invalid_attribute: "Invalid attribute in for loop. Valid attributes are limit and offset"

lib/liquid/parse_context.rb

@@ -12,6 +12,18 @@ module Liquid
       @locale   = @template_options[:locale] ||= I18n.new
       @warnings = []
 
+      # constructing new StringScanner in Lexer, Tokenizer, etc is expensive
+      # This StringScanner will be shared by all of them
+      @string_scanner = StringScanner.new("")
+
+      @expression_cache = if options[:expression_cache].nil?
+        {}
+      elsif options[:expression_cache].respond_to?(:[]) && options[:expression_cache].respond_to?(:[]=)
+        options[:expression_cache]
+      elsif options[:expression_cache]
+        {}
+      end
+
       self.depth   = 0
       self.partial = false
     end
@@ -24,12 +36,22 @@ module Liquid
       Liquid::BlockBody.new
     end
 
-    def new_tokenizer(markup, start_line_number: nil, for_liquid_tag: false)
-      Tokenizer.new(markup, line_number: start_line_number, for_liquid_tag: for_liquid_tag)
+    def new_parser(input)
+      @string_scanner.string = input
+      Parser.new(@string_scanner)
+    end
+
+    def new_tokenizer(source, start_line_number: nil, for_liquid_tag: false)
+      Tokenizer.new(
+        source: source,
+        string_scanner: @string_scanner,
+        line_number: start_line_number,
+        for_liquid_tag: for_liquid_tag,
+      )
     end
 
     def parse_expression(markup)
-      Expression.parse(markup)
+      Expression.parse(markup, @string_scanner, @expression_cache)
     end
 
     def partial=(value)

lib/liquid/parser.rb

@@ -3,8 +3,8 @@
 module Liquid
   class Parser
     def initialize(input)
-      l       = Lexer.new(input)
-      @tokens = l.tokenize
+      ss = input.is_a?(StringScanner) ? input : StringScanner.new(input)
+      @tokens = Lexer.tokenize(ss)
       @p      = 0 # pointer to current location
     end
 
@@ -48,7 +48,7 @@ module Liquid
 
     def expression
       token = @tokens[@p]
-      case token[0]
+      expr = case token[0]
       when :id
         str = consume
         str << variable_lookups
@@ -60,12 +60,41 @@ module Liquid
       when :string, :number
         consume
       when :open_round
-        consume
-        first = expression
-        consume(:dotdot)
+        consume_round_parentheses(token)
+      else
+        raise SyntaxError, "#{token} is not a valid expression"
+      end
+      if look(:comparison)
+        operator = consume(:comparison)
+        left     = expr
+        right    = expression
+
+        "#{left} #{operator} #{right}"
+      elsif look(:boolean_operator)
+        operator = consume(:boolean_operator)
+        left     = expr
+        right    = expression
+        if look(:close_round)
+          "(#{left} #{operator} #{right})"
+        else
+          "#{left} #{operator} #{right}"
+        end
+      else
+        expr
+      end
+    end
+
+    def consume_round_parentheses(token)
+      consume
+      first = expression
+      dotdot_token = consume?(:dotdot)
+      if dotdot_token
         last = expression
         consume(:close_round)
         "(#{first}..#{last})"
+      elsif look(:close_round)
+        consume(:close_round)
+        first
       else
         raise SyntaxError, "#{token} is not a valid expression"
       end

lib/liquid/range_lookup.rb

@@ -2,9 +2,9 @@
 
 module Liquid
   class RangeLookup
-    def self.parse(start_markup, end_markup)
-      start_obj = Expression.parse(start_markup)
-      end_obj   = Expression.parse(end_markup)
+    def self.parse(start_markup, end_markup, string_scanner, cache = nil)
+      start_obj = Expression.parse(start_markup, string_scanner, cache)
+      end_obj   = Expression.parse(end_markup, string_scanner, cache)
       if start_obj.respond_to?(:evaluate) || end_obj.respond_to?(:evaluate)
         new(start_obj, end_obj)
       else

lib/liquid/standardfilters.rb

@@ -64,7 +64,7 @@ module Liquid
     # @liquid_syntax string | downcase
     # @liquid_return [string]
     def downcase(input)
-      input.to_s.downcase
+      Utils.to_s(input).downcase
     end
 
     # @liquid_public_docs
@@ -75,7 +75,7 @@ module Liquid
     # @liquid_syntax string | upcase
     # @liquid_return [string]
     def upcase(input)
-      input.to_s.upcase
+      Utils.to_s(input).upcase
     end
 
     # @liquid_public_docs
@@ -86,7 +86,7 @@ module Liquid
     # @liquid_syntax string | capitalize
     # @liquid_return [string]
     def capitalize(input)
-      input.to_s.capitalize
+      Utils.to_s(input).capitalize
     end
 
     # @liquid_public_docs
@@ -97,7 +97,7 @@ module Liquid
     # @liquid_syntax string | escape
     # @liquid_return [string]
     def escape(input)
-      CGI.escapeHTML(input.to_s) unless input.nil?
+      CGI.escapeHTML(Utils.to_s(input)) unless input.nil?
     end
     alias_method :h, :escape
 
@@ -109,7 +109,7 @@ module Liquid
     # @liquid_syntax string | escape_once
     # @liquid_return [string]
     def escape_once(input)
-      input.to_s.gsub(HTML_ESCAPE_ONCE_REGEXP, HTML_ESCAPE)
+      Utils.to_s(input).gsub(HTML_ESCAPE_ONCE_REGEXP, HTML_ESCAPE)
     end
 
     # @liquid_public_docs
@@ -124,7 +124,7 @@ module Liquid
     # @liquid_syntax string | url_encode
     # @liquid_return [string]
     def url_encode(input)
-      CGI.escape(input.to_s) unless input.nil?
+      CGI.escape(Utils.to_s(input)) unless input.nil?
     end
 
     # @liquid_public_docs
@@ -138,7 +138,7 @@ module Liquid
     def url_decode(input)
       return if input.nil?
 
-      result = CGI.unescape(input.to_s)
+      result = CGI.unescape(Utils.to_s(input))
       raise Liquid::ArgumentError, "invalid byte sequence in #{result.encoding}" unless result.valid_encoding?
 
       result
@@ -152,7 +152,7 @@ module Liquid
     # @liquid_syntax string | base64_encode
     # @liquid_return [string]
     def base64_encode(input)
-      Base64.strict_encode64(input.to_s)
+      Base64.strict_encode64(Utils.to_s(input))
     end
 
     # @liquid_public_docs
@@ -163,7 +163,7 @@ module Liquid
     # @liquid_syntax string | base64_decode
     # @liquid_return [string]
     def base64_decode(input)
-      input = input.to_s
+      input = Utils.to_s(input)
       StandardFilters.try_coerce_encoding(Base64.strict_decode64(input), encoding: input.encoding)
     rescue ::ArgumentError
       raise Liquid::ArgumentError, "invalid base64 provided to base64_decode"
@@ -177,7 +177,7 @@ module Liquid
     # @liquid_syntax string | base64_url_safe_encode
     # @liquid_return [string]
     def base64_url_safe_encode(input)
-      Base64.urlsafe_encode64(input.to_s)
+      Base64.urlsafe_encode64(Utils.to_s(input))
     end
 
     # @liquid_public_docs
@@ -188,7 +188,7 @@ module Liquid
     # @liquid_syntax string | base64_url_safe_decode
     # @liquid_return [string]
     def base64_url_safe_decode(input)
-      input = input.to_s
+      input = Utils.to_s(input)
       StandardFilters.try_coerce_encoding(Base64.urlsafe_decode64(input), encoding: input.encoding)
     rescue ::ArgumentError
       raise Liquid::ArgumentError, "invalid base64 provided to base64_url_safe_decode"
@@ -212,7 +212,7 @@ module Liquid
         if input.is_a?(Array)
           input.slice(offset, length) || []
         else
-          input.to_s.slice(offset, length) || ''
+          Utils.to_s(input).slice(offset, length) || ''
         end
       rescue RangeError
         if I64_RANGE.cover?(length) && I64_RANGE.cover?(offset)
@@ -236,10 +236,10 @@ module Liquid
     # @liquid_return [string]
     def truncate(input, length = 50, truncate_string = "...")
       return if input.nil?
-      input_str = input.to_s
+      input_str = Utils.to_s(input)
       length    = Utils.to_integer(length)
 
-      truncate_string_str = truncate_string.to_s
+      truncate_string_str = Utils.to_s(truncate_string)
 
       l = length - truncate_string_str.length
       l = 0 if l < 0
@@ -263,7 +263,7 @@ module Liquid
     # @liquid_return [string]
     def truncatewords(input, words = 15, truncate_string = "...")
       return if input.nil?
-      input = input.to_s
+      input = Utils.to_s(input)
       words = Utils.to_integer(words)
       words = 1 if words <= 0
 
@@ -277,7 +277,8 @@ module Liquid
       return input if wordlist.length <= words
 
       wordlist.pop
-      wordlist.join(" ").concat(truncate_string.to_s)
+      truncate_string = Utils.to_s(truncate_string)
+      wordlist.join(" ").concat(truncate_string)
     end
 
     # @liquid_public_docs
@@ -288,7 +289,9 @@ module Liquid
     # @liquid_syntax string | split: string
     # @liquid_return [array[string]]
     def split(input, pattern)
-      input.to_s.split(pattern.to_s)
+      pattern = Utils.to_s(pattern)
+      input = Utils.to_s(input)
+      input.split(pattern)
     end
 
     # @liquid_public_docs
@@ -299,7 +302,8 @@ module Liquid
     # @liquid_syntax string | strip
     # @liquid_return [string]
     def strip(input)
-      input.to_s.strip
+      input = Utils.to_s(input)
+      input.strip
     end
 
     # @liquid_public_docs
@@ -310,7 +314,8 @@ module Liquid
     # @liquid_syntax string | lstrip
     # @liquid_return [string]
     def lstrip(input)
-      input.to_s.lstrip
+      input = Utils.to_s(input)
+      input.lstrip
     end
 
     # @liquid_public_docs
@@ -321,7 +326,8 @@ module Liquid
     # @liquid_syntax string | rstrip
     # @liquid_return [string]
     def rstrip(input)
-      input.to_s.rstrip
+      input = Utils.to_s(input)
+      input.rstrip
     end
 
     # @liquid_public_docs
@@ -332,8 +338,9 @@ module Liquid
     # @liquid_syntax string | strip_html
     # @liquid_return [string]
     def strip_html(input)
+      input = Utils.to_s(input)
       empty  = ''
-      result = input.to_s.gsub(STRIP_HTML_BLOCKS, empty)
+      result = input.gsub(STRIP_HTML_BLOCKS, empty)
       result.gsub!(STRIP_HTML_TAGS, empty)
       result
     end
@@ -346,7 +353,8 @@ module Liquid
     # @liquid_syntax string | strip_newlines
     # @liquid_return [string]
     def strip_newlines(input)
-      input.to_s.gsub(/\r?\n/, '')
+      input = Utils.to_s(input)
+      input.gsub(/\r?\n/, '')
     end
 
     # @liquid_public_docs
@@ -357,6 +365,7 @@ module Liquid
     # @liquid_syntax array | join
     # @liquid_return [string]
     def join(input, glue = ' ')
+      glue = Utils.to_s(glue)
       InputIterator.new(input, context).join(glue)
     end
 
@@ -424,29 +433,59 @@ module Liquid
     # @liquid_syntax array | where: string, string
     # @liquid_return [array[untyped]]
     def where(input, property, target_value = nil)
-      ary = InputIterator.new(input, context)
+      filter_array(input, property, target_value) { |ary, &block| ary.select(&block) }
+    end
 
-      if ary.empty?
-        []
-      elsif target_value.nil?
-        ary.select do |item|
-          item[property]
-        rescue TypeError
-          raise_property_error(property)
-        rescue NoMethodError
-          return nil unless item.respond_to?(:[])
-          raise
-        end
-      else
-        ary.select do |item|
-          item[property] == target_value
-        rescue TypeError
-          raise_property_error(property)
-        rescue NoMethodError
-          return nil unless item.respond_to?(:[])
-          raise
-        end
-      end
+    # @liquid_public_docs
+    # @liquid_type filter
+    # @liquid_category array
+    # @liquid_summary
+    #   Filters an array to exclude items with a specific property value.
+    # @liquid_description
+    #   This requires you to provide both the property name and the associated value.
+    # @liquid_syntax array | reject: string, string
+    # @liquid_return [array[untyped]]
+    def reject(input, property, target_value = nil)
+      filter_array(input, property, target_value) { |ary, &block| ary.reject(&block) }
+    end
+
+    # @liquid_public_docs
+    # @liquid_type filter
+    # @liquid_category array
+    # @liquid_summary
+    #   Tests if any item in an array has a specific property value.
+    # @liquid_description
+    #   This requires you to provide both the property name and the associated value.
+    # @liquid_syntax array | has: string, string
+    # @liquid_return [boolean]
+    def has(input, property, target_value = nil)
+      filter_array(input, property, target_value, false) { |ary, &block| ary.any?(&block) }
+    end
+
+    # @liquid_public_docs
+    # @liquid_type filter
+    # @liquid_category array
+    # @liquid_summary
+    #   Returns the first item in an array with a specific property value.
+    # @liquid_description
+    #   This requires you to provide both the property name and the associated value.
+    # @liquid_syntax array | find: string, string
+    # @liquid_return [untyped]
+    def find(input, property, target_value = nil)
+      filter_array(input, property, target_value, nil) { |ary, &block| ary.find(&block) }
+    end
+
+    # @liquid_public_docs
+    # @liquid_type filter
+    # @liquid_category array
+    # @liquid_summary
+    #   Returns the index of the first item in an array with a specific property value.
+    # @liquid_description
+    #   This requires you to provide both the property name and the associated value.
+    # @liquid_syntax array | find_index: string, string
+    # @liquid_return [number]
+    def find_index(input, property, target_value = nil)
+      filter_array(input, property, target_value, nil) { |ary, &block| ary.find_index(&block) }
     end
 
     # @liquid_public_docs
@@ -543,7 +582,10 @@ module Liquid
     # @liquid_syntax string | replace: string, string
     # @liquid_return [string]
     def replace(input, string, replacement = '')
-      input.to_s.gsub(string.to_s, replacement.to_s)
+      string = Utils.to_s(string)
+      replacement = Utils.to_s(replacement)
+      input = Utils.to_s(input)
+      input.gsub(string, replacement)
     end
 
     # @liquid_public_docs
@@ -554,7 +596,10 @@ module Liquid
     # @liquid_syntax string | replace_first: string, string
     # @liquid_return [string]
     def replace_first(input, string, replacement = '')
-      input.to_s.sub(string.to_s, replacement.to_s)
+      string = Utils.to_s(string)
+      replacement = Utils.to_s(replacement)
+      input = Utils.to_s(input)
+      input.sub(string, replacement)
     end
 
     # @liquid_public_docs
@@ -565,9 +610,9 @@ module Liquid
     # @liquid_syntax string | replace_last: string, string
     # @liquid_return [string]
     def replace_last(input, string, replacement)
-      input = input.to_s
-      string = string.to_s
-      replacement = replacement.to_s
+      input = Utils.to_s(input)
+      string = Utils.to_s(string)
+      replacement = Utils.to_s(replacement)
 
       start_index = input.rindex(string)
 
@@ -619,7 +664,9 @@ module Liquid
     # @liquid_syntax string | append: string
     # @liquid_return [string]
     def append(input, string)
-      input.to_s + string.to_s
+      input = Utils.to_s(input)
+      string = Utils.to_s(string)
+      input + string
     end
 
     # @liquid_public_docs
@@ -648,7 +695,9 @@ module Liquid
     # @liquid_syntax string | prepend: string
     # @liquid_return [string]
     def prepend(input, string)
-      string.to_s + input.to_s
+      input = Utils.to_s(input)
+      string = Utils.to_s(string)
+      string + input
     end
 
     # @liquid_public_docs
@@ -659,7 +708,8 @@ module Liquid
     # @liquid_syntax string | newline_to_br
     # @liquid_return [string]
     def newline_to_br(input)
-      input.to_s.gsub(/\r?\n/, "<br />\n")
+      input = Utils.to_s(input)
+      input.gsub(/\r?\n/, "<br />\n")
     end
 
     # Reformat a date using Ruby's core Time#strftime( string ) -> string
@@ -694,11 +744,12 @@ module Liquid
     #
     #   See also: http://www.ruby-doc.org/core/Time.html#method-i-strftime
     def date(input, format)
-      return input if format.to_s.empty?
+      str_format = Utils.to_s(format)
+      return input if str_format.empty?
 
       return input unless (date = Utils.to_date(input))
 
-      date.strftime(format.to_s)
+      date.strftime(str_format)
     end
 
     # @liquid_public_docs
@@ -918,6 +969,25 @@ module Liquid
 
     attr_reader :context
 
+    def filter_array(input, property, target_value, default_value = [], &block)
+      ary = InputIterator.new(input, context)
+
+      return default_value if ary.empty?
+
+      block.call(ary) do |item|
+        if target_value.nil?
+          item[property]
+        else
+          item[property] == target_value
+        end
+      rescue TypeError
+        raise_property_error(property)
+      rescue NoMethodError
+        return nil unless item.respond_to?(:[])
+        raise
+      end
+    end
+
     def raise_property_error(property)
       raise Liquid::ArgumentError, "cannot select the property '#{property}'"
     end
@@ -968,7 +1038,18 @@ module Liquid
       end
 
       def join(glue)
-        to_a.join(glue.to_s)
+        first = true
+        output = +""
+        each do |item|
+          if first
+            first = false
+          else
+            output << glue
+          end
+
+          output << Liquid::Utils.to_s(item)
+        end
+        output
       end
 
       def concat(args)

lib/liquid/tags.rb

@@ -19,6 +19,7 @@ require_relative "tags/comment"
 require_relative "tags/raw"
 require_relative "tags/render"
 require_relative "tags/cycle"
+require_relative "tags/doc"
 
 module Liquid
   module Tags
@@ -42,6 +43,7 @@ module Liquid
       'if' => If,
       'echo' => Echo,
       'tablerow' => TableRow,
+      'doc' => Doc,
     }.freeze
   end
 end

lib/liquid/tags/cycle.rb

@@ -68,7 +68,13 @@ module Liquid
     def variables_from_string(markup)
       markup.split(',').collect do |var|
         var =~ /\s*(#{QuotedFragment})\s*/o
-        Regexp.last_match(1) ? parse_expression(Regexp.last_match(1)) : nil
+        next unless Regexp.last_match(1)
+
+        # Expression Parser returns cached objects, and we need to dup them to
+        # start the cycle over for each new cycle call.
+        # Liquid-C does not have a cache, so we don't need to dup the object.
+        var = parse_expression(Regexp.last_match(1))
+        var.is_a?(VariableLookup) ? var.dup : var
       end.compact
     end
 

lib/liquid/tags/doc.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Liquid
+  # @liquid_public_docs
+  # @liquid_type tag
+  # @liquid_category syntax
+  # @liquid_name doc
+  # @liquid_summary
+  #   Documents template elements with annotations.
+  # @liquid_description
+  #   The `doc` tag allows developers to include documentation within Liquid
+  #   templates. Any content inside `doc` tags is not rendered or outputted.
+  #   Liquid code inside will be parsed but not executed. This facilitates
+  #   tooling support for features like code completion, linting, and inline
+  #   documentation.
+  # @liquid_syntax
+  #   {% doc %}
+  #     Renders a message.
+  #
+  #     @param {string} foo - A string value.
+  #     @param {string} [bar] - An optional string value.
+  #
+  #     @example
+  #     {% render 'message', foo: 'Hello', bar: 'World' %}
+  #   {% enddoc %}
+  #   {{ foo }}, {{ bar }}!
+  class Doc < Block
+    NO_UNEXPECTED_ARGS = /\A\s*\z/
+
+    def initialize(tag_name, markup, parse_context)
+      super
+      ensure_valid_markup(tag_name, markup, parse_context)
+    end
+
+    def parse(tokens)
+      while (token = tokens.shift)
+        tag_name = token =~ BlockBody::FullTokenPossiblyInvalid && Regexp.last_match(2)
+
+        raise_nested_doc_error if tag_name == @tag_name
+
+        if tag_name == block_delimiter
+          parse_context.trim_whitespace = (token[-3] == WhitespaceControl)
+          return
+        end
+      end
+
+      raise_tag_never_closed(block_name)
+    end
+
+    def render_to_output_buffer(_context, output)
+      output
+    end
+
+    def blank?
+      true
+    end
+
+    def nodelist
+      []
+    end
+
+    private
+
+    def ensure_valid_markup(tag_name, markup, parse_context)
+      unless NO_UNEXPECTED_ARGS.match?(markup)
+        raise SyntaxError, parse_context.locale.t("errors.syntax.block_tag_unexpected_args", tag: tag_name)
+      end
+    end
+
+    def raise_nested_doc_error
+      raise SyntaxError, parse_context.locale.t("errors.syntax.doc_invalid_nested")
+    end
+  end
+end

lib/liquid/tags/for.rb

@@ -88,7 +88,7 @@ module Liquid
     end
 
     def strict_parse(markup)
-      p = Parser.new(markup)
+      p = @parse_context.new_parser(markup)
       @variable_name = p.consume(:id)
       raise SyntaxError, options[:locale].t("errors.syntax.for_invalid_in") unless p.id?('in')
 

lib/liquid/tags/if.rb

@@ -102,7 +102,7 @@ module Liquid
     end
 
     def strict_parse(markup)
-      p = Parser.new(markup)
+      p = @parse_context.new_parser(markup)
       condition = parse_binary_comparisons(p)
       p.consume(:end_of_string)
       condition

lib/liquid/tokenizer.rb

@@ -1,20 +1,43 @@
 # frozen_string_literal: true
 
+require "strscan"
+
 module Liquid
   class Tokenizer
     attr_reader :line_number, :for_liquid_tag
 
-    def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
-      @source         = source
-      @line_number    = line_number || (line_numbers ? 1 : nil)
+    TAG_END = /%\}/
+    TAG_OR_VARIABLE_START = /\{[\{\%]/
+    NEWLINE = /\n/
+
+    OPEN_CURLEY = "{".ord
+    CLOSE_CURLEY = "}".ord
+    PERCENTAGE = "%".ord
+
+    def initialize(
+      source:,
+      string_scanner:,
+      line_numbers: false,
+      line_number: nil,
+      for_liquid_tag: false
+    )
+      @line_number = line_number || (line_numbers ? 1 : nil)
       @for_liquid_tag = for_liquid_tag
-      @offset         = 0
-      @tokens         = tokenize
+      @source = source.to_s.to_str
+      @offset = 0
+      @tokens = []
+
+      if @source
+        @ss = string_scanner
+        @ss.string = @source
+        tokenize
+      end
     end
 
     def shift
       token = @tokens[@offset]
-      return nil unless token
+
+      return unless token
 
       @offset += 1
 
@@ -28,18 +51,111 @@ module Liquid
     private
 
     def tokenize
-      return [] if @source.empty?
-
-      return @source.split("\n") if @for_liquid_tag
-
-      tokens = @source.split(TemplateParser)
-
-      # removes the rogue empty element at the beginning of the array
-      if tokens[0]&.empty?
-        @offset += 1
+      if @for_liquid_tag
+        @tokens = @source.split("\n")
+      else
+        @tokens << shift_normal until @ss.eos?
       end
 
-      tokens
+      @source = nil
+      @ss = nil
+    end
+
+    def shift_normal
+      token = next_token
+
+      return unless token
+
+      token
+    end
+
+    def next_token
+      # possible states: :text, :tag, :variable
+      byte_a = @ss.peek_byte
+
+      if byte_a == OPEN_CURLEY
+        @ss.scan_byte
+
+        byte_b = @ss.peek_byte
+
+        if byte_b == PERCENTAGE
+          @ss.scan_byte
+          return next_tag_token
+        elsif byte_b == OPEN_CURLEY
+          @ss.scan_byte
+          return next_variable_token
+        end
+
+        @ss.pos -= 1
+      end
+
+      next_text_token
+    end
+
+    def next_text_token
+      start = @ss.pos
+
+      unless @ss.skip_until(TAG_OR_VARIABLE_START)
+        token = @ss.rest
+        @ss.terminate
+        return token
+      end
+
+      pos = @ss.pos -= 2
+      @source.byteslice(start, pos - start)
+    rescue ::ArgumentError => e
+      if e.message == "invalid byte sequence in #{@ss.string.encoding}"
+        raise SyntaxError, "Invalid byte sequence in #{@ss.string.encoding}"
+      else
+        raise
+      end
+    end
+
+    def next_variable_token
+      start = @ss.pos - 2
+
+      byte_a = byte_b = @ss.scan_byte
+
+      while byte_b
+        byte_a = @ss.scan_byte while byte_a && (byte_a != CLOSE_CURLEY && byte_a != OPEN_CURLEY)
+
+        break unless byte_a
+
+        if @ss.eos?
+          return byte_a == CLOSE_CURLEY ? @source.byteslice(start, @ss.pos - start) : "{{"
+        end
+
+        byte_b = @ss.scan_byte
+
+        if byte_a == CLOSE_CURLEY
+          if byte_b == CLOSE_CURLEY
+            return @source.byteslice(start, @ss.pos - start)
+          elsif byte_b != CLOSE_CURLEY
+            @ss.pos -= 1
+            return @source.byteslice(start, @ss.pos - start)
+          end
+        elsif byte_a == OPEN_CURLEY && byte_b == PERCENTAGE
+          return next_tag_token_with_start(start)
+        end
+
+        byte_a = byte_b
+      end
+
+      "{{"
+    end
+
+    def next_tag_token
+      start = @ss.pos - 2
+      if (len = @ss.skip_until(TAG_END))
+        @source.byteslice(start, len + 2)
+      else
+        "{%"
+      end
+    end
+
+    def next_tag_token_with_start(start)
+      @ss.skip_until(TAG_END)
+      @source.byteslice(start, @ss.pos - start)
     end
   end
 end

lib/liquid/utils.rb

@@ -89,5 +89,101 @@ module Liquid
       # Otherwise return the object itself
       obj
     end
+
+    def self.to_s(obj, seen = {})
+      case obj
+      when Hash
+        # If the custom hash implementation overrides `#to_s`, use their
+        # custom implementation. Otherwise we use Liquid's default
+        # implementation.
+        if obj.class.instance_method(:to_s) == HASH_TO_S_METHOD
+          hash_inspect(obj, seen)
+        else
+          obj.to_s
+        end
+      when Array
+        array_inspect(obj, seen)
+      else
+        obj.to_s
+      end
+    end
+
+    def self.inspect(obj, seen = {})
+      case obj
+      when Hash
+        # If the custom hash implementation overrides `#inspect`, use their
+        # custom implementation. Otherwise we use Liquid's default
+        # implementation.
+        if obj.class.instance_method(:inspect) == HASH_INSPECT_METHOD
+          hash_inspect(obj, seen)
+        else
+          obj.inspect
+        end
+      when Array
+        array_inspect(obj, seen)
+      else
+        obj.inspect
+      end
+    end
+
+    def self.array_inspect(arr, seen = {})
+      if seen[arr.object_id]
+        return "[...]"
+      end
+
+      seen[arr.object_id] = true
+      str = +"["
+      cursor = 0
+      len = arr.length
+
+      while cursor < len
+        if cursor > 0
+          str << ", "
+        end
+
+        item_str = inspect(arr[cursor], seen)
+        str << item_str
+        cursor += 1
+      end
+
+      str << "]"
+      str
+    ensure
+      seen.delete(arr.object_id)
+    end
+
+    def self.hash_inspect(hash, seen = {})
+      if seen[hash.object_id]
+        return "{...}"
+      end
+      seen[hash.object_id] = true
+
+      str = +"{"
+      first = true
+      hash.each do |key, value|
+        if first
+          first = false
+        else
+          str << ", "
+        end
+
+        key_str = inspect(key, seen)
+        str << key_str
+        str << "=>"
+
+        value_str = inspect(value, seen)
+        str << value_str
+      end
+      str << "}"
+      str
+    ensure
+      seen.delete(hash.object_id)
+    end
+
+    HASH_TO_S_METHOD = Hash.instance_method(:to_s)
+    private_constant :HASH_TO_S_METHOD
+
+    HASH_INSPECT_METHOD = Hash.instance_method(:inspect)
+    private_constant :HASH_INSPECT_METHOD
   end
 end

lib/liquid/variable.rb

@@ -61,7 +61,7 @@ module Liquid
 
     def strict_parse(markup)
       @filters = []
-      p = Parser.new(markup)
+      p = @parse_context.new_parser(markup)
 
       return if p.look(:end_of_string)
 
@@ -107,8 +107,8 @@ module Liquid
         obj.each do |o|
           render_obj_to_output(o, output)
         end
-      when
-        output << obj.to_s
+      else
+        output << Liquid::Utils.to_s(obj)
       end
     end
 

lib/liquid/variable_lookup.rb

@@ -6,16 +6,20 @@ module Liquid
 
     attr_reader :name, :lookups
 
-    def self.parse(markup)
-      new(markup)
+    def self.parse(markup, string_scanner = StringScanner.new(""), cache = nil)
+      new(markup, string_scanner, cache)
     end
 
-    def initialize(markup)
+    def initialize(markup, string_scanner = StringScanner.new(""), cache = nil)
       lookups = markup.scan(VariableParser)
 
       name = lookups.shift
       if name&.start_with?('[') && name&.end_with?(']')
-        name = Expression.parse(name[1..-2])
+        name = Expression.parse(
+          name[1..-2],
+          string_scanner,
+          cache,
+        )
       end
       @name = name
 
@@ -25,7 +29,11 @@ module Liquid
       @lookups.each_index do |i|
         lookup = lookups[i]
         if lookup&.start_with?('[') && lookup&.end_with?(']')
-          lookups[i] = Expression.parse(lookup[1..-2])
+          lookups[i] = Expression.parse(
+            lookup[1..-2],
+            string_scanner,
+            cache,
+          )
         elsif COMMAND_METHODS.include?(lookup)
           @command_flags |= 1 << i
         end

lib/liquid/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module Liquid
-  VERSION = "5.6.0.rc1"
+  VERSION = "5.8.1"
 end

liquid.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |s|
   s.summary     = "A secure, non-evaling end user template engine with aesthetic markup."
   s.authors     = ["Tobias Lütke"]
   s.email       = ["[email protected]"]
-  s.homepage    = "http://www.liquidmarkup.org"
+  s.homepage    = "https://shopify.github.io/liquid/"
   s.license     = "MIT"
   # s.description = "A secure, non-evaling end user template engine with aesthetic markup."
 
@@ -28,7 +28,7 @@ Gem::Specification.new do |s|
 
   s.require_path = "lib"
 
-  s.add_dependency("strscan")
+  s.add_dependency("strscan", ">= 3.1.1")
   s.add_dependency("bigdecimal")
 
   s.add_development_dependency('rake', '~> 13.0')

performance/benchmark.rb

@@ -9,14 +9,17 @@ Liquid::Environment.default.error_mode = ARGV.first.to_sym if ARGV.first
 profiler = ThemeRunner.new
 
 Benchmark.ips do |x|
-  x.time   = 10
-  x.warmup = 5
+  x.time   = 20
+  x.warmup = 10
 
   puts
   puts "Running benchmark for #{x.time} seconds (with #{x.warmup} seconds warmup)."
   puts
 
-  x.report("parse:") { profiler.compile }
-  x.report("render:") { profiler.render }
-  x.report("parse & render:") { profiler.run }
+  phase = ENV["PHASE"] || "all"
+
+  x.report("tokenize:") { profiler.tokenize } if phase == "all" || phase == "tokenize"
+  x.report("parse:") { profiler.compile } if phase == "all" || phase == "parse"
+  x.report("render:") { profiler.render } if phase == "all" || phase == "render"
+  x.report("parse & render:") { profiler.run } if phase == "all" || phase == "run"
 end

performance/theme_runner.rb

@@ -48,6 +48,19 @@ class ThemeRunner
     end
   end
 
+  # `tokenize` will just test the tokenizen portion of liquid without any templates
+  def tokenize
+    ss = StringScanner.new("")
+    @tests.each do |test_hash|
+      tokenizer = Liquid::Tokenizer.new(
+        source: test_hash[:liquid],
+        string_scanner: ss,
+        line_numbers: true,
+      )
+      while tokenizer.shift; end
+    end
+  end
+
   # `run` is called to benchmark rendering and compiling at the same time
   def run
     each_test do |liquid, layout, assigns, page_template, template_name|

performance/unit/expression_benchmark.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "benchmark/ips"
+
+# benchmark liquid lexing
+
+require 'liquid'
+
+RubyVM::YJIT.enable
+
+STRING_MARKUPS = [
+  "\"foo\"",
+  "\"fooooooooooo\"",
+  "\"foooooooooooooooooooooooooooooo\"",
+  "'foo'",
+  "'fooooooooooo'",
+  "'foooooooooooooooooooooooooooooo'",
+]
+
+VARIABLE_MARKUPS = [
+  "article",
+  "article.title",
+  "article.title.size",
+  "very_long_variable_name_2024_11_05",
+  "very_long_variable_name_2024_11_05.size",
+]
+
+NUMBER_MARKUPS = [
+  "0",
+  "35",
+  "1241891024912849",
+  "3.5",
+  "3.51214128409128",
+  "12381902839.123819283910283",
+  "123.456.789",
+  "-123",
+  "-12.33",
+  "-405.231",
+  "-0",
+  "0",
+  "0.0",
+  "0.0000000000000000000000",
+  "0.00000000001",
+]
+
+RANGE_MARKUPS = [
+  "(1..30)",
+  "(1...30)",
+  "(1..30..5)",
+  "(1.0...30.0)",
+  "(1.........30)",
+  "(1..foo)",
+  "(foo..30)",
+  "(foo..bar)",
+  "(foo...bar...100)",
+  "(foo...bar...100.0)",
+]
+
+LITERAL_MARKUPS = [
+  nil,
+  'nil',
+  'null',
+  '',
+  'true',
+  'false',
+  'blank',
+  'empty',
+]
+
+MARKUPS = {
+  "string" => STRING_MARKUPS,
+  "literal" => LITERAL_MARKUPS,
+  "variable" => VARIABLE_MARKUPS,
+  "number" => NUMBER_MARKUPS,
+  "range" => RANGE_MARKUPS,
+}
+
+Benchmark.ips do |x|
+  x.config(time: 5, warmup: 5)
+
+  MARKUPS.each do |type, markups|
+    x.report("Liquid::Expression#parse: #{type}") do
+      markups.each do |markup|
+        Liquid::Expression.parse(markup)
+      end
+    end
+  end
+
+  x.report("Liquid::Expression#parse: all") do
+    MARKUPS.values.flatten.each do |markup|
+      Liquid::Expression.parse(markup)
+    end
+  end
+end

performance/unit/lexer_benchmark.rb

@@ -29,31 +29,12 @@ EXPRESSIONS = [
   "foo | default: -1",
 ]
 
-EXPRESSIONS.each do |expr|
-  lexer_1_result = Liquid::Lexer1.new(expr).tokenize
-  lexer_2_result = Liquid::Lexer2.new(expr).tokenize
-
-  next if lexer_1_result == lexer_2_result
-
-  warn "Lexer1 and Lexer2 results are different for expression: #{expr}"
-  warn "expected: #{lexer_1_result}"
-  warn "got: #{lexer_2_result}"
-  abort
-end
-
 Benchmark.ips do |x|
   x.config(time: 10, warmup: 5)
 
-  x.report("Liquid::Lexer1#tokenize") do
+  x.report("Liquid::Lexer#tokenize") do
     EXPRESSIONS.each do |expr|
-      l = Liquid::Lexer1.new(expr)
-      l.tokenize
-    end
-  end
-
-  x.report("Liquid::Lexer2#tokenize") do
-    EXPRESSIONS.each do |expr|
-      l = Liquid::Lexer2.new(expr)
+      l = Liquid::Lexer.new(expr)
       l.tokenize
     end
   end

test/integration/expression_test.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'test_helper'
+require 'lru_redux'
 
 class ExpressionTest < Minitest::Test
   def test_keyword_literals
@@ -13,6 +14,7 @@ class ExpressionTest < Minitest::Test
     assert_template_result("double quoted", '{{"double quoted"}}')
     assert_template_result("spaced", "{{ 'spaced' }}")
     assert_template_result("spaced2", "{{ 'spaced2' }}")
+    assert_template_result("emoji🔥", "{{ 'emoji🔥' }}")
   end
 
   def test_int
@@ -22,8 +24,18 @@ class ExpressionTest < Minitest::Test
   end
 
   def test_float
+    assert_template_result("-17.42", "{{ -17.42 }}")
     assert_template_result("2.5", "{{ 2.5 }}")
+    assert_expression_result(0.0, "0.....5")
+    assert_expression_result(0.0, "-0..1")
     assert_expression_result(1.5, "1.5")
+
+    # this is a unfortunate quirky behavior of Liquid
+    result = Expression.parse(".5")
+    assert_kind_of(Liquid::VariableLookup, result)
+
+    result = Expression.parse("-.5")
+    assert_kind_of(Liquid::VariableLookup, result)
   end
 
   def test_range
@@ -40,6 +52,101 @@ class ExpressionTest < Minitest::Test
     )
   end
 
+  def test_quirky_negative_sign_expression_markup
+    result = Expression.parse("-", nil)
+    assert(result.is_a?(VariableLookup))
+    assert_equal("-", result.name)
+
+    # for this template, the expression markup is "-"
+    assert_template_result(
+      "",
+      "{{ - 'theme.css' - }}",
+    )
+  end
+
+  def test_expression_cache
+    skip("Liquid-C does not support Expression caching") if defined?(Liquid::C) && Liquid::C.enabled
+
+    cache = {}
+    template = <<~LIQUID
+      {% assign x = 1 %}
+      {{ x }}
+      {% assign x = 2 %}
+      {{ x }}
+      {% assign y = 1 %}
+      {{ y }}
+    LIQUID
+
+    Liquid::Template.parse(template, expression_cache: cache).render
+
+    assert_equal(
+      ["1", "2", "x", "y"],
+      cache.to_a.map { _1[0] }.sort,
+    )
+  end
+
+  def test_expression_cache_with_true_boolean
+    skip("Liquid-C does not support Expression caching") if defined?(Liquid::C) && Liquid::C.enabled
+
+    template = <<~LIQUID
+      {% assign x = 1 %}
+      {{ x }}
+      {% assign x = 2 %}
+      {{ x }}
+      {% assign y = 1 %}
+      {{ y }}
+    LIQUID
+
+    parse_context = ParseContext.new(expression_cache: true)
+
+    Liquid::Template.parse(template, parse_context).render
+
+    cache = parse_context.instance_variable_get(:@expression_cache)
+
+    assert_equal(
+      ["1", "2", "x", "y"],
+      cache.to_a.map { _1[0] }.sort,
+    )
+  end
+
+  def test_expression_cache_with_lru_redux
+    skip("Liquid-C does not support Expression caching") if defined?(Liquid::C) && Liquid::C.enabled
+
+    cache = LruRedux::Cache.new(10)
+    template = <<~LIQUID
+      {% assign x = 1 %}
+      {{ x }}
+      {% assign x = 2 %}
+      {{ x }}
+      {% assign y = 1 %}
+      {{ y }}
+    LIQUID
+
+    Liquid::Template.parse(template, expression_cache: cache).render
+
+    assert_equal(
+      ["1", "2", "x", "y"],
+      cache.to_a.map { _1[0] }.sort,
+    )
+  end
+
+  def test_disable_expression_cache
+    skip("Liquid-C does not support Expression caching") if defined?(Liquid::C) && Liquid::C.enabled
+
+    template = <<~LIQUID
+      {% assign x = 1 %}
+      {{ x }}
+      {% assign x = 2 %}
+      {{ x }}
+      {% assign y = 1 %}
+      {{ y }}
+    LIQUID
+
+    parse_context = Liquid::ParseContext.new(expression_cache: false)
+    Liquid::Template.parse(template, parse_context).render
+    assert(parse_context.instance_variable_get(:@expression_cache).nil?)
+  end
+
   private
 
   def assert_expression_result(expect, markup, **assigns)

test/integration/hash_rendering_test.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class HashRenderingTest < Minitest::Test
+  def test_render_empty_hash
+    assert_template_result("{}", "{{ my_hash }}", { "my_hash" => {} })
+  end
+
+  def test_render_hash_with_string_keys_and_values
+    assert_template_result("{\"key1\"=>\"value1\", \"key2\"=>\"value2\"}", "{{ my_hash }}", { "my_hash" => { "key1" => "value1", "key2" => "value2" } })
+  end
+
+  def test_render_hash_with_symbol_keys_and_integer_values
+    assert_template_result("{:key1=>1, :key2=>2}", "{{ my_hash }}", { "my_hash" => { key1: 1, key2: 2 } })
+  end
+
+  def test_render_nested_hash
+    assert_template_result("{\"outer\"=>{\"inner\"=>\"value\"}}", "{{ my_hash }}", { "my_hash" => { "outer" => { "inner" => "value" } } })
+  end
+
+  def test_render_hash_with_array_values
+    assert_template_result("{\"numbers\"=>[1, 2, 3]}", "{{ my_hash }}", { "my_hash" => { "numbers" => [1, 2, 3] } })
+  end
+
+  def test_render_recursive_hash
+    recursive_hash = { "self" => {} }
+    recursive_hash["self"]["self"] = recursive_hash
+    assert_template_result("{\"self\"=>{\"self\"=>{...}}}", "{{ my_hash }}", { "my_hash" => recursive_hash })
+  end
+
+  def test_hash_with_downcase_filter
+    assert_template_result("{\"key\"=>\"value\", \"anotherkey\"=>\"anothervalue\"}", "{{ my_hash | downcase }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
+  end
+
+  def test_hash_with_upcase_filter
+    assert_template_result("{\"KEY\"=>\"VALUE\", \"ANOTHERKEY\"=>\"ANOTHERVALUE\"}", "{{ my_hash | upcase }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
+  end
+
+  def test_hash_with_strip_filter
+    assert_template_result("{\"Key\"=>\"Value\", \"AnotherKey\"=>\"AnotherValue\"}", "{{ my_hash | strip }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
+  end
+
+  def test_hash_with_escape_filter
+    assert_template_result("{&quot;Key&quot;=&gt;&quot;Value&quot;, &quot;AnotherKey&quot;=&gt;&quot;AnotherValue&quot;}", "{{ my_hash | escape }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
+  end
+
+  def test_hash_with_url_encode_filter
+    assert_template_result("%7B%22Key%22%3D%3E%22Value%22%2C+%22AnotherKey%22%3D%3E%22AnotherValue%22%7D", "{{ my_hash | url_encode }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
+  end
+
+  def test_hash_with_strip_html_filter
+    assert_template_result("{\"Key\"=>\"Value\", \"AnotherKey\"=>\"AnotherValue\"}", "{{ my_hash | strip_html }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
+  end
+
+  def test_hash_with_truncate__20_filter
+    assert_template_result("{\"Key\"=>\"Value\", ...", "{{ my_hash | truncate: 20 }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
+  end
+
+  def test_hash_with_replace___key____replaced_key__filter
+    assert_template_result("{\"Key\"=>\"Value\", \"AnotherKey\"=>\"AnotherValue\"}", "{{ my_hash | replace: 'key', 'replaced_key' }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
+  end
+
+  def test_hash_with_append____appended_text__filter
+    assert_template_result("{\"Key\"=>\"Value\", \"AnotherKey\"=>\"AnotherValue\"} appended text", "{{ my_hash | append: ' appended text' }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
+  end
+
+  def test_hash_with_prepend___prepended_text___filter
+    assert_template_result("prepended text {\"Key\"=>\"Value\", \"AnotherKey\"=>\"AnotherValue\"}", "{{ my_hash | prepend: 'prepended text ' }}", { "my_hash" => { "Key" => "Value", "AnotherKey" => "AnotherValue" } })
+  end
+
+  def test_render_hash_with_array_values_empty
+    assert_template_result("{\"numbers\"=>[]}", "{{ my_hash }}", { "my_hash" => { "numbers" => [] } })
+  end
+
+  def test_render_hash_with_array_values_hash
+    assert_template_result("{\"numbers\"=>[{:foo=>42}]}", "{{ my_hash }}", { "my_hash" => { "numbers" => [{ foo: 42 }] } })
+  end
+
+  def test_join_filter_with_hash
+    array = [{ "key1" => "value1" }, { "key2" => "value2" }]
+    glue = { "lol" => "wut" }
+    assert_template_result("{\"key1\"=>\"value1\"}{\"lol\"=>\"wut\"}{\"key2\"=>\"value2\"}", "{{ my_array | join: glue }}", { "my_array" => array, "glue" => glue })
+  end
+
+  def test_render_hash_with_hash_key
+    assert_template_result("{{\"foo\"=>\"bar\"}=>42}", "{{ my_hash }}", { "my_hash" => { Hash["foo" => "bar"] => 42 } })
+  end
+
+  def test_rendering_hash_with_custom_to_s_method_uses_custom_to_s
+    my_hash = Class.new(Hash) do
+      def to_s
+        "kewl"
+      end
+    end.new
+
+    assert_template_result("kewl", "{{ my_hash }}", { "my_hash" => my_hash })
+  end
+
+  def test_rendering_hash_without_custom_to_s_uses_default_inspect
+    my_hash = Class.new(Hash).new
+    my_hash[:foo] = :bar
+
+    assert_template_result("{:foo=>:bar}", "{{ my_hash }}", { "my_hash" => my_hash })
+  end
+end

test/integration/parsing_quirks_test.rb

@@ -134,6 +134,14 @@ class ParsingQuirksTest < Minitest::Test
 
   def test_incomplete_expression
     with_error_mode(:lax) do
+      assert_template_result("false", "{{ false - }}")
+      assert_template_result("false", "{{ false > }}")
+      assert_template_result("false", "{{ false < }}")
+      assert_template_result("false", "{{ false = }}")
+      assert_template_result("false", "{{ false ! }}")
+      assert_template_result("false", "{{ false 1 }}")
+      assert_template_result("false", "{{ false a }}")
+
       assert_template_result("false", "{% liquid assign foo = false -\n%}{{ foo }}")
       assert_template_result("false", "{% liquid assign foo = false >\n%}{{ foo }}")
       assert_template_result("false", "{% liquid assign foo = false <\n%}{{ foo }}")

test/integration/standard_filter_test.rb

@@ -133,6 +133,18 @@ class StandardFiltersTest < Minitest::Test
     assert_equal([], @filters.slice(input, -(1 << 63), 6))
   end
 
+  def test_find_on_empty_array
+    assert_nil(@filters.find([], 'foo', 'bar'))
+  end
+
+  def test_find_index_on_empty_array
+    assert_nil(@filters.find_index([], 'foo', 'bar'))
+  end
+
+  def test_has_on_empty_array
+    refute(@filters.has([], 'foo', 'bar'))
+  end
+
   def test_truncate
     assert_equal('1234...', @filters.truncate('1234567890', 7))
     assert_equal('1234567890', @filters.truncate('1234567890', 20))
@@ -281,6 +293,16 @@ class StandardFiltersTest < Minitest::Test
     assert_equal('1121314', @filters.join([1, 2, 3, 4], 1))
   end
 
+  def test_join_calls_to_liquid_on_each_element
+    drop = Class.new(Liquid::Drop) do
+      def to_liquid
+        'i did it'
+      end
+    end
+
+    assert_equal('i did it, i did it', @filters.join([drop.new, drop.new], ", "))
+  end
+
   def test_sort
     assert_equal([1, 2, 3, 4], @filters.sort([4, 3, 2, 1]))
     assert_equal([{ "a" => 1 }, { "a" => 2 }, { "a" => 3 }, { "a" => 4 }], @filters.sort([{ "a" => 4 }, { "a" => 3 }, { "a" => 1 }, { "a" => 2 }], "a"))
@@ -827,21 +849,216 @@ class StandardFiltersTest < Minitest::Test
     assert_template_result('abc', "{{ 'abc' | date: '%D' }}")
   end
 
-  def test_where
-    input = [
+  def test_reject
+    array = [
       { "handle" => "alpha", "ok" => true },
       { "handle" => "beta", "ok" => false },
       { "handle" => "gamma", "ok" => false },
       { "handle" => "delta", "ok" => true },
     ]
 
-    expectation = [
+    template = "{{ array | reject: 'ok' | map: 'handle' | join: ' ' }}"
+    expected_output = "beta gamma"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_reject_with_value
+    array = [
       { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
       { "handle" => "delta", "ok" => true },
     ]
 
-    assert_equal(expectation, @filters.where(input, "ok", true))
-    assert_equal(expectation, @filters.where(input, "ok"))
+    template = "{{ array | reject: 'ok', true | map: 'handle' | join: ' ' }}"
+    expected_output = "beta gamma"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_reject_with_false_value
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | reject: 'ok', false | map: 'handle' | join: ' ' }}"
+    expected_output = "alpha delta"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_has
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => false },
+    ]
+
+    expected_output = "true"
+
+    assert_template_result(expected_output, "{{ array | has: 'ok' }}", { "array" => array })
+    assert_template_result(expected_output, "{{ array | has: 'ok', true }}", { "array" => array })
+  end
+
+  def test_has_when_does_not_have_it
+    array = [
+      { "handle" => "alpha", "ok" => false },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => false },
+    ]
+
+    expected_output = "false"
+
+    assert_template_result(expected_output, "{{ array | has: 'ok' }}", { "array" => array })
+    assert_template_result(expected_output, "{{ array | has: 'ok', true }}", { "array" => array })
+  end
+
+  def test_has_with_empty_arrays
+    template = <<~LIQUID
+      {%- assign has_product = products | has: 'title.content', 'Not found' -%}
+      {%- unless has_product -%}
+        Product not found.
+      {%- endunless -%}
+    LIQUID
+    expected_output = "Product not found."
+
+    assert_template_result(expected_output, template, { "products" => [] })
+  end
+
+  def test_has_with_false_value
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | has: 'ok', false }}"
+    expected_output = "true"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_has_with_false_value_when_does_not_have_it
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => true },
+      { "handle" => "gamma", "ok" => true },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | has: 'ok', false }}"
+    expected_output = "false"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_find_with_value
+    products = [
+      { "title" => "Pro goggles",    "price" => 1299 },
+      { "title" => "Thermal gloves", "price" => 1499 },
+      { "title" => "Alpine jacket",  "price" => 3999 },
+      { "title" => "Mountain boots", "price" => 3899 },
+      { "title" => "Safety helmet",  "price" => 1999 }
+    ]
+
+    template = <<~LIQUID
+      {%- assign product = products | find: 'price', 3999 -%}
+      {{- product.title -}}
+    LIQUID
+    expected_output = "Alpine jacket"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_find_with_empty_arrays
+    template = <<~LIQUID
+      {%- assign product = products | find: 'title.content', 'Not found' -%}
+      {%- unless product -%}
+        Product not found.
+      {%- endunless -%}
+    LIQUID
+    expected_output = "Product not found."
+
+    assert_template_result(expected_output, template, { "products" => [] })
+  end
+
+  def test_find_index_with_value
+    products = [
+      { "title" => "Pro goggles",    "price" => 1299 },
+      { "title" => "Thermal gloves", "price" => 1499 },
+      { "title" => "Alpine jacket",  "price" => 3999 },
+      { "title" => "Mountain boots", "price" => 3899 },
+      { "title" => "Safety helmet",  "price" => 1999 }
+    ]
+
+    template = <<~LIQUID
+      {%- assign index = products | find_index: 'price', 3999 -%}
+      {{- index -}}
+    LIQUID
+    expected_output = "2"
+
+    assert_template_result(expected_output, template, { "products" => products })
+  end
+
+  def test_find_index_with_empty_arrays
+    template = <<~LIQUID
+      {%- assign index = products | find_index: 'title.content', 'Not found' -%}
+      {%- unless index -%}
+        Index not found.
+      {%- endunless -%}
+    LIQUID
+    expected_output = "Index not found."
+
+    assert_template_result(expected_output, template, { "products" => [] })
+  end
+
+  def test_where
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | where: 'ok' | map: 'handle' | join: ' ' }}"
+    expected_output = "alpha delta"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_where_with_value
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | where: 'ok', true | map: 'handle' | join: ' ' }}"
+    expected_output = "alpha delta"
+
+    assert_template_result(expected_output, template, { "array" => array })
+  end
+
+  def test_where_with_false_value
+    array = [
+      { "handle" => "alpha", "ok" => true },
+      { "handle" => "beta", "ok" => false },
+      { "handle" => "gamma", "ok" => false },
+      { "handle" => "delta", "ok" => true },
+    ]
+
+    template = "{{ array | where: 'ok', false | map: 'handle' | join: ' ' }}"
+    expected_output = "beta gamma"
+
+    assert_template_result(expected_output, template, { "array" => array })
   end
 
   def test_where_string_keys

test/integration/tags/cycle_tag_test.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class CycleTagTest < Minitest::Test
+  def test_simple_cycle
+    template = <<~LIQUID
+      {%- cycle '1', '2', '3' -%}
+      {%- cycle '1', '2', '3' -%}
+      {%- cycle '1', '2', '3' -%}
+    LIQUID
+
+    assert_template_result("123", template)
+  end
+
+  def test_simple_cycle_inside_for_loop
+    template = <<~LIQUID
+      {%- for i in (1..3) -%}
+        {% cycle '1', '2', '3' %}
+      {%- endfor -%}
+    LIQUID
+
+    assert_template_result("123", template)
+  end
+
+  def test_cycle_with_variables_inside_for_loop
+    template = <<~LIQUID
+      {%- assign a = 1 -%}
+      {%- assign b = 2 -%}
+      {%- assign c = 3 -%}
+      {%- for i in (1..3) -%}
+        {% cycle a, b, c %}
+      {%- endfor -%}
+    LIQUID
+
+    assert_template_result("123", template)
+  end
+
+  def test_cycle_tag_always_resets_cycle
+    template = <<~LIQUID
+      {%- assign a = "1" -%}
+      {%- cycle a, "2" -%}
+      {%- cycle a, "2" -%}
+    LIQUID
+
+    assert_template_result("11", template)
+  end
+end

test/integration/tags/raw_tag_test.rb

@@ -16,6 +16,7 @@ class RawTagTest < Minitest::Test
     assert_template_result('>{{ test }}<', '> {%- raw -%}{{ test }}{%- endraw -%} <')
     assert_template_result("> inner  <", "> {%- raw -%} inner {%- endraw %} <")
     assert_template_result("> inner <", "> {%- raw -%} inner {%- endraw -%} <")
+    assert_template_result("{Hello}", "{% raw %}{{% endraw %}Hello{% raw %}}{% endraw %}")
   end
 
   def test_open_tag_in_raw

test/test_boolean_helper.rb

@@ -0,0 +1,61 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+module Minitest
+  module Assertions
+    include Liquid
+    def assert_with_lax_parsing(template, expected_output, context = {})
+      prev_error_mode = Liquid::Environment.default.error_mode
+      Liquid::Environment.default.error_mode = :lax
+
+      begin
+        actual_output = Liquid::Template.parse(template).render(context)
+      rescue StandardError => e
+        actual_output = e.message
+      ensure
+        Liquid::Environment.default.error_mode = prev_error_mode
+      end
+
+      assert_equal(expected_output.strip, actual_output.strip)
+    end
+
+    def assert_parity(liquid_expression, expected_result, args = {})
+      assert_condition(liquid_expression, expected_result, args)
+      assert_expression(liquid_expression, expected_result, args)
+    end
+
+    def assert_expression(liquid_expression, expected_result, args = {})
+      assert_parity_scenario(:expression, "{{ #{liquid_expression} }}", expected_result, args)
+    end
+
+    def assert_condition(liquid_condition, expected_result, args = {})
+      assert_parity_scenario(:condition, "{% if #{liquid_condition} %}true{% else %}false{% endif %}", expected_result, args)
+    end
+
+    def assert_parity_scenario(kind, template, exp_output, args = {})
+      act_output = Liquid::Template.parse(template).render(args)
+
+      assert_equal(exp_output, act_output, <<~ERROR_MESSAGE)
+        #{kind.to_s.capitalize} template failure:
+        ---
+        #{template}
+        ---
+        args: #{args.inspect}
+      ERROR_MESSAGE
+    end
+  end
+end
+
+class LinkDrop < Liquid::Drop
+  attr_accessor :levels, :links, :title, :type, :url
+
+  def initialize(levels: nil, links: nil, title: nil, type: nil, url: nil)
+    super()
+
+    @levels = levels
+    @links = links
+    @title = title
+    @type = type
+    @url = url
+  end
+end

test/test_helper.rb

@@ -15,11 +15,6 @@ if (env_mode = ENV['LIQUID_PARSER_MODE'])
 end
 Liquid::Environment.default.error_mode = mode
 
-if ENV['LIQUID_C'] == '1'
-  puts "-- LIQUID C"
-  require 'liquid/c'
-end
-
 if Minitest.const_defined?('Test')
   # We're on Minitest 5+. Nothing to do here.
 else

test/unit/block_unit_test.rb

@@ -47,12 +47,18 @@ class BlockUnitTest < Minitest::Test
     )
   end
 
-  def test_with_block
+  def test_comment_tag_with_block
     template = Liquid::Template.parse("  {% comment %} {% endcomment %} ")
     assert_equal([String, Comment, String], block_types(template.root.nodelist))
     assert_equal(3, template.root.nodelist.size)
   end
 
+  def test_doc_tag_with_block
+    template = Liquid::Template.parse("  {% doc %} {% enddoc %} ")
+    assert_equal([String, Doc, String], block_types(template.root.nodelist))
+    assert_equal(3, template.root.nodelist.size)
+  end
+
   private
 
   def block_types(nodelist)

test/unit/boolean_precedence_unit_test.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+require 'test_boolean_helper'
+
+class BooleanPrecedenceUnitTest < Minitest::Test
+  include Liquid
+
+  def test_basic_boolean_parenthesized_expressions
+    assert_parity("false and (false or true)", "false")
+    assert_parity("true and (false or true)", "true")
+    assert_parity("(true and false) or true", "true")
+    assert_parity("(false and true) or false", "false")
+  end
+
+  def test_nested_boolean_parentheses
+    assert_parity("(false and (true or false)) or true", "true")
+    assert_parity("true and (false or (true and true))", "true")
+    assert_parity("(true and (false or false)) or false", "false")
+  end
+
+  def test_multiple_operations_with_consistent_operators
+    assert_parity("(true and true) and (false or true)", "true")
+    assert_parity("(false or false) or (true and false)", "false")
+  end
+
+  def test_parentheses_changing_default_precedence
+    # Default precedence: (true and false) or true
+    assert_parity("true and false or true", "true")
+    # With parentheses: true and (false or true)
+    assert_parity("true and (false or true)", "true")
+
+    # Default precedence: false or (true and true)
+    assert_parity("false or true and true", "true")
+    # With parentheses: (false or true) and true
+    assert_parity("(false or true) and true", "true")
+  end
+
+  def test_boolean_parentheses_with_variables
+    assert_parity("(a or b) and c", "true", { "a" => true, "b" => false, "c" => true })
+    assert_parity("(a or b) and c", "false", { "a" => true, "b" => false, "c" => false })
+    assert_parity("a and (b or c)", "true", { "a" => true, "b" => false, "c" => true })
+    assert_parity("a and (b or c)", "false", { "a" => false, "b" => true, "c" => true })
+  end
+
+  def test_comparison_operators_inside_parentheses
+    assert_parity("(1 > 0) and (2 < 3)", "true")
+    assert_parity("(1 < 0) or (2 > 3)", "false")
+    assert_parity("true and (1 == 1)", "true")
+    assert_parity("false or (2 != 2)", "false")
+  end
+
+  def test_complex_nested_boolean_expressions
+    assert_parity("((true and false) or (false and true)) or ((false or true) and (true or false))", "true")
+    assert_parity("((true and true) or (false and false)) and ((true or false) and (false or true))", "true")
+  end
+
+  def test_not_operator_with_parentheses
+    # Testing how 'not' interacts with parentheses
+    assert_parity("not (true or false)", "false")
+    assert_parity("not (false and true)", "true")
+    assert_parity("(not false) and true", "true")
+    assert_parity("(not true) or false", "false")
+    assert_parity("not (not true)", "true")
+  end
+
+  def test_nil_values_with_boolean_precedence
+    # How nil values interact with boolean expressions and parentheses
+    assert_parity("nil and (true or false)", "false")
+    assert_parity("(nil or true) and false", "false")
+    assert_parity("(nil and nil) or true", "true")
+    assert_parity("true and (nil or false)", "false")
+  end
+
+  def test_mixed_primitive_types_with_parentheses
+    # Testing how different types interact in boolean expressions with parentheses
+    assert_parity("('' or 0) and true", "true")
+    assert_parity("(true and 'string') or false", "true")
+    assert_parity("(false or '') and 1", "false")
+    assert_parity("(nil or false) and 'text'", "false")
+  end
+
+  def test_triple_operator_precedence
+    # Testing three different operators with different parenthesizing
+    assert_parity("true or false and true or false", "true") # default precedence
+    assert_parity("true or (false and true) or false", "true")
+    assert_parity("(true or false) and (true or false)", "true")
+    assert_parity("((true or false) and true) or false", "true")
+    assert_parity("true or (false and (true or false))", "true")
+  end
+
+  def test_undefined_variables_with_parentheses
+    # How undefined variables behave with parentheses
+    assert_parity("(undefined_var or true) and false", "false")
+    assert_parity("true and (undefined_var or false)", "false")
+    assert_parity("(undefined_var and true) or true", "true")
+    assert_parity("false or (undefined_var and false)", "false")
+  end
+
+  def test_comparison_chaining_with_parentheses
+    # Testing how comparison chains work with parentheses
+    assert_parity("(1 < 2) and (2 < 3) and (3 < 4)", "true")
+    assert_parity("(1 < 2) and ((2 > 3) or (3 < 4))", "true")
+    assert_parity(
+      "(a > b) or ((c < d) and (e == f))",
+      "true",
+      { "a" => 5, "b" => 3, "c" => 1, "d" => 2, "e" => 7, "f" => 7 },
+    )
+    assert_parity(
+      "(a > b) or ((c < d) and (e == f))",
+      "false",
+      { "a" => 3, "b" => 5, "c" => 2, "d" => 1, "e" => 7, "f" => 8 },
+    )
+  end
+
+  def test_deeply_nested_expressions
+    # Testing very deep nesting to ensure parser handles it correctly
+    assert_parity("(((true and true) or (false and false)) and ((true or false) and (true)))", "true")
+    assert_parity(
+      "(((a or b) and c) or (d and (e or f)))",
+      "true",
+      { "a" => false, "b" => true, "c" => true, "d" => true, "e" => true, "f" => false },
+    )
+  end
+
+  def test_malformed_parentheses
+    # Unbalanced parentheses - missing closing parenthesis
+    template = "{% if (true and false %}true{% else %}false{% endif %}"
+    assert_raises(Liquid::SyntaxError) { Liquid::Template.parse(template) }
+
+    # Unbalanced parentheses - missing opening parenthesis
+    template = "{% if true and false) %}true{% else %}false{% endif %}"
+    assert_raises(Liquid::SyntaxError) { Liquid::Template.parse(template) }
+
+    # Empty parentheses
+    template = "{% if () %}true{% else %}false{% endif %}"
+    assert_raises(Liquid::SyntaxError) { Liquid::Template.parse(template) }
+
+    # Consecutive opening parentheses without operators
+    template = "{% if ((true) %}true{% else %}false{% endif %}"
+    assert_raises(Liquid::SyntaxError) { Liquid::Template.parse(template) }
+
+    # Consecutive closing parentheses without proper opening
+    template = "{% if (true)) %}true{% else %}false{% endif %}"
+    assert_raises(Liquid::SyntaxError) { Liquid::Template.parse(template) }
+
+    # Parentheses with missing operand
+    template = "{% if (and true) %}true{% else %}false{% endif %}"
+    assert_raises(Liquid::SyntaxError) { Liquid::Template.parse(template) }
+
+    # Operator followed immediately by closing parenthesis
+    template = "{% if (true and) %}true{% else %}false{% endif %}"
+    assert_raises(Liquid::SyntaxError) { Liquid::Template.parse(template) }
+
+    # Nested malformed parentheses
+    template = "{% if (true and (false or true) %}true{% else %}false{% endif %}"
+    assert_raises(Liquid::SyntaxError) { Liquid::Template.parse(template) }
+
+    # Double parentheses with no content between them
+    template = "{% if true and (()) %}true{% else %}false{% endif %}"
+    assert_raises(Liquid::SyntaxError) { Liquid::Template.parse(template) }
+
+    # Misplaced parentheses around operators
+    template = "{% if true (and) false %}true{% else %}false{% endif %}"
+    assert_raises(Liquid::SyntaxError) { Liquid::Template.parse(template) }
+
+    # Parentheses at wrong position in expression
+    template = "{% if true) and (false %}true{% else %}false{% endif %}"
+    assert_raises(Liquid::SyntaxError) { Liquid::Template.parse(template) }
+  end
+end

test/unit/boolean_unit_test.rb

@@ -0,0 +1,491 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+require 'test_boolean_helper'
+
+class BooleanUnitTest < Minitest::Test
+  include Liquid
+  def test_simple_boolean_comparison
+    assert_parity("1 > 0", "true")
+    assert_parity("1 < 0", "false")
+  end
+
+  def test_boolean_and_operator
+    assert_parity("true and true", "true")
+    assert_parity("true and false", "false")
+  end
+
+  def test_boolean_or_operator
+    assert_parity("true or false", "true")
+    assert_parity("false or false", "false")
+  end
+
+  def test_operator_precedence
+    assert_parity("false and false or true", "false")
+  end
+
+  def test_complex_boolean_expressions
+    assert_parity("true and true and true", "true")
+    assert_parity("true and false and true", "false")
+    assert_parity("false or false or true", "true")
+  end
+
+  def test_boolean_with_variables
+    assert_parity("a and b", "true", { "a" => true, "b" => true })
+    assert_parity("a and b", "false", { "a" => true, "b" => false })
+    assert_parity("a or b", "true", { "a" => false, "b" => true })
+    assert_parity("a or b", "false", { "a" => false, "b" => false })
+  end
+
+  def test_nil_equals_nil
+    assert_parity("nil == nil", "true")
+  end
+
+  def test_nil_not_equals_nil
+    assert_parity("nil != nil", "false")
+  end
+
+  def test_nil_not_equals_empty_string
+    assert_parity("nil == ''", "false")
+    assert_parity("nil != ''", "true")
+  end
+
+  def test_undefined_variable_in_comparisons
+    assert_parity("undefined_var == nil", "true")
+    assert_parity("undefined_var != nil", "false")
+  end
+
+  def test_undefined_variable_compared_to_empty_string
+    assert_parity("undefined_var == ''", "false")
+    assert_parity("undefined_var != ''", "true")
+  end
+
+  def test_boolean_variable_in_comparisons
+    assert_parity("t == true", "true", { "t" => true })
+    assert_parity("f == false", "true", { "f" => false })
+  end
+
+  def test_boolean_variable_compared_to_nil
+    assert_parity("t == nil", "false", { "t" => true })
+    assert_parity("f == nil", "false", { "f" => false })
+    assert_parity("f != nil", "true", { "f" => false })
+  end
+
+  def test_nil_and_undefined_variables_in_boolean_expressions
+    assert_parity("x == undefined_var", "true", { "x" => nil })
+    assert_parity("x != undefined_var", "false", { "x" => nil })
+  end
+
+  def test_nil_literal_in_or_expression
+    assert_parity("nil or true", "true")
+  end
+
+  def test_nil_variable_in_or_expression
+    assert_parity("x or false", "false", { "x" => nil })
+  end
+
+  def test_mixed_boolean_expressions
+    assert_parity("a > b and c < d", "true",  { "a" => 99, "b" => 0, "c" => 0,  "d" => 99 })
+    assert_parity("a > b and c < d", "false", { "a" => 99, "b" => 0, "c" => 99, "d" => 0  })
+  end
+
+  def test_boolean_assignment_shorthand
+    template = Liquid::Template.parse("{% assign lazy_load = media_position > 1 %}{{ lazy_load }}")
+    assert_equal("false", template.render("media_position" => 1))
+    assert_equal("true",  template.render("media_position" => 2))
+  end
+
+  def test_equality_operators_with_integer_literals
+    assert_expression("1", "1")
+    assert_expression("1 == 1", "true")
+    assert_expression("1 != 1", "false")
+    assert_expression("1 == 2", "false")
+    assert_expression("1 != 2", "true")
+  end
+
+  def test_equality_operators_with_stirng_literals
+    assert_expression("'hello'", "hello")
+    assert_expression("'hello' == 'hello'", "true")
+    assert_expression("'hello' != 'hello'", "false")
+    assert_expression("'hello' == 'world'", "false")
+    assert_expression("'hello' != 'world'", "true")
+  end
+
+  def test_equality_operators_with_float_literals
+    assert_expression("1.5", "1.5")
+    assert_expression("1.5 == 1.5", "true")
+    assert_expression("1.5 != 1.5", "false")
+    assert_expression("1.5 == 2.5", "false")
+    assert_expression("1.5 != 2.5", "true")
+  end
+
+  def test_equality_operators_with_nil_literals
+    assert_expression("nil", "")
+    assert_expression("nil == nil", "true")
+    assert_expression("nil != nil", "false")
+    assert_expression("null == nil", "true")
+    assert_expression("null != nil", "false")
+  end
+
+  def test_equality_operators_with_boolean_literals
+    assert_expression("true", "true")
+    assert_expression("false", "false")
+    assert_expression("true == true", "true")
+    assert_expression("true != true", "false")
+    assert_expression("false == false", "true")
+    assert_expression("false != false", "false")
+    assert_expression("true == false", "false")
+    assert_expression("true != false", "true")
+  end
+
+  def test_equality_operators_with_empty_literals
+    assert_expression("empty", "")
+    assert_expression("empty == ''", "true")
+    assert_expression("empty == empty", "true")
+    assert_expression("empty != empty", "false")
+    assert_expression("blank == blank", "true")
+    assert_expression("blank != blank", "false")
+    assert_expression("empty == blank", "true")
+    assert_expression("empty != blank", "false")
+  end
+
+  def test_nil_renders_as_empty_string
+    # No parity needed here. This is to ensure expressions rendered with {{ }}
+    # will still render as an empty string to preserve pre-existing behavior.
+    assert_expression("nil", "")
+    assert_expression("x", "", { "x" => nil })
+    assert_parity_scenario(:expression, "hello {{ x }}", "hello ", { "x" => nil })
+  end
+
+  def test_nil_comparison_with_blank
+    assert_parity("nil_value == blank", "false")
+    assert_parity("nil_value != blank", "true")
+    assert_parity("undefined != blank", "true")
+    assert_parity("undefined == blank", "false")
+  end
+
+  def test_if_with_variables
+    assert_parity("value", "true",  { "value" => true })
+    assert_parity("value", "false", { "value" => false })
+  end
+
+  def test_nil_variable_in_and_expression
+    assert_condition("x and true", "false", { "x" => nil })
+    assert_condition("true and x", "false", { "x" => nil })
+
+    assert_expression("x and true", "",     { "x" => nil })
+    assert_expression("true and x", "",     { "x" => nil })
+  end
+
+  def test_boolean_variable_in_and_expression
+    assert_parity("true and x", "false", { "x" => false })
+    assert_parity("x and true", "false", { "x" => false })
+
+    assert_parity("true and x", "true",  { "x" => true })
+    assert_parity("x and true", "true",  { "x" => true })
+
+    assert_parity("true or x", "true",   { "x" => false })
+    assert_parity("x or true", "true",   { "x" => false })
+
+    assert_parity("true or x", "true",   { "x" => true })
+    assert_parity("x or true", "true",   { "x" => true })
+  end
+
+  def test_multi_variable_boolean_nil_and_expression
+    assert_condition("x and y", "false", { "x" => nil, "y" => true })
+    assert_condition("y and x", "false", { "x" => true, "y" => nil })
+
+    assert_expression("x and y", "",     { "x" => nil, "y" => true })
+    assert_expression("y and x", "",     { "x" => true, "y" => nil })
+  end
+
+  def test_multi_truthy_variables_and_expressions
+    assert_condition("x or y", "true",   { "x" => nil, "y" => "hello" })
+    assert_condition("y or x", "true",   { "x" => "hello", "y" => nil })
+
+    assert_expression("x or y", "hello", { "x" => nil, "y" => "hello" })
+    assert_expression("y or x", "hello", { "x" => "hello", "y" => nil })
+  end
+
+  def test_multi_variable_boolean_nil_or_expression
+    assert_parity("x or y", "true", { "x" => nil, "y" => true })
+    assert_parity("y or x", "true", { "x" => true, "y" => nil })
+  end
+
+  def test_links_not_blank_with_drop_returns_true_for_all_cases
+    link = LinkDrop.new(
+      levels: 0,
+      links: [
+        LinkDrop.new(levels: 1, links: [], title: "About", type: "page_link", url: "/pages/about"),
+        LinkDrop.new(levels: 1, links: [], title: "Contact", type: "page_link", url: "/pages/contact"),
+      ],
+      title: "Main Menu",
+      type: "menu",
+      url: nil,
+    )
+
+    template = <<~LIQUID
+      {%- if link.links != blank -%}
+        true
+      {%- else -%}
+        false
+      {%- endif -%}
+    LIQUID
+
+    act_output = Liquid::Template.parse(template).render({ "link" => link })
+    assert_equal("true", act_output)
+
+    act_output = Liquid::Template.parse(template).render({ "link" => link.tap { |l| l.links = [] } })
+    assert_equal("true", act_output)
+
+    act_output = Liquid::Template.parse(template).render({ "link" => link.tap { |l| l.links = nil } })
+    assert_equal("true", act_output)
+
+    act_output = Liquid::Template.parse(template).render({ "link" => LinkDrop.new })
+    assert_equal("true", act_output)
+  end
+
+  def test_links_truthy_with_drop_returns_false_for_nil_and_empty_drop
+    link = LinkDrop.new(
+      levels: 0,
+      links: [
+        LinkDrop.new(levels: 1, links: [], title: "About", type: "page_link", url: "/pages/about"),
+        LinkDrop.new(levels: 1, links: [], title: "Contact", type: "page_link", url: "/pages/contact"),
+      ],
+      title: "Main Menu",
+      type: "menu",
+      url: nil,
+    )
+
+    template = <<~LIQUID
+      {%- if link.links -%}
+        true
+      {%- else -%}
+        false
+      {%- endif -%}
+    LIQUID
+
+    act_output = Liquid::Template.parse(template).render({ "link" => link })
+    assert_equal("true", act_output)
+
+    act_output = Liquid::Template.parse(template).render({ "link" => link.tap { |l| l.links = [] } })
+    assert_equal("true", act_output)
+
+    act_output = Liquid::Template.parse(template).render({ "link" => link.tap { |l| l.links = nil } })
+    assert_equal("false", act_output)
+
+    act_output = Liquid::Template.parse(template).render({ "link" => {} })
+    assert_equal("false", act_output)
+  end
+
+  def test_links_not_blank_with_hash_returns_true_for_all_cases
+    link = {
+      "levels" => 0,
+      "links" => [
+        {
+          "levels" => 1,
+          "links" => [],
+          "title" => { "text" => "About" },
+          "type" => "page_link",
+          "url" => "/pages/about",
+        },
+        {
+          "levels" => 1,
+          "links" => [],
+          "title" => { "text" => "Contact" },
+          "type" => "page_link",
+          "url" => "/pages/contact",
+        },
+      ],
+      "title" => { "text" => "Main Menu" },
+      "type" => "menu",
+      "url" => nil,
+    }
+
+    template = <<~LIQUID
+      {%- if link.links != blank -%}
+        true
+      {%- else -%}
+        false
+      {%- endif -%}
+    LIQUID
+
+    act_output = Liquid::Template.parse(template).render({ "link" => link })
+    assert_equal("true", act_output)
+
+    act_output = Liquid::Template.parse(template).render({ "link" => { **link, "links" => [] } })
+    assert_equal("true", act_output)
+
+    act_output = Liquid::Template.parse(template).render({ "link" => { **link, "links" => nil } })
+    assert_equal("true", act_output)
+
+    act_output = Liquid::Template.parse(template).render({ "link" => {} })
+    assert_equal("true", act_output)
+  end
+
+  def test_links_truthy_with_hash_returns_false_for_nil_and_empty_hash
+    link = {
+      "levels" => 0,
+      "links" => [
+        {
+          "levels" => 1,
+          "links" => [],
+          "title" => { "text" => "About" },
+          "type" => "page_link",
+          "url" => "/pages/about",
+        },
+        {
+          "levels" => 1,
+          "links" => [],
+          "title" => { "text" => "Contact" },
+          "type" => "page_link",
+          "url" => "/pages/contact",
+        },
+      ],
+      "title" => { "text" => "Main Menu" },
+      "type" => "menu",
+      "url" => nil,
+    }
+
+    template = <<~LIQUID
+      {%- if link.links -%}
+        true
+      {%- else -%}
+        false
+      {%- endif -%}
+    LIQUID
+
+    act_output = Liquid::Template.parse(template).render({ "link" => link })
+    assert_equal("true", act_output)
+
+    act_output = Liquid::Template.parse(template).render({ "link" => { **link, "links" => [] } })
+    assert_equal("true", act_output)
+
+    act_output = Liquid::Template.parse(template).render({ "link" => { **link, "links" => nil } })
+    assert_equal("false", act_output)
+
+    act_output = Liquid::Template.parse(template).render({ "link" => {} })
+    assert_equal("false", act_output)
+  end
+
+  def test_conditions_with_boolean_operators_without_whitespace_around_operator
+    template = <<~LIQUID
+      <option variant_id="{{ variant.id }}" {% if current_variant.id==variant.id %}selected{%- endif -%}>{{ variant.title }}</option>
+    LIQUID
+
+    context = {
+      "variant" => {
+        "id" => 420,
+        "title" => "Default Title",
+      },
+      "current_variant" => {
+        "id" => 420,
+      },
+    }
+
+    # Expected output
+    # Note: Ideally we would like the whitespace around the boolean operator to be optional.
+    # So the more correct expected output would be:
+    #
+    # <option variant_id="420" selected>Default Title</option>
+    #
+    # However, the existing behaviour in liquid-ruby is that the whitespace is required around the boolean operator.
+    expected_lax_output = <<~HTML
+      <option variant_id="420" >Default Title</option>
+    HTML
+
+    expected_strict_output = <<~HTML
+      <option variant_id="420" selected>Default Title</option>
+    HTML
+
+    # This bugged output only happens in lax mode.
+    assert_with_lax_parsing(template, expected_lax_output, context)
+
+    # Default test parsing mode (strict) works as properly expected
+    assert_equal(expected_strict_output.delete("\n"), actual_strict_output.delete("\n"))
+  end
+
+  # TESTING INCORRECT BEHAVIOUR OF LIQUID-RUBY
+  # If liquid-vm fails this test, we should change it.
+  def test_boolean_conditional_with_json_filter
+    # Define the Liquid template to test
+    template = <<~LIQUID
+      {{ template.name == 'index' | json }}
+    LIQUID
+
+    # Define the context for the template where the template name is 'index'
+    context = {
+      "template" => {
+        "name" => "product",
+      },
+    }
+
+    # Expected output
+    # Note: I dont know what is the correct output here but this is the liquid-ruby 'main' output.
+    #
+    # It feels incorrect but I dont know whats better
+    expected_output = "product"
+
+    # Render the template with the context
+    actual_parsed_template = Liquid::Template.parse(template)
+    actual_output = actual_parsed_template.render(context)
+
+    # Assert that the actual output matches the expected output
+    assert_equal(expected_output, actual_output.strip)
+  end
+
+  # TESTING INCORRECT BEHAVIOUR OF LIQUID-RUBY
+  # If liquid-vm fails this test, we should change it.
+  def test_chained_conditional_with_object_contains
+    # Define the Liquid template to test
+    template = <<~LIQUID
+      {{ settings.prefilter_status and template contains 'collection' }}
+    LIQUID
+
+    # Test with context containing 'collection'
+    context_with_collection = {
+      "template" => {
+        "name" => "collection",
+      },
+      "settings" => {
+        "prefilter_status" => true,
+      },
+    }
+    # NOTE: This is a bug that liquid-ruby `main` output returns the first value.
+    assert_with_lax_parsing(template, "true", context_with_collection)
+
+    # Test with context not containing 'collection'
+    context_without_collection = {
+      "template" => {
+        "name" => "not-collection",
+      },
+      "settings" => {
+        "prefilter_status" => true,
+      },
+    }
+    # NOTE: This is a bug that liquid-ruby `main` output returns the first value.
+    assert_with_lax_parsing(template, "true", context_without_collection)
+  end
+
+  # TESTING INCORRECT BEHAVIOUR OF LIQUID-RUBY
+  # If liquid-vm fails this test, we should change it.
+  def test_assign_boolean_expression_to_variable
+    template = <<~LIQUID
+      {%- liquid
+        assign is_preview_mode = content_for_header contains "foo" or content_for_header contains "bar"
+        echo is_preview_mode
+      -%}
+    LIQUID
+
+    context = { "content_for_header" => "Some content" }
+
+    # Expected output
+    # This value should be "false" but it is the value of the variable from the failed expression.
+    assert_template_result("Some content", template, context)
+
+    # This following validation should only be supported with our changes. It is the short-hand for the above template.
+    # The validation for it is the expected correct output.
+    template = Liquid::Template.parse("{% assign is_preview_mode = content_for_header contains 'foo' or content_for_header contains 'bar' %}{{ is_preview_mode }}")
+    assert_equal("false", template.render(context))
+  end
+end

test/unit/environment_filter_test.rb

@@ -36,14 +36,14 @@ class EnvironmentFilterTest < Minitest::Test
     assert_equal("public", strainer.invoke("public_filter"))
   end
 
-  def test_stainer_raises_argument_error
+  def test_strainer_raises_argument_error
     strainer = @environment.create_strainer(@context)
     assert_raises(Liquid::ArgumentError) do
       strainer.invoke("public_filter", 1)
     end
   end
 
-  def test_stainer_argument_error_contains_backtrace
+  def test_strainer_argument_error_contains_backtrace
     strainer = @environment.create_strainer(@context)
 
     exception = assert_raises(Liquid::ArgumentError) do
@@ -54,8 +54,9 @@ class EnvironmentFilterTest < Minitest::Test
       /\ALiquid error: wrong number of arguments \((1 for 0|given 1, expected 0)\)\z/,
       exception.message,
     )
+
     source = AccessScopeFilters.instance_method(:public_filter).source_location
-    assert_equal(source.map(&:to_s), exception.backtrace[0].split(':')[0..1])
+    assert_equal(source[0..1].map(&:to_s), exception.backtrace[0].split(':')[0..1])
   end
 
   def test_strainer_only_invokes_public_filter_methods

test/unit/expression/logical_expression_test.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+require 'test_boolean_helper'
+
+class LogicalExpressionTest < Minitest::Test
+  include Liquid
+
+  def setup
+    @ss = StringScanner.new("")
+    @cache = {}
+  end
+
+  def test_logical_detection
+    assert(Expression::LogicalExpression.logical?("foo and bar"))
+    assert(Expression::LogicalExpression.logical?("foo or bar"))
+    assert(Expression::LogicalExpression.logical?("true and false"))
+    assert(Expression::LogicalExpression.logical?("1 or 0"))
+
+    refute(Expression::LogicalExpression.logical?("foo"))
+    refute(Expression::LogicalExpression.logical?("1 == 1"))
+    refute(Expression::LogicalExpression.logical?("a contains b"))
+    refute(Expression::LogicalExpression.logical?("not foo"))
+  end
+
+  def test_parenthesized_logical_detection
+    assert(Expression::LogicalExpression.logical?("a and (b or c)"))
+    assert(Expression::LogicalExpression.logical?("(a or b) and c"))
+  end
+
+  def test_boolean_operator_detection
+    assert(Expression::LogicalExpression.boolean_operator?("and"))
+    assert(Expression::LogicalExpression.boolean_operator?("or"))
+
+    refute(Expression::LogicalExpression.boolean_operator?("not"))
+    refute(Expression::LogicalExpression.boolean_operator?("=="))
+    refute(Expression::LogicalExpression.boolean_operator?("contains"))
+    refute(Expression::LogicalExpression.boolean_operator?("foo"))
+  end
+
+  def test_basic_parsing
+    result = Expression::LogicalExpression.parse("true and false", @ss, @cache)
+    assert_instance_of(Condition, result)
+
+    result = Expression::LogicalExpression.parse("a or b", @ss, @cache)
+    assert_instance_of(Condition, result)
+  end
+
+  def test_parsing_with_different_expressions
+    # Test with simple variable expressions
+    result = Expression::LogicalExpression.parse("var1 and var2", @ss, @cache)
+    assert_instance_of(Condition, result)
+
+    # Test with comparison expressions
+    result = Expression::LogicalExpression.parse("a == 1 and b != 2", @ss, @cache)
+    assert_instance_of(Condition, result)
+  end
+
+  def test_parsing_complex_expressions
+    # Test with nested logical expressions
+    result = Expression::LogicalExpression.parse("a and b or c", @ss, @cache)
+    assert_instance_of(Condition, result)
+
+    result = Expression::LogicalExpression.parse("a or b and c", @ss, @cache)
+    assert_instance_of(Condition, result)
+  end
+
+  def test_parsing_parenthesized_expressions
+    result = Expression::LogicalExpression.parse("(a and b) or c", @ss, @cache)
+    assert_instance_of(Condition, result)
+
+    result = Expression::LogicalExpression.parse("a and (b or c)", @ss, @cache)
+    assert_instance_of(Condition, result)
+
+    # Test with complex expressions
+    result = Expression::LogicalExpression.parse("(a or b) and (c or d)", @ss, @cache)
+    assert_instance_of(Condition, result)
+  end
+
+  def test_evaluation_of_parsed_expressions
+    context = Liquid::Context.new(
+      "a" => true,
+      "b" => false,
+      "c" => true,
+      "d" => false,
+    )
+
+    # Test simple logical expressions
+    expr = Expression::LogicalExpression.parse("a and c", @ss, @cache)
+    assert_equal(true, expr.evaluate(context))
+
+    expr = Expression::LogicalExpression.parse("a and b", @ss, @cache)
+    assert_equal(false, expr.evaluate(context))
+
+    expr = Expression::LogicalExpression.parse("b or c", @ss, @cache)
+    assert_equal(true, expr.evaluate(context))
+
+    expr = Expression::LogicalExpression.parse("b or d", @ss, @cache)
+    assert_equal(false, expr.evaluate(context))
+  end
+
+  def test_evaluation_of_complex_expressions
+    context = Liquid::Context.new(
+      "a" => true,
+      "b" => false,
+      "c" => true,
+      "d" => false,
+    )
+
+    # Test complex logical expressions
+    expr = Expression::LogicalExpression.parse("a and b or c", @ss, @cache)
+    assert_equal(true, expr.evaluate(context))
+  end
+
+  def test_evaluation_of_parenthesized_expressions
+    context = Liquid::Context.new(
+      "a" => true,
+      "b" => false,
+      "c" => true,
+      "d" => false,
+    )
+
+    expr = Expression::LogicalExpression.parse("a and (b or d)", @ss, @cache)
+    assert_equal(false, expr.evaluate(context))
+
+    expr = Expression::LogicalExpression.parse("(a or b) and (c or d)", @ss, @cache)
+    assert_equal(true, expr.evaluate(context))
+
+    expr = Expression::LogicalExpression.parse("(a or b) and (b or d)", @ss, @cache)
+    assert_equal(false, expr.evaluate(context))
+  end
+
+  def test_precedence_rules
+    context = Liquid::Context.new(
+      "a" => true,
+      "b" => false,
+      "c" => true,
+    )
+
+    # Test precedence rules (AND has higher precedence than OR)
+    # This should be interpreted as: a and (b or c)
+    expr1 = Expression::LogicalExpression.parse("a and b or c", @ss, @cache)
+    assert_equal(true, expr1.evaluate(context))
+
+    # Change context to make the expressions evaluate differently
+    context = Liquid::Context.new(
+      "a" => false,
+      "b" => false,
+      "c" => true,
+    )
+
+    # With these values, "a and (b or c)" would be false
+    expr1 = Expression::LogicalExpression.parse("a and b or c", @ss, @cache)
+    assert_equal(false, expr1.evaluate(context))
+  end
+
+  def test_precedence_with_parentheses
+    context = Liquid::Context.new(
+      "a" => true,
+      "b" => false,
+      "c" => true,
+    )
+
+    # This should be interpreted as: (a and b) or c
+    expr2 = Expression::LogicalExpression.parse("(a and b) or c", @ss, @cache)
+    assert_equal(true, expr2.evaluate(context))
+
+    # Change context to make the expressions evaluate differently
+    context = Liquid::Context.new(
+      "a" => false,
+      "b" => false,
+      "c" => true,
+    )
+
+    # But "(a and b) or c" would be true
+    expr2 = Expression::LogicalExpression.parse("(a and b) or c", @ss, @cache)
+    assert_equal(true, expr2.evaluate(context))
+  end
+
+  def test_integration_with_if_tag
+    # Test that our expressions work properly in actual templates
+    assert_template_result("true", "{% if true and true %}true{% else %}false{% endif %}")
+    assert_template_result("false", "{% if true and false %}true{% else %}false{% endif %}")
+    assert_template_result("true", "{% if false or true %}true{% else %}false{% endif %}")
+    assert_template_result("false", "{% if false or false %}true{% else %}false{% endif %}")
+  end
+
+  def test_integration_with_parenthesized_if_tag
+    # Test with parenthesized expressions
+    assert_template_result("true", "{% if (true and false) or true %}true{% else %}false{% endif %}")
+    assert_template_result("false", "{% if true and (false or false) %}true{% else %}false{% endif %}")
+    assert_template_result("true", "{% if true and (false or true) %}true{% else %}false{% endif %}")
+  end
+
+  def test_integration_with_variables
+    # Test with variables
+    template = "{% if a and b %}true{% else %}false{% endif %}"
+    assert_template_result("true", template, { "a" => true, "b" => true })
+    assert_template_result("false", template, { "a" => true, "b" => false })
+
+    template = "{% if a or b %}true{% else %}false{% endif %}"
+    assert_template_result("true", template, { "a" => true, "b" => false })
+    assert_template_result("false", template, { "a" => false, "b" => false })
+  end
+
+  def test_integration_with_parenthesized_variables
+    # Test with parenthesized expressions
+    template = "{% if (a and b) or c %}true{% else %}false{% endif %}"
+    assert_template_result("true", template, { "a" => true, "b" => true, "c" => false })
+    assert_template_result("true", template, { "a" => false, "b" => false, "c" => true })
+    assert_template_result("false", template, { "a" => false, "b" => false, "c" => false })
+
+    template = "{% if a and (b or c) %}true{% else %}false{% endif %}"
+    assert_template_result("true", template, { "a" => true, "b" => true, "c" => false })
+    assert_template_result("true", template, { "a" => true, "b" => false, "c" => true })
+    assert_template_result("false", template, { "a" => true, "b" => false, "c" => false })
+    assert_template_result("false", template, { "a" => false, "b" => true, "c" => true })
+  end
+end

test/unit/lexer_unit_test.rb

@@ -131,9 +131,91 @@ class LexerUnitTest < Minitest::Test
     assert_equal([[:id, "false"], [:number, "1"], [:end_of_string]], tokenize("false 1"))
   end
 
+  def test_error_with_invalid_utf8
+    error = assert_raises(SyntaxError) do
+      tokenize("\x00\xff")
+    end
+    assert_equal(
+      'Liquid syntax error: Invalid byte sequence in UTF-8',
+      error.message,
+    )
+  end
+
+  def test_boolean_and_operator
+    exp = [
+      [:id, "true"],
+      [:boolean_operator, "and"],
+      [:id, "false"],
+      [:end_of_string],
+    ]
+    act = tokenize("true and false")
+    assert_equal(exp, act)
+  end
+
+  def test_boolean_or_operator
+    exp = [
+      [:id, "false"],
+      [:boolean_operator, "or"],
+      [:id, "true"],
+      [:end_of_string],
+    ]
+    act = tokenize("false or true")
+    assert_equal(exp, act)
+  end
+
+  def test_boolean_operators_in_complex_expressions
+    exp = [
+      [:id, "a"],
+      [:boolean_operator, "and"],
+      [:id, "b"],
+      [:boolean_operator, "or"],
+      [:id, "c"],
+      [:end_of_string],
+    ]
+    act = tokenize("a and b or c")
+    assert_equal(exp, act)
+  end
+
+  def test_boolean_operators_with_comparisons
+    exp = [
+      [:id, "a"],
+      [:comparison, ">"],
+      [:number, "5"],
+      [:boolean_operator, "and"],
+      [:id, "b"],
+      [:comparison, "<"],
+      [:number, "10"],
+      [:end_of_string],
+    ]
+    act = tokenize("a > 5 and b < 10")
+    assert_equal(exp, act)
+  end
+
+  def test_boolean_operators_as_property_names
+    exp = [
+      [:id, "obj"],
+      [:dot, "."],
+      [:id, "and"],
+      [:dot, "."],
+      [:id, "property"],
+      [:end_of_string],
+    ]
+    act = tokenize("obj.and.property")
+    assert_equal(exp, act)
+
+    exp = [
+      [:id, "obj"],
+      [:dot, "."],
+      [:id, "or"],
+      [:end_of_string],
+    ]
+    act = tokenize("obj.or")
+    assert_equal(exp, act)
+  end
+
   private
 
   def tokenize(input)
-    Lexer.new(input).tokenize
+    Lexer.tokenize(StringScanner.new(input))
   end
 end

test/unit/parser_unit_test.rb

@@ -6,20 +6,20 @@ class ParserUnitTest < Minitest::Test
   include Liquid
 
   def test_consume
-    p = Parser.new("wat: 7")
+    p = new_parser("wat: 7")
     assert_equal('wat', p.consume(:id))
     assert_equal(':', p.consume(:colon))
     assert_equal('7', p.consume(:number))
   end
 
   def test_jump
-    p = Parser.new("wat: 7")
+    p = new_parser("wat: 7")
     p.jump(2)
     assert_equal('7', p.consume(:number))
   end
 
   def test_consume?
-    p = Parser.new("wat: 7")
+    p = new_parser("wat: 7")
     assert_equal('wat', p.consume?(:id))
     assert_equal(false, p.consume?(:dot))
     assert_equal(':', p.consume(:colon))
@@ -27,7 +27,7 @@ class ParserUnitTest < Minitest::Test
   end
 
   def test_id?
-    p = Parser.new("wat 6 Peter Hegemon")
+    p = new_parser("wat 6 Peter Hegemon")
     assert_equal('wat', p.id?('wat'))
     assert_equal(false, p.id?('endgame'))
     assert_equal('6', p.consume(:number))
@@ -36,7 +36,7 @@ class ParserUnitTest < Minitest::Test
   end
 
   def test_look
-    p = Parser.new("wat 6 Peter Hegemon")
+    p = new_parser("wat 6 Peter Hegemon")
     assert_equal(true, p.look(:id))
     assert_equal('wat', p.consume(:id))
     assert_equal(false, p.look(:comparison))
@@ -46,12 +46,12 @@ class ParserUnitTest < Minitest::Test
   end
 
   def test_expressions
-    p = Parser.new("hi.there hi?[5].there? hi.there.bob")
+    p = new_parser("hi.there hi?[5].there? hi.there.bob")
     assert_equal('hi.there', p.expression)
     assert_equal('hi?[5].there?', p.expression)
     assert_equal('hi.there.bob', p.expression)
 
-    p = Parser.new("567 6.0 'lol' \"wut\"")
+    p = new_parser("567 6.0 'lol' \"wut\"")
     assert_equal('567', p.expression)
     assert_equal('6.0', p.expression)
     assert_equal("'lol'", p.expression)
@@ -59,7 +59,7 @@ class ParserUnitTest < Minitest::Test
   end
 
   def test_ranges
-    p = Parser.new("(5..7) (1.5..9.6) (young..old) (hi[5].wat..old)")
+    p = new_parser("(5..7) (1.5..9.6) (young..old) (hi[5].wat..old)")
     assert_equal('(5..7)', p.expression)
     assert_equal('(1.5..9.6)', p.expression)
     assert_equal('(young..old)', p.expression)
@@ -67,7 +67,7 @@ class ParserUnitTest < Minitest::Test
   end
 
   def test_arguments
-    p = Parser.new("filter: hi.there[5], keyarg: 7")
+    p = new_parser("filter: hi.there[5], keyarg: 7")
     assert_equal('filter', p.consume(:id))
     assert_equal(':', p.consume(:colon))
     assert_equal('hi.there[5]', p.argument)
@@ -77,8 +77,14 @@ class ParserUnitTest < Minitest::Test
 
   def test_invalid_expression
     assert_raises(SyntaxError) do
-      p = Parser.new("==")
+      p = new_parser("==")
       p.expression
     end
   end
+
+  private
+
+  def new_parser(str)
+    Parser.new(StringScanner.new(str))
+  end
 end

test/unit/range_unit_test.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class RangeUnitTest < Minitest::Test
+  include Liquid
+
+  def test_basic_range_creation
+    assert_template_result("1 2 3 4 5", "{% for i in (1..5) %}{{ i }} {% endfor %}")
+  end
+
+  def test_range_with_variables
+    assert_template_result("3 4 5", "{% assign start = 3 %}{% for i in (start..5) %}{{ i }} {% endfor %}")
+    assert_template_result("1 2 3", "{% assign end = 3 %}{% for i in (1..end) %}{{ i }} {% endfor %}")
+    assert_template_result("2 3 4", "{% assign start = 2 %}{% assign end = 4 %}{% for i in (start..end) %}{{ i }} {% endfor %}")
+  end
+
+  def test_range_with_whitespace
+    assert_template_result("1 2 3", "{% for i in ( 1 .. 3 ) %}{{ i }} {% endfor %}")
+    assert_template_result("1 2 3", "{% for i in (1 .. 3) %}{{ i }} {% endfor %}")
+  end
+
+  def test_range_with_expressions
+    assert_template_result("3 4 5", "{% assign x = 1 %}{% assign start = x | plus: 2 %}{% for i in (start..5) %}{{ i }} {% endfor %}")
+    assert_template_result("1 2 3", "{% assign x = 2 %}{% assign end = x | plus: 1 %}{% for i in (1..end) %}{{ i }} {% endfor %}")
+  end
+
+  def test_range_with_literals_in_iteration
+    assert_template_result("1 2 3 4 5", "{% for i in (1..5) %}{{ i }} {% endfor %}")
+  end
+
+  def test_range_size_and_first_last
+    assert_template_result("5", "{{ (1..5) | size }}")
+    assert_template_result("1", "{{ (1..5) | first }}")
+    assert_template_result("5", "{{ (1..5) | last }}")
+  end
+
+  def test_empty_ranges
+    assert_template_result("", "{% for i in (5..1) %}{{ i }}{% endfor %}")
+  end
+
+  def test_ranges_in_conditionals
+    assert_template_result("yes", "{% if 3 >= (1..5) %}no{% else %}yes{% endif %}")
+    assert_template_result("yes", "{% if (1..5) contains 3 %}yes{% else %}no{% endif %}")
+    assert_template_result("no", "{% if (1..5) contains 6 %}yes{% else %}no{% endif %}")
+  end
+
+  def test_range_with_negative_numbers
+    assert_template_result("-3 -2 -1 0", "{% for i in (-3..0) %}{{ i }} {% endfor %}")
+  end
+
+  def test_range_with_floats
+    # Liquid doesn't support float ranges, should either error or not iterate
+    template = "{% for i in (1.5..3.5) %}{{ i }} {% endfor %}"
+    # Floats are rounded down to the nearest integer
+    assert_template_result("1 2 3", template)
+  end
+
+  # def test_ranges_with_calculated_endpoints
+  #   assert_template_result(
+  #     "3 4 5",
+  #     "{% assign start = 1 %}{% assign end = 7 %}{% for i in (start | plus: 2 .. end | minus: 2) %}{{ i }} {% endfor %}",
+  #   )
+  # end
+
+  def test_malformed_ranges
+    # Missing start value
+    assert_raises(Liquid::SyntaxError) { Liquid::Template.parse("{% for i in (..5) %}{{ i }}{% endfor %}") }
+    # Missing end value
+    assert_raises(Liquid::SyntaxError) { Liquid::Template.parse("{% for i in (1..) %}{{ i }}{% endfor %}") }
+    # Missing both values
+    assert_raises(Liquid::SyntaxError) { Liquid::Template.parse("{% for i in (..) %}{{ i }}{% endfor %}") }
+    # Wrong syntax (no parentheses)
+    assert_raises(Liquid::SyntaxError) { Liquid::Template.parse("{% for i in 1..5 %}{{ i }}{% endfor %}") }
+    # Unbalanced parentheses
+    assert_raises(Liquid::SyntaxError) { Liquid::Template.parse("{% for i in (1..5 %}{{ i }}{% endfor %}") }
+    # Invalid characters in range
+    assert_raises(Liquid::SyntaxError) { Liquid::Template.parse("{% for i in (#..@) %}{{ i }}{% endfor %}") }
+    # Invalid range
+    assert_raises(Liquid::SyntaxError) { Liquid::Template.parse("{% assign start = 1 %}{% assign end = 7 %}{% for i in (start | plus: 2 .. end | minus: 2) %}{{ i }} {% endfor %}") }
+  end
+
+  def test_ranges_with_strings_and_variables
+    assert_template_result(
+      "3 4 5",
+      "{% assign range = (3..5) %}{% for i in range %}{{ i }} {% endfor %}",
+    )
+    assert_template_result(
+      "4 5 6",
+      "{% assign start = 4 %}{% assign range = (start..6) %}{% for i in range %}{{ i }} {% endfor %}",
+    )
+  end
+
+  def test_ranges_with_limit_and_offset
+    assert_template_result(
+      "2 3",
+      "{% for i in (1..5) limit:2 offset:1 %}{{ i }} {% endfor %}",
+    )
+    assert_template_result(
+      "3 4 5",
+      "{% for i in (1..5) offset:2 %}{{ i }} {% endfor %}",
+    )
+    assert_template_result(
+      "1 2",
+      "{% for i in (1..5) limit:2 %}{{ i }} {% endfor %}",
+    )
+  end
+
+  def test_reversed_ranges
+    assert_template_result(
+      "5 4 3 2 1",
+      "{% for i in (1..5) reversed %}{{ i }} {% endfor %}",
+    )
+  end
+
+  def test_variable_ranges_with_reversed
+    assert_template_result(
+      "4 3 2 1",
+      "{% assign num = 4 %}{% for i in (1..num) reversed %}{{ i }} {% endfor %}",
+    )
+  end
+
+  def test_assigned_ranges_with_reversed
+    assert_template_result(
+      "5 4 3 2 1",
+      "{% assign range = (1..5) %}{% for i in range reversed %}{{ i }} {% endfor %}",
+    )
+  end
+
+  private
+
+  def assert_template_result(expected, template, assigns = {})
+    assert_equal(expected, Liquid::Template.parse(template).render!(assigns).strip)
+  end
+end

test/unit/tag_unit_test.rb

@@ -6,18 +6,18 @@ class TagUnitTest < Minitest::Test
   include Liquid
 
   def test_tag
-    tag = Tag.parse('tag', "", Tokenizer.new(""), ParseContext.new)
+    tag = Tag.parse('tag', "", new_tokenizer, ParseContext.new)
     assert_equal('liquid::tag', tag.name)
     assert_equal('', tag.render(Context.new))
   end
 
   def test_return_raw_text_of_tag
-    tag = Tag.parse("long_tag", "param1, param2, param3", Tokenizer.new(""), ParseContext.new)
+    tag = Tag.parse("long_tag", "param1, param2, param3", new_tokenizer, ParseContext.new)
     assert_equal("long_tag param1, param2, param3", tag.raw)
   end
 
   def test_tag_name_should_return_name_of_the_tag
-    tag = Tag.parse("some_tag", "", Tokenizer.new(""), ParseContext.new)
+    tag = Tag.parse("some_tag", "", new_tokenizer, ParseContext.new)
     assert_equal('some_tag', tag.tag_name)
   end
 
@@ -26,7 +26,16 @@ class TagUnitTest < Minitest::Test
   end
 
   def test_tag_render_to_output_buffer_nil_value
-    custom_tag = CustomTag.parse("some_tag", "", Tokenizer.new(""), ParseContext.new)
+    custom_tag = CustomTag.parse("some_tag", "", new_tokenizer, ParseContext.new)
     assert_equal('some string', custom_tag.render_to_output_buffer(Context.new, "some string"))
   end
+
+  private
+
+  def new_tokenizer
+    Tokenizer.new(
+      source: "",
+      string_scanner: StringScanner.new(""),
+    )
+  end
 end

test/unit/tags/doc_tag_unit_test.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class DocTagUnitTest < Minitest::Test
+  def test_doc_tag
+    template = <<~LIQUID.chomp
+      {% doc %}
+        Renders loading-spinner.
+
+        @param {string} foo - some foo
+        @param {string} [bar] - optional bar
+
+        @example
+        {% render 'loading-spinner', foo: 'foo' %}
+        {% render 'loading-spinner', foo: 'foo', bar: 'bar' %}
+      {% enddoc %}
+    LIQUID
+
+    assert_template_result('', template)
+  end
+
+  def test_doc_tag_does_not_support_extra_arguments
+    error = assert_raises(Liquid::SyntaxError) do
+      template = <<~LIQUID.chomp
+        {% doc extra %}
+        {% enddoc %}
+      LIQUID
+
+      Liquid::Template.parse(template)
+    end
+
+    exp_error = "Liquid syntax error: Syntax Error in 'doc' - Valid syntax: {% doc %}{% enddoc %}"
+    act_error = error.message
+
+    assert_equal(exp_error, act_error)
+  end
+
+  def test_doc_tag_must_support_valid_tags
+    assert_match_syntax_error("Liquid syntax error (line 1): 'doc' tag was never closed", '{% doc %} foo')
+    assert_match_syntax_error("Liquid syntax error (line 1): Syntax Error in 'doc' - Valid syntax: {% doc %}{% enddoc %}", '{% doc } foo {% enddoc %}')
+    assert_match_syntax_error("Liquid syntax error (line 1): Syntax Error in 'doc' - Valid syntax: {% doc %}{% enddoc %}", '{% doc } foo %}{% enddoc %}')
+  end
+
+  def test_doc_tag_ignores_liquid_nodes
+    template = <<~LIQUID.chomp
+      {% doc %}
+        {% if true %}
+        {% if ... %}
+        {%- for ? -%}
+        {% while true %}
+        {%
+          unless if
+        %}
+        {% endcase %}
+      {% enddoc %}
+    LIQUID
+
+    assert_template_result('', template)
+  end
+
+  def test_doc_tag_ignores_unclosed_liquid_tags
+    template = <<~LIQUID.chomp
+      {% doc %}
+        {% if true %}
+      {% enddoc %}
+    LIQUID
+
+    assert_template_result('', template)
+  end
+
+  def test_doc_tag_does_not_allow_nested_docs
+    error = assert_raises(Liquid::SyntaxError) do
+      template = <<~LIQUID.chomp
+        {% doc %}
+          {% doc %}
+            {% doc %}
+        {% enddoc %}
+      LIQUID
+
+      Liquid::Template.parse(template)
+    end
+
+    exp_error = "Liquid syntax error: Syntax Error in 'doc' - Nested doc tags are not allowed"
+    act_error = error.message
+
+    assert_equal(exp_error, act_error)
+  end
+
+  def test_doc_tag_ignores_nested_raw_tags
+    template = <<~LIQUID.chomp
+      {% doc %}
+        {% raw %}
+      {% enddoc %}
+    LIQUID
+
+    assert_template_result('', template)
+  end
+
+  def test_doc_tag_ignores_unclosed_assign
+    template = <<~LIQUID.chomp
+      {% doc %}
+        {% assign foo = "1"
+      {% enddoc %}
+    LIQUID
+
+    assert_template_result('', template)
+  end
+
+  def test_doc_tag_ignores_malformed_syntax
+    template = <<~LIQUID.chomp
+      {% doc %}
+      {% {{ {%- enddoc %}
+    LIQUID
+
+    assert_template_result('', template)
+  end
+
+  def test_doc_tag_preserves_error_line_numbers
+    template = Liquid::Template.parse(<<~LIQUID.chomp, line_numbers: true)
+      {% doc %}
+        {% if true %}
+      {% enddoc %}
+      {{ errors.standard_error }}
+    LIQUID
+
+    expected = <<~TEXT.chomp
+
+      Liquid error (line 4): standard error
+    TEXT
+
+    assert_equal(expected, template.render('errors' => ErrorDrop.new))
+  end
+
+  def test_doc_tag_whitespace_control
+    # Basic whitespace control
+    assert_template_result("Hello!", "      {%- doc -%}123{%- enddoc -%}Hello!")
+    assert_template_result("Hello!", "{%- doc -%}123{%- enddoc -%}     Hello!")
+    assert_template_result("Hello!", "      {%- doc -%}123{%- enddoc -%}     Hello!")
+    assert_template_result("Hello!", <<~LIQUID.chomp)
+      {%- doc %}Whitespace control!{% enddoc -%}
+      Hello!
+    LIQUID
+  end
+
+  def test_doc_tag_delimiter_handling
+    assert_template_result('', <<~LIQUID.chomp)
+      {% if true %}
+        {% doc %}
+          {% docEXTRA %}wut{% enddocEXTRA %}xyz
+        {% enddoc %}
+      {% endif %}
+    LIQUID
+
+    assert_template_result('', "{% doc %}123{% enddoc xyz %}")
+    assert_template_result('', "{% doc %}123{% enddoc\txyz %}")
+    assert_template_result('', "{% doc %}123{% enddoc\nxyz %}")
+    assert_template_result('', "{% doc %}123{% enddoc\n   xyz  enddoc %}")
+  end
+
+  def test_doc_tag_visitor
+    template_source = '{% doc %}{% enddoc %}'
+
+    assert_equal(
+      [Liquid::Doc],
+      visit(template_source),
+    )
+  end
+
+  private
+
+  def traversal(template)
+    ParseTreeVisitor
+      .for(Template.parse(template).root)
+      .add_callback_for(Liquid::Doc) do |tag|
+        tag_class = tag.class
+        tag_class
+      end
+  end
+
+  def visit(template)
+    traversal(template).visit.flatten.compact
+  end
+end

test/unit/template_unit_test.rb

@@ -35,4 +35,15 @@ class TemplateUnitTest < Minitest::Test
   def test_template_inheritance
     assert_equal("foo", TemplateSubclass.parse("foo").render)
   end
+
+  def test_invalid_utf8
+    input = "\xff\x00"
+    error = assert_raises(SyntaxError) do
+      Liquid::Tokenizer.new(source: input, string_scanner: StringScanner.new(input))
+    end
+    assert_equal(
+      'Liquid syntax error: Invalid byte sequence in UTF-8',
+      error.message,
+    )
+  end
 end

test/unit/tokenizer_unit_test.rb

@@ -6,6 +6,7 @@ class TokenizerTest < Minitest::Test
   def test_tokenize_strings
     assert_equal([' '], tokenize(' '))
     assert_equal(['hello world'], tokenize('hello world'))
+    assert_equal(['{}'], tokenize('{}'))
   end
 
   def test_tokenize_variables
@@ -30,6 +31,23 @@ class TokenizerTest < Minitest::Test
     assert_equal([1, 1, 3], tokenize_line_numbers(" {{\n funk \n}} "))
   end
 
+  def test_tokenize_with_nil_source_returns_empty_array
+    assert_equal([], tokenize(nil))
+  end
+
+  def test_incomplete_curly_braces
+    assert_equal(["{{.}", " "], tokenize('{{.} '))
+    assert_equal(["{{}", "%}"], tokenize('{{}%}'))
+    assert_equal(["{{}}", "}"], tokenize('{{}}}'))
+  end
+
+  def test_unmatching_start_and_end
+    assert_equal(["{{%}"], tokenize('{{%}'))
+    assert_equal(["{{%%%}}"], tokenize('{{%%%}}'))
+    assert_equal(["{%", "}}"], tokenize('{%}}'))
+    assert_equal(["{%%}", "}"], tokenize('{%%}}'))
+  end
+
   private
 
   def new_tokenizer(source, parse_context: Liquid::ParseContext.new, start_line_number: nil)