wip

Gemfile

@@ -26,3 +26,7 @@ group :test do
     gem 'liquid-c', github: 'Shopify/liquid-c', ref: 'main'
   end
 end
+
+gem "vernier", "~> 1.0"
+
+gem "debug", "~> 1.9"

Gemfile.lock

@@ -16,6 +16,13 @@ GEM
   specs:
     ast (2.4.2)
     benchmark-ips (2.13.0)
+    debug (1.9.2)
+      irb (~> 1.10)
+      reline (>= 0.3.8)
+    io-console (0.7.2)
+    irb (1.13.1)
+      rdoc (>= 4.0.0)
+      reline (>= 0.4.2)
     json (2.7.2)
     language_server-protocol (3.17.0.3)
     memory_profiler (1.0.1)
@@ -24,10 +31,16 @@ GEM
     parser (3.3.0.5)
       ast (~> 2.4.1)
       racc
+    psych (5.1.2)
+      stringio
     racc (1.7.3)
     rainbow (3.1.1)
     rake (13.2.1)
+    rdoc (6.7.0)
+      psych (>= 4.0.0)
     regexp_parser (2.9.0)
+    reline (0.5.8)
+      io-console (~> 0.5)
     rexml (3.2.6)
     rubocop (1.61.0)
       json (~> 2.3)
@@ -49,15 +62,18 @@ GEM
       rubocop (~> 1.44)
     ruby-progressbar (1.13.0)
     stackprof (0.2.26)
+    stringio (3.1.0)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
     unicode-display_width (2.5.0)
+    vernier (1.0.1)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   benchmark-ips
+  debug (~> 1.9)
   liquid!
   liquid-c!
   memory_profiler
@@ -68,6 +84,7 @@ DEPENDENCIES
   rubocop-shopify (~> 2.12.0)
   stackprof
   terminal-table
+  vernier (~> 1.0)
 
 BUNDLED WITH
    2.5.7

example/liquid2profile

@@ -0,0 +1,96 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+if %x(which samply).empty?
+  print "samply is not installed. Would you like to install it? [y/n] "
+  answer = $stdin.gets.chomp
+  if answer.downcase == "y"
+    puts "Installing samply..."
+    system("cargo install --git https://github.com/mstange/samply samply")
+  else
+    puts "samply is required for this script to run. Exiting..."
+    exit
+  end
+end
+
+options = {
+  db: File.expand_path("../bench/benchmarks/basic/db.yml", __dir__),
+  n_runs: 50000,
+  samply_run: true,
+  yjit_perfmap: false,
+  samply_output: "profile.json",
+  vernier: false,
+}
+
+optparse = OptionParser.new do |opts|
+  opts.on("--db [DB]", "use the provided db file instead of the default") do |db|
+    options[:db] = db
+  end
+  opts.on("-n [N]", "number of runs") do |n|
+    options[:n_runs] = n.to_i
+  end
+  opts.on("--skip-samply-run", "skip running samply") do
+    options[:samply_run] = false
+  end
+  opts.on("--yjit-perfmap", "enable yjit perfmap") do
+    options[:yjit_perfmap] = true
+  end
+  opts.on("--samply-output [OUTPUT]", "output file for samply") do |output|
+    options[:samply_output] = output
+  end
+  opts.on("--vernier", "enable vernier") do
+    options[:vernier] = true
+  end
+end
+optparse.parse!
+
+if options[:samply_run] && !options[:vernier]
+  samply_output = options[:samply_output]
+  if options[:yjit_perfmap]
+    exec(
+      "samply",
+      "record",
+      "--output",
+      samply_output,
+      "ruby",
+      "--yjit",
+      "--yjit-perf",
+      __FILE__,
+      "--skip-samply-run",
+      *ARGV,
+    )
+  else
+    exec("samply", "record", "--output", samply_output, "bundle", "exec", __FILE__, "--skip-samply-run", *ARGV)
+  end
+end
+
+lib = File.expand_path("../lib", __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+n_runs = options[:n_runs]
+code = ARGF.read
+
+require "liquid"
+# require "liquid/c"
+
+cpu_time_start = Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID)
+
+if options[:vernier]
+  require "vernier"
+  Vernier.trace(out: "profile.json") do
+    n_runs.times do
+      Liquid::Template.parse(code)
+    end
+  end
+else
+  n_runs.times do
+    Liquid::Template.parse(code)
+  end
+end
+
+cpu_time_end = Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID)
+cpu_time = cpu_time_end - cpu_time_start
+puts "CPU time: #{cpu_time}"

lib/liquid.rb

@@ -49,6 +49,7 @@ module Liquid
   self.cache_classes = true
 end
 
+require "debug"
 require "liquid/version"
 require 'liquid/parse_tree_visitor'
 require 'liquid/lexer'
@@ -62,9 +63,9 @@ require 'liquid/errors'
 require 'liquid/interrupts'
 require 'liquid/strainer_template'
 require 'liquid/strainer_factory'
+require 'liquid/parser_switching'
 require 'liquid/expression'
 require 'liquid/context'
-require 'liquid/parser_switching'
 require 'liquid/tag'
 require 'liquid/tag/disabler'
 require 'liquid/tag/disableable'

lib/liquid/block_body.rb

@@ -246,8 +246,10 @@ module Liquid
     end
 
     def create_variable(token, parse_context)
-      if token =~ ContentOfVariable
-        markup = Regexp.last_match(1)
+      if token.end_with?("}}")
+        start_markup = token[2] == WhitespaceControl ? 3 : 2
+        end_markup = token[-3] == WhitespaceControl ? -3 : -2
+        markup = token[start_markup...end_markup]
         return Variable.new(markup, parse_context)
       end
       BlockBody.raise_missing_variable_terminator(token, parse_context)

lib/liquid/expression.rb

@@ -20,7 +20,31 @@ module Liquid
     # malicious input as described in https://github.com/Shopify/liquid/issues/1357
     RANGES_REGEX         = /\A\(\s*(?>(\S+)\s*\.\.)\s*(\S+)\s*\)\z/
 
-    def self.parse(markup)
+    include ParserSwitching
+
+    def self.parse(markup, parse_context)
+      new(markup, parse_context)
+    end
+
+    private_class_method def self.new(markup, parse_context)
+      obj = allocate
+      obj.instance_variable_set(:@markup, markup)
+      obj.instance_variable_set(:@parse_context, parse_context)
+      if !parse_context.nil?
+        obj.strict_parse_with_error_mode_fallback(markup)
+      else
+        obj.lax_parse(markup)
+      end
+    end
+
+    def strict_parse(markup)
+      return nil unless markup
+
+      p = Parser.new(markup)
+      p.expression
+    end
+
+    def lax_parse(markup)
       return nil unless markup
 
       markup = markup.strip
@@ -33,7 +57,7 @@ module Liquid
       when INTEGERS_REGEX
         Regexp.last_match(1).to_i
       when RANGES_REGEX
-        RangeLookup.parse(Regexp.last_match(1), Regexp.last_match(2))
+        RangeLookup.parse(Regexp.last_match(1), Regexp.last_match(2), parse_context)
       when FLOATS_REGEX
         Regexp.last_match(1).to_f
       else

lib/liquid/lexer.rb

@@ -1,58 +1,180 @@
 # frozen_string_literal: true
 
-require "strscan"
 module Liquid
+  class StringScanner
+    def initialize(string)
+      @len = string.length
+      @string = string.freeze
+      @buffer = IO::Buffer.for(@string)
+      @pos    = 0
+    end
+
+    def eos?
+      @pos >= @len
+    end
+
+    def peek(n = 0)
+      return if @pos + n >= @len
+      @buffer.get_value(:U8, @pos + n)
+    end
+
+    def match(str)
+      return if @pos + str.length > @len
+
+      if (@buffer.slice(@pos, str.length) <=> IO::Buffer.for(str)) == 0
+        advance(str.length)
+      end
+    end
+
+    def match_until(char)
+      pos = 1
+      pos += 1 while peek(pos) != char
+      if peek(pos) == char
+        advance(pos + 1)
+      end
+    end
+
+    def advance(n = 1)
+      original_pos = @pos
+      @pos += n
+      @string[original_pos, n]
+    end
+
+    def space?(c)
+      return false unless c
+      c == 32 || c == 9 || c == 10 || c == 13
+    end
+
+    def skip_spaces
+      @pos += 1 while @pos < @len && space?(@buffer.get_value(:U8, @pos))
+    end
+  end
+
   class Lexer
     SPECIALS = {
-      '|' => :pipe,
-      '.' => :dot,
-      ':' => :colon,
-      ',' => :comma,
-      '[' => :open_square,
-      ']' => :close_square,
-      '(' => :open_round,
-      ')' => :close_round,
-      '?' => :question,
-      '-' => :dash,
+      '|'.ord => :pipe,
+      '.'.ord => :dot,
+      ':'.ord => :colon,
+      ','.ord => :comma,
+      '['.ord => :open_square,
+      ']'.ord => :close_square,
+      '('.ord => :open_round,
+      ')'.ord => :close_round,
+      '?'.ord => :question,
+      '-'.ord => :dash,
     }.freeze
-    IDENTIFIER            = /[a-zA-Z_][\w-]*\??/
-    SINGLE_STRING_LITERAL = /'[^\']*'/
-    DOUBLE_STRING_LITERAL = /"[^\"]*"/
-    STRING_LITERAL        = Regexp.union(SINGLE_STRING_LITERAL, DOUBLE_STRING_LITERAL)
-    NUMBER_LITERAL        = /-?\d+(\.\d+)?/
-    DOTDOT                = /\.\./
-    COMPARISON_OPERATOR   = /==|!=|<>|<=?|>=?|contains(?=\s)/
-    WHITESPACE_OR_NOTHING = /\s*/
+
+    LESS_THAN = '<'.ord
+    GREATER_THAN = '>'.ord
+    EQUALS = '='.ord
+    EXCLAMATION = '!'.ord
+    QUOTE = '"'.ord
+    APOSTROPHE = "'".ord
+    DASH = '-'.ord
+    DOT = '.'.ord
+    UNDERSCORE = '_'.ord
+    QUESTION_MARK = '?'.ord
 
     def initialize(input)
       @ss = StringScanner.new(input)
     end
 
+    def digit?(char)
+      return false unless char
+      char >= 48 && char <= 57
+    end
+
+    def alpha?(char)
+      return false unless char
+      char >= 65 && char <= 90 || char >= 97 && char <= 122
+    end
+
+    def identifier?(char)
+      return false unless char
+      digit?(char) || alpha?(char) || char == UNDERSCORE || char == DASH
+    end
+
     def tokenize
       @output = []
 
       until @ss.eos?
-        @ss.skip(WHITESPACE_OR_NOTHING)
+        @ss.skip_spaces
         break if @ss.eos?
-        tok      = if (t = @ss.scan(COMPARISON_OPERATOR))
-          [:comparison, t]
-        elsif (t = @ss.scan(STRING_LITERAL))
-          [:string, t]
-        elsif (t = @ss.scan(NUMBER_LITERAL))
-          [:number, t]
-        elsif (t = @ss.scan(IDENTIFIER))
-          [:id, t]
-        elsif (t = @ss.scan(DOTDOT))
-          [:dotdot, t]
-        else
-          c     = @ss.getch
-          if (s = SPECIALS[c])
-            [s, c]
-          else
-            raise SyntaxError, "Unexpected character #{c}"
+
+        next_char = @ss.peek
+        case next_char
+        when LESS_THAN
+          @output << [:comparison, @ss.match("<=") || @ss.match("<>") || @ss.match("<")]
+          next
+        when GREATER_THAN
+          @output << [:comparison, @ss.match(">=") || @ss.match(">")]
+          next
+        when EQUALS
+          if (match = @ss.match("=="))
+            @output << [:comparison, match]
+            next
+          end
+        when EXCLAMATION
+          if (match = @ss.match("!="))
+            @output << [:comparison, match]
+            next
+          end
+        when DOT
+          if (match = @ss.match(".."))
+            @output << [:dotdot, match]
+            next
           end
         end
-        @output << tok
+
+        if (match = @ss.match("contains"))
+          @output << [:comparison, match]
+          next
+        end
+
+        if next_char == APOSTROPHE || next_char == QUOTE
+          if (str = @ss.match_until(next_char))
+            @output << [:string, str]
+            next
+          end
+        end
+
+        if next_char == DASH || digit?(next_char)
+          peek = 1
+          has_dot = false
+          while (peeked = @ss.peek(peek))
+            if !has_dot && peeked == DOT
+              has_dot = true
+            elsif !digit?(peeked)
+              break
+            end
+            peek += 1
+          end
+          peek -= 1
+
+          if @ss.peek(peek) == DOT
+            peek -= 1
+          end
+
+          if @ss.peek(peek) != DASH
+            @output << [:number, @ss.advance(peek)]
+            next
+          end
+        end
+
+        if alpha?(next_char) || next_char == UNDERSCORE
+          peek = 1
+          peek += 1 while identifier?(@ss.peek(peek))
+          peek += 1 if @ss.peek(peek) == QUESTION_MARK
+          @output << [:id, @ss.advance(peek)]
+          next
+        end
+
+        if (special = SPECIALS[next_char])
+          @output << [special, @ss.advance]
+          next
+        else
+          raise SyntaxError, "Unexpected character #{next_char.chr}"
+        end
       end
 
       @output << [:end_of_string]

lib/liquid/parse_context.rb

@@ -28,7 +28,7 @@ module Liquid
     end
 
     def parse_expression(markup)
-      Expression.parse(markup)
+      Expression.parse(markup, self)
     end
 
     def partial=(value)

lib/liquid/parser.rb

@@ -2,6 +2,9 @@
 
 module Liquid
   class Parser
+    Kwarg = Struct.new(:name, :value)
+    Arg = Struct.new(:value)
+
     def initialize(input)
       l       = Lexer.new(input)
       @tokens = l.tokenize
@@ -50,8 +53,15 @@ module Liquid
       token = @tokens[@p]
       case token[0]
       when :id
-        str = consume
-        str << variable_lookups
+        name = consume
+        lookups = variable_lookups
+        command_flags = 0
+        lookups.each_index do |i|
+          if VariableLookup::COMMAND_METHODS.include?(lookups[i])
+            @command_flags |= 1 << i
+          end
+        end
+        VariableLookup.new_with(name, lookups, command_flags)
       when :open_square
         str = consume
         str << expression
@@ -72,31 +82,32 @@ module Liquid
     end
 
     def argument
-      str = +""
       # might be a keyword argument (identifier: expression)
       if look(:id) && look(:colon, 1)
-        str << consume << consume << ' '
+        name = consume(:id)
+        consume(:colon)
+        value = expression
+        Kwarg.new(name, value)
+      else
+        Arg.new(expression)
       end
-
-      str << expression
-      str
     end
 
     def variable_lookups
-      str = +""
+      lookups = []
       loop do
         if look(:open_square)
-          str << consume
-          str << expression
-          str << consume(:close_square)
+          consume
+          lookups << expression
+          consume(:close_square)
         elsif look(:dot)
-          str << consume
-          str << consume(:id)
+          consume
+          lookups << consume(:id)
         else
           break
         end
       end
-      str
+      lookups
     end
   end
 end

lib/liquid/parser_switching.rb

@@ -33,6 +33,7 @@ module Liquid
     def strict_parse_with_error_context(markup)
       strict_parse(markup)
     rescue SyntaxError => e
+      puts e
       e.line_number    = line_number
       e.markup_context = markup_context(markup)
       raise e

lib/liquid/range_lookup.rb

@@ -2,9 +2,9 @@
 
 module Liquid
   class RangeLookup
-    def self.parse(start_markup, end_markup)
-      start_obj = Expression.parse(start_markup)
-      end_obj   = Expression.parse(end_markup)
+    def self.parse(start_markup, end_markup, parse_context = nil)
+      start_obj = Expression.parse(start_markup, parse_context)
+      end_obj   = Expression.parse(end_markup, parse_context)
       if start_obj.respond_to?(:evaluate) || end_obj.respond_to?(:evaluate)
         new(start_obj, end_obj)
       else

lib/liquid/tags/for.rb

@@ -92,8 +92,7 @@ module Liquid
       @variable_name = p.consume(:id)
       raise SyntaxError, options[:locale].t("errors.syntax.for_invalid_in") unless p.id?('in')
 
-      collection_name  = p.expression
-      @collection_name = parse_expression(collection_name)
+      @collection_name = p.expression
 
       @name     = "#{@variable_name}-#{collection_name}"
       @reversed = p.id?('reversed')
@@ -104,7 +103,7 @@ module Liquid
           raise SyntaxError, options[:locale].t("errors.syntax.for_invalid_attribute")
         end
         p.consume(:colon)
-        set_attribute(attribute, p.expression)
+        set_attribute(attribute, p.expression, do_parse: false)
       end
       p.consume(:end_of_string)
     end
@@ -174,16 +173,18 @@ module Liquid
       output
     end
 
-    def set_attribute(key, expr)
+    def set_attribute(key, expr, do_parse: true)
       case key
       when 'offset'
         @from = if expr == 'continue'
           :continue
-        else
+        elsif do_parse
           parse_expression(expr)
+        else
+          expr
         end
       when 'limit'
-        @limit = parse_expression(expr)
+        @limit = do_parse ? parse_expression(expr) : expr
       end
     end
 

lib/liquid/tags/if.rb

@@ -120,9 +120,9 @@ module Liquid
     end
 
     def parse_comparison(p)
-      a = parse_expression(p.expression)
+      a = p.expression
       if (op = p.consume?(:comparison))
-        b = parse_expression(p.expression)
+        b = p.expression
         Condition.new(a, op, b)
       else
         Condition.new(a)

lib/liquid/variable.rb

@@ -65,11 +65,11 @@ module Liquid
 
       return if p.look(:end_of_string)
 
-      @name = parse_context.parse_expression(p.expression)
+      @name = p.expression
       while p.consume?(:pipe)
         filtername = p.consume(:id)
         filterargs = p.consume?(:colon) ? parse_filterargs(p) : []
-        @filters << parse_filter_expressions(filtername, filterargs)
+        @filters << parse_strict_filter_expressions(filtername, filterargs)
       end
       p.consume(:end_of_string)
     end
@@ -132,6 +132,22 @@ module Liquid
       result
     end
 
+    def parse_strict_filter_expressions(filter_name, args)
+      filter_args  = []
+      keyword_args = nil
+      args.each do |a|
+        if a.is_a?(Liquid::Parser::Kwarg)
+          keyword_args ||= {}
+          keyword_args[a.name] = a.value
+        else
+          filter_args << a.value
+        end
+      end
+      result = [filter_name, filter_args]
+      result << keyword_args if keyword_args
+      result
+    end
+
     def evaluate_filter_expressions(context, filter_args, filter_kwargs)
       parsed_args = filter_args.map { |expr| context.evaluate(expr) }
       if filter_kwargs

lib/liquid/variable_lookup.rb

@@ -6,16 +6,24 @@ module Liquid
 
     attr_reader :name, :lookups
 
-    def self.parse(markup)
-      new(markup)
+    def self.parse(markup, parse_context = nil)
+      new(markup, parse_context)
     end
 
-    def initialize(markup)
+    def self.new_with(name, lookups, command_flags)
+      instance = allocate
+      instance.instance_variable_set(:@name, name)
+      instance.instance_variable_set(:@lookups, lookups)
+      instance.instance_variable_set(:@command_flags, command_flags)
+      instance
+    end
+
+    def initialize(markup, parse_context = nil)
       lookups = markup.scan(VariableParser)
 
       name = lookups.shift
       if name&.start_with?('[') && name&.end_with?(']')
-        name = Expression.parse(name[1..-2])
+        name = Expression.parse(name[1..-2], parse_context)
       end
       @name = name
 
@@ -25,7 +33,7 @@ module Liquid
       @lookups.each_index do |i|
         lookup = lookups[i]
         if lookup&.start_with?('[') && lookup&.end_with?(']')
-          lookups[i] = Expression.parse(lookup[1..-2])
+          lookups[i] = Expression.parse(lookup[1..-2], parse_context)
         elsif COMMAND_METHODS.include?(lookup)
           @command_flags |= 1 << i
         end

product.liquid

@@ -0,0 +1,34 @@
+<div id="product-left">
+  {% for image in product.images %}<div id="product-image">
+    <a href="{{ image | product_img_url: 'large' }}" rel="lightbox[images]" title="{{ product.title | escape }}"><img src="{{ image | product_img_url: 'medium' }}" alt="{{ product.title | escape }}" /></a>
+  </div>
+  {% endfor %}
+</div>
+<div id="product-right">
+  <h1>{{ product.title }}</h1>
+  {{ product.description }}
+
+  {% if product.available %}
+  <form action="/cart/add" method="post">
+
+    <div id="product-variants">
+      <div id="price-field"></div>
+
+      <select id="product-select" name='id'>
+        {% for variant in product.variants %}
+          <option value="{{ variant.id }}">{{ variant.title }} - {{ variant.price | money }}</option>
+        {% endfor %}
+      </select>
+    </div>
+
+    <input type="image" src="purchase.png" name="add" value="Purchase" id="purchase" />
+  </form>
+  {% else %}
+    <p class="bold-red">This product is temporarily unavailable</p>
+  {% endif %}
+
+  <div id="product-details">
+    <strong>Continue Shopping</strong><br />
+    Browse more {{ product.type }} or additional {{ product.vendor }} products.
+  </div>
+</div>