wip

example/liquid2profile

@@ -74,6 +74,7 @@ n_runs = options[:n_runs]
 code = ARGF.read
 
 require "liquid"
+# require "liquid/c"
 
 cpu_time_start = Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID)
 

lib/liquid.rb

@@ -63,9 +63,9 @@ require 'liquid/errors'
 require 'liquid/interrupts'
 require 'liquid/strainer_template'
 require 'liquid/strainer_factory'
+require 'liquid/parser_switching'
 require 'liquid/expression'
 require 'liquid/context'
-require 'liquid/parser_switching'
 require 'liquid/tag'
 require 'liquid/tag/disabler'
 require 'liquid/tag/disableable'

lib/liquid/block_body.rb

@@ -246,8 +246,10 @@ module Liquid
     end
 
     def create_variable(token, parse_context)
-      if token =~ ContentOfVariable
-        markup = Regexp.last_match(1)
+      if token.end_with?("}}")
+        start_markup = token[2] == WhitespaceControl ? 3 : 2
+        end_markup = token[-3] == WhitespaceControl ? -3 : -2
+        markup = token[start_markup...end_markup]
         return Variable.new(markup, parse_context)
       end
       BlockBody.raise_missing_variable_terminator(token, parse_context)

lib/liquid/expression.rb

@@ -20,7 +20,31 @@ module Liquid
     # malicious input as described in https://github.com/Shopify/liquid/issues/1357
     RANGES_REGEX         = /\A\(\s*(?>(\S+)\s*\.\.)\s*(\S+)\s*\)\z/
 
-    def self.parse(markup)
+    include ParserSwitching
+
+    def self.parse(markup, parse_context)
+      new(markup, parse_context)
+    end
+
+    private_class_method def self.new(markup, parse_context)
+      obj = allocate
+      obj.instance_variable_set(:@markup, markup)
+      obj.instance_variable_set(:@parse_context, parse_context)
+      if !parse_context.nil?
+        obj.strict_parse_with_error_mode_fallback(markup)
+      else
+        obj.lax_parse(markup)
+      end
+    end
+
+    def strict_parse(markup)
+      return nil unless markup
+
+      p = Parser.new(markup)
+      p.expression
+    end
+
+    def lax_parse(markup)
       return nil unless markup
 
       markup = markup.strip
@@ -33,7 +57,7 @@ module Liquid
       when INTEGERS_REGEX
         Regexp.last_match(1).to_i
       when RANGES_REGEX
-        RangeLookup.parse(Regexp.last_match(1), Regexp.last_match(2))
+        RangeLookup.parse(Regexp.last_match(1), Regexp.last_match(2), parse_context)
       when FLOATS_REGEX
         Regexp.last_match(1).to_f
       else

lib/liquid/lexer.rb

@@ -1,58 +1,180 @@
 # frozen_string_literal: true
 
-require "strscan"
 module Liquid
+  class StringScanner
+    def initialize(string)
+      @len = string.length
+      @string = string.freeze
+      @buffer = IO::Buffer.for(@string)
+      @pos    = 0
+    end
+
+    def eos?
+      @pos >= @len
+    end
+
+    def peek(n = 0)
+      return if @pos + n >= @len
+      @buffer.get_value(:U8, @pos + n)
+    end
+
+    def match(str)
+      return if @pos + str.length > @len
+
+      if (@buffer.slice(@pos, str.length) <=> IO::Buffer.for(str)) == 0
+        advance(str.length)
+      end
+    end
+
+    def match_until(char)
+      pos = 1
+      pos += 1 while peek(pos) != char
+      if peek(pos) == char
+        advance(pos + 1)
+      end
+    end
+
+    def advance(n = 1)
+      original_pos = @pos
+      @pos += n
+      @string[original_pos, n]
+    end
+
+    def space?(c)
+      return false unless c
+      c == 32 || c == 9 || c == 10 || c == 13
+    end
+
+    def skip_spaces
+      @pos += 1 while @pos < @len && space?(@buffer.get_value(:U8, @pos))
+    end
+  end
+
   class Lexer
     SPECIALS = {
-      '|' => :pipe,
-      '.' => :dot,
-      ':' => :colon,
-      ',' => :comma,
-      '[' => :open_square,
-      ']' => :close_square,
-      '(' => :open_round,
-      ')' => :close_round,
-      '?' => :question,
-      '-' => :dash,
+      '|'.ord => :pipe,
+      '.'.ord => :dot,
+      ':'.ord => :colon,
+      ','.ord => :comma,
+      '['.ord => :open_square,
+      ']'.ord => :close_square,
+      '('.ord => :open_round,
+      ')'.ord => :close_round,
+      '?'.ord => :question,
+      '-'.ord => :dash,
     }.freeze
-    IDENTIFIER            = /[a-zA-Z_][\w-]*\??/
-    SINGLE_STRING_LITERAL = /'[^\']*'/
-    DOUBLE_STRING_LITERAL = /"[^\"]*"/
-    STRING_LITERAL        = Regexp.union(SINGLE_STRING_LITERAL, DOUBLE_STRING_LITERAL)
-    NUMBER_LITERAL        = /-?\d+(\.\d+)?/
-    DOTDOT                = /\.\./
-    COMPARISON_OPERATOR   = /==|!=|<>|<=?|>=?|contains(?=\s)/
-    WHITESPACE_OR_NOTHING = /\s*/
+
+    LESS_THAN = '<'.ord
+    GREATER_THAN = '>'.ord
+    EQUALS = '='.ord
+    EXCLAMATION = '!'.ord
+    QUOTE = '"'.ord
+    APOSTROPHE = "'".ord
+    DASH = '-'.ord
+    DOT = '.'.ord
+    UNDERSCORE = '_'.ord
+    QUESTION_MARK = '?'.ord
 
     def initialize(input)
       @ss = StringScanner.new(input)
     end
 
+    def digit?(char)
+      return false unless char
+      char >= 48 && char <= 57
+    end
+
+    def alpha?(char)
+      return false unless char
+      char >= 65 && char <= 90 || char >= 97 && char <= 122
+    end
+
+    def identifier?(char)
+      return false unless char
+      digit?(char) || alpha?(char) || char == UNDERSCORE || char == DASH
+    end
+
     def tokenize
       @output = []
 
       until @ss.eos?
-        @ss.skip(WHITESPACE_OR_NOTHING)
+        @ss.skip_spaces
         break if @ss.eos?
-        tok      = if (t = @ss.scan(COMPARISON_OPERATOR))
-          [:comparison, t]
-        elsif (t = @ss.scan(STRING_LITERAL))
-          [:string, t]
-        elsif (t = @ss.scan(NUMBER_LITERAL))
-          [:number, t]
-        elsif (t = @ss.scan(IDENTIFIER))
-          [:id, t]
-        elsif (t = @ss.scan(DOTDOT))
-          [:dotdot, t]
-        else
-          c     = @ss.getch
-          if (s = SPECIALS[c])
-            [s, c]
-          else
-            raise SyntaxError, "Unexpected character #{c}"
+
+        next_char = @ss.peek
+        case next_char
+        when LESS_THAN
+          @output << [:comparison, @ss.match("<=") || @ss.match("<>") || @ss.match("<")]
+          next
+        when GREATER_THAN
+          @output << [:comparison, @ss.match(">=") || @ss.match(">")]
+          next
+        when EQUALS
+          if (match = @ss.match("=="))
+            @output << [:comparison, match]
+            next
+          end
+        when EXCLAMATION
+          if (match = @ss.match("!="))
+            @output << [:comparison, match]
+            next
+          end
+        when DOT
+          if (match = @ss.match(".."))
+            @output << [:dotdot, match]
+            next
           end
         end
-        @output << tok
+
+        if (match = @ss.match("contains"))
+          @output << [:comparison, match]
+          next
+        end
+
+        if next_char == APOSTROPHE || next_char == QUOTE
+          if (str = @ss.match_until(next_char))
+            @output << [:string, str]
+            next
+          end
+        end
+
+        if next_char == DASH || digit?(next_char)
+          peek = 1
+          has_dot = false
+          while (peeked = @ss.peek(peek))
+            if !has_dot && peeked == DOT
+              has_dot = true
+            elsif !digit?(peeked)
+              break
+            end
+            peek += 1
+          end
+          peek -= 1
+
+          if @ss.peek(peek) == DOT
+            peek -= 1
+          end
+
+          if @ss.peek(peek) != DASH
+            @output << [:number, @ss.advance(peek)]
+            next
+          end
+        end
+
+        if alpha?(next_char) || next_char == UNDERSCORE
+          peek = 1
+          peek += 1 while identifier?(@ss.peek(peek))
+          peek += 1 if @ss.peek(peek) == QUESTION_MARK
+          @output << [:id, @ss.advance(peek)]
+          next
+        end
+
+        if (special = SPECIALS[next_char])
+          @output << [special, @ss.advance]
+          next
+        else
+          raise SyntaxError, "Unexpected character #{next_char.chr}"
+        end
       end
 
       @output << [:end_of_string]

lib/liquid/parse_context.rb

@@ -28,7 +28,7 @@ module Liquid
     end
 
     def parse_expression(markup)
-      Expression.parse(markup)
+      Expression.parse(markup, self)
     end
 
     def partial=(value)

lib/liquid/parser.rb

@@ -2,6 +2,9 @@
 
 module Liquid
   class Parser
+    Kwarg = Struct.new(:name, :value)
+    Arg = Struct.new(:value)
+
     def initialize(input)
       l       = Lexer.new(input)
       @tokens = l.tokenize
@@ -50,8 +53,15 @@ module Liquid
       token = @tokens[@p]
       case token[0]
       when :id
-        str = consume
-        str << variable_lookups
+        name = consume
+        lookups = variable_lookups
+        command_flags = 0
+        lookups.each_index do |i|
+          if VariableLookup::COMMAND_METHODS.include?(lookups[i])
+            @command_flags |= 1 << i
+          end
+        end
+        VariableLookup.new_with(name, lookups, command_flags)
       when :open_square
         str = consume
         str << expression
@@ -72,31 +82,32 @@ module Liquid
     end
 
     def argument
-      str = +""
       # might be a keyword argument (identifier: expression)
       if look(:id) && look(:colon, 1)
-        str << consume << consume << ' '
+        name = consume(:id)
+        consume(:colon)
+        value = expression
+        Kwarg.new(name, value)
+      else
+        Arg.new(expression)
       end
-
-      str << expression
-      str
     end
 
     def variable_lookups
-      str = +""
+      lookups = []
       loop do
         if look(:open_square)
-          str << consume
-          str << expression
-          str << consume(:close_square)
+          consume
+          lookups << expression
+          consume(:close_square)
         elsif look(:dot)
-          str << consume
-          str << consume(:id)
+          consume
+          lookups << consume(:id)
         else
           break
         end
       end
-      str
+      lookups
     end
   end
 end

lib/liquid/parser_switching.rb

@@ -33,6 +33,7 @@ module Liquid
     def strict_parse_with_error_context(markup)
       strict_parse(markup)
     rescue SyntaxError => e
+      puts e
       e.line_number    = line_number
       e.markup_context = markup_context(markup)
       raise e

lib/liquid/range_lookup.rb

@@ -2,9 +2,9 @@
 
 module Liquid
   class RangeLookup
-    def self.parse(start_markup, end_markup)
-      start_obj = Expression.parse(start_markup)
-      end_obj   = Expression.parse(end_markup)
+    def self.parse(start_markup, end_markup, parse_context = nil)
+      start_obj = Expression.parse(start_markup, parse_context)
+      end_obj   = Expression.parse(end_markup, parse_context)
       if start_obj.respond_to?(:evaluate) || end_obj.respond_to?(:evaluate)
         new(start_obj, end_obj)
       else

lib/liquid/tags/for.rb

@@ -92,8 +92,7 @@ module Liquid
       @variable_name = p.consume(:id)
       raise SyntaxError, options[:locale].t("errors.syntax.for_invalid_in") unless p.id?('in')
 
-      collection_name  = p.expression
-      @collection_name = parse_expression(collection_name)
+      @collection_name = p.expression
 
       @name     = "#{@variable_name}-#{collection_name}"
       @reversed = p.id?('reversed')
@@ -104,7 +103,7 @@ module Liquid
           raise SyntaxError, options[:locale].t("errors.syntax.for_invalid_attribute")
         end
         p.consume(:colon)
-        set_attribute(attribute, p.expression)
+        set_attribute(attribute, p.expression, do_parse: false)
       end
       p.consume(:end_of_string)
     end
@@ -174,16 +173,18 @@ module Liquid
       output
     end
 
-    def set_attribute(key, expr)
+    def set_attribute(key, expr, do_parse: true)
       case key
       when 'offset'
         @from = if expr == 'continue'
           :continue
-        else
+        elsif do_parse
           parse_expression(expr)
+        else
+          expr
         end
       when 'limit'
-        @limit = parse_expression(expr)
+        @limit = do_parse ? parse_expression(expr) : expr
       end
     end
 

lib/liquid/tags/if.rb

@@ -120,9 +120,9 @@ module Liquid
     end
 
     def parse_comparison(p)
-      a = parse_expression(p.expression)
+      a = p.expression
       if (op = p.consume?(:comparison))
-        b = parse_expression(p.expression)
+        b = p.expression
         Condition.new(a, op, b)
       else
         Condition.new(a)

lib/liquid/variable.rb

@@ -65,11 +65,11 @@ module Liquid
 
       return if p.look(:end_of_string)
 
-      @name = parse_context.parse_expression(p.expression)
+      @name = p.expression
       while p.consume?(:pipe)
         filtername = p.consume(:id)
         filterargs = p.consume?(:colon) ? parse_filterargs(p) : []
-        @filters << parse_filter_expressions(filtername, filterargs)
+        @filters << parse_strict_filter_expressions(filtername, filterargs)
       end
       p.consume(:end_of_string)
     end
@@ -132,6 +132,22 @@ module Liquid
       result
     end
 
+    def parse_strict_filter_expressions(filter_name, args)
+      filter_args  = []
+      keyword_args = nil
+      args.each do |a|
+        if a.is_a?(Liquid::Parser::Kwarg)
+          keyword_args ||= {}
+          keyword_args[a.name] = a.value
+        else
+          filter_args << a.value
+        end
+      end
+      result = [filter_name, filter_args]
+      result << keyword_args if keyword_args
+      result
+    end
+
     def evaluate_filter_expressions(context, filter_args, filter_kwargs)
       parsed_args = filter_args.map { |expr| context.evaluate(expr) }
       if filter_kwargs

lib/liquid/variable_lookup.rb

@@ -6,16 +6,24 @@ module Liquid
 
     attr_reader :name, :lookups
 
-    def self.parse(markup)
-      new(markup)
+    def self.parse(markup, parse_context = nil)
+      new(markup, parse_context)
     end
 
-    def initialize(markup)
+    def self.new_with(name, lookups, command_flags)
+      instance = allocate
+      instance.instance_variable_set(:@name, name)
+      instance.instance_variable_set(:@lookups, lookups)
+      instance.instance_variable_set(:@command_flags, command_flags)
+      instance
+    end
+
+    def initialize(markup, parse_context = nil)
       lookups = markup.scan(VariableParser)
 
       name = lookups.shift
       if name&.start_with?('[') && name&.end_with?(']')
-        name = Expression.parse(name[1..-2])
+        name = Expression.parse(name[1..-2], parse_context)
       end
       @name = name
 
@@ -25,7 +33,7 @@ module Liquid
       @lookups.each_index do |i|
         lookup = lookups[i]
         if lookup&.start_with?('[') && lookup&.end_with?(']')
-          lookups[i] = Expression.parse(lookup[1..-2])
+          lookups[i] = Expression.parse(lookup[1..-2], parse_context)
         elsif COMMAND_METHODS.include?(lookup)
           @command_flags |= 1 << i
         end