mirror of
https://github.com/Shopify/liquid.git
synced 2026-09-16 09:20:42 -07:00
use StringScanner to improve Expression Parsing and Tokenizer
This commit is contained in:
Michael Go
+105
-1
@@ -1,7 +1,9 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require "lru_redux"
|
||||
|
||||
module Liquid
|
||||
class Expression
|
||||
class Expression1
|
||||
LITERALS = {
|
||||
nil => nil,
|
||||
'nil' => nil,
|
||||
@@ -45,4 +47,106 @@ module Liquid
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
class Expression2
|
||||
LITERALS = {
|
||||
nil => nil,
|
||||
'nil' => nil,
|
||||
'null' => nil,
|
||||
'' => nil,
|
||||
'true' => true,
|
||||
'false' => false,
|
||||
'blank' => '',
|
||||
'empty' => ''
|
||||
}.freeze
|
||||
|
||||
DOT = ".".ord
|
||||
ZERO = "0".ord
|
||||
NINE = "9".ord
|
||||
DASH = "-".ord
|
||||
|
||||
# Use an atomic group (?>...) to avoid pathological backtracing from
|
||||
# malicious input as described in https://github.com/Shopify/liquid/issues/1357
|
||||
RANGES_REGEX = /\A\(\s*(?>(\S+)\s*\.\.)\s*(\S+)\s*\)\z/
|
||||
CACHE = LruRedux::Cache.new(1_000_000) # most themes would have less than 2,000 unique expression
|
||||
|
||||
class << self
|
||||
def string_scanner
|
||||
@ss ||= StringScanner.new("")
|
||||
end
|
||||
|
||||
def parse(markup)
|
||||
return unless markup
|
||||
|
||||
markup = markup.strip # markup can be a frozen string
|
||||
|
||||
return CACHE[markup] if CACHE.key?(markup)
|
||||
|
||||
CACHE[markup] = inner_parse(markup)
|
||||
end
|
||||
|
||||
def inner_parse(markup)
|
||||
if (markup.start_with?('"') && markup.end_with?('"')) ||
|
||||
(markup.start_with?("'") && markup.end_with?("'"))
|
||||
return markup[1..-2]
|
||||
elsif (markup.start_with?("(") && markup.end_with?(")")) && markup =~ RANGES_REGEX
|
||||
return RangeLookup.parse(Regexp.last_match(1), Regexp.last_match(2))
|
||||
end
|
||||
|
||||
return LITERALS[markup] if LITERALS.key?(markup)
|
||||
|
||||
if (num = parse_number(markup))
|
||||
num
|
||||
else
|
||||
VariableLookup.parse(markup)
|
||||
end
|
||||
end
|
||||
|
||||
def parse_number(markup)
|
||||
ss = string_scanner
|
||||
ss.string = markup
|
||||
|
||||
is_integer = true
|
||||
last_dot_pos = nil
|
||||
num_end_pos = nil
|
||||
|
||||
# the first byte must be a digit, a period, or a dash
|
||||
byte = ss.scan_byte
|
||||
|
||||
return false if byte != DASH && byte != DOT && (byte < ZERO || byte > NINE)
|
||||
|
||||
while (byte = ss.scan_byte)
|
||||
return false if byte != DOT && (byte < ZERO || byte > NINE)
|
||||
|
||||
# we found our number and now we are just scanning the rest of the string
|
||||
next if num_end_pos
|
||||
|
||||
if byte == DOT
|
||||
if is_integer == false
|
||||
num_end_pos = ss.pos - 1
|
||||
else
|
||||
is_integer = false
|
||||
last_dot_pos = ss.pos
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
num_end_pos = markup.length if ss.eos?
|
||||
|
||||
return markup.to_i if is_integer
|
||||
|
||||
if num_end_pos
|
||||
# number ends with a number "123.123"
|
||||
markup.byteslice(0, num_end_pos).to_f
|
||||
elsif last_dot_pos
|
||||
markup.byteslice(0, last_dot_pos).to_f
|
||||
else
|
||||
# we should never reach this point
|
||||
false
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Expression = StringScanner.instance_methods.include?(:scan_byte) ? Expression2 : Expression1
|
||||
end
|
||||
|
||||
+11
-3
@@ -92,6 +92,7 @@ module Liquid
|
||||
SINGLE_COMPARISON_TOKENS = [].tap do |table|
|
||||
table["<".ord] = COMPARISON_LESS_THAN
|
||||
table[">".ord] = COMPARISON_GREATER_THAN
|
||||
table.freeze
|
||||
end
|
||||
|
||||
TWO_CHARS_COMPARISON_JUMP_TABLE = [].tap do |table|
|
||||
@@ -103,18 +104,17 @@ module Liquid
|
||||
sub_table["=".ord] = COMPARISION_NOT_EQUAL
|
||||
sub_table.freeze
|
||||
end
|
||||
table.freeze
|
||||
end
|
||||
|
||||
COMPARISON_JUMP_TABLE = [].tap do |table|
|
||||
table["<".ord] = [].tap do |sub_table|
|
||||
sub_table["=".ord] = COMPARISON_LESS_THAN_OR_EQUAL
|
||||
sub_table[">".ord] = COMPARISON_NOT_EQUAL_ALT
|
||||
RUBY_WHITESPACE.each { |c| sub_table[c.ord] = COMPARISON_LESS_THAN }
|
||||
sub_table.freeze
|
||||
end
|
||||
table[">".ord] = [].tap do |sub_table|
|
||||
sub_table["=".ord] = COMPARISON_GREATER_THAN_OR_EQUAL
|
||||
RUBY_WHITESPACE.each { |c| sub_table[c.ord] = COMPARISON_GREATER_THAN }
|
||||
sub_table.freeze
|
||||
end
|
||||
table.freeze
|
||||
@@ -157,8 +157,15 @@ module Liquid
|
||||
table.freeze
|
||||
end
|
||||
|
||||
class << self
|
||||
def string_scanner
|
||||
@string_scanner ||= StringScanner.new("")
|
||||
end
|
||||
end
|
||||
|
||||
def initialize(input)
|
||||
@ss = StringScanner.new(input)
|
||||
@ss = Lexer2.string_scanner
|
||||
@ss.string = input
|
||||
end
|
||||
|
||||
# rubocop:disable Metrics/BlockNesting
|
||||
@@ -233,5 +240,6 @@ module Liquid
|
||||
end
|
||||
end
|
||||
|
||||
# Remove this once we can depend on strscan >= 3.1.1
|
||||
Lexer = StringScanner.instance_methods.include?(:scan_byte) ? Lexer2 : Lexer1
|
||||
end
|
||||
|
||||
+153
-1
@@ -1,7 +1,9 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require "strscan"
|
||||
|
||||
module Liquid
|
||||
class Tokenizer
|
||||
class Tokenizer1
|
||||
attr_reader :line_number, :for_liquid_tag
|
||||
|
||||
def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
|
||||
@@ -42,4 +44,154 @@ module Liquid
|
||||
tokens
|
||||
end
|
||||
end
|
||||
|
||||
class Tokenizer2
|
||||
attr_reader :line_number, :for_liquid_tag
|
||||
|
||||
TAG_END = /%\}/
|
||||
TAG_OR_VARIABLE_START = /\{[\{\%]/
|
||||
NEWLINE = /\n/
|
||||
|
||||
OPEN_CURLEY = "{".ord
|
||||
CLOSE_CURLEY = "}".ord
|
||||
PERCENTAGE = "%".ord
|
||||
|
||||
class << self
|
||||
def string_scanner
|
||||
@string_scanner ||= StringScanner.new("")
|
||||
end
|
||||
end
|
||||
|
||||
def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
|
||||
@line_number = line_number || (line_numbers ? 1 : nil)
|
||||
@for_liquid_tag = for_liquid_tag
|
||||
@source = source
|
||||
@offset = 0
|
||||
@tokens = []
|
||||
tokenize
|
||||
end
|
||||
|
||||
def shift
|
||||
token = @tokens[@offset]
|
||||
|
||||
return unless token
|
||||
|
||||
@offset += 1
|
||||
|
||||
if @line_number
|
||||
@line_number += @for_liquid_tag ? 1 : token.count("\n")
|
||||
end
|
||||
|
||||
token
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def tokenize
|
||||
if @for_liquid_tag
|
||||
@tokens = @source.split("\n")
|
||||
else
|
||||
@ss = Tokenizer2.string_scanner
|
||||
@ss.string = @source
|
||||
@tokens << shift_normal until @ss.eos?
|
||||
end
|
||||
|
||||
@ss = nil
|
||||
@source = nil
|
||||
end
|
||||
|
||||
def shift_normal
|
||||
token = next_token
|
||||
|
||||
return unless token
|
||||
|
||||
token
|
||||
end
|
||||
|
||||
def next_token
|
||||
# possible states: :text, :tag, :variable
|
||||
byte_a = @ss.peek_byte
|
||||
|
||||
if byte_a == OPEN_CURLEY
|
||||
@ss.scan_byte
|
||||
|
||||
byte_b = @ss.peek_byte
|
||||
|
||||
if byte_b == PERCENTAGE
|
||||
@ss.scan_byte
|
||||
return next_tag_token
|
||||
elsif byte_b == OPEN_CURLEY
|
||||
@ss.scan_byte
|
||||
return next_variable_token
|
||||
end
|
||||
|
||||
@ss.pos -= 1
|
||||
end
|
||||
|
||||
next_text_token
|
||||
end
|
||||
|
||||
def next_text_token
|
||||
start = @ss.pos
|
||||
|
||||
unless @ss.skip_until(TAG_OR_VARIABLE_START)
|
||||
token = @ss.rest
|
||||
@ss.terminate
|
||||
return token
|
||||
end
|
||||
|
||||
pos = @ss.pos -= 2
|
||||
@source.byteslice(start, pos - start)
|
||||
end
|
||||
|
||||
def next_variable_token
|
||||
start = @ss.pos - 2
|
||||
|
||||
byte_a = byte_b = @ss.scan_byte
|
||||
|
||||
while byte_b
|
||||
byte_a = @ss.scan_byte while byte_a && (byte_a != CLOSE_CURLEY && byte_a != OPEN_CURLEY)
|
||||
|
||||
break unless byte_a
|
||||
|
||||
if @ss.eos?
|
||||
return byte_a == CLOSE_CURLEY ? @source.byteslice(start, @ss.pos - start) : "{{"
|
||||
end
|
||||
|
||||
byte_b = @ss.scan_byte
|
||||
|
||||
if byte_a == CLOSE_CURLEY
|
||||
if byte_b == CLOSE_CURLEY
|
||||
return @source.byteslice(start, @ss.pos - start)
|
||||
elsif byte_b != CLOSE_CURLEY
|
||||
@ss.pos -= 1
|
||||
return @source.byteslice(start, @ss.pos - start)
|
||||
end
|
||||
elsif byte_a == OPEN_CURLEY && byte_b == PERCENTAGE
|
||||
return next_tag_token_with_start(start)
|
||||
end
|
||||
|
||||
byte_a = byte_b
|
||||
end
|
||||
|
||||
"{{"
|
||||
end
|
||||
|
||||
def next_tag_token
|
||||
start = @ss.pos - 2
|
||||
if (len = @ss.skip_until(TAG_END))
|
||||
@source.byteslice(start, len + 2)
|
||||
else
|
||||
"{%"
|
||||
end
|
||||
end
|
||||
|
||||
def next_tag_token_with_start(start)
|
||||
@ss.skip_until(TAG_END)
|
||||
@source.byteslice(start, @ss.pos - start)
|
||||
end
|
||||
end
|
||||
|
||||
# Remove this once we can depend on strscan >= 3.1.1
|
||||
Tokenizer = StringScanner.instance_methods.include?(:scan_byte) ? Tokenizer2 : Tokenizer1
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user