Labyricorn/liquid
1
0
Fork 0
mirror of https://github.com/Shopify/liquid.git synced 2026-09-12 23:40:45 -07:00
Code Issues Packages Projects Releases Wiki Activity

Adds ByteTables, moves cursor load order, consolidates byte constants

Browse Source 
Adds ByteTables (lib/liquid/byte_tables.rb): four frozen 256-entry boolean
lookup arrays replacing inline byte-range comparisons throughout:
  IDENT_START, IDENT_CONT, DIGIT, WHITESPACE

Moves require 'liquid/cursor' to immediately after byte_tables in liquid.rb.
Cursor has zero Liquid dependencies; loading it early lets every subsequent
file reference Cursor:: constants directly.

Removes all local byte-constant definitions that duplicated Cursor::
  tokenizer.rb   OPEN_CURLEY / CLOSE_CURLEY / PERCENTAGE
  block_body.rb  OPEN_CURLEY_BYTE / PERCENT_BYTE / DASH_BYTE / CLOSE_CURLEY_BYTE
  expression.rb  DOT / DASH / ZERO / NINE / INTEGER_REGEX / FLOAT_REGEX

Replaces inline byte-range comparisons with ByteTables lookups in:
  cursor.rb, variable_lookup.rb, expression.rb, standardfilters.rb

Removes incidental dead code alongside the constant consolidation:
  tokenizer.rb: require 'strscan', unused string_scanner: param,
                @ss = nil, .to_s.to_str, 'tokenize if @source' guard
  block_body.rb: require 'English'
This commit is contained in:
Chris Pak
2026-04-04 22:09:14 -07:00
parent d9c42fd2eb
commit 03e5e29b0b
8 changed files with 120 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/liquid.rb
+2 -1
View File
@@ -52,6 +52,8 @@ end
require "liquid/version"
require "liquid/deprecations"
require "liquid/const"
require 'liquid/byte_tables'
require 'liquid/cursor'
require 'liquid/standardfilters'
require 'liquid/file_system'
require 'liquid/parser_switching'
@@ -83,7 +85,6 @@ require 'liquid/expression'
require 'liquid/template'
require 'liquid/condition'
require 'liquid/utils'
require 'liquid/cursor'
require 'liquid/tokenizer'
require 'liquid/parse_context'
require 'liquid/partial_cache'
lib/liquid/block_body.rb
+6 -11
View File
@@ -1,6 +1,5 @@
# frozen_string_literal: true
require 'English'
module Liquid
class BlockBody
@@ -124,8 +123,6 @@ module Liquid
end
end
OPEN_CURLEY_BYTE = 123 # '{'.ord
PERCENT_BYTE = 37 # '%'.ord
# Fast check if string is whitespace-only (replaces WhitespaceOrNothing regex)
BLANK_STRING_REGEX = /\A\s*\z/
@@ -139,9 +136,9 @@ module Liquid
next if token.empty?
first_byte = token.getbyte(0)
if first_byte == OPEN_CURLEY_BYTE
if first_byte == Cursor::LCURLY
second_byte = token.getbyte(1)
if second_byte == PERCENT_BYTE
if second_byte == Cursor::PCT
whitespace_handler(token, parse_context)
cursor = parse_context.cursor
tag_name = cursor.parse_tag_token(token)
@@ -168,7 +165,7 @@ module Liquid
new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
@blank &&= new_tag.blank?
@nodelist << new_tag
elsif second_byte == OPEN_CURLEY_BYTE
elsif second_byte == Cursor::LCURLY
whitespace_handler(token, parse_context)
@nodelist << create_variable(token, parse_context)
@blank = false
@@ -195,10 +192,9 @@ module Liquid
yield nil, nil
end
DASH_BYTE = 45 # '-'.ord
def whitespace_handler(token, parse_context)
if token.getbyte(2) == DASH_BYTE
if token.getbyte(2) == Cursor::DASH
previous_token = @nodelist.last
if previous_token.is_a?(String)
first_byte = previous_token.getbyte(0)
@@ -208,7 +204,7 @@ module Liquid
end
end
end
parse_context.trim_whitespace = (token.getbyte(token.bytesize - 3) == DASH_BYTE)
parse_context.trim_whitespace = (token.getbyte(token.bytesize - 3) == Cursor::DASH)
end
def blank?
@@ -270,11 +266,10 @@ module Liquid
BlockBody.render_node(context, output, node)
end
CLOSE_CURLEY_BYTE = 125 # '}'.ord
def create_variable(token, parse_context)
len = token.bytesize
if len >= 4 && token.getbyte(len - 1) == CLOSE_CURLEY_BYTE && token.getbyte(len - 2) == CLOSE_CURLEY_BYTE
if len >= 4 && token.getbyte(len - 1) == Cursor::RCURLY && token.getbyte(len - 2) == Cursor::RCURLY
markup = parse_context.cursor.parse_variable_token(token)
return Variable.new(markup, parse_context)
end
lib/liquid/byte_tables.rb
+40
View File
@@ -0,0 +1,40 @@
# frozen_string_literal: true
module Liquid
# Pre-computed 256-entry boolean lookup tables for byte classification.
# Built once at load time; used as TABLE[byte] — a single array index
# instead of 3-5 comparison operators per check.
#
# Performance: neutral to slightly faster vs. chained comparisons.
# Readability: replaces expressions like
# (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || b == 95
# with the intent-revealing
# ByteTables::IDENT_START[b]
module ByteTables
# [a-zA-Z_] — valid first byte of an identifier
IDENT_START = Array.new(256, false).tap { |t|
(97..122).each { |b| t[b] = true } # a-z
(65..90).each { |b| t[b] = true } # A-Z
t[95] = true # _
}.freeze
# [a-zA-Z0-9_-] — valid continuation byte of an identifier
IDENT_CONT = Array.new(256, false).tap { |t|
(97..122).each { |b| t[b] = true } # a-z
(65..90).each { |b| t[b] = true } # A-Z
(48..57).each { |b| t[b] = true } # 0-9
t[95] = true # _
t[45] = true # -
}.freeze
# [0-9] — ASCII digit
DIGIT = Array.new(256, false).tap { |t|
(48..57).each { |b| t[b] = true }
}.freeze
# [ \t\n\r\f] — ASCII whitespace
WHITESPACE = Array.new(256, false).tap { |t|
[32, 9, 10, 13, 12].each { |b| t[b] = true } # space, tab, \n, \r, \f
}.freeze
end
end
lib/liquid/cursor.rb
+2 -2
View File
@@ -238,11 +238,11 @@ module Liquid
b = token.getbyte(pos)
if b == HASH
pos += 1
elsif b && ((b >= 97 && b <= 122) || (b >= 65 && b <= 90) || b == USCORE)
elsif b && ByteTables::IDENT_START[b]
pos += 1
while pos < len
b = token.getbyte(pos)
break unless (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || (b >= 48 && b <= 57) || b == USCORE || b == DASH
break unless ByteTables::IDENT_CONT[b]
pos += 1
end
pos += 1 if pos < len && token.getbyte(pos) == QMARK
lib/liquid/expression.rb
+11 -19
View File
@@ -16,16 +16,9 @@ module Liquid
'-' => VariableLookup.parse("-", nil).freeze,
}.freeze
DOT = ".".ord
ZERO = "0".ord
NINE = "9".ord
DASH = "-".ord
# Use an atomic group (?>...) to avoid pathological backtracing from
# malicious input as described in https://github.com/Shopify/liquid/issues/1357
RANGES_REGEX = /\A\(\s*(?>(\S+)\s*\.\.)\s*(\S+)\s*\)\z/
INTEGER_REGEX = /\A(-?\d+)\z/
FLOAT_REGEX = /\A(-?\d+)\.\d+\z/
class << self
def safe_parse(parser, ss = StringScanner.new(""), cache = nil)
@@ -37,11 +30,10 @@ module Liquid
# Only strip if there's leading/trailing whitespace (avoids allocation)
first_byte = markup.getbyte(0)
if first_byte == 32 || first_byte == 9 || first_byte == 10 || first_byte == 13 # space, tab, \n, \r
if first_byte && ByteTables::WHITESPACE[first_byte]
markup = markup.strip
else
last_byte = markup.getbyte(markup.bytesize - 1)
markup = markup.strip if last_byte == 32 || last_byte == 9 || last_byte == 10 || last_byte == 13
elsif first_byte
markup = markup.strip if ByteTables::WHITESPACE[markup.getbyte(markup.bytesize - 1)]
end
if (markup.start_with?('"') && markup.end_with?('"')) ||
@@ -85,15 +77,15 @@ module Liquid
# Quick reject: first byte must be digit or dash
pos = 0
first = markup.getbyte(pos)
if first == DASH
if first == Cursor::DASH
pos += 1
return false if pos >= len
b = markup.getbyte(pos)
return false if b < ZERO || b > NINE
return false unless ByteTables::DIGIT[b]
pos += 1
elsif first >= ZERO && first <= NINE
elsif ByteTables::DIGIT[first]
pos += 1
else
return false
@@ -102,7 +94,7 @@ module Liquid
# Scan digits
while pos < len
b = markup.getbyte(pos)
break if b < ZERO || b > NINE
break unless ByteTables::DIGIT[b]
pos += 1
end
@@ -113,14 +105,14 @@ module Liquid
end
# Check for dot (float)
if markup.getbyte(pos) == DOT
if markup.getbyte(pos) == Cursor::DOT
dot_pos = pos
pos += 1
# Must have at least one digit after dot
digit_after_dot = pos
while pos < len
b = markup.getbyte(pos)
break if b < ZERO || b > NINE
break unless ByteTables::DIGIT[b]
pos += 1
end
@@ -133,9 +125,9 @@ module Liquid
# Return the float portion up to second dot
while pos < len
b = markup.getbyte(pos)
if b == DOT
if b == Cursor::DOT
return markup.byteslice(0, pos).to_f
elsif b < ZERO || b > NINE
elsif !ByteTables::DIGIT[b]
return false
end
lib/liquid/standardfilters.rb
+3 -3
View File
@@ -286,7 +286,7 @@ module Liquid
# Skip leading whitespace
while pos < len
b = input.getbyte(pos)
break unless b == 32 || b == 9 || b == 10 || b == 13 || b == 12
break unless ByteTables::WHITESPACE[b]
pos += 1
end
@@ -297,7 +297,7 @@ module Liquid
# Skip non-whitespace chars (word body)
while pos < len
b = input.getbyte(pos)
break if b == 32 || b == 9 || b == 10 || b == 13 || b == 12
break if ByteTables::WHITESPACE[b]
pos += 1
end
@@ -317,7 +317,7 @@ module Liquid
# Skip whitespace between words
while pos < len
b = input.getbyte(pos)
break unless b == 32 || b == 9 || b == 10 || b == 13 || b == 12
break unless ByteTables::WHITESPACE[b]
pos += 1
end
end
lib/liquid/tokenizer.rb
+54 -68
View File
@@ -1,29 +1,23 @@
# frozen_string_literal: true
require "strscan"
module Liquid
class Tokenizer
attr_reader :line_number, :for_liquid_tag
OPEN_CURLEY = "{".ord
CLOSE_CURLEY = "}".ord
PERCENTAGE = "%".ord
def initialize(
source:,
string_scanner:,
string_scanner: nil,
line_numbers: false,
line_number: nil,
for_liquid_tag: false
)
@line_number = line_number || (line_numbers ? 1 : nil)
@for_liquid_tag = for_liquid_tag
@source = source.to_s.to_str
@source = source.to_s
@offset = 0
@tokens = []
tokenize if @source
tokenize
end
def shift
@@ -50,11 +44,10 @@ module Liquid
end
@source = nil
@ss = nil
end
# Fast tokenizer using String#index instead of StringScanner regex.
# String#index is ~40% faster for finding { delimiters.
# Fast tokenizer using String#byteindex instead of StringScanner regex.
# String#byteindex is ~40% faster for finding { delimiters.
def tokenize_fast
src = @source
unless src.valid_encoding?
@@ -76,7 +69,7 @@ module Liquid
next_byte = idx + 1 < len ? src.getbyte(idx + 1) : nil
if next_byte == PERCENTAGE # {%
if next_byte == Cursor::PCT # {%
# Emit text before tag
@tokens << src.byteslice(pos, idx - pos) if idx > pos
@@ -86,65 +79,14 @@ module Liquid
@tokens << src.byteslice(idx, close + 2 - idx)
pos = close + 2
else
# Emit malformed token to propagate a missing-terminator error in the parser
@tokens << "{%"
pos = idx + 2
end
elsif next_byte == OPEN_CURLEY # {{
# Emit text before variable
elsif next_byte == Cursor::LCURLY # {{
# Emit text before variable, then scan for the closing }}.
@tokens << src.byteslice(pos, idx - pos) if idx > pos
# Scan variable token — matches original tokenizer's byte-by-byte logic:
# Find } or {, then check next byte for }}/{% nesting
scan_pos = idx + 2
found = false
while scan_pos < len
b = src.getbyte(scan_pos)
if b == CLOSE_CURLEY # }
if scan_pos + 1 >= len
# } at end of string — emit token up to here
@tokens << src.byteslice(idx, scan_pos + 1 - idx)
pos = scan_pos + 1
found = true
break
end
b2 = src.getbyte(scan_pos + 1)
if b2 == CLOSE_CURLEY
# Found }} — close variable
@tokens << src.byteslice(idx, scan_pos + 2 - idx)
pos = scan_pos + 2
found = true
break
else
# } followed by non-} — emit token up to here (matches original: @ss.pos -= 1)
@tokens << src.byteslice(idx, scan_pos + 1 - idx)
pos = scan_pos + 1
found = true
break
end
elsif b == OPEN_CURLEY
if scan_pos + 1 < len && src.getbyte(scan_pos + 1) == PERCENTAGE
# Found {% inside {{ — scan to %} and emit as one token
close = src.byteindex('%}', scan_pos + 2)
if close
@tokens << src.byteslice(idx, close + 2 - idx)
pos = close + 2
else
@tokens << src.byteslice(idx, len - idx)
pos = len
end
found = true
break
end
scan_pos += 1
else
scan_pos += 1
end
end
unless found
@tokens << "{{"
pos = idx + 2
end
pos = scan_variable_token(src, idx, len)
else
# Lone '{' — not the start of a tag or variable.
# Find the next '{{' or '{%' to know where this text token ends.
@@ -162,5 +104,49 @@ module Liquid
end
end
end
# Scans a {{ ... }} variable token starting at `idx` in `src`.
# Emits the token to @tokens and returns the new position after the token.
# Handles }}, single }, and embedded {% ... %} (nested tag inside variable).
private def scan_variable_token(src, idx, len)
# Byte-by-byte scan: find } or {, then inspect the next byte.
scan_pos = idx + 2
while scan_pos < len
b = src.getbyte(scan_pos)
if b == Cursor::RCURLY # }
if scan_pos + 1 >= len
# } at end of string — emit token up to here
@tokens << src.byteslice(idx, scan_pos + 1 - idx)
return scan_pos + 1
end
b2 = src.getbyte(scan_pos + 1)
if b2 == Cursor::RCURLY
# Found }} — close variable
@tokens << src.byteslice(idx, scan_pos + 2 - idx)
return scan_pos + 2
else
# } followed by non-} — emit token up to here (matches original: @ss.pos -= 1)
@tokens << src.byteslice(idx, scan_pos + 1 - idx)
return scan_pos + 1
end
elsif b == Cursor::LCURLY && scan_pos + 1 < len && src.getbyte(scan_pos + 1) == Cursor::PCT
# Found {% inside {{ — scan to %} and emit as one token
close = src.byteindex('%}', scan_pos + 2)
if close
@tokens << src.byteslice(idx, close + 2 - idx)
return close + 2
else
@tokens << src.byteslice(idx, len - idx)
return len
end
else
scan_pos += 1
end
end
# Reached end without finding }} — malformed
@tokens << "{{"
idx + 2
end
end
end
lib/liquid/variable_lookup.rb
+2 -2
View File
@@ -48,12 +48,12 @@ module Liquid
end
elsif byte == 46 # '.'
pos += 1
elsif (byte >= 97 && byte <= 122) || (byte >= 65 && byte <= 90) || (byte >= 48 && byte <= 57) || byte == 95 || byte == 45 # \w or -
elsif ByteTables::IDENT_CONT[byte] # [\w-]
start = pos
pos += 1
while pos < len
b = markup.getbyte(pos)
break unless (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || (b >= 48 && b <= 57) || b == 95 || b == 45
break unless ByteTables::IDENT_CONT[b]
pos += 1
end
# Check trailing '?'