From 03e5e29b0bd329d3988d12fafae819b5f8d6a7c6 Mon Sep 17 00:00:00 2001 From: Chris Pak Date: Sat, 4 Apr 2026 20:12:53 -0700 Subject: [PATCH] Adds ByteTables, moves cursor load order, consolidates byte constants Adds ByteTables (lib/liquid/byte_tables.rb): four frozen 256-entry boolean lookup arrays replacing inline byte-range comparisons throughout: IDENT_START, IDENT_CONT, DIGIT, WHITESPACE Moves require 'liquid/cursor' to immediately after byte_tables in liquid.rb. Cursor has zero Liquid dependencies; loading it early lets every subsequent file reference Cursor:: constants directly. Removes all local byte-constant definitions that duplicated Cursor:: tokenizer.rb OPEN_CURLEY / CLOSE_CURLEY / PERCENTAGE block_body.rb OPEN_CURLEY_BYTE / PERCENT_BYTE / DASH_BYTE / CLOSE_CURLEY_BYTE expression.rb DOT / DASH / ZERO / NINE / INTEGER_REGEX / FLOAT_REGEX Replaces inline byte-range comparisons with ByteTables lookups in: cursor.rb, variable_lookup.rb, expression.rb, standardfilters.rb Removes incidental dead code alongside the constant consolidation: tokenizer.rb: require 'strscan', unused string_scanner: param, @ss = nil, .to_s.to_str, 'tokenize if @source' guard block_body.rb: require 'English' --- lib/liquid.rb | 3 +- lib/liquid/block_body.rb | 17 ++--- lib/liquid/byte_tables.rb | 40 +++++++++++ lib/liquid/cursor.rb | 4 +- lib/liquid/expression.rb | 30 +++------ lib/liquid/standardfilters.rb | 6 +- lib/liquid/tokenizer.rb | 122 +++++++++++++++------------------- lib/liquid/variable_lookup.rb | 4 +- 8 files changed, 120 insertions(+), 106 deletions(-) create mode 100644 lib/liquid/byte_tables.rb diff --git a/lib/liquid.rb b/lib/liquid.rb index 14b02d26..cfdb88d5 100644 --- a/lib/liquid.rb +++ b/lib/liquid.rb @@ -52,6 +52,8 @@ end require "liquid/version" require "liquid/deprecations" require "liquid/const" +require 'liquid/byte_tables' +require 'liquid/cursor' require 'liquid/standardfilters' require 'liquid/file_system' require 'liquid/parser_switching' @@ -83,7 +85,6 @@ require 'liquid/expression' require 'liquid/template' require 'liquid/condition' require 'liquid/utils' -require 'liquid/cursor' require 'liquid/tokenizer' require 'liquid/parse_context' require 'liquid/partial_cache' diff --git a/lib/liquid/block_body.rb b/lib/liquid/block_body.rb index eb14aa2c..dbca1382 100644 --- a/lib/liquid/block_body.rb +++ b/lib/liquid/block_body.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -require 'English' module Liquid class BlockBody @@ -124,8 +123,6 @@ module Liquid end end - OPEN_CURLEY_BYTE = 123 # '{'.ord - PERCENT_BYTE = 37 # '%'.ord # Fast check if string is whitespace-only (replaces WhitespaceOrNothing regex) BLANK_STRING_REGEX = /\A\s*\z/ @@ -139,9 +136,9 @@ module Liquid next if token.empty? first_byte = token.getbyte(0) - if first_byte == OPEN_CURLEY_BYTE + if first_byte == Cursor::LCURLY second_byte = token.getbyte(1) - if second_byte == PERCENT_BYTE + if second_byte == Cursor::PCT whitespace_handler(token, parse_context) cursor = parse_context.cursor tag_name = cursor.parse_tag_token(token) @@ -168,7 +165,7 @@ module Liquid new_tag = tag.parse(tag_name, markup, tokenizer, parse_context) @blank &&= new_tag.blank? @nodelist << new_tag - elsif second_byte == OPEN_CURLEY_BYTE + elsif second_byte == Cursor::LCURLY whitespace_handler(token, parse_context) @nodelist << create_variable(token, parse_context) @blank = false @@ -195,10 +192,9 @@ module Liquid yield nil, nil end - DASH_BYTE = 45 # '-'.ord def whitespace_handler(token, parse_context) - if token.getbyte(2) == DASH_BYTE + if token.getbyte(2) == Cursor::DASH previous_token = @nodelist.last if previous_token.is_a?(String) first_byte = previous_token.getbyte(0) @@ -208,7 +204,7 @@ module Liquid end end end - parse_context.trim_whitespace = (token.getbyte(token.bytesize - 3) == DASH_BYTE) + parse_context.trim_whitespace = (token.getbyte(token.bytesize - 3) == Cursor::DASH) end def blank? @@ -270,11 +266,10 @@ module Liquid BlockBody.render_node(context, output, node) end - CLOSE_CURLEY_BYTE = 125 # '}'.ord def create_variable(token, parse_context) len = token.bytesize - if len >= 4 && token.getbyte(len - 1) == CLOSE_CURLEY_BYTE && token.getbyte(len - 2) == CLOSE_CURLEY_BYTE + if len >= 4 && token.getbyte(len - 1) == Cursor::RCURLY && token.getbyte(len - 2) == Cursor::RCURLY markup = parse_context.cursor.parse_variable_token(token) return Variable.new(markup, parse_context) end diff --git a/lib/liquid/byte_tables.rb b/lib/liquid/byte_tables.rb new file mode 100644 index 00000000..392960a3 --- /dev/null +++ b/lib/liquid/byte_tables.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Liquid + # Pre-computed 256-entry boolean lookup tables for byte classification. + # Built once at load time; used as TABLE[byte] — a single array index + # instead of 3-5 comparison operators per check. + # + # Performance: neutral to slightly faster vs. chained comparisons. + # Readability: replaces expressions like + # (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || b == 95 + # with the intent-revealing + # ByteTables::IDENT_START[b] + module ByteTables + # [a-zA-Z_] — valid first byte of an identifier + IDENT_START = Array.new(256, false).tap { |t| + (97..122).each { |b| t[b] = true } # a-z + (65..90).each { |b| t[b] = true } # A-Z + t[95] = true # _ + }.freeze + + # [a-zA-Z0-9_-] — valid continuation byte of an identifier + IDENT_CONT = Array.new(256, false).tap { |t| + (97..122).each { |b| t[b] = true } # a-z + (65..90).each { |b| t[b] = true } # A-Z + (48..57).each { |b| t[b] = true } # 0-9 + t[95] = true # _ + t[45] = true # - + }.freeze + + # [0-9] — ASCII digit + DIGIT = Array.new(256, false).tap { |t| + (48..57).each { |b| t[b] = true } + }.freeze + + # [ \t\n\r\f] — ASCII whitespace + WHITESPACE = Array.new(256, false).tap { |t| + [32, 9, 10, 13, 12].each { |b| t[b] = true } # space, tab, \n, \r, \f + }.freeze + end +end diff --git a/lib/liquid/cursor.rb b/lib/liquid/cursor.rb index 0d0bd4cf..97603f79 100644 --- a/lib/liquid/cursor.rb +++ b/lib/liquid/cursor.rb @@ -238,11 +238,11 @@ module Liquid b = token.getbyte(pos) if b == HASH pos += 1 - elsif b && ((b >= 97 && b <= 122) || (b >= 65 && b <= 90) || b == USCORE) + elsif b && ByteTables::IDENT_START[b] pos += 1 while pos < len b = token.getbyte(pos) - break unless (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || (b >= 48 && b <= 57) || b == USCORE || b == DASH + break unless ByteTables::IDENT_CONT[b] pos += 1 end pos += 1 if pos < len && token.getbyte(pos) == QMARK diff --git a/lib/liquid/expression.rb b/lib/liquid/expression.rb index c5fca063..7b0ae148 100644 --- a/lib/liquid/expression.rb +++ b/lib/liquid/expression.rb @@ -16,16 +16,9 @@ module Liquid '-' => VariableLookup.parse("-", nil).freeze, }.freeze - DOT = ".".ord - ZERO = "0".ord - NINE = "9".ord - DASH = "-".ord - # Use an atomic group (?>...) to avoid pathological backtracing from # malicious input as described in https://github.com/Shopify/liquid/issues/1357 RANGES_REGEX = /\A\(\s*(?>(\S+)\s*\.\.)\s*(\S+)\s*\)\z/ - INTEGER_REGEX = /\A(-?\d+)\z/ - FLOAT_REGEX = /\A(-?\d+)\.\d+\z/ class << self def safe_parse(parser, ss = StringScanner.new(""), cache = nil) @@ -37,11 +30,10 @@ module Liquid # Only strip if there's leading/trailing whitespace (avoids allocation) first_byte = markup.getbyte(0) - if first_byte == 32 || first_byte == 9 || first_byte == 10 || first_byte == 13 # space, tab, \n, \r + if first_byte && ByteTables::WHITESPACE[first_byte] markup = markup.strip - else - last_byte = markup.getbyte(markup.bytesize - 1) - markup = markup.strip if last_byte == 32 || last_byte == 9 || last_byte == 10 || last_byte == 13 + elsif first_byte + markup = markup.strip if ByteTables::WHITESPACE[markup.getbyte(markup.bytesize - 1)] end if (markup.start_with?('"') && markup.end_with?('"')) || @@ -85,15 +77,15 @@ module Liquid # Quick reject: first byte must be digit or dash pos = 0 first = markup.getbyte(pos) - if first == DASH + if first == Cursor::DASH pos += 1 return false if pos >= len b = markup.getbyte(pos) - return false if b < ZERO || b > NINE + return false unless ByteTables::DIGIT[b] pos += 1 - elsif first >= ZERO && first <= NINE + elsif ByteTables::DIGIT[first] pos += 1 else return false @@ -102,7 +94,7 @@ module Liquid # Scan digits while pos < len b = markup.getbyte(pos) - break if b < ZERO || b > NINE + break unless ByteTables::DIGIT[b] pos += 1 end @@ -113,14 +105,14 @@ module Liquid end # Check for dot (float) - if markup.getbyte(pos) == DOT + if markup.getbyte(pos) == Cursor::DOT dot_pos = pos pos += 1 # Must have at least one digit after dot digit_after_dot = pos while pos < len b = markup.getbyte(pos) - break if b < ZERO || b > NINE + break unless ByteTables::DIGIT[b] pos += 1 end @@ -133,9 +125,9 @@ module Liquid # Return the float portion up to second dot while pos < len b = markup.getbyte(pos) - if b == DOT + if b == Cursor::DOT return markup.byteslice(0, pos).to_f - elsif b < ZERO || b > NINE + elsif !ByteTables::DIGIT[b] return false end diff --git a/lib/liquid/standardfilters.rb b/lib/liquid/standardfilters.rb index 4cbdc8ed..0564f12b 100644 --- a/lib/liquid/standardfilters.rb +++ b/lib/liquid/standardfilters.rb @@ -286,7 +286,7 @@ module Liquid # Skip leading whitespace while pos < len b = input.getbyte(pos) - break unless b == 32 || b == 9 || b == 10 || b == 13 || b == 12 + break unless ByteTables::WHITESPACE[b] pos += 1 end @@ -297,7 +297,7 @@ module Liquid # Skip non-whitespace chars (word body) while pos < len b = input.getbyte(pos) - break if b == 32 || b == 9 || b == 10 || b == 13 || b == 12 + break if ByteTables::WHITESPACE[b] pos += 1 end @@ -317,7 +317,7 @@ module Liquid # Skip whitespace between words while pos < len b = input.getbyte(pos) - break unless b == 32 || b == 9 || b == 10 || b == 13 || b == 12 + break unless ByteTables::WHITESPACE[b] pos += 1 end end diff --git a/lib/liquid/tokenizer.rb b/lib/liquid/tokenizer.rb index 5bbceaa7..ba3e0da0 100644 --- a/lib/liquid/tokenizer.rb +++ b/lib/liquid/tokenizer.rb @@ -1,29 +1,23 @@ # frozen_string_literal: true -require "strscan" - module Liquid class Tokenizer attr_reader :line_number, :for_liquid_tag - OPEN_CURLEY = "{".ord - CLOSE_CURLEY = "}".ord - PERCENTAGE = "%".ord - def initialize( source:, - string_scanner:, + string_scanner: nil, line_numbers: false, line_number: nil, for_liquid_tag: false ) @line_number = line_number || (line_numbers ? 1 : nil) @for_liquid_tag = for_liquid_tag - @source = source.to_s.to_str + @source = source.to_s @offset = 0 @tokens = [] - tokenize if @source + tokenize end def shift @@ -50,11 +44,10 @@ module Liquid end @source = nil - @ss = nil end - # Fast tokenizer using String#index instead of StringScanner regex. - # String#index is ~40% faster for finding { delimiters. + # Fast tokenizer using String#byteindex instead of StringScanner regex. + # String#byteindex is ~40% faster for finding { delimiters. def tokenize_fast src = @source unless src.valid_encoding? @@ -76,7 +69,7 @@ module Liquid next_byte = idx + 1 < len ? src.getbyte(idx + 1) : nil - if next_byte == PERCENTAGE # {% + if next_byte == Cursor::PCT # {% # Emit text before tag @tokens << src.byteslice(pos, idx - pos) if idx > pos @@ -86,65 +79,14 @@ module Liquid @tokens << src.byteslice(idx, close + 2 - idx) pos = close + 2 else + # Emit malformed token to propagate a missing-terminator error in the parser @tokens << "{%" pos = idx + 2 end - elsif next_byte == OPEN_CURLEY # {{ - # Emit text before variable + elsif next_byte == Cursor::LCURLY # {{ + # Emit text before variable, then scan for the closing }}. @tokens << src.byteslice(pos, idx - pos) if idx > pos - - # Scan variable token — matches original tokenizer's byte-by-byte logic: - # Find } or {, then check next byte for }}/{% nesting - scan_pos = idx + 2 - found = false - while scan_pos < len - b = src.getbyte(scan_pos) - if b == CLOSE_CURLEY # } - if scan_pos + 1 >= len - # } at end of string — emit token up to here - @tokens << src.byteslice(idx, scan_pos + 1 - idx) - pos = scan_pos + 1 - found = true - break - end - b2 = src.getbyte(scan_pos + 1) - if b2 == CLOSE_CURLEY - # Found }} — close variable - @tokens << src.byteslice(idx, scan_pos + 2 - idx) - pos = scan_pos + 2 - found = true - break - else - # } followed by non-} — emit token up to here (matches original: @ss.pos -= 1) - @tokens << src.byteslice(idx, scan_pos + 1 - idx) - pos = scan_pos + 1 - found = true - break - end - elsif b == OPEN_CURLEY - if scan_pos + 1 < len && src.getbyte(scan_pos + 1) == PERCENTAGE - # Found {% inside {{ — scan to %} and emit as one token - close = src.byteindex('%}', scan_pos + 2) - if close - @tokens << src.byteslice(idx, close + 2 - idx) - pos = close + 2 - else - @tokens << src.byteslice(idx, len - idx) - pos = len - end - found = true - break - end - scan_pos += 1 - else - scan_pos += 1 - end - end - - unless found - @tokens << "{{" - pos = idx + 2 - end + pos = scan_variable_token(src, idx, len) else # Lone '{' — not the start of a tag or variable. # Find the next '{{' or '{%' to know where this text token ends. @@ -162,5 +104,49 @@ module Liquid end end end + + # Scans a {{ ... }} variable token starting at `idx` in `src`. + # Emits the token to @tokens and returns the new position after the token. + # Handles }}, single }, and embedded {% ... %} (nested tag inside variable). + private def scan_variable_token(src, idx, len) + # Byte-by-byte scan: find } or {, then inspect the next byte. + scan_pos = idx + 2 + while scan_pos < len + b = src.getbyte(scan_pos) + if b == Cursor::RCURLY # } + if scan_pos + 1 >= len + # } at end of string — emit token up to here + @tokens << src.byteslice(idx, scan_pos + 1 - idx) + return scan_pos + 1 + end + b2 = src.getbyte(scan_pos + 1) + if b2 == Cursor::RCURLY + # Found }} — close variable + @tokens << src.byteslice(idx, scan_pos + 2 - idx) + return scan_pos + 2 + else + # } followed by non-} — emit token up to here (matches original: @ss.pos -= 1) + @tokens << src.byteslice(idx, scan_pos + 1 - idx) + return scan_pos + 1 + end + elsif b == Cursor::LCURLY && scan_pos + 1 < len && src.getbyte(scan_pos + 1) == Cursor::PCT + # Found {% inside {{ — scan to %} and emit as one token + close = src.byteindex('%}', scan_pos + 2) + if close + @tokens << src.byteslice(idx, close + 2 - idx) + return close + 2 + else + @tokens << src.byteslice(idx, len - idx) + return len + end + else + scan_pos += 1 + end + end + + # Reached end without finding }} — malformed + @tokens << "{{" + idx + 2 + end end end diff --git a/lib/liquid/variable_lookup.rb b/lib/liquid/variable_lookup.rb index 6fcf6e6c..3f9b6cf3 100644 --- a/lib/liquid/variable_lookup.rb +++ b/lib/liquid/variable_lookup.rb @@ -48,12 +48,12 @@ module Liquid end elsif byte == 46 # '.' pos += 1 - elsif (byte >= 97 && byte <= 122) || (byte >= 65 && byte <= 90) || (byte >= 48 && byte <= 57) || byte == 95 || byte == 45 # \w or - + elsif ByteTables::IDENT_CONT[byte] # [\w-] start = pos pos += 1 while pos < len b = markup.getbyte(pos) - break unless (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || (b >= 48 && b <= 57) || b == 95 || b == 45 + break unless ByteTables::IDENT_CONT[b] pos += 1 end # Check trailing '?'