Adds ByteTables, moves cursor load order, consolidates byte constants

Adds ByteTables (lib/liquid/byte_tables.rb): four frozen 256-entry boolean
lookup arrays replacing inline byte-range comparisons throughout:
  IDENT_START, IDENT_CONT, DIGIT, WHITESPACE

Moves require 'liquid/cursor' to immediately after byte_tables in liquid.rb.
Cursor has zero Liquid dependencies; loading it early lets every subsequent
file reference Cursor:: constants directly.

Removes all local byte-constant definitions that duplicated Cursor::
  tokenizer.rb   OPEN_CURLEY / CLOSE_CURLEY / PERCENTAGE
  block_body.rb  OPEN_CURLEY_BYTE / PERCENT_BYTE / DASH_BYTE / CLOSE_CURLEY_BYTE
  expression.rb  DOT / DASH / ZERO / NINE / INTEGER_REGEX / FLOAT_REGEX

Replaces inline byte-range comparisons with ByteTables lookups in:
  cursor.rb, variable_lookup.rb, expression.rb, standardfilters.rb

Removes incidental dead code alongside the constant consolidation:
  tokenizer.rb: require 'strscan', unused string_scanner: param,
                @ss = nil, .to_s.to_str, 'tokenize if @source' guard
  block_body.rb: require 'English'

lib/liquid.rb

@@ -52,6 +52,8 @@ end
 require "liquid/version"
 require "liquid/deprecations"
 require "liquid/const"
+require 'liquid/byte_tables'
+require 'liquid/cursor'
 require 'liquid/standardfilters'
 require 'liquid/file_system'
 require 'liquid/parser_switching'
@@ -83,7 +85,6 @@ require 'liquid/expression'
 require 'liquid/template'
 require 'liquid/condition'
 require 'liquid/utils'
-require 'liquid/cursor'
 require 'liquid/tokenizer'
 require 'liquid/parse_context'
 require 'liquid/partial_cache'

lib/liquid/block_body.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'English'
 
 module Liquid
   class BlockBody
@@ -124,8 +123,6 @@ module Liquid
       end
     end
 
-    OPEN_CURLEY_BYTE = 123 # '{'.ord
-    PERCENT_BYTE = 37 # '%'.ord
 
     # Fast check if string is whitespace-only (replaces WhitespaceOrNothing regex)
     BLANK_STRING_REGEX = /\A\s*\z/
@@ -139,9 +136,9 @@ module Liquid
         next if token.empty?
 
         first_byte = token.getbyte(0)
-        if first_byte == OPEN_CURLEY_BYTE
+        if first_byte == Cursor::LCURLY
           second_byte = token.getbyte(1)
-          if second_byte == PERCENT_BYTE
+          if second_byte == Cursor::PCT
             whitespace_handler(token, parse_context)
             cursor = parse_context.cursor
             tag_name = cursor.parse_tag_token(token)
@@ -168,7 +165,7 @@ module Liquid
             new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
             @blank &&= new_tag.blank?
             @nodelist << new_tag
-          elsif second_byte == OPEN_CURLEY_BYTE
+          elsif second_byte == Cursor::LCURLY
             whitespace_handler(token, parse_context)
             @nodelist << create_variable(token, parse_context)
             @blank = false
@@ -195,10 +192,9 @@ module Liquid
       yield nil, nil
     end
 
-    DASH_BYTE = 45 # '-'.ord
 
     def whitespace_handler(token, parse_context)
-      if token.getbyte(2) == DASH_BYTE
+      if token.getbyte(2) == Cursor::DASH
         previous_token = @nodelist.last
         if previous_token.is_a?(String)
           first_byte = previous_token.getbyte(0)
@@ -208,7 +204,7 @@ module Liquid
           end
         end
       end
-      parse_context.trim_whitespace = (token.getbyte(token.bytesize - 3) == DASH_BYTE)
+      parse_context.trim_whitespace = (token.getbyte(token.bytesize - 3) == Cursor::DASH)
     end
 
     def blank?
@@ -270,11 +266,10 @@ module Liquid
       BlockBody.render_node(context, output, node)
     end
 
-    CLOSE_CURLEY_BYTE = 125 # '}'.ord
 
     def create_variable(token, parse_context)
       len = token.bytesize
-      if len >= 4 && token.getbyte(len - 1) == CLOSE_CURLEY_BYTE && token.getbyte(len - 2) == CLOSE_CURLEY_BYTE
+      if len >= 4 && token.getbyte(len - 1) == Cursor::RCURLY && token.getbyte(len - 2) == Cursor::RCURLY
         markup = parse_context.cursor.parse_variable_token(token)
         return Variable.new(markup, parse_context)
       end

lib/liquid/byte_tables.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Liquid
+  # Pre-computed 256-entry boolean lookup tables for byte classification.
+  # Built once at load time; used as TABLE[byte] — a single array index
+  # instead of 3-5 comparison operators per check.
+  #
+  # Performance: neutral to slightly faster vs. chained comparisons.
+  # Readability: replaces expressions like
+  #   (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || b == 95
+  # with the intent-revealing
+  #   ByteTables::IDENT_START[b]
+  module ByteTables
+    # [a-zA-Z_] — valid first byte of an identifier
+    IDENT_START = Array.new(256, false).tap { |t|
+      (97..122).each { |b| t[b] = true }  # a-z
+      (65..90).each  { |b| t[b] = true }  # A-Z
+      t[95] = true                          # _
+    }.freeze
+
+    # [a-zA-Z0-9_-] — valid continuation byte of an identifier
+    IDENT_CONT = Array.new(256, false).tap { |t|
+      (97..122).each { |b| t[b] = true }  # a-z
+      (65..90).each  { |b| t[b] = true }  # A-Z
+      (48..57).each  { |b| t[b] = true }  # 0-9
+      t[95] = true                          # _
+      t[45] = true                          # -
+    }.freeze
+
+    # [0-9] — ASCII digit
+    DIGIT = Array.new(256, false).tap { |t|
+      (48..57).each { |b| t[b] = true }
+    }.freeze
+
+    # [ \t\n\r\f] — ASCII whitespace
+    WHITESPACE = Array.new(256, false).tap { |t|
+      [32, 9, 10, 13, 12].each { |b| t[b] = true }  # space, tab, \n, \r, \f
+    }.freeze
+  end
+end

lib/liquid/cursor.rb

@@ -238,11 +238,11 @@ module Liquid
       b = token.getbyte(pos)
       if b == HASH
         pos += 1
-      elsif b && ((b >= 97 && b <= 122) || (b >= 65 && b <= 90) || b == USCORE)
+      elsif b && ByteTables::IDENT_START[b]
         pos += 1
         while pos < len
           b = token.getbyte(pos)
-          break unless (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || (b >= 48 && b <= 57) || b == USCORE || b == DASH
+          break unless ByteTables::IDENT_CONT[b]
           pos += 1
         end
         pos += 1 if pos < len && token.getbyte(pos) == QMARK

lib/liquid/expression.rb

@@ -16,16 +16,9 @@ module Liquid
       '-' => VariableLookup.parse("-", nil).freeze,
     }.freeze
 
-    DOT = ".".ord
-    ZERO = "0".ord
-    NINE = "9".ord
-    DASH = "-".ord
-
     # Use an atomic group (?>...) to avoid pathological backtracing from
     # malicious input as described in https://github.com/Shopify/liquid/issues/1357
     RANGES_REGEX = /\A\(\s*(?>(\S+)\s*\.\.)\s*(\S+)\s*\)\z/
-    INTEGER_REGEX = /\A(-?\d+)\z/
-    FLOAT_REGEX = /\A(-?\d+)\.\d+\z/
 
     class << self
       def safe_parse(parser, ss = StringScanner.new(""), cache = nil)
@@ -37,11 +30,10 @@ module Liquid
 
         # Only strip if there's leading/trailing whitespace (avoids allocation)
         first_byte = markup.getbyte(0)
-        if first_byte == 32 || first_byte == 9 || first_byte == 10 || first_byte == 13 # space, tab, \n, \r
+        if first_byte && ByteTables::WHITESPACE[first_byte]
           markup = markup.strip
-        else
+        elsif first_byte
-          last_byte = markup.getbyte(markup.bytesize - 1)
+          markup = markup.strip if ByteTables::WHITESPACE[markup.getbyte(markup.bytesize - 1)]
-          markup = markup.strip if last_byte == 32 || last_byte == 9 || last_byte == 10 || last_byte == 13
         end
 
         if (markup.start_with?('"') && markup.end_with?('"')) ||
@@ -85,15 +77,15 @@ module Liquid
         # Quick reject: first byte must be digit or dash
         pos = 0
         first = markup.getbyte(pos)
-        if first == DASH
+        if first == Cursor::DASH
           pos += 1
           return false if pos >= len
 
           b = markup.getbyte(pos)
-          return false if b < ZERO || b > NINE
+          return false unless ByteTables::DIGIT[b]
 
           pos += 1
-        elsif first >= ZERO && first <= NINE
+        elsif ByteTables::DIGIT[first]
           pos += 1
         else
           return false
@@ -102,7 +94,7 @@ module Liquid
         # Scan digits
         while pos < len
           b = markup.getbyte(pos)
-          break if b < ZERO || b > NINE
+          break unless ByteTables::DIGIT[b]
 
           pos += 1
         end
@@ -113,14 +105,14 @@ module Liquid
         end
 
         # Check for dot (float)
-        if markup.getbyte(pos) == DOT
+        if markup.getbyte(pos) == Cursor::DOT
           dot_pos = pos
           pos += 1
           # Must have at least one digit after dot
           digit_after_dot = pos
           while pos < len
             b = markup.getbyte(pos)
-            break if b < ZERO || b > NINE
+            break unless ByteTables::DIGIT[b]
 
             pos += 1
           end
@@ -133,9 +125,9 @@ module Liquid
             # Return the float portion up to second dot
             while pos < len
               b = markup.getbyte(pos)
-              if b == DOT
+              if b == Cursor::DOT
                 return markup.byteslice(0, pos).to_f
-              elsif b < ZERO || b > NINE
+              elsif !ByteTables::DIGIT[b]
                 return false
               end
 

lib/liquid/standardfilters.rb

@@ -286,7 +286,7 @@ module Liquid
       # Skip leading whitespace
       while pos < len
         b = input.getbyte(pos)
-        break unless b == 32 || b == 9 || b == 10 || b == 13 || b == 12
+        break unless ByteTables::WHITESPACE[b]
         pos += 1
       end
 
@@ -297,7 +297,7 @@ module Liquid
         # Skip non-whitespace chars (word body)
         while pos < len
           b = input.getbyte(pos)
-          break if b == 32 || b == 9 || b == 10 || b == 13 || b == 12
+          break if ByteTables::WHITESPACE[b]
           pos += 1
         end
 
@@ -317,7 +317,7 @@ module Liquid
         # Skip whitespace between words
         while pos < len
           b = input.getbyte(pos)
-          break unless b == 32 || b == 9 || b == 10 || b == 13 || b == 12
+          break unless ByteTables::WHITESPACE[b]
           pos += 1
         end
       end

lib/liquid/tokenizer.rb

@@ -1,29 +1,23 @@
 # frozen_string_literal: true
 
-require "strscan"
-
 module Liquid
   class Tokenizer
     attr_reader :line_number, :for_liquid_tag
 
-    OPEN_CURLEY = "{".ord
-    CLOSE_CURLEY = "}".ord
-    PERCENTAGE = "%".ord
-
     def initialize(
       source:,
-      string_scanner:,
+      string_scanner: nil,
       line_numbers: false,
       line_number: nil,
       for_liquid_tag: false
     )
       @line_number = line_number || (line_numbers ? 1 : nil)
       @for_liquid_tag = for_liquid_tag
-      @source = source.to_s.to_str
+      @source = source.to_s
       @offset = 0
       @tokens = []
 
-      tokenize if @source
+      tokenize
     end
 
     def shift
@@ -50,11 +44,10 @@ module Liquid
       end
 
       @source = nil
-      @ss = nil
     end
 
-    # Fast tokenizer using String#index instead of StringScanner regex.
+    # Fast tokenizer using String#byteindex instead of StringScanner regex.
-    # String#index is ~40% faster for finding { delimiters.
+    # String#byteindex is ~40% faster for finding { delimiters.
     def tokenize_fast
       src = @source
       unless src.valid_encoding?
@@ -76,7 +69,7 @@ module Liquid
 
         next_byte = idx + 1 < len ? src.getbyte(idx + 1) : nil
 
-        if next_byte == PERCENTAGE # {%
+        if next_byte == Cursor::PCT # {%
           # Emit text before tag
           @tokens << src.byteslice(pos, idx - pos) if idx > pos
 
@@ -86,65 +79,14 @@ module Liquid
             @tokens << src.byteslice(idx, close + 2 - idx)
             pos = close + 2
           else
+            # Emit malformed token to propagate a missing-terminator error in the parser
             @tokens << "{%"
             pos = idx + 2
           end
-        elsif next_byte == OPEN_CURLEY # {{
+        elsif next_byte == Cursor::LCURLY # {{
-          # Emit text before variable
+          # Emit text before variable, then scan for the closing }}.
           @tokens << src.byteslice(pos, idx - pos) if idx > pos
-
+          pos = scan_variable_token(src, idx, len)
-          # Scan variable token — matches original tokenizer's byte-by-byte logic:
-          # Find } or {, then check next byte for }}/{% nesting
-          scan_pos = idx + 2
-          found = false
-          while scan_pos < len
-            b = src.getbyte(scan_pos)
-            if b == CLOSE_CURLEY # }
-              if scan_pos + 1 >= len
-                # } at end of string — emit token up to here
-                @tokens << src.byteslice(idx, scan_pos + 1 - idx)
-                pos = scan_pos + 1
-                found = true
-                break
-              end
-              b2 = src.getbyte(scan_pos + 1)
-              if b2 == CLOSE_CURLEY
-                # Found }} — close variable
-                @tokens << src.byteslice(idx, scan_pos + 2 - idx)
-                pos = scan_pos + 2
-                found = true
-                break
-              else
-                # } followed by non-} — emit token up to here (matches original: @ss.pos -= 1)
-                @tokens << src.byteslice(idx, scan_pos + 1 - idx)
-                pos = scan_pos + 1
-                found = true
-                break
-              end
-            elsif b == OPEN_CURLEY
-              if scan_pos + 1 < len && src.getbyte(scan_pos + 1) == PERCENTAGE
-                # Found {% inside {{ — scan to %} and emit as one token
-                close = src.byteindex('%}', scan_pos + 2)
-                if close
-                  @tokens << src.byteslice(idx, close + 2 - idx)
-                  pos = close + 2
-                else
-                  @tokens << src.byteslice(idx, len - idx)
-                  pos = len
-                end
-                found = true
-                break
-              end
-              scan_pos += 1
-            else
-              scan_pos += 1
-            end
-          end
-
-          unless found
-            @tokens << "{{"
-            pos = idx + 2
-          end
         else
           # Lone '{' — not the start of a tag or variable.
           # Find the next '{{' or '{%' to know where this text token ends.
@@ -162,5 +104,49 @@ module Liquid
         end
       end
     end
+
+    # Scans a {{ ... }} variable token starting at `idx` in `src`.
+    # Emits the token to @tokens and returns the new position after the token.
+    # Handles }}, single }, and embedded {% ... %} (nested tag inside variable).
+    private def scan_variable_token(src, idx, len)
+      # Byte-by-byte scan: find } or {, then inspect the next byte.
+      scan_pos = idx + 2
+      while scan_pos < len
+        b = src.getbyte(scan_pos)
+        if b == Cursor::RCURLY # }
+          if scan_pos + 1 >= len
+            # } at end of string — emit token up to here
+            @tokens << src.byteslice(idx, scan_pos + 1 - idx)
+            return scan_pos + 1
+          end
+          b2 = src.getbyte(scan_pos + 1)
+          if b2 == Cursor::RCURLY
+            # Found }} — close variable
+            @tokens << src.byteslice(idx, scan_pos + 2 - idx)
+            return scan_pos + 2
+          else
+            # } followed by non-} — emit token up to here (matches original: @ss.pos -= 1)
+            @tokens << src.byteslice(idx, scan_pos + 1 - idx)
+            return scan_pos + 1
+          end
+        elsif b == Cursor::LCURLY && scan_pos + 1 < len && src.getbyte(scan_pos + 1) == Cursor::PCT
+          # Found {% inside {{ — scan to %} and emit as one token
+          close = src.byteindex('%}', scan_pos + 2)
+          if close
+            @tokens << src.byteslice(idx, close + 2 - idx)
+            return close + 2
+          else
+            @tokens << src.byteslice(idx, len - idx)
+            return len
+          end
+        else
+          scan_pos += 1
+        end
+      end
+
+      # Reached end without finding }} — malformed
+      @tokens << "{{"
+      idx + 2
+    end
   end
 end

lib/liquid/variable_lookup.rb

@@ -48,12 +48,12 @@ module Liquid
           end
         elsif byte == 46 # '.'
           pos += 1
-        elsif (byte >= 97 && byte <= 122) || (byte >= 65 && byte <= 90) || (byte >= 48 && byte <= 57) || byte == 95 || byte == 45 # \w or -
+        elsif ByteTables::IDENT_CONT[byte] # [\w-]
           start = pos
           pos += 1
           while pos < len
             b = markup.getbyte(pos)
-            break unless (b >= 97 && b <= 122) || (b >= 65 && b <= 90) || (b >= 48 && b <= 57) || b == 95 || b == 45
+            break unless ByteTables::IDENT_CONT[b]
             pos += 1
           end
           # Check trailing '?'