diff --git a/autoresearch.jsonl b/autoresearch.jsonl index 8287d4b5..a7194b13 100644 --- a/autoresearch.jsonl +++ b/autoresearch.jsonl @@ -12,3 +12,4 @@ {"run":11,"commit":"c09e722","metric":3889,"metrics":{"parse_µs":2770,"render_µs":1119,"allocations":24766},"status":"discard","description":"TruthyCondition for simple if checks: -115 allocs but YJIT polymorphism at evaluate call site hurts speed","timestamp":1773349649377,"segment":0} {"run":12,"commit":"c09e722","metric":4150,"metrics":{"parse_µs":2769,"render_µs":1381,"allocations":24881},"status":"discard","description":"Index loop for filters: YJIT optimizes each+destructure better than manual indexing","timestamp":1773349699285,"segment":0} {"run":13,"commit":"b7ae55f","metric":3556,"metrics":{"parse_µs":2388,"render_µs":1168,"allocations":24882},"status":"keep","description":"Replace StringScanner tokenizer with String#byteindex — 12% faster parse, no regex overhead for delimiter finding","timestamp":1773349875890,"segment":0} +{"run":14,"commit":"e25f2f1","metric":3464,"metrics":{"parse_µs":2335,"render_µs":1129,"allocations":24882},"status":"keep","description":"Confirmation run: byteindex tokenizer consistently 3,400-3,600µs","timestamp":1773349889465,"segment":0} diff --git a/lib/liquid/tokenizer.rb b/lib/liquid/tokenizer.rb index 54b56a9a..59b4c47e 100644 --- a/lib/liquid/tokenizer.rb +++ b/lib/liquid/tokenizer.rb @@ -6,10 +6,6 @@ module Liquid class Tokenizer attr_reader :line_number, :for_liquid_tag - TAG_END = /%\}/ - TAG_OR_VARIABLE_START = /\{[\{\%]/ - NEWLINE = /\n/ - OPEN_CURLEY = "{".ord CLOSE_CURLEY = "}".ord PERCENTAGE = "%".ord @@ -27,11 +23,7 @@ module Liquid @offset = 0 @tokens = [] - if @source - @ss = string_scanner - @ss.string = @source - tokenize - end + tokenize if @source end def shift