perf(parser): memoize createTrie to avoid rebuilding tries per Tokenizer (#911)

The Tokenizer constructor calls createTrie(operators) and
createTrie(literalValues) on every instantiation, and liquidjs builds a
fresh Tokenizer per output/tag while parsing. On typical templates this
rebuilt the same prefix-tries dozens of times and showed up as a large
share of parse CPU in profiling.

Memoize createTrie with a module-level WeakMap keyed on the input object.
The inputs (operators, literalValues) are stable references and the trie
is only ever read afterward (via matchTrie), never mutated, so caching by
reference is behavior-preserving. WeakMap (not Map) lets short-lived,
per-instance operator objects and their tries be garbage collected.
This commit is contained in:
Max Medve
2026-06-22 20:11:06 +08:00
committed by GitHub
parent 956b51ea95
commit 3a0d80d1f4
2 changed files with 18 additions and 0 deletions
+8
View File
@@ -522,6 +522,14 @@ describe('Tokenizer', function () {
expect(new Tokenizer('contains b').matchTrie(opTrie)).toBe(8)
})
})
describe('#createTrie()', function () {
it('should return the same trie for the same input', () => {
expect(createTrie(defaultOperators)).toBe(createTrie(defaultOperators))
})
it('should return distinct tries for distinct inputs', () => {
expect(createTrie({ foo: 1 })).not.toBe(createTrie({ foo: 1 }))
})
})
describe('#readLiquidTagTokens', () => {
it('should read newline terminated tokens', () => {
const tokenizer = new Tokenizer('echo \'hello\'')
+10
View File
@@ -10,7 +10,16 @@ export type Trie<T> = {
needBoundary?: true
} & Record<string, any>
// Tries are built once per input object and reused: the Tokenizer rebuilds them
// on every instantiation, but `input` (operators/literalValues) is a stable
// reference. WeakMap-keying by `input` lets short-lived operator objects (and
// their tries) be garbage collected. The returned trie is treated as read-only
// by callers (matchTrie only reads it); do not mutate it.
const trieCache = new WeakMap<TrieInput<any>, Trie<any>>()
export function createTrie<T = any> (input: TrieInput<T>): Trie<T> {
const cached = trieCache.get(input)
if (cached) return cached
const trie: Trie<T> = {}
for (const [name, data] of Object.entries(input)) {
let node = trie
@@ -29,5 +38,6 @@ export function createTrie<T = any> (input: TrieInput<T>): Trie<T> {
node.data = data
node.end = true
}
trieCache.set(input, trie)
return trie
}