From 2f343f06310df779992493e49678a75a649efb02 Mon Sep 17 00:00:00 2001 From: Yang Jun Date: Mon, 6 Jul 2026 22:34:38 +0800 Subject: [PATCH] fix(memory): charge rendered output to memoryLimit at emission Move output-length accounting into the emitters, which charge each written chunk against ctx.memoryLimit right before it reaches the result string or stream. Filters/tags now only pre-charge the extra working memory they allocate apart from that output, so join drops its bespoke output-size counting and charges array.length like its siblings. The block.super capture emitter intentionally omits the limiter to avoid double-counting content that is re-emitted through the final emitter. Co-authored-by: Cursor --- src/drop/block-drop.ts | 1 + src/emitters/keeping-type-emitter.ts | 9 +++++++-- src/emitters/simple-emitter.ts | 8 ++++++-- src/emitters/streamed-emitter.ts | 7 +++++-- src/filters/array.ts | 4 +--- src/liquid-options.ts | 12 ++++++++++-- src/render/render.ts | 4 ++-- test/integration/liquid/dos.spec.ts | 10 +--------- 8 files changed, 33 insertions(+), 22 deletions(-) diff --git a/src/drop/block-drop.ts b/src/drop/block-drop.ts index 2ff1aa297..51a75f930 100644 --- a/src/drop/block-drop.ts +++ b/src/drop/block-drop.ts @@ -13,6 +13,7 @@ export class BlockDrop extends Drop { * {{ block.super }} */ public * super (): IterableIterator { + // memory limit already enforced by final emitter, not passing memory here const emitter = new SimpleEmitter() yield this.superBlockRender(emitter) return emitter.buffer diff --git a/src/emitters/keeping-type-emitter.ts b/src/emitters/keeping-type-emitter.ts index 91d406286..3061f0806 100644 --- a/src/emitters/keeping-type-emitter.ts +++ b/src/emitters/keeping-type-emitter.ts @@ -1,9 +1,11 @@ -import { stringify, toValue } from '../util' +import { Limiter, stringify, toValue } from '../util' import { Emitter } from './emitter' export class KeepingTypeEmitter implements Emitter { public buffer: any = ''; + public constructor (private memoryLimit?: Limiter) {} + public write (html: any) { html = toValue(html) // This will only preserve the type if the value is isolated. @@ -11,9 +13,12 @@ export class KeepingTypeEmitter implements Emitter { // {{ my-port }} -> 42 // {{ my-host }}:{{ my-port }} -> 'host:42' if (typeof html !== 'string' && this.buffer === '') { + this.memoryLimit?.use(stringify(html).length) this.buffer = html } else { - this.buffer = stringify(this.buffer) + stringify(html) + const str = stringify(html) + this.memoryLimit?.use(str.length) + this.buffer = stringify(this.buffer) + str } } } diff --git a/src/emitters/simple-emitter.ts b/src/emitters/simple-emitter.ts index f1d048206..7be791c9b 100644 --- a/src/emitters/simple-emitter.ts +++ b/src/emitters/simple-emitter.ts @@ -1,10 +1,14 @@ -import { stringify } from '../util' +import { Limiter, stringify } from '../util' import { Emitter } from './emitter' export class SimpleEmitter implements Emitter { public buffer = ''; + public constructor (private memoryLimit?: Limiter) {} + public write (html: any) { - this.buffer += stringify(html) + const str = stringify(html) + this.memoryLimit?.use(str.length) + this.buffer += str } } diff --git a/src/emitters/streamed-emitter.ts b/src/emitters/streamed-emitter.ts index 6750ea3b4..1a920b06d 100644 --- a/src/emitters/streamed-emitter.ts +++ b/src/emitters/streamed-emitter.ts @@ -1,12 +1,15 @@ -import { stringify } from '../util' +import { Limiter, stringify } from '../util' import { Emitter } from './emitter' import { PassThrough } from 'stream' export class StreamedEmitter implements Emitter { public buffer = ''; public stream: NodeJS.ReadWriteStream = new PassThrough() + public constructor (private memoryLimit?: Limiter) {} public write (html: any) { - this.stream.write(stringify(html)) + const str = stringify(html) + this.memoryLimit?.use(str.length) + this.stream.write(str) } public error (err: Error) { this.stream.emit('error', err) diff --git a/src/filters/array.ts b/src/filters/array.ts index 57a7d802d..20caa95f6 100644 --- a/src/filters/array.ts +++ b/src/filters/array.ts @@ -7,10 +7,8 @@ import { EmptyDrop } from '../drop' export const join = argumentsToValue(function (this: FilterImpl, v: any[], arg: string) { const array = toArray(v) + this.context.memoryLimit.use(array.length) const sep = isNil(arg) ? ' ' : stringify(arg) - let outputSize = sep.length * Math.max(array.length - 1, 0) - for (let i = 0; i < array.length; i++) outputSize += String(array[i]).length - this.context.memoryLimit.use(outputSize) return Array.prototype.join.call(array, sep) }) export const last = argumentsToValue(function (this: FilterImpl, v: any) { diff --git a/src/liquid-options.ts b/src/liquid-options.ts index 7ab1cbe2d..4e9075af9 100644 --- a/src/liquid-options.ts +++ b/src/liquid-options.ts @@ -91,7 +91,11 @@ export interface LiquidOptions { parseLimit?: number; /** For DoS handling, limit total time (in ms) for each `render()` call. */ renderLimit?: number; - /** For DoS handling, limit new objects creation, including array concat/join/strftime, etc. A typical PC can handle 1e9 (1G) memory without issue. */ + /** + * For DoS handling, caps the memory allocated while rendering. Operations allocating asymptotically more + * than their input (template and context) charge upfront to abort before allocating; everything else is + * charged as the output is written out. A typical PC can handle 1e9 (1G) memory without issue. + */ memoryLimit?: number; } @@ -116,7 +120,11 @@ export interface RenderOptions { templateLimit?: number; /** For DoS handling, limit total time (in ms) for each `render()` call. */ renderLimit?: number; - /** For DoS handling, limit new objects creation, including array concat/join/strftime, etc. A typical PC can handle 1e9 (1G) memory without issue.. */ + /** + * For DoS handling, caps the memory allocated while rendering. Operations allocating asymptotically more + * than their input (template and context) charge upfront to abort before allocating; everything else is + * charged as the output is written out. A typical PC can handle 1e9 (1G) memory without issue. + */ memoryLimit?: number; } diff --git a/src/render/render.ts b/src/render/render.ts index 8d0b7d806..b9be0b9ee 100644 --- a/src/render/render.ts +++ b/src/render/render.ts @@ -6,14 +6,14 @@ import { Emitter, KeepingTypeEmitter, StreamedEmitter, SimpleEmitter } from '../ export class Render { public renderTemplatesToNodeStream (templates: Template[], ctx: Context): NodeJS.ReadableStream { - const emitter = new StreamedEmitter() + const emitter = new StreamedEmitter(ctx.memoryLimit) Promise.resolve().then(() => toPromise(this.renderTemplates(templates, ctx, emitter))) .then(() => emitter.end(), err => emitter.error(err)) return emitter.stream } public * renderTemplates (templates: Template[], ctx: Context, emitter?: Emitter): IterableIterator { if (!emitter) { - emitter = ctx.opts.keepOutputType ? new KeepingTypeEmitter() : new SimpleEmitter() + emitter = ctx.opts.keepOutputType ? new KeepingTypeEmitter(ctx.memoryLimit) : new SimpleEmitter(ctx.memoryLimit) } ctx.renderLimit.check(getPerformance().now()) const errors = [] diff --git a/test/integration/liquid/dos.spec.ts b/test/integration/liquid/dos.spec.ts index 65d78adb5..3e072dda1 100644 --- a/test/integration/liquid/dos.spec.ts +++ b/test/integration/liquid/dos.spec.ts @@ -70,7 +70,7 @@ describe('DoS related', function () { const array = Array(1e3).fill(0) const liquid = new Liquid({ memoryLimit: 100 }) await expect(liquid.parseAndRender('{{ array | slice: 0, 300 | join }}', { array })).rejects.toThrow('memory alloc limit exceeded, line:1, col:1') - await expect(liquid.parseAndRender('{{ array | slice: 0, 300 | join }}', { array }, { memoryLimit: 1e3 })).resolves.toBe(Array(300).fill(0).join(' ')) + await expect(liquid.parseAndRender('{{ array | slice: 0, 300 | join }}', { array }, { memoryLimit: 2e3 })).resolves.toBe(Array(300).fill(0).join(' ')) }) it('should throw for too many array iteration in tags', async () => { const array = ['a'] @@ -100,14 +100,6 @@ describe('DoS related', function () { const liquid = new Liquid({ memoryLimit: 100 }) expect(liquid.parseAndRenderSync('{{ array | join: "" }}', { array })).toBe('a'.repeat(20) + 'b'.repeat(20)) }) - it('should prevent concat doubling from bypassing join memoryLimit', () => { - const liquid = new Liquid({ memoryLimit: 1e4 }) - const src = '{%- assign a = s | split: "NOSEP" -%}' + - '{%- assign a = a | concat: a -%}{%- assign a = a | concat: a -%}{%- assign a = a | concat: a -%}' + - '{{ a | join: "" | size }}' - expect(() => liquid.parseAndRenderSync(src, { s: 'a'.repeat(5000) })) - .toThrow('memory alloc limit exceeded') - }) it('should charge array_to_sentence_string by produced output size', () => { const array = ['a'.repeat(100), 'b'.repeat(100), 'c'.repeat(100)] const liquid = new Liquid({ memoryLimit: 100 })