diff --git a/benchmark/memory.js b/benchmark/memory.js index 244f4ef66..256782b39 100644 --- a/benchmark/memory.js +++ b/benchmark/memory.js @@ -28,10 +28,12 @@ function html () { for (let i = 0; i < SAMPLE_COUNT; i++) { templates.push(engine.parse(str)) } - global.gc() + const diff1 = (getHeapStatistics().used_heap_size - base) / SAMPLE_COUNT + console.log(`${h(str.length)} lorem-html before GC x ${h(diff1)}/tpl (${SAMPLE_COUNT} instances sampled)`) - const diff = (getHeapStatistics().used_heap_size - base) / SAMPLE_COUNT - console.log(`${h(str.length)} HTML template x ${h(diff)}/instance (${SAMPLE_COUNT} instances sampled)`) + global.gc() + const diff2 = (getHeapStatistics().used_heap_size - base) / SAMPLE_COUNT + console.log(`${h(str.length)} lorem-html after GC x ${h(diff2)}/tpl (${SAMPLE_COUNT} instances sampled)`) } function todolist () { @@ -44,10 +46,12 @@ function todolist () { for (let i = 0; i < SAMPLE_COUNT; i++) { templates.push(engine.parse(str)) } - global.gc() + const diff1 = (getHeapStatistics().used_heap_size - base) / SAMPLE_COUNT + console.log(`${h(str.length)} todolist before GC x ${h(diff1)}/tpl (${SAMPLE_COUNT} instances sampled)`) - const diff = (getHeapStatistics().used_heap_size - base) / SAMPLE_COUNT - console.log(`${h(str.length)} Todo template x ${h(diff)}/instance (${SAMPLE_COUNT} instances sampled)`) + global.gc() + const diff2 = (getHeapStatistics().used_heap_size - base) / SAMPLE_COUNT + console.log(`${h(str.length)} todolist after GC x ${h(diff2)}/tpl (${SAMPLE_COUNT} instances sampled)`) } function h (size) { diff --git a/rollup.config.ts b/rollup.config.ts index 3856c2478..614fca1de 100644 --- a/rollup.config.ts +++ b/rollup.config.ts @@ -49,11 +49,6 @@ const esm = { delimiters: ['', ''], './fs/node': './fs/browser' }), - replace({ - include: './src/parser/token.ts', - delimiters: ['', ''], - './flatten/node': './flatten/browser' - }), typescript({ tsconfigOverride: { include: [ 'src' ], @@ -83,11 +78,6 @@ const umd = { delimiters: ['', ''], './fs/node': './fs/browser' }), - replace({ - include: './src/parser/token.ts', - delimiters: ['', ''], - './flatten/node': './flatten/browser' - }), typescript({ tsconfigOverride: { include: [ 'src' ], @@ -116,11 +106,6 @@ const min = { delimiters: ['', ''], './fs/node': './fs/browser' }), - replace({ - include: './src/parser/token.ts', - delimiters: ['', ''], - './flatten/node': './flatten/browser' - }), typescript({ tsconfigOverride: { include: [ 'src' ], diff --git a/src/context/context.ts b/src/context/context.ts index d38e40491..3263eb307 100644 --- a/src/context/context.ts +++ b/src/context/context.ts @@ -80,6 +80,7 @@ export class Context { private parseProp (str: string) { str = String(str) const seq: string[] = [] + const push = () => name.length && (seq.push(name), (name = '')) let name = '' let j let i = 0 @@ -120,11 +121,6 @@ export class Context { throw new TypeError(`invalid path:"${str}"`) } return seq - - function push () { - if (name.length) seq.push(name) - name = '' - } } } diff --git a/src/parser/filter-token.ts b/src/parser/filter-token.ts index bf8805e0b..316c83a4f 100644 --- a/src/parser/filter-token.ts +++ b/src/parser/filter-token.ts @@ -12,6 +12,5 @@ export class FilterToken extends Token { file?: string ) { super(raw, input, line, col, file) - this.type = 'filter' } } diff --git a/src/parser/flatten/browser.ts b/src/parser/flatten/browser.ts deleted file mode 100644 index 6ffb38bce..000000000 --- a/src/parser/flatten/browser.ts +++ /dev/null @@ -1,3 +0,0 @@ -export function flatten (str: string) { - return str -} diff --git a/src/parser/flatten/node.ts b/src/parser/flatten/node.ts deleted file mode 100644 index e14c2dccd..000000000 --- a/src/parser/flatten/node.ts +++ /dev/null @@ -1,10 +0,0 @@ -/** - * This function forces a string to be re-instantiated in memory using a flat representation instead of a graph - * of concatenated strings. This is an optimization to reduce the memory footprint of token string fragments after - * they are parsed. - * This optimization targets the V8 javascript engine and only works on Node.js. - * @param {string} str - */ -export function flatten (str: string) { - return Buffer.from(str).toString() -} diff --git a/src/parser/html-token.ts b/src/parser/html-token.ts index c87b0deda..89ccf6eea 100644 --- a/src/parser/html-token.ts +++ b/src/parser/html-token.ts @@ -3,10 +3,9 @@ import { Token } from './token' export class HTMLToken extends Token { public constructor (str: string, input: string, line: number, col: number, file?: string) { super(str, input, line, col, file) - this.type = 'html' this.content = str } public static is (token: Token): token is HTMLToken { - return token.type === 'html' + return token instanceof HTMLToken } } diff --git a/src/parser/output-token.ts b/src/parser/output-token.ts index 2b30e6ef4..6145f235f 100644 --- a/src/parser/output-token.ts +++ b/src/parser/output-token.ts @@ -13,9 +13,8 @@ export class OutputToken extends DelimitedToken { file?: string ) { super(raw, value, input, line, pos, options.trimOutputLeft, options.trimOutputRight, file) - this.type = 'output' } public static is (token: Token): token is OutputToken { - return token.type === 'output' + return token instanceof OutputToken } } diff --git a/src/parser/substr.ts b/src/parser/substr.ts new file mode 100644 index 000000000..b56c0c144 --- /dev/null +++ b/src/parser/substr.ts @@ -0,0 +1,19 @@ +export class Substr { + constructor ( + public str: string, + public begin: number, + public end: number = begin + ) {} + size () { + return this.end - this.begin + } + toString () { + return this.str.slice(this.begin, this.end) + } + first () { + return this.str[this.begin] + } + last () { + return this.str[this.end - 1] + } +} diff --git a/src/parser/tag-token.ts b/src/parser/tag-token.ts index 4f2189d96..b1811dd1f 100644 --- a/src/parser/tag-token.ts +++ b/src/parser/tag-token.ts @@ -17,7 +17,6 @@ export class TagToken extends DelimitedToken { file?: string ) { super(raw, value, input, line, pos, options.trimTagLeft, options.trimTagRight, file) - this.type = 'tag' const match = this.content.match(lexical.tagLine) if (!match) { throw new TokenizationError(`illegal tag syntax`, this) @@ -26,6 +25,6 @@ export class TagToken extends DelimitedToken { this.args = match[2] } public static is (token: Token): token is TagToken { - return token.type === 'tag' + return token instanceof TagToken } } diff --git a/src/parser/token.ts b/src/parser/token.ts index 6657a3201..6e05fd463 100644 --- a/src/parser/token.ts +++ b/src/parser/token.ts @@ -1,18 +1,14 @@ -import { flatten } from './flatten/node' - export class Token { public trimLeft = false public trimRight = false - public type = 'notset' - public raw: string public content: string - public constructor (raw: string, + public constructor ( + public raw: string, public input: string, public line: number, public col: number, public file?: string ) { - this.raw = flatten(raw) this.content = raw } } diff --git a/src/parser/tokenizer.ts b/src/parser/tokenizer.ts index ed7558220..4a4a1282e 100644 --- a/src/parser/tokenizer.ts +++ b/src/parser/tokenizer.ts @@ -1,4 +1,5 @@ import { whiteSpaceCtrl } from './whitespace-ctrl' +import { Substr } from './substr' import { FilterArg } from './filter-arg' import { FilterToken } from './filter-token' import { ellipsis } from '../util/underscore' @@ -32,15 +33,16 @@ export class Tokenizer { * readExpression (): IterableIterator { while (this.p < this.N) { - let val = this.readValue() - if (val) { - yield val + const operand = this.readValue() + if (operand.size()) { + yield operand.toString() continue } this.readBlank() - while (OPERATOR & this.peekType()) val += this.read() - if (val) { - yield val + const operator = new Substr(this.input, this.p) + while (OPERATOR & this.peekType()) operator.end = this.read() + if (operator.size()) { + yield operator.toString() continue } this.read() @@ -63,7 +65,7 @@ export class Tokenizer { readFilterToken (): FilterToken | null { this.readTo('|') const begin = this.p - const name = this.readVariable() + const name = this.readVariable().toString() if (!name) return null const args = [] this.readBlank() @@ -85,9 +87,9 @@ export class Tokenizer { this.readBlank() if (this.peek() === ':') { this.read() - return [key, this.readValue()] + return [key.toString(), this.readValue().toString()] } - return key + return key.toString() } readTokens (): Token[] { @@ -102,27 +104,27 @@ export class Tokenizer { readToken (): Token { const { tagDelimiterLeft, outputDelimiterLeft } = this.options - if (this.peekWord(tagDelimiterLeft.length) === tagDelimiterLeft) return this.readTagToken() - if (this.peekWord(outputDelimiterLeft.length) === outputDelimiterLeft) return this.readOutputToken() + if (this.matchWord(tagDelimiterLeft)) return this.readTagToken() + if (this.matchWord(outputDelimiterLeft)) return this.readOutputToken() return this.readHTMLToken() } readHTMLToken (): HTMLToken { - let html = '' + const html = new Substr(this.input, this.p) while (this.p < this.N) { const { tagDelimiterLeft, outputDelimiterLeft } = this.options - if (this.peekWord(tagDelimiterLeft.length) === tagDelimiterLeft) break - if (this.peekWord(outputDelimiterLeft.length) === outputDelimiterLeft) break - html += this.read() + if (this.matchWord(tagDelimiterLeft)) break + if (this.matchWord(outputDelimiterLeft)) break + html.end = this.read() } - return new HTMLToken(html, this.input, this.line, this.col, this.file) + return new HTMLToken(html.toString(), this.input, this.line, this.col, this.file) } readTagToken (): TagToken { const { line, col, file, input, options } = this const { tagDelimiterLeft, tagDelimiterRight } = options - const buffer = this.readTo(tagDelimiterRight) - if (buffer.slice(-tagDelimiterRight.length) !== tagDelimiterRight) { + const buffer = this.readTo(tagDelimiterRight).toString() + if (!this.reverseMatchWord(tagDelimiterRight, buffer)) { throw new TokenizationError( `tag "${ellipsis(buffer, 16)}" not closed`, new Token(buffer, input, line, col, file) @@ -135,8 +137,8 @@ export class Tokenizer { readOutputToken (): OutputToken { const { line, col, file, input, options } = this const { outputDelimiterLeft, outputDelimiterRight } = options - const buffer = this.readTo(outputDelimiterRight) - if (buffer.slice(-outputDelimiterRight.length) !== outputDelimiterRight) { + const buffer = this.readTo(outputDelimiterRight).toString() + if (!this.reverseMatchWord(outputDelimiterRight, buffer)) { throw new TokenizationError( `output "${ellipsis(buffer, 16)}" not closed`, new Token(buffer, input, line, col, file) @@ -146,10 +148,10 @@ export class Tokenizer { return new OutputToken(buffer, value, input, line, col, options, file) } - readVariable () { + readVariable (): Substr { this.readBlank() - let ans = '' - while (this.peekType() & VARIABLE) ans += this.read() + const ans = new Substr(this.input, this.p) + while (this.peekType() & VARIABLE) ans.end = this.read() return ans } @@ -165,39 +167,40 @@ export class Tokenizer { readHash () { this.readBlank() if (this.peek() === ',') this.read() - const name = this.readVariable() + const name = this.readVariable().toString() if (!name) return null this.readBlank() let value = '' if (this.peek() === ':') { this.read() - value = this.readValue() + value = this.readValue().toString() } return [name, value] } - readPropertyAccess () { + readPropertyAccess (): Substr { this.readBlank() - let ans = '' + const ans = new Substr(this.input, this.p) let nested = 0 while (this.p < this.N) { const c = this.peek() const code = this.peekType() if (c === '[') { - ans += this.read() + this.readValue() + this.read() + ans.end = this.readValue().end nested++ } else if (c === ']') { if (!nested) break - ans += this.read() + ans.end = this.read() nested-- } else if (c === '.') { if (this.peekType(1) & VARIABLE) { - ans += this.read() - ans += this.readVariable() + this.read() + ans.end = this.readVariable().end } else break } else if (code & VARIABLE) { - ans += this.read() + ans.end = this.read() } else { if (nested) this.read() else break @@ -205,54 +208,57 @@ export class Tokenizer { } return ans } - readTo (end: string) { - let ans = '' + readTo (end: string): Substr { + const ans = new Substr(this.input, this.p) while (this.p < this.N) { - ans += this.read() - if (ans.slice(-end.length) === end) break + ans.end = this.read() + if (this.reverseMatchWord(end)) break } return ans } - readValue () { + readValue (): Substr { let val = this.readQuoted() - if (val) return val + if (val.size()) return val val = this.readBoolean() - if (val) return val + if (val.size()) return val val = this.readPropertyAccess() - if (val) return val + if (val.size()) return val return this.readRange() } - readRange () { + readRange (): Substr { this.readBlank() - if (this.peek() !== '(') return '' - let ans = this.read() - ans += this.readValue() - ans += this.read(2) - ans += this.readValue() - ans += this.read() + const ans = new Substr(this.input, this.p) + if (this.peek() !== '(') return ans + this.read() + this.readValue() + this.read(2) + this.readValue() + ans.end = this.read() return ans } - readBoolean () { + readBoolean (): Substr { this.readBlank() - if (this.peekWord(4) === 'true' && !(this.peekType(4) & VARIABLE)) return this.read(4) - if (this.peekWord(5) === 'false' && !(this.peekType(5) & VARIABLE)) return this.read(5) - return '' + const ans = new Substr(this.input, this.p) + if (this.matchWord('true') && !(this.peekType(4) & VARIABLE)) ans.end = this.read(4) + else if (this.matchWord('false') && !(this.peekType(5) & VARIABLE)) ans.end = this.read(5) + return ans } - readQuoted () { + readQuoted (): Substr { this.readBlank() - if (!(this.peekType() & QUOTE)) return '' - let ans = this.read() + const ans = new Substr(this.input, this.p) + if (!(this.peekType() & QUOTE)) return ans + ans.end = this.read() let escaped = false while (this.p < this.N) { - const c = this.read() - ans += c - if (c === ans[0] && !escaped) break + ans.end = this.read() + if (ans.last() === ans.first() && !escaped) break if (escaped) escaped = false - else if (c === '\\') escaped = true + else if (ans.last() === '\\') escaped = true } return ans } - read (n = 1): string { + read (n = 1): number { + if (n > 1) this.read(n - 1) const c = this.input[this.p++] if (c === '\n') { this.line++ @@ -260,10 +266,21 @@ export class Tokenizer { } else { this.col++ } - return n === 1 ? c : c + this.read(n - 1) + return this.p } - peekWord (n: number) { - return this.input.substr(this.p, n) + matchWord (word: string) { + for (let i = 0; i < word.length; i++) { + if (word[i] !== this.input[this.p + i]) return false + } + return true + } + reverseMatchWord (word: string, buffer?: string) { + const str = buffer || this.input + const end = buffer === undefined ? this.p : buffer.length + for (let i = 0; i < word.length; i++) { + if (word[word.length - 1 - i] !== str[end - 1 - i]) return false + } + return true } peekType (n = 0) { return +TYPES[this.input.charCodeAt(this.p + n)] diff --git a/src/template/value.ts b/src/template/value.ts index 47396aad7..cd2b525d0 100644 --- a/src/template/value.ts +++ b/src/template/value.ts @@ -13,7 +13,7 @@ export class Value { */ public constructor (str: string, private readonly filterMap: FilterMap) { const tokenizer = new Tokenizer(str) - this.initial = tokenizer.readValue() + this.initial = tokenizer.readValue().toString() this.filters = tokenizer.readFilterTokens().map(({ name, args }) => new Filter(name, this.filterMap.get(name), args)) } public * value (ctx: Context) { diff --git a/test/unit/parser/tokenizer.ts b/test/unit/parser/tokenizer.ts index d9740c139..261a9d898 100644 --- a/test/unit/parser/tokenizer.ts +++ b/test/unit/parser/tokenizer.ts @@ -6,21 +6,21 @@ import { HTMLToken } from '../../../src/parser/html-token' describe('Tokenize', function () { it('should read quoted', () => { - expect(new Tokenizer('"foo" ff').readQuoted()).to.equal('"foo"') - expect(new Tokenizer(' "foo"ff').readQuoted()).to.equal('"foo"') + expect(new Tokenizer('"foo" ff').readQuoted().toString()).to.equal('"foo"') + expect(new Tokenizer(' "foo"ff').readQuoted().toString()).to.equal('"foo"') }) it('should read property access', () => { - expect(new Tokenizer('a[ b][ "c d" ]').readPropertyAccess()).to.equal('a[b]["c d"]') - expect(new Tokenizer('a.b[c[d.e]]').readPropertyAccess()).to.equal('a.b[c[d.e]]') + expect(new Tokenizer('a[ b][ "c d" ]').readPropertyAccess().toString()).to.equal('a[ b][ "c d" ]') + expect(new Tokenizer('a.b[c[d.e]]').readPropertyAccess().toString()).to.equal('a.b[c[d.e]]') }) it('should read value', () => { - expect(new Tokenizer('2.33.2').readValue()).to.equal('2.33.2') - expect(new Tokenizer('"foo"a').readValue()).to.equal('"foo"') - expect(new Tokenizer('a[b]["c d"]').readValue()).to.equal('a[b]["c d"]') + expect(new Tokenizer('2.33.2').readValue().toString()).to.equal('2.33.2') + expect(new Tokenizer('"foo"a').readValue().toString()).to.equal('"foo"') + expect(new Tokenizer('a[b]["c d"]').readValue().toString()).to.equal('a[b]["c d"]') }) it('should read hash', () => { expect(new Tokenizer('foo: 3').readHash()).to.deep.equal(['foo', '3']) - expect(new Tokenizer(', foo: a[ "bar"]').readHash()).to.deep.equal(['foo', 'a["bar"]']) + expect(new Tokenizer(', foo: a[ "bar"]').readHash()).to.deep.equal(['foo', 'a[ "bar"]']) }) it('should read hashs', () => { expect(new Tokenizer(', limit: 3 reverse offset:off').readHashes()) diff --git a/test/unit/render/render.ts b/test/unit/render/render.ts index 78deef159..b51efcfe4 100644 --- a/test/unit/render/render.ts +++ b/test/unit/render/render.ts @@ -14,7 +14,7 @@ describe('render', function () { describe('.renderTemplates()', function () { it('should render html', async function () { const scope = new Context() - const token = { type: 'html', content: '

' } as Token + const token = { content: '

' } as Token const html = await toThenable(render.renderTemplates([new HTML(token)], scope)) return expect(html).to.equal('

') }) diff --git a/test/unit/template/tag.ts b/test/unit/template/tag.ts index f3104644a..195fe3c07 100644 --- a/test/unit/template/tag.ts +++ b/test/unit/template/tag.ts @@ -16,7 +16,6 @@ describe('Tag', function () { it('should call tag.render', async function () { const spy = sinon.spy() const token = { - type: 'tag', content: 'foo', args: '', name: 'foo'