perf: remove transient strings to reduce memory

benchmark/memory.js

@@ -28,10 +28,12 @@ function html () {
   for (let i = 0; i < SAMPLE_COUNT; i++) {
     templates.push(engine.parse(str))
   }
-  global.gc()
+  const diff1 = (getHeapStatistics().used_heap_size - base) / SAMPLE_COUNT
+  console.log(`${h(str.length)} lorem-html before GC x ${h(diff1)}/tpl (${SAMPLE_COUNT} instances sampled)`)
 
-  const diff = (getHeapStatistics().used_heap_size - base) / SAMPLE_COUNT
+  global.gc()
-  console.log(`${h(str.length)} HTML template x ${h(diff)}/instance (${SAMPLE_COUNT} instances sampled)`)
+  const diff2 = (getHeapStatistics().used_heap_size - base) / SAMPLE_COUNT
+  console.log(`${h(str.length)} lorem-html after GC x ${h(diff2)}/tpl (${SAMPLE_COUNT} instances sampled)`)
 }
 
 function todolist () {
@@ -44,10 +46,12 @@ function todolist () {
   for (let i = 0; i < SAMPLE_COUNT; i++) {
     templates.push(engine.parse(str))
   }
-  global.gc()
+  const diff1 = (getHeapStatistics().used_heap_size - base) / SAMPLE_COUNT
+  console.log(`${h(str.length)} todolist before GC x ${h(diff1)}/tpl (${SAMPLE_COUNT} instances sampled)`)
 
-  const diff = (getHeapStatistics().used_heap_size - base) / SAMPLE_COUNT
+  global.gc()
-  console.log(`${h(str.length)} Todo template x ${h(diff)}/instance (${SAMPLE_COUNT} instances sampled)`)
+  const diff2 = (getHeapStatistics().used_heap_size - base) / SAMPLE_COUNT
+  console.log(`${h(str.length)} todolist after GC x ${h(diff2)}/tpl (${SAMPLE_COUNT} instances sampled)`)
 }
 
 function h (size) {

rollup.config.ts

@@ -49,11 +49,6 @@ const esm = {
       delimiters: ['', ''],
       './fs/node': './fs/browser'
     }),
-    replace({
-      include: './src/parser/token.ts',
-      delimiters: ['', ''],
-      './flatten/node': './flatten/browser'
-    }),
     typescript({
       tsconfigOverride: {
         include: [ 'src' ],
@@ -83,11 +78,6 @@ const umd = {
       delimiters: ['', ''],
       './fs/node': './fs/browser'
     }),
-    replace({
-      include: './src/parser/token.ts',
-      delimiters: ['', ''],
-      './flatten/node': './flatten/browser'
-    }),
     typescript({
       tsconfigOverride: {
         include: [ 'src' ],
@@ -116,11 +106,6 @@ const min = {
       delimiters: ['', ''],
       './fs/node': './fs/browser'
     }),
-    replace({
-      include: './src/parser/token.ts',
-      delimiters: ['', ''],
-      './flatten/node': './flatten/browser'
-    }),
     typescript({
       tsconfigOverride: {
         include: [ 'src' ],

src/context/context.ts

@@ -80,6 +80,7 @@ export class Context {
   private parseProp (str: string) {
     str = String(str)
     const seq: string[] = []
+    const push = () => name.length && (seq.push(name), (name = ''))
     let name = ''
     let j
     let i = 0
@@ -120,11 +121,6 @@ export class Context {
       throw new TypeError(`invalid path:"${str}"`)
     }
     return seq
-
-    function push () {
-      if (name.length) seq.push(name)
-      name = ''
-    }
   }
 }
 

src/parser/filter-token.ts

@@ -12,6 +12,5 @@ export class FilterToken extends Token {
     file?: string
   ) {
     super(raw, input, line, col, file)
-    this.type = 'filter'
   }
 }

src/parser/flatten/browser.ts

@@ -1,3 +0,0 @@
-export function flatten (str: string) {
-  return str
-}

src/parser/flatten/node.ts

@@ -1,10 +0,0 @@
-/**
- * This function forces a string to be re-instantiated in memory using a flat representation instead of a graph
- * of concatenated strings. This is an optimization to reduce the memory footprint of token string fragments after
- * they are parsed.
- * This optimization targets the V8 javascript engine and only works on Node.js.
- * @param {string} str
- */
-export function flatten (str: string) {
-  return Buffer.from(str).toString()
-}

src/parser/html-token.ts

@@ -3,10 +3,9 @@ import { Token } from './token'
 export class HTMLToken extends Token {
   public constructor (str: string, input: string, line: number, col: number, file?: string) {
     super(str, input, line, col, file)
-    this.type = 'html'
     this.content = str
   }
   public static is (token: Token): token is HTMLToken {
-    return token.type === 'html'
+    return token instanceof HTMLToken
   }
 }

src/parser/output-token.ts

@@ -13,9 +13,8 @@ export class OutputToken extends DelimitedToken {
     file?: string
   ) {
     super(raw, value, input, line, pos, options.trimOutputLeft, options.trimOutputRight, file)
-    this.type = 'output'
   }
   public static is (token: Token): token is OutputToken {
-    return token.type === 'output'
+    return token instanceof OutputToken
   }
 }

src/parser/substr.ts

@@ -0,0 +1,19 @@
+export class Substr {
+  constructor (
+    public str: string,
+    public begin: number,
+    public end: number = begin
+  ) {}
+  size () {
+    return this.end - this.begin
+  }
+  toString () {
+    return this.str.slice(this.begin, this.end)
+  }
+  first () {
+    return this.str[this.begin]
+  }
+  last () {
+    return this.str[this.end - 1]
+  }
+}

src/parser/tag-token.ts

@@ -17,7 +17,6 @@ export class TagToken extends DelimitedToken {
     file?: string
   ) {
     super(raw, value, input, line, pos, options.trimTagLeft, options.trimTagRight, file)
-    this.type = 'tag'
     const match = this.content.match(lexical.tagLine)
     if (!match) {
       throw new TokenizationError(`illegal tag syntax`, this)
@@ -26,6 +25,6 @@ export class TagToken extends DelimitedToken {
     this.args = match[2]
   }
   public static is (token: Token): token is TagToken {
-    return token.type === 'tag'
+    return token instanceof TagToken
   }
 }

src/parser/token.ts

@@ -1,18 +1,14 @@
-import { flatten } from './flatten/node'
-
 export class Token {
   public trimLeft = false
   public trimRight = false
-  public type = 'notset'
-  public raw: string
   public content: string
-  public constructor (raw: string,
+  public constructor (
+    public raw: string,
     public input: string,
     public line: number,
     public col: number,
     public file?: string
   ) {
-    this.raw = flatten(raw)
     this.content = raw
   }
 }

src/parser/tokenizer.ts

@@ -1,4 +1,5 @@
 import { whiteSpaceCtrl } from './whitespace-ctrl'
+import { Substr } from './substr'
 import { FilterArg } from './filter-arg'
 import { FilterToken } from './filter-token'
 import { ellipsis } from '../util/underscore'
@@ -32,15 +33,16 @@ export class Tokenizer {
 
   * readExpression (): IterableIterator<string> {
     while (this.p < this.N) {
-      let val = this.readValue()
+      const operand = this.readValue()
-      if (val) {
+      if (operand.size()) {
-        yield val
+        yield operand.toString()
         continue
       }
       this.readBlank()
-      while (OPERATOR & this.peekType()) val += this.read()
+      const operator = new Substr(this.input, this.p)
-      if (val) {
+      while (OPERATOR & this.peekType()) operator.end = this.read()
-        yield val
+      if (operator.size()) {
+        yield operator.toString()
         continue
       }
       this.read()
@@ -63,7 +65,7 @@ export class Tokenizer {
   readFilterToken (): FilterToken | null {
     this.readTo('|')
     const begin = this.p
-    const name = this.readVariable()
+    const name = this.readVariable().toString()
     if (!name) return null
     const args = []
     this.readBlank()
@@ -85,9 +87,9 @@ export class Tokenizer {
     this.readBlank()
     if (this.peek() === ':') {
       this.read()
-      return [key, this.readValue()]
+      return [key.toString(), this.readValue().toString()]
     }
-    return key
+    return key.toString()
   }
 
   readTokens (): Token[] {
@@ -102,27 +104,27 @@ export class Tokenizer {
 
   readToken (): Token {
     const { tagDelimiterLeft, outputDelimiterLeft } = this.options
-    if (this.peekWord(tagDelimiterLeft.length) === tagDelimiterLeft) return this.readTagToken()
+    if (this.matchWord(tagDelimiterLeft)) return this.readTagToken()
-    if (this.peekWord(outputDelimiterLeft.length) === outputDelimiterLeft) return this.readOutputToken()
+    if (this.matchWord(outputDelimiterLeft)) return this.readOutputToken()
     return this.readHTMLToken()
   }
 
   readHTMLToken (): HTMLToken {
-    let html = ''
+    const html = new Substr(this.input, this.p)
     while (this.p < this.N) {
       const { tagDelimiterLeft, outputDelimiterLeft } = this.options
-      if (this.peekWord(tagDelimiterLeft.length) === tagDelimiterLeft) break
+      if (this.matchWord(tagDelimiterLeft)) break
-      if (this.peekWord(outputDelimiterLeft.length) === outputDelimiterLeft) break
+      if (this.matchWord(outputDelimiterLeft)) break
-      html += this.read()
+      html.end = this.read()
     }
-    return new HTMLToken(html, this.input, this.line, this.col, this.file)
+    return new HTMLToken(html.toString(), this.input, this.line, this.col, this.file)
   }
 
   readTagToken (): TagToken {
     const { line, col, file, input, options } = this
     const { tagDelimiterLeft, tagDelimiterRight } = options
-    const buffer = this.readTo(tagDelimiterRight)
+    const buffer = this.readTo(tagDelimiterRight).toString()
-    if (buffer.slice(-tagDelimiterRight.length) !== tagDelimiterRight) {
+    if (!this.reverseMatchWord(tagDelimiterRight, buffer)) {
       throw new TokenizationError(
         `tag "${ellipsis(buffer, 16)}" not closed`,
         new Token(buffer, input, line, col, file)
@@ -135,8 +137,8 @@ export class Tokenizer {
   readOutputToken (): OutputToken {
     const { line, col, file, input, options } = this
     const { outputDelimiterLeft, outputDelimiterRight } = options
-    const buffer = this.readTo(outputDelimiterRight)
+    const buffer = this.readTo(outputDelimiterRight).toString()
-    if (buffer.slice(-outputDelimiterRight.length) !== outputDelimiterRight) {
+    if (!this.reverseMatchWord(outputDelimiterRight, buffer)) {
       throw new TokenizationError(
         `output "${ellipsis(buffer, 16)}" not closed`,
         new Token(buffer, input, line, col, file)
@@ -146,10 +148,10 @@ export class Tokenizer {
     return new OutputToken(buffer, value, input, line, col, options, file)
   }
 
-  readVariable () {
+  readVariable (): Substr {
     this.readBlank()
-    let ans = ''
+    const ans = new Substr(this.input, this.p)
-    while (this.peekType() & VARIABLE) ans += this.read()
+    while (this.peekType() & VARIABLE) ans.end = this.read()
     return ans
   }
 
@@ -165,39 +167,40 @@ export class Tokenizer {
   readHash () {
     this.readBlank()
     if (this.peek() === ',') this.read()
-    const name = this.readVariable()
+    const name = this.readVariable().toString()
     if (!name) return null
 
     this.readBlank()
     let value = ''
     if (this.peek() === ':') {
       this.read()
-      value = this.readValue()
+      value = this.readValue().toString()
     }
     return [name, value]
   }
 
-  readPropertyAccess () {
+  readPropertyAccess (): Substr {
     this.readBlank()
-    let ans = ''
+    const ans = new Substr(this.input, this.p)
     let nested = 0
     while (this.p < this.N) {
       const c = this.peek()
       const code = this.peekType()
       if (c === '[') {
-        ans += this.read() + this.readValue()
+        this.read()
+        ans.end = this.readValue().end
         nested++
       } else if (c === ']') {
         if (!nested) break
-        ans += this.read()
+        ans.end = this.read()
         nested--
       } else if (c === '.') {
         if (this.peekType(1) & VARIABLE) {
-          ans += this.read()
+          this.read()
-          ans += this.readVariable()
+          ans.end = this.readVariable().end
         } else break
       } else if (code & VARIABLE) {
-        ans += this.read()
+        ans.end = this.read()
       } else {
         if (nested) this.read()
         else break
@@ -205,54 +208,57 @@ export class Tokenizer {
     }
     return ans
   }
-  readTo (end: string) {
+  readTo (end: string): Substr {
-    let ans = ''
+    const ans = new Substr(this.input, this.p)
     while (this.p < this.N) {
-      ans += this.read()
+      ans.end = this.read()
-      if (ans.slice(-end.length) === end) break
+      if (this.reverseMatchWord(end)) break
     }
     return ans
   }
-  readValue () {
+  readValue (): Substr {
     let val = this.readQuoted()
-    if (val) return val
+    if (val.size()) return val
     val = this.readBoolean()
-    if (val) return val
+    if (val.size()) return val
     val = this.readPropertyAccess()
-    if (val) return val
+    if (val.size()) return val
     return this.readRange()
   }
-  readRange () {
+  readRange (): Substr {
     this.readBlank()
-    if (this.peek() !== '(') return ''
+    const ans = new Substr(this.input, this.p)
-    let ans = this.read()
+    if (this.peek() !== '(') return ans
-    ans += this.readValue()
+    this.read()
-    ans += this.read(2)
+    this.readValue()
-    ans += this.readValue()
+    this.read(2)
-    ans += this.read()
+    this.readValue()
+    ans.end = this.read()
     return ans
   }
-  readBoolean () {
+  readBoolean (): Substr {
     this.readBlank()
-    if (this.peekWord(4) === 'true' && !(this.peekType(4) & VARIABLE)) return this.read(4)
+    const ans = new Substr(this.input, this.p)
-    if (this.peekWord(5) === 'false' && !(this.peekType(5) & VARIABLE)) return this.read(5)
+    if (this.matchWord('true') && !(this.peekType(4) & VARIABLE)) ans.end = this.read(4)
-    return ''
+    else if (this.matchWord('false') && !(this.peekType(5) & VARIABLE)) ans.end = this.read(5)
+    return ans
   }
-  readQuoted () {
+  readQuoted (): Substr {
     this.readBlank()
-    if (!(this.peekType() & QUOTE)) return ''
+    const ans = new Substr(this.input, this.p)
-    let ans = this.read()
+    if (!(this.peekType() & QUOTE)) return ans
+    ans.end = this.read()
     let escaped = false
     while (this.p < this.N) {
-      const c = this.read()
+      ans.end = this.read()
-      ans += c
+      if (ans.last() === ans.first() && !escaped) break
-      if (c === ans[0] && !escaped) break
       if (escaped) escaped = false
-      else if (c === '\\') escaped = true
+      else if (ans.last() === '\\') escaped = true
     }
     return ans
   }
-  read (n = 1): string {
+  read (n = 1): number {
+    if (n > 1) this.read(n - 1)
     const c = this.input[this.p++]
     if (c === '\n') {
       this.line++
@@ -260,10 +266,21 @@ export class Tokenizer {
     } else {
       this.col++
     }
-    return n === 1 ? c : c + this.read(n - 1)
+    return this.p
   }
-  peekWord (n: number) {
+  matchWord (word: string) {
-    return this.input.substr(this.p, n)
+    for (let i = 0; i < word.length; i++) {
+      if (word[i] !== this.input[this.p + i]) return false
+    }
+    return true
+  }
+  reverseMatchWord (word: string, buffer?: string) {
+    const str = buffer || this.input
+    const end = buffer === undefined ? this.p : buffer.length
+    for (let i = 0; i < word.length; i++) {
+      if (word[word.length - 1 - i] !== str[end - 1 - i]) return false
+    }
+    return true
   }
   peekType (n = 0) {
     return +TYPES[this.input.charCodeAt(this.p + n)]

src/template/value.ts

@@ -13,7 +13,7 @@ export class Value {
    */
   public constructor (str: string, private readonly filterMap: FilterMap) {
     const tokenizer = new Tokenizer(str)
-    this.initial = tokenizer.readValue()
+    this.initial = tokenizer.readValue().toString()
     this.filters = tokenizer.readFilterTokens().map(({ name, args }) => new Filter(name, this.filterMap.get(name), args))
   }
   public * value (ctx: Context) {

test/unit/parser/tokenizer.ts

@@ -6,21 +6,21 @@ import { HTMLToken } from '../../../src/parser/html-token'
 
 describe('Tokenize', function () {
   it('should read quoted', () => {
-    expect(new Tokenizer('"foo" ff').readQuoted()).to.equal('"foo"')
+    expect(new Tokenizer('"foo" ff').readQuoted().toString()).to.equal('"foo"')
-    expect(new Tokenizer(' "foo"ff').readQuoted()).to.equal('"foo"')
+    expect(new Tokenizer(' "foo"ff').readQuoted().toString()).to.equal('"foo"')
   })
   it('should read property access', () => {
-    expect(new Tokenizer('a[ b][ "c d" ]').readPropertyAccess()).to.equal('a[b]["c d"]')
+    expect(new Tokenizer('a[ b][ "c d" ]').readPropertyAccess().toString()).to.equal('a[ b][ "c d" ]')
-    expect(new Tokenizer('a.b[c[d.e]]').readPropertyAccess()).to.equal('a.b[c[d.e]]')
+    expect(new Tokenizer('a.b[c[d.e]]').readPropertyAccess().toString()).to.equal('a.b[c[d.e]]')
   })
   it('should read value', () => {
-    expect(new Tokenizer('2.33.2').readValue()).to.equal('2.33.2')
+    expect(new Tokenizer('2.33.2').readValue().toString()).to.equal('2.33.2')
-    expect(new Tokenizer('"foo"a').readValue()).to.equal('"foo"')
+    expect(new Tokenizer('"foo"a').readValue().toString()).to.equal('"foo"')
-    expect(new Tokenizer('a[b]["c d"]').readValue()).to.equal('a[b]["c d"]')
+    expect(new Tokenizer('a[b]["c d"]').readValue().toString()).to.equal('a[b]["c d"]')
   })
   it('should read hash', () => {
     expect(new Tokenizer('foo: 3').readHash()).to.deep.equal(['foo', '3'])
-    expect(new Tokenizer(', foo: a[ "bar"]').readHash()).to.deep.equal(['foo', 'a["bar"]'])
+    expect(new Tokenizer(', foo: a[ "bar"]').readHash()).to.deep.equal(['foo', 'a[ "bar"]'])
   })
   it('should read hashs', () => {
     expect(new Tokenizer(', limit: 3 reverse offset:off').readHashes())

test/unit/render/render.ts

@@ -14,7 +14,7 @@ describe('render', function () {
   describe('.renderTemplates()', function () {
     it('should render html', async function () {
       const scope = new Context()
-      const token = { type: 'html', content: '<p>' } as Token
+      const token = { content: '<p>' } as Token
       const html = await toThenable(render.renderTemplates([new HTML(token)], scope))
       return expect(html).to.equal('<p>')
     })

test/unit/template/tag.ts

@@ -16,7 +16,6 @@ describe('Tag', function () {
   it('should call tag.render', async function () {
     const spy = sinon.spy()
     const token = {
-      type: 'tag',
       content: 'foo',
       args: '',
       name: 'foo'