Labyricorn/liquidjs
1
0
Fork 0
mirror of https://github.com/harttle/liquidjs.git synced 2026-09-15 04:10:40 -07:00
Code Issues Packages Projects Releases Wiki Activity

perf: add string flattening to reduce retained memory (node only)

Browse Source 
add a memory footprint benchmark
This commit is contained in:
Patrick Malouin
2019-10-10 09:35:15 +08:00
committed by Jun Yang
parent 7bae5bcb79
commit 3ad512c51c
7 changed files with 102 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/parser/tokenizer.ts
+7 -6
View File
@@ -5,6 +5,7 @@ import { Token } from './token'
import { OutputToken } from './output-token'
import { TokenizationError } from '../util/error'
import { NormalizedFullOptions, applyDefault } from '../liquid-options'
import { flatten } from './flatten/node'
enum ParseState { HTML, OUTPUT, TAG }
@@ -36,7 +37,7 @@ export class Tokenizer {
}
if (state === ParseState.HTML) {
if (input.substr(p, outputDelimiterLeft.length) === outputDelimiterLeft) {
if (buffer) tokens.push(new HTMLToken(buffer, input, line, col, file))
if (buffer) tokens.push(new HTMLToken(flatten(buffer), input, line, col, file))
buffer = outputDelimiterLeft
line = curLine
col = p - lineBegin + 1
@@ -44,7 +45,7 @@ export class Tokenizer {
state = ParseState.OUTPUT
continue
} else if (input.substr(p, tagDelimiterLeft.length) === tagDelimiterLeft) {
if (buffer) tokens.push(new HTMLToken(buffer, input, line, col, file))
if (buffer) tokens.push(new HTMLToken(flatten(buffer), input, line, col, file))
buffer = tagDelimiterLeft
line = curLine
col = p - lineBegin + 1
@@ -57,7 +58,7 @@ export class Tokenizer {
input.substr(p, outputDelimiterRight.length) === outputDelimiterRight
) {
buffer += outputDelimiterRight
tokens.push(new OutputToken(buffer, buffer.slice(outputDelimiterLeft.length, -outputDelimiterRight.length), input, line, col, this.options, file))
tokens.push(new OutputToken(flatten(buffer), buffer.slice(outputDelimiterLeft.length, -outputDelimiterRight.length), input, line, col, this.options, file))
p += outputDelimiterRight.length
buffer = ''
line = curLine
@@ -66,7 +67,7 @@ export class Tokenizer {
continue
} else if (input.substr(p, tagDelimiterRight.length) === tagDelimiterRight) {
buffer += tagDelimiterRight
tokens.push(new TagToken(buffer, buffer.slice(tagDelimiterLeft.length, -tagDelimiterRight.length), input, line, col, this.options, file))
tokens.push(new TagToken(flatten(buffer), buffer.slice(tagDelimiterLeft.length, -tagDelimiterRight.length), input, line, col, this.options, file))
p += tagDelimiterRight.length
buffer = ''
line = curLine
@@ -81,10 +82,10 @@ export class Tokenizer {
const str = buffer.length > 16 ? buffer.slice(0, 13) + '...' : buffer
throw new TokenizationError(
`${t} "${str}" not closed`,
new Token(buffer, input, line, col, file)
new Token(flatten(buffer), input, line, col, file)
)
}
if (buffer) tokens.push(new HTMLToken(buffer, input, line, col, file))
if (buffer) tokens.push(new HTMLToken(flatten(buffer), input, line, col, file))
whiteSpaceCtrl(tokens, this.options)
return tokens