perf: introduce AST to avoid reparse

This commit is contained in:
harttle
2020-03-15 02:51:25 +08:00
committed by Jun Yang
parent 3b58f1c3f6
commit d2d6a38235
96 changed files with 1553 additions and 1168 deletions
+1 -1
View File
@@ -31,7 +31,7 @@ function slice<T> (v: T[], begin: number, length = 1): T[] {
function where<T extends object> (this: FilterImpl, arr: T[], property: string, expected?: any): T[] {
return arr.filter(obj => {
const value = this.context.getFromScope(obj, property)
const value = this.context.getFromScope(obj, property.split('.'))
return expected === undefined ? isTruthy(value) : value === expected
})
}
+7 -9
View File
@@ -1,15 +1,13 @@
import { assert } from '../../util/assert'
import { identifier } from '../../parser/lexical'
import { TagImplOptions, TagToken, Context } from '../../types'
const re = new RegExp(`(${identifier.source})\\s*=([^]*)`)
import { Tokenizer, assert, TagImplOptions, TagToken, Context } from '../../types'
export default {
parse: function (token: TagToken) {
const match = token.args.match(re) as RegExpMatchArray
assert(match, `illegal token ${token.raw}`)
this.key = match[1]
this.value = match[2]
const tokenizer = new Tokenizer(token.args)
this.key = tokenizer.readWord().content
tokenizer.skipBlank()
assert(tokenizer.peek() === '=', () => `illegal token ${token.getText()}`)
tokenizer.advance()
this.value = tokenizer.remaining()
},
render: function * (ctx: Context) {
ctx.bottom()[this.key] = yield this.liquid._evalValue(this.value, ctx)
+3 -3
View File
@@ -1,8 +1,8 @@
import BlockMode from '../../context/block-mode'
import { ParseStream, TagToken, Token, Template, Context, TagImplOptions, Emitter } from '../../types'
import { ParseStream, TagToken, TopLevelToken, Template, Context, TagImplOptions, Emitter } from '../../types'
export default {
parse: function (token: TagToken, remainTokens: Token[]) {
parse: function (token: TagToken, remainTokens: TopLevelToken[]) {
const match = /\w+/.exec(token.args)
this.block = match ? match[0] : ''
this.tpls = [] as Template[]
@@ -10,7 +10,7 @@ export default {
.on('tag:endblock', () => stream.stop())
.on('template', (tpl: Template) => this.tpls.push(tpl))
.on('end', () => {
throw new Error(`tag ${token.raw} not closed`)
throw new Error(`tag ${token.getText()} not closed`)
})
stream.start()
},
+6 -10
View File
@@ -1,22 +1,18 @@
import { assert } from '../../util/assert'
import { identifier } from '../../parser/lexical'
import { Template, Context, TagImplOptions, TagToken, Token } from '../../types'
const re = new RegExp(`(${identifier.source})`)
import { Tokenizer, assert, Template, Context, TagImplOptions, TagToken, TopLevelToken } from '../../types'
export default {
parse: function (tagToken: TagToken, remainTokens: Token[]) {
const match = tagToken.args.match(re) as RegExpMatchArray
assert(match, `${tagToken.args} not valid identifier`)
parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
const tokenizer = new Tokenizer(tagToken.args)
this.variable = tokenizer.readWord().content
assert(this.variable, () => `${tagToken.args} not valid identifier`)
this.variable = match[1]
this.templates = []
const stream = this.liquid.parser.parseStream(remainTokens)
stream.on('tag:endcapture', () => stream.stop())
.on('template', (tpl: Template) => this.templates.push(tpl))
.on('end', () => {
throw new Error(`tag ${tagToken.raw} not closed`)
throw new Error(`tag ${tagToken.getText()} not closed`)
})
stream.start()
},
+3 -3
View File
@@ -1,7 +1,7 @@
import { Expression, Emitter, TagToken, Token, Context, Template, TagImplOptions, ParseStream } from '../../types'
import { Expression, Emitter, TagToken, TopLevelToken, Context, Template, TagImplOptions, ParseStream } from '../../types'
export default {
parse: function (tagToken: TagToken, remainTokens: Token[]) {
parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
this.cond = tagToken.args
this.cases = []
this.elseTemplates = []
@@ -18,7 +18,7 @@ export default {
.on('tag:endcase', () => stream.stop())
.on('template', (tpl: Template) => p.push(tpl))
.on('end', () => {
throw new Error(`tag ${tagToken.raw} not closed`)
throw new Error(`tag ${tagToken.getText()} not closed`)
})
stream.start()
+4 -4
View File
@@ -1,16 +1,16 @@
import { TagToken } from '../../parser/tag-token'
import { Token } from '../../parser/token'
import { TagToken } from '../../tokens/tag-token'
import { TopLevelToken } from '../../tokens/toplevel-token'
import { TagImplOptions } from '../../template/tag/tag-impl-options'
export default {
parse: function (tagToken: TagToken, remainTokens: Token[]) {
parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
const stream = this.liquid.parser.parseStream(remainTokens)
stream
.on('token', (token: TagToken) => {
if (token.name === 'endcomment') stream.stop()
})
.on('end', () => {
throw new Error(`tag ${tagToken.raw} not closed`)
throw new Error(`tag ${tagToken.getText()} not closed`)
})
stream.start()
}
+20 -16
View File
@@ -1,28 +1,32 @@
import { assert } from '../../util/assert'
import { value as rValue } from '../../parser/lexical'
import { Emitter, Expression, TagToken, Context, TagImplOptions } from '../../types'
const groupRE = new RegExp(`^(?:(${rValue.source})\\s*:\\s*)?(.*)$`)
const candidatesRE = new RegExp(rValue.source, 'g')
import { evalToken, Emitter, TagToken, Context, TagImplOptions } from '../../types'
import { Tokenizer } from '../../parser/tokenizer'
export default {
parse: function (tagToken: TagToken) {
let match: RegExpExecArray | null = groupRE.exec(tagToken.args) as RegExpExecArray
assert(match, `illegal tag: ${tagToken.raw}`)
this.group = new Expression(match[1])
const candidates = match[2]
const tokenizer = new Tokenizer(tagToken.args)
const group = tokenizer.readValue()
tokenizer.skipBlank()
this.candidates = []
while ((match = candidatesRE.exec(candidates))) {
this.candidates.push(match[0])
if (group) {
if (tokenizer.peek() === ':') {
this.group = group
tokenizer.advance()
} else this.candidates.push(group)
}
assert(this.candidates.length, `empty candidates: ${tagToken.raw}`)
while (!tokenizer.end()) {
const value = tokenizer.readValue()
if (value) this.candidates.push(value)
tokenizer.readTo(',')
}
assert(this.candidates.length, () => `empty candidates: ${tagToken.getText()}`)
},
render: function * (ctx: Context, emitter: Emitter) {
const group = yield this.group.value(ctx)
render: function (ctx: Context, emitter: Emitter) {
const group = evalToken(this.group, ctx)
const fingerprint = `cycle:${group}:` + this.candidates.join(',')
const groups = ctx.getRegister('cycle')
let idx = groups[fingerprint]
@@ -34,7 +38,7 @@ export default {
const candidate = this.candidates[idx]
idx = (idx + 1) % this.candidates.length
groups[fingerprint] = idx
const html = yield new Expression(candidate).value(ctx)
const html = evalToken(candidate, ctx)
emitter.write(html)
}
} as TagImplOptions
+3 -6
View File
@@ -1,13 +1,10 @@
import { assert } from '../../util/assert'
import { identifier } from '../../parser/lexical'
import { Emitter, TagToken, Context, TagImplOptions } from '../../types'
import { Tokenizer, Emitter, TagToken, Context, TagImplOptions } from '../../types'
import { isNumber, stringify } from '../../util/underscore'
export default {
parse: function (token: TagToken) {
const match = token.args.match(identifier) as RegExpMatchArray
assert(match, `illegal identifier ${token.args}`)
this.variable = match[0]
const tokenizer = new Tokenizer(token.args)
this.variable = tokenizer.readWord().content
},
render: function (context: Context, emitter: Emitter) {
const scope = context.environments
+17 -15
View File
@@ -1,21 +1,24 @@
import { Emitter, TagToken, Token, Context, Template, TagImplOptions, ParseStream } from '../../types'
import { assert, Tokenizer, evalToken, Emitter, TagToken, TopLevelToken, Context, Template, TagImplOptions, ParseStream } from '../../types'
import { toCollection } from '../../util/collection'
import { Expression } from '../../render/expression'
import { assert } from '../../util/assert'
import { identifier, value } from '../../parser/lexical'
import { ForloopDrop } from '../../drop/forloop-drop'
import { Hash } from '../../template/tag/hash'
const re = new RegExp(`^(${identifier.source})\\s+in\\s+(${value.source})`)
export default {
type: 'block',
parse: function (tagToken: TagToken, remainTokens: Token[]) {
const match = re.exec(tagToken.args) as RegExpExecArray
assert(match, `illegal tag: ${tagToken.raw}`)
this.variable = match[1]
this.collection = match[2]
this.hash = new Hash(tagToken.args.slice(match[0].length))
parse: function (token: TagToken, remainTokens: TopLevelToken[]) {
const toknenizer = new Tokenizer(token.args)
const variable = toknenizer.readWord()
const inStr = toknenizer.readWord()
const collection = toknenizer.readValue()
assert(
variable.size() && inStr.content === 'in' && collection,
() => `illegal tag: ${token.getText()}`
)
this.variable = variable.content
this.collection = collection
this.hash = new Hash(toknenizer.remaining())
this.templates = []
this.elseTemplates = []
@@ -26,15 +29,14 @@ export default {
.on('tag:endfor', () => stream.stop())
.on('template', (tpl: Template) => p.push(tpl))
.on('end', () => {
throw new Error(`tag ${tagToken.raw} not closed`)
throw new Error(`tag ${token.getText()} not closed`)
})
stream.start()
},
render: function * (ctx: Context, emitter: Emitter) {
const r = this.liquid.renderer
let collection = yield new Expression(this.collection).value(ctx)
collection = toCollection(collection)
let collection = toCollection(evalToken(this.collection, ctx))
if (!collection.length) {
yield r.renderTemplates(this.elseTemplates, ctx, emitter)
+3 -3
View File
@@ -1,7 +1,7 @@
import { Emitter, isTruthy, Expression, TagToken, Token, Context, Template, TagImplOptions, ParseStream } from '../../types'
import { Emitter, isTruthy, Expression, TagToken, TopLevelToken, Context, Template, TagImplOptions, ParseStream } from '../../types'
export default {
parse: function (tagToken: TagToken, remainTokens: Token[]) {
parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
this.branches = []
this.elseTemplates = []
@@ -21,7 +21,7 @@ export default {
.on('tag:endif', () => stream.stop())
.on('template', (tpl: Template) => p.push(tpl))
.on('end', () => {
throw new Error(`tag ${tagToken.raw} not closed`)
throw new Error(`tag ${tagToken.getText()} not closed`)
})
stream.start()
+24 -18
View File
@@ -1,35 +1,41 @@
import { assert } from '../../util/assert'
import { Expression, Hash, Emitter, TagToken, Context, TagImplOptions } from '../../types'
import { quoted, value, quotedLine } from '../../parser/lexical'
import { assert, evalQuotedToken, TypeGuards, Tokenizer, evalToken, Hash, Emitter, TagToken, Context, TagImplOptions } from '../../types'
import BlockMode from '../../context/block-mode'
const rFile = new RegExp(`^(${quoted.source}|[^\\s,]+)(?:\\s+with\\s+(${value.source}))?`)
export default {
parse: function (token: TagToken) {
const match = rFile.exec(token.args)
if (!match) {
throw new Error(`illegal argument "${token.args}"`)
}
this.file = match[1]
this.hash = new Hash(token.args.slice(match[0].length))
this.withVar = match[2]
const args = token.args
const tokenizer = new Tokenizer(args)
this.file = this.liquid.options.dynamicPartials
? tokenizer.readValue()
: tokenizer.readFileName()
assert(this.file, () => `illegal argument "${token.args}"`)
const begin = tokenizer.p
const withStr = tokenizer.readWord()
if (withStr.content === 'with') {
tokenizer.skipBlank()
if (tokenizer.peek() !== ':') {
this.withVar = tokenizer.readValue()
} else tokenizer.p = begin
} else tokenizer.p = begin
this.hash = new Hash(tokenizer.remaining())
},
render: function * (ctx: Context, emitter: Emitter) {
const { liquid, hash, withVar, file } = this
const { renderer } = liquid
const filepath = ctx.opts.dynamicPartials
? (quotedLine.exec(file)
? yield renderer.renderTemplates(liquid.parse(file.slice(1, -1)), ctx)
: yield new Expression(file).value(ctx))
: file
assert(filepath, `illegal filename "${file}":"${filepath}"`)
? (TypeGuards.isQuotedToken(file)
? yield renderer.renderTemplates(liquid.parse(evalQuotedToken(file)), ctx)
: yield evalToken(file, ctx))
: file.getText()
assert(filepath, () => `illegal filename "${file}":"${filepath}"`)
const saved = ctx.saveRegister('blocks', 'blockMode')
ctx.setRegister('blocks', {})
ctx.setRegister('blockMode', BlockMode.OUTPUT)
const scope = yield hash.render(ctx)
if (withVar) scope[filepath] = yield new Expression(withVar).evaluate(ctx)
if (withVar) scope[filepath] = evalToken(withVar, ctx)
const templates = yield liquid._parseFile(filepath, ctx.opts, ctx.sync)
ctx.push(scope)
yield renderer.renderTemplates(templates, ctx, emitter)
+3 -6
View File
@@ -1,13 +1,10 @@
import { assert } from '../../util/assert'
import { identifier } from '../../parser/lexical'
import { isNumber, stringify } from '../../util/underscore'
import { Emitter, TagToken, Context, TagImplOptions } from '../../types'
import { Tokenizer, Emitter, TagToken, Context, TagImplOptions } from '../../types'
export default {
parse: function (token: TagToken) {
const match = token.args.match(identifier)
assert(match, `illegal identifier ${token.args}`)
this.variable = match![0]
const tokenizer = new Tokenizer(token.args)
this.variable = tokenizer.readWord().content
},
render: function (context: Context, emitter: Emitter) {
const scope = context.environments
+13 -17
View File
@@ -1,29 +1,25 @@
import { assert } from '../../util/assert'
import { quotedLine, quoted } from '../../parser/lexical'
import { Emitter, Hash, Expression, TagToken, Token, Context, TagImplOptions } from '../../types'
import { assert, evalQuotedToken, TypeGuards, evalToken, Tokenizer, Emitter, Hash, TagToken, TopLevelToken, Context, TagImplOptions } from '../../types'
import BlockMode from '../../context/block-mode'
const rFile = new RegExp(`^(${quoted.source}|[^\\s,]+)`)
export default {
parse: function (token: TagToken, remainTokens: Token[]) {
const match = rFile.exec(token.args)
if (!match) {
throw new Error(`illegal argument "${token.args}"`)
}
this.file = match[1]
this.hash = new Hash(token.args.slice(match[0].length))
parse: function (token: TagToken, remainTokens: TopLevelToken[]) {
const tokenizer = new Tokenizer(token.args)
const file = this.liquid.options.dynamicPartials ? tokenizer.readValue() : tokenizer.readFileName()
assert(file, () => `illegal argument "${token.args}"`)
this.file = file
this.hash = new Hash(tokenizer.remaining())
this.tpls = this.liquid.parser.parse(remainTokens)
},
render: function * (ctx: Context, emitter: Emitter) {
const { liquid, hash, file } = this
const { renderer } = liquid
const filepath = ctx.opts.dynamicPartials
? (quotedLine.exec(file)
? yield renderer.renderTemplates(liquid.parse(file.slice(1, -1)), ctx)
: yield new Expression(file).value(ctx))
: this.file
assert(filepath, `illegal filename "${file}":"${filepath}"`)
? (TypeGuards.isQuotedToken(file)
? yield renderer.renderTemplates(liquid.parse(evalQuotedToken(file)), ctx)
: evalToken(this.file, ctx))
: file.getText()
assert(filepath, () => `illegal filename "${file.getText()}":"${filepath}"`)
// render the remaining tokens immediately
ctx.setRegister('blockMode', BlockMode.STORE)
+4 -4
View File
@@ -1,7 +1,7 @@
import { TagToken, Token, TagImplOptions } from '../../types'
import { TagToken, TopLevelToken, TagImplOptions } from '../../types'
export default {
parse: function (tagToken: TagToken, remainTokens: Token[]) {
parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
this.tokens = []
const stream = this.liquid.parser.parseStream(remainTokens)
@@ -11,11 +11,11 @@ export default {
else this.tokens.push(token)
})
.on('end', () => {
throw new Error(`tag ${tagToken.raw} not closed`)
throw new Error(`tag ${tagToken.getText()} not closed`)
})
stream.start()
},
render: function () {
return this.tokens.map((token: Token) => token.raw).join('')
return this.tokens.map((token: TopLevelToken) => token.getText()).join('')
}
} as TagImplOptions
+45 -32
View File
@@ -1,56 +1,69 @@
import { assert } from '../../util/assert'
import { ForloopDrop } from '../../drop/forloop-drop'
import { toCollection } from '../../util/collection'
import { Expression, Hash, Emitter, TagToken, Context, TagImplOptions } from '../../types'
import { identifier, value, quoted, quotedLine } from '../../parser/lexical'
const rFile = new RegExp(`^(${quoted.source}|[^\\s,]+)`)
const rWith = new RegExp(`^\\s+with\\s+(${value.source})(?:\\s+as\\s+(${identifier.source}))?`)
const rFor = new RegExp(`^\\s+for\\s+(${value.source})\\s+as\\s+(${identifier.source})`)
import { evalQuotedToken, TypeGuards, Tokenizer, evalToken, Hash, Emitter, TagToken, Context, TagImplOptions } from '../../types'
export default {
parse: function (token: TagToken) {
let args = token.args
let match = rFile.exec(args)
const args = token.args
const tokenizer = new Tokenizer(args)
this.file = this.liquid.options.dynamicPartials
? tokenizer.readValue()
: tokenizer.readFileName()
assert(this.file, () => `illegal argument "${token.args}"`)
assert(match, `illegal argument "${token.args}"`)
this.file = match![1]
args = args.substr(match![0].length)
while (!tokenizer.end()) {
tokenizer.skipBlank()
const begin = tokenizer.p
const keyword = tokenizer.readWord()
if (keyword.content === 'with' || keyword.content === 'for') {
tokenizer.skipBlank()
if (tokenizer.peek() !== ':') {
const value = tokenizer.readValue()
if (value) {
const beforeAs = tokenizer.p
const asStr = tokenizer.readWord()
let alias
if (asStr.content === 'as') alias = tokenizer.readWord()
else tokenizer.p = beforeAs
while (true) {
if ((match = rWith.exec(args))) {
this.withVar = match[1]
this.withAs = match[2]
args = args.substr(match[0].length)
} else if ((match = rFor.exec(args))) {
this.forVar = match[1]
this.forAs = match[2]
args = args.substr(match[0].length)
} else break
this[keyword.content] = { value, alias: alias && alias.content }
tokenizer.skipBlank()
if (tokenizer.peek() === ',') tokenizer.advance()
continue
}
}
}
tokenizer.p = begin
break
}
this.hash = new Hash(args)
this.hash = new Hash(tokenizer.remaining())
},
render: function * (ctx: Context, emitter: Emitter) {
const { liquid, withVar, withAs, forVar, forAs, file, hash } = this
const { liquid, file, hash } = this
const { renderer } = liquid
const filepath = ctx.opts.dynamicPartials
? (quotedLine.exec(file)
? yield renderer.renderTemplates(liquid.parse(file.slice(1, -1)), ctx)
: yield new Expression(file).value(ctx))
: this.file
assert(filepath, `illegal filename "${file}":"${filepath}"`)
? (TypeGuards.isQuotedToken(file)
? yield renderer.renderTemplates(liquid.parse(evalQuotedToken(file)), ctx)
: evalToken(file, ctx))
: file.getText()
assert(filepath, () => `illegal filename "${file.getText()}":"${filepath}"`)
const childCtx = new Context({}, ctx.opts, ctx.sync)
const scope = yield hash.render(ctx)
if (withVar) scope[withAs || filepath] = yield new Expression(withVar).evaluate(ctx)
if (this['with']) {
const { value, alias } = this['with']
scope[alias || filepath] = evalToken(value, ctx)
}
childCtx.push(scope)
if (forVar) {
let collection = yield new Expression(forVar).value(ctx)
if (this['for']) {
const { value, alias } = this['for']
let collection = evalToken(value, ctx)
collection = toCollection(collection)
scope['forloop'] = new ForloopDrop(collection.length)
for (const item of collection) {
scope[forAs] = item
scope[alias] = item
const templates = yield liquid._parseFile(filepath, childCtx.opts, childCtx.sync)
yield renderer.renderTemplates(templates, childCtx, emitter)
scope.forloop.next()
+15 -15
View File
@@ -1,21 +1,21 @@
import { assert } from '../../util/assert'
import { toCollection } from '../../util/collection'
import { Expression, Emitter, Hash, TagToken, Token, Context, Template, TagImplOptions, ParseStream } from '../../types'
import { identifier, value } from '../../parser/lexical'
import { assert, evalToken, Emitter, Hash, TagToken, TopLevelToken, Context, Template, TagImplOptions, ParseStream } from '../../types'
import { TablerowloopDrop } from '../../drop/tablerowloop-drop'
const re = new RegExp(`^(${identifier.source})\\s+in\\s+` +
`(${value.source})`)
import { Tokenizer } from '../../parser/tokenizer'
export default {
parse: function (tagToken: TagToken, remainTokens: Token[]) {
const match = re.exec(tagToken.args) as RegExpExecArray
assert(match, `illegal tag: ${tagToken.raw}`)
parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
const tokenizer = new Tokenizer(tagToken.args)
this.variable = match[1]
this.collection = match[2]
this.variable = tokenizer.readWord()
tokenizer.skipBlank()
const tmp = tokenizer.readWord()
assert(tmp && tmp.content === 'in', () => `illegal tag: ${tagToken.getText()}`)
this.collection = tokenizer.readValue()
this.hash = new Hash(tokenizer.remaining())
this.templates = []
this.hash = new Hash(tagToken.args.slice(match[0].length))
let p
const stream: ParseStream = this.liquid.parser.parseStream(remainTokens)
@@ -23,14 +23,14 @@ export default {
.on('tag:endtablerow', () => stream.stop())
.on('template', (tpl: Template) => p.push(tpl))
.on('end', () => {
throw new Error(`tag ${tagToken.raw} not closed`)
throw new Error(`tag ${tagToken.getText()} not closed`)
})
stream.start()
},
render: function * (ctx: Context, emitter: Emitter) {
let collection = toCollection(yield new Expression(this.collection).value(ctx))
let collection = toCollection(evalToken(this.collection, ctx))
const hash = yield this.hash.render(ctx)
const offset = hash.offset || 0
const limit = (hash.limit === undefined) ? collection.length : hash.limit
@@ -44,7 +44,7 @@ export default {
ctx.push(scope)
for (let idx = 0; idx < collection.length; idx++, tablerowloop.next()) {
scope[this.variable] = collection[idx]
scope[this.variable.content] = collection[idx]
if (tablerowloop.col0() === 0) {
if (tablerowloop.row() !== 1) emitter.write('</tr>')
emitter.write(`<tr class="row${tablerowloop.row()}">`)
+4 -4
View File
@@ -1,7 +1,7 @@
import { Emitter, Expression, isFalsy, ParseStream, Context, TagImplOptions, Token, TagToken } from '../../types'
import { TopLevelToken, Template, Emitter, Expression, isFalsy, ParseStream, Context, TagImplOptions, Token, TagToken } from '../../types'
export default {
parse: function (tagToken: TagToken, remainTokens: Token[]) {
parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
this.templates = []
this.elseTemplates = []
let p
@@ -12,9 +12,9 @@ export default {
})
.on('tag:else', () => (p = this.elseTemplates))
.on('tag:endunless', () => stream.stop())
.on('template', tpl => p.push(tpl))
.on('template', (tpl: Template) => p.push(tpl))
.on('end', () => {
throw new Error(`tag ${tagToken.raw} not closed`)
throw new Error(`tag ${tagToken.getText()} not closed`)
})
stream.start()