Labyricorn/liquidjs
1
0
Fork 0
mirror of https://github.com/harttle/liquidjs.git synced 2026-09-18 05:50:43 -07:00
Code Issues Packages Projects Releases Wiki Activity

perf: introduce AST to avoid reparse

Browse Source
This commit is contained in:
harttle
2020-03-15 02:51:25 +08:00
committed by Jun Yang
parent 3b58f1c3f6
commit d2d6a38235
96 changed files with 1553 additions and 1168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/builtin/filters/array.ts
+1 -1
View File
@@ -31,7 +31,7 @@ function slice<T> (v: T[], begin: number, length = 1): T[] {
function where<T extends object> (this: FilterImpl, arr: T[], property: string, expected?: any): T[] {
return arr.filter(obj => {
const value = this.context.getFromScope(obj, property)
const value = this.context.getFromScope(obj, property.split('.'))
return expected === undefined ? isTruthy(value) : value === expected
})
}
src/builtin/tags/assign.ts
+7 -9
View File
@@ -1,15 +1,13 @@
import { assert } from '../../util/assert'
import { identifier } from '../../parser/lexical'
import { TagImplOptions, TagToken, Context } from '../../types'
const re = new RegExp(`(${identifier.source})\\s*=([^]*)`)
import { Tokenizer, assert, TagImplOptions, TagToken, Context } from '../../types'
export default {
parse: function (token: TagToken) {
const match = token.args.match(re) as RegExpMatchArray
assert(match, `illegal token ${token.raw}`)
this.key = match[1]
this.value = match[2]
const tokenizer = new Tokenizer(token.args)
this.key = tokenizer.readWord().content
tokenizer.skipBlank()
assert(tokenizer.peek() === '=', () => `illegal token ${token.getText()}`)
tokenizer.advance()
this.value = tokenizer.remaining()
},
render: function * (ctx: Context) {
ctx.bottom()[this.key] = yield this.liquid._evalValue(this.value, ctx)
src/builtin/tags/block.ts
+3 -3
View File
@@ -1,8 +1,8 @@
import BlockMode from '../../context/block-mode'
import { ParseStream, TagToken, Token, Template, Context, TagImplOptions, Emitter } from '../../types'
import { ParseStream, TagToken, TopLevelToken, Template, Context, TagImplOptions, Emitter } from '../../types'
export default {
parse: function (token: TagToken, remainTokens: Token[]) {
parse: function (token: TagToken, remainTokens: TopLevelToken[]) {
const match = /\w+/.exec(token.args)
this.block = match ? match[0] : ''
this.tpls = [] as Template[]
@@ -10,7 +10,7 @@ export default {
.on('tag:endblock', () => stream.stop())
.on('template', (tpl: Template) => this.tpls.push(tpl))
.on('end', () => {
throw new Error(`tag ${token.raw} not closed`)
throw new Error(`tag ${token.getText()} not closed`)
})
stream.start()
},
src/builtin/tags/capture.ts
+6 -10
View File
@@ -1,22 +1,18 @@
import { assert } from '../../util/assert'
import { identifier } from '../../parser/lexical'
import { Template, Context, TagImplOptions, TagToken, Token } from '../../types'
const re = new RegExp(`(${identifier.source})`)
import { Tokenizer, assert, Template, Context, TagImplOptions, TagToken, TopLevelToken } from '../../types'
export default {
parse: function (tagToken: TagToken, remainTokens: Token[]) {
const match = tagToken.args.match(re) as RegExpMatchArray
assert(match, `${tagToken.args} not valid identifier`)
parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
const tokenizer = new Tokenizer(tagToken.args)
this.variable = tokenizer.readWord().content
assert(this.variable, () => `${tagToken.args} not valid identifier`)
this.variable = match[1]
this.templates = []
const stream = this.liquid.parser.parseStream(remainTokens)
stream.on('tag:endcapture', () => stream.stop())
.on('template', (tpl: Template) => this.templates.push(tpl))
.on('end', () => {
throw new Error(`tag ${tagToken.raw} not closed`)
throw new Error(`tag ${tagToken.getText()} not closed`)
})
stream.start()
},
src/builtin/tags/case.ts
+3 -3
View File
@@ -1,7 +1,7 @@
import { Expression, Emitter, TagToken, Token, Context, Template, TagImplOptions, ParseStream } from '../../types'
import { Expression, Emitter, TagToken, TopLevelToken, Context, Template, TagImplOptions, ParseStream } from '../../types'
export default {
parse: function (tagToken: TagToken, remainTokens: Token[]) {
parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
this.cond = tagToken.args
this.cases = []
this.elseTemplates = []
@@ -18,7 +18,7 @@ export default {
.on('tag:endcase', () => stream.stop())
.on('template', (tpl: Template) => p.push(tpl))
.on('end', () => {
throw new Error(`tag ${tagToken.raw} not closed`)
throw new Error(`tag ${tagToken.getText()} not closed`)
})
stream.start()
src/builtin/tags/comment.ts
+4 -4
View File
@@ -1,16 +1,16 @@
import { TagToken } from '../../parser/tag-token'
import { Token } from '../../parser/token'
import { TagToken } from '../../tokens/tag-token'
import { TopLevelToken } from '../../tokens/toplevel-token'
import { TagImplOptions } from '../../template/tag/tag-impl-options'
export default {
parse: function (tagToken: TagToken, remainTokens: Token[]) {
parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
const stream = this.liquid.parser.parseStream(remainTokens)
stream
.on('token', (token: TagToken) => {
if (token.name === 'endcomment') stream.stop()
})
.on('end', () => {
throw new Error(`tag ${tagToken.raw} not closed`)
throw new Error(`tag ${tagToken.getText()} not closed`)
})
stream.start()
}
src/builtin/tags/cycle.ts
+20 -16
View File
@@ -1,28 +1,32 @@
import { assert } from '../../util/assert'
import { value as rValue } from '../../parser/lexical'
import { Emitter, Expression, TagToken, Context, TagImplOptions } from '../../types'
const groupRE = new RegExp(`^(?:(${rValue.source})\\s*:\\s*)?(.*)$`)
const candidatesRE = new RegExp(rValue.source, 'g')
import { evalToken, Emitter, TagToken, Context, TagImplOptions } from '../../types'
import { Tokenizer } from '../../parser/tokenizer'
export default {
parse: function (tagToken: TagToken) {
let match: RegExpExecArray | null = groupRE.exec(tagToken.args) as RegExpExecArray
assert(match, `illegal tag: ${tagToken.raw}`)
this.group = new Expression(match[1])
const candidates = match[2]
const tokenizer = new Tokenizer(tagToken.args)
const group = tokenizer.readValue()
tokenizer.skipBlank()
this.candidates = []
while ((match = candidatesRE.exec(candidates))) {
this.candidates.push(match[0])
if (group) {
if (tokenizer.peek() === ':') {
this.group = group
tokenizer.advance()
} else this.candidates.push(group)
}
assert(this.candidates.length, `empty candidates: ${tagToken.raw}`)
while (!tokenizer.end()) {
const value = tokenizer.readValue()
if (value) this.candidates.push(value)
tokenizer.readTo(',')
}
assert(this.candidates.length, () => `empty candidates: ${tagToken.getText()}`)
},
render: function * (ctx: Context, emitter: Emitter) {
const group = yield this.group.value(ctx)
render: function (ctx: Context, emitter: Emitter) {
const group = evalToken(this.group, ctx)
const fingerprint = `cycle:${group}:` + this.candidates.join(',')
const groups = ctx.getRegister('cycle')
let idx = groups[fingerprint]
@@ -34,7 +38,7 @@ export default {
const candidate = this.candidates[idx]
idx = (idx + 1) % this.candidates.length
groups[fingerprint] = idx
const html = yield new Expression(candidate).value(ctx)
const html = evalToken(candidate, ctx)
emitter.write(html)
}
} as TagImplOptions
src/builtin/tags/decrement.ts
+3 -6
View File
@@ -1,13 +1,10 @@
import { assert } from '../../util/assert'
import { identifier } from '../../parser/lexical'
import { Emitter, TagToken, Context, TagImplOptions } from '../../types'
import { Tokenizer, Emitter, TagToken, Context, TagImplOptions } from '../../types'
import { isNumber, stringify } from '../../util/underscore'
export default {
parse: function (token: TagToken) {
const match = token.args.match(identifier) as RegExpMatchArray
assert(match, `illegal identifier ${token.args}`)
this.variable = match[0]
const tokenizer = new Tokenizer(token.args)
this.variable = tokenizer.readWord().content
},
render: function (context: Context, emitter: Emitter) {
const scope = context.environments
src/builtin/tags/for.ts
+17 -15
View File
@@ -1,21 +1,24 @@
import { Emitter, TagToken, Token, Context, Template, TagImplOptions, ParseStream } from '../../types'
import { assert, Tokenizer, evalToken, Emitter, TagToken, TopLevelToken, Context, Template, TagImplOptions, ParseStream } from '../../types'
import { toCollection } from '../../util/collection'
import { Expression } from '../../render/expression'
import { assert } from '../../util/assert'
import { identifier, value } from '../../parser/lexical'
import { ForloopDrop } from '../../drop/forloop-drop'
import { Hash } from '../../template/tag/hash'
const re = new RegExp(`^(${identifier.source})\\s+in\\s+(${value.source})`)
export default {
type: 'block',
parse: function (tagToken: TagToken, remainTokens: Token[]) {
const match = re.exec(tagToken.args) as RegExpExecArray
assert(match, `illegal tag: ${tagToken.raw}`)
this.variable = match[1]
this.collection = match[2]
this.hash = new Hash(tagToken.args.slice(match[0].length))
parse: function (token: TagToken, remainTokens: TopLevelToken[]) {
const toknenizer = new Tokenizer(token.args)
const variable = toknenizer.readWord()
const inStr = toknenizer.readWord()
const collection = toknenizer.readValue()
assert(
variable.size() && inStr.content === 'in' && collection,
() => `illegal tag: ${token.getText()}`
)
this.variable = variable.content
this.collection = collection
this.hash = new Hash(toknenizer.remaining())
this.templates = []
this.elseTemplates = []
@@ -26,15 +29,14 @@ export default {
.on('tag:endfor', () => stream.stop())
.on('template', (tpl: Template) => p.push(tpl))
.on('end', () => {
throw new Error(`tag ${tagToken.raw} not closed`)
throw new Error(`tag ${token.getText()} not closed`)
})
stream.start()
},
render: function * (ctx: Context, emitter: Emitter) {
const r = this.liquid.renderer
let collection = yield new Expression(this.collection).value(ctx)
collection = toCollection(collection)
let collection = toCollection(evalToken(this.collection, ctx))
if (!collection.length) {
yield r.renderTemplates(this.elseTemplates, ctx, emitter)
src/builtin/tags/if.ts
+3 -3
View File
@@ -1,7 +1,7 @@
import { Emitter, isTruthy, Expression, TagToken, Token, Context, Template, TagImplOptions, ParseStream } from '../../types'
import { Emitter, isTruthy, Expression, TagToken, TopLevelToken, Context, Template, TagImplOptions, ParseStream } from '../../types'
export default {
parse: function (tagToken: TagToken, remainTokens: Token[]) {
parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
this.branches = []
this.elseTemplates = []
@@ -21,7 +21,7 @@ export default {
.on('tag:endif', () => stream.stop())
.on('template', (tpl: Template) => p.push(tpl))
.on('end', () => {
throw new Error(`tag ${tagToken.raw} not closed`)
throw new Error(`tag ${tagToken.getText()} not closed`)
})
stream.start()
src/builtin/tags/include.ts
+24 -18
View File
@@ -1,35 +1,41 @@
import { assert } from '../../util/assert'
import { Expression, Hash, Emitter, TagToken, Context, TagImplOptions } from '../../types'
import { quoted, value, quotedLine } from '../../parser/lexical'
import { assert, evalQuotedToken, TypeGuards, Tokenizer, evalToken, Hash, Emitter, TagToken, Context, TagImplOptions } from '../../types'
import BlockMode from '../../context/block-mode'
const rFile = new RegExp(`^(${quoted.source}|[^\\s,]+)(?:\\s+with\\s+(${value.source}))?`)
export default {
parse: function (token: TagToken) {
const match = rFile.exec(token.args)
if (!match) {
throw new Error(`illegal argument "${token.args}"`)
}
this.file = match[1]
this.hash = new Hash(token.args.slice(match[0].length))
this.withVar = match[2]
const args = token.args
const tokenizer = new Tokenizer(args)
this.file = this.liquid.options.dynamicPartials
? tokenizer.readValue()
: tokenizer.readFileName()
assert(this.file, () => `illegal argument "${token.args}"`)
const begin = tokenizer.p
const withStr = tokenizer.readWord()
if (withStr.content === 'with') {
tokenizer.skipBlank()
if (tokenizer.peek() !== ':') {
this.withVar = tokenizer.readValue()
} else tokenizer.p = begin
} else tokenizer.p = begin
this.hash = new Hash(tokenizer.remaining())
},
render: function * (ctx: Context, emitter: Emitter) {
const { liquid, hash, withVar, file } = this
const { renderer } = liquid
const filepath = ctx.opts.dynamicPartials
? (quotedLine.exec(file)
? yield renderer.renderTemplates(liquid.parse(file.slice(1, -1)), ctx)
: yield new Expression(file).value(ctx))
: file
assert(filepath, `illegal filename "${file}":"${filepath}"`)
? (TypeGuards.isQuotedToken(file)
? yield renderer.renderTemplates(liquid.parse(evalQuotedToken(file)), ctx)
: yield evalToken(file, ctx))
: file.getText()
assert(filepath, () => `illegal filename "${file}":"${filepath}"`)
const saved = ctx.saveRegister('blocks', 'blockMode')
ctx.setRegister('blocks', {})
ctx.setRegister('blockMode', BlockMode.OUTPUT)
const scope = yield hash.render(ctx)
if (withVar) scope[filepath] = yield new Expression(withVar).evaluate(ctx)
if (withVar) scope[filepath] = evalToken(withVar, ctx)
const templates = yield liquid._parseFile(filepath, ctx.opts, ctx.sync)
ctx.push(scope)
yield renderer.renderTemplates(templates, ctx, emitter)
src/builtin/tags/increment.ts
+3 -6
View File
@@ -1,13 +1,10 @@
import { assert } from '../../util/assert'
import { identifier } from '../../parser/lexical'
import { isNumber, stringify } from '../../util/underscore'
import { Emitter, TagToken, Context, TagImplOptions } from '../../types'
import { Tokenizer, Emitter, TagToken, Context, TagImplOptions } from '../../types'
export default {
parse: function (token: TagToken) {
const match = token.args.match(identifier)
assert(match, `illegal identifier ${token.args}`)
this.variable = match![0]
const tokenizer = new Tokenizer(token.args)
this.variable = tokenizer.readWord().content
},
render: function (context: Context, emitter: Emitter) {
const scope = context.environments
src/builtin/tags/layout.ts
+13 -17
View File
@@ -1,29 +1,25 @@
import { assert } from '../../util/assert'
import { quotedLine, quoted } from '../../parser/lexical'
import { Emitter, Hash, Expression, TagToken, Token, Context, TagImplOptions } from '../../types'
import { assert, evalQuotedToken, TypeGuards, evalToken, Tokenizer, Emitter, Hash, TagToken, TopLevelToken, Context, TagImplOptions } from '../../types'
import BlockMode from '../../context/block-mode'
const rFile = new RegExp(`^(${quoted.source}|[^\\s,]+)`)
export default {
parse: function (token: TagToken, remainTokens: Token[]) {
const match = rFile.exec(token.args)
if (!match) {
throw new Error(`illegal argument "${token.args}"`)
}
this.file = match[1]
this.hash = new Hash(token.args.slice(match[0].length))
parse: function (token: TagToken, remainTokens: TopLevelToken[]) {
const tokenizer = new Tokenizer(token.args)
const file = this.liquid.options.dynamicPartials ? tokenizer.readValue() : tokenizer.readFileName()
assert(file, () => `illegal argument "${token.args}"`)
this.file = file
this.hash = new Hash(tokenizer.remaining())
this.tpls = this.liquid.parser.parse(remainTokens)
},
render: function * (ctx: Context, emitter: Emitter) {
const { liquid, hash, file } = this
const { renderer } = liquid
const filepath = ctx.opts.dynamicPartials
? (quotedLine.exec(file)
? yield renderer.renderTemplates(liquid.parse(file.slice(1, -1)), ctx)
: yield new Expression(file).value(ctx))
: this.file
assert(filepath, `illegal filename "${file}":"${filepath}"`)
? (TypeGuards.isQuotedToken(file)
? yield renderer.renderTemplates(liquid.parse(evalQuotedToken(file)), ctx)
: evalToken(this.file, ctx))
: file.getText()
assert(filepath, () => `illegal filename "${file.getText()}":"${filepath}"`)
// render the remaining tokens immediately
ctx.setRegister('blockMode', BlockMode.STORE)
src/builtin/tags/raw.ts
+4 -4
View File
@@ -1,7 +1,7 @@
import { TagToken, Token, TagImplOptions } from '../../types'
import { TagToken, TopLevelToken, TagImplOptions } from '../../types'
export default {
parse: function (tagToken: TagToken, remainTokens: Token[]) {
parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
this.tokens = []
const stream = this.liquid.parser.parseStream(remainTokens)
@@ -11,11 +11,11 @@ export default {
else this.tokens.push(token)
})
.on('end', () => {
throw new Error(`tag ${tagToken.raw} not closed`)
throw new Error(`tag ${tagToken.getText()} not closed`)
})
stream.start()
},
render: function () {
return this.tokens.map((token: Token) => token.raw).join('')
return this.tokens.map((token: TopLevelToken) => token.getText()).join('')
}
} as TagImplOptions
src/builtin/tags/render.ts
+45 -32
View File
@@ -1,56 +1,69 @@
import { assert } from '../../util/assert'
import { ForloopDrop } from '../../drop/forloop-drop'
import { toCollection } from '../../util/collection'
import { Expression, Hash, Emitter, TagToken, Context, TagImplOptions } from '../../types'
import { identifier, value, quoted, quotedLine } from '../../parser/lexical'
const rFile = new RegExp(`^(${quoted.source}|[^\\s,]+)`)
const rWith = new RegExp(`^\\s+with\\s+(${value.source})(?:\\s+as\\s+(${identifier.source}))?`)
const rFor = new RegExp(`^\\s+for\\s+(${value.source})\\s+as\\s+(${identifier.source})`)
import { evalQuotedToken, TypeGuards, Tokenizer, evalToken, Hash, Emitter, TagToken, Context, TagImplOptions } from '../../types'
export default {
parse: function (token: TagToken) {
let args = token.args
let match = rFile.exec(args)
const args = token.args
const tokenizer = new Tokenizer(args)
this.file = this.liquid.options.dynamicPartials
? tokenizer.readValue()
: tokenizer.readFileName()
assert(this.file, () => `illegal argument "${token.args}"`)
assert(match, `illegal argument "${token.args}"`)
this.file = match![1]
args = args.substr(match![0].length)
while (!tokenizer.end()) {
tokenizer.skipBlank()
const begin = tokenizer.p
const keyword = tokenizer.readWord()
if (keyword.content === 'with' || keyword.content === 'for') {
tokenizer.skipBlank()
if (tokenizer.peek() !== ':') {
const value = tokenizer.readValue()
if (value) {
const beforeAs = tokenizer.p
const asStr = tokenizer.readWord()
let alias
if (asStr.content === 'as') alias = tokenizer.readWord()
else tokenizer.p = beforeAs
while (true) {
if ((match = rWith.exec(args))) {
this.withVar = match[1]
this.withAs = match[2]
args = args.substr(match[0].length)
} else if ((match = rFor.exec(args))) {
this.forVar = match[1]
this.forAs = match[2]
args = args.substr(match[0].length)
} else break
this[keyword.content] = { value, alias: alias && alias.content }
tokenizer.skipBlank()
if (tokenizer.peek() === ',') tokenizer.advance()
continue
}
}
}
tokenizer.p = begin
break
}
this.hash = new Hash(args)
this.hash = new Hash(tokenizer.remaining())
},
render: function * (ctx: Context, emitter: Emitter) {
const { liquid, withVar, withAs, forVar, forAs, file, hash } = this
const { liquid, file, hash } = this
const { renderer } = liquid
const filepath = ctx.opts.dynamicPartials
? (quotedLine.exec(file)
? yield renderer.renderTemplates(liquid.parse(file.slice(1, -1)), ctx)
: yield new Expression(file).value(ctx))
: this.file
assert(filepath, `illegal filename "${file}":"${filepath}"`)
? (TypeGuards.isQuotedToken(file)
? yield renderer.renderTemplates(liquid.parse(evalQuotedToken(file)), ctx)
: evalToken(file, ctx))
: file.getText()
assert(filepath, () => `illegal filename "${file.getText()}":"${filepath}"`)
const childCtx = new Context({}, ctx.opts, ctx.sync)
const scope = yield hash.render(ctx)
if (withVar) scope[withAs || filepath] = yield new Expression(withVar).evaluate(ctx)
if (this['with']) {
const { value, alias } = this['with']
scope[alias || filepath] = evalToken(value, ctx)
}
childCtx.push(scope)
if (forVar) {
let collection = yield new Expression(forVar).value(ctx)
if (this['for']) {
const { value, alias } = this['for']
let collection = evalToken(value, ctx)
collection = toCollection(collection)
scope['forloop'] = new ForloopDrop(collection.length)
for (const item of collection) {
scope[forAs] = item
scope[alias] = item
const templates = yield liquid._parseFile(filepath, childCtx.opts, childCtx.sync)
yield renderer.renderTemplates(templates, childCtx, emitter)
scope.forloop.next()
src/builtin/tags/tablerow.ts
+15 -15
View File
@@ -1,21 +1,21 @@
import { assert } from '../../util/assert'
import { toCollection } from '../../util/collection'
import { Expression, Emitter, Hash, TagToken, Token, Context, Template, TagImplOptions, ParseStream } from '../../types'
import { identifier, value } from '../../parser/lexical'
import { assert, evalToken, Emitter, Hash, TagToken, TopLevelToken, Context, Template, TagImplOptions, ParseStream } from '../../types'
import { TablerowloopDrop } from '../../drop/tablerowloop-drop'
const re = new RegExp(`^(${identifier.source})\\s+in\\s+` +
`(${value.source})`)
import { Tokenizer } from '../../parser/tokenizer'
export default {
parse: function (tagToken: TagToken, remainTokens: Token[]) {
const match = re.exec(tagToken.args) as RegExpExecArray
assert(match, `illegal tag: ${tagToken.raw}`)
parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
const tokenizer = new Tokenizer(tagToken.args)
this.variable = match[1]
this.collection = match[2]
this.variable = tokenizer.readWord()
tokenizer.skipBlank()
const tmp = tokenizer.readWord()
assert(tmp && tmp.content === 'in', () => `illegal tag: ${tagToken.getText()}`)
this.collection = tokenizer.readValue()
this.hash = new Hash(tokenizer.remaining())
this.templates = []
this.hash = new Hash(tagToken.args.slice(match[0].length))
let p
const stream: ParseStream = this.liquid.parser.parseStream(remainTokens)
@@ -23,14 +23,14 @@ export default {
.on('tag:endtablerow', () => stream.stop())
.on('template', (tpl: Template) => p.push(tpl))
.on('end', () => {
throw new Error(`tag ${tagToken.raw} not closed`)
throw new Error(`tag ${tagToken.getText()} not closed`)
})
stream.start()
},
render: function * (ctx: Context, emitter: Emitter) {
let collection = toCollection(yield new Expression(this.collection).value(ctx))
let collection = toCollection(evalToken(this.collection, ctx))
const hash = yield this.hash.render(ctx)
const offset = hash.offset || 0
const limit = (hash.limit === undefined) ? collection.length : hash.limit
@@ -44,7 +44,7 @@ export default {
ctx.push(scope)
for (let idx = 0; idx < collection.length; idx++, tablerowloop.next()) {
scope[this.variable] = collection[idx]
scope[this.variable.content] = collection[idx]
if (tablerowloop.col0() === 0) {
if (tablerowloop.row() !== 1) emitter.write('</tr>')
emitter.write(`<tr class="row${tablerowloop.row()}">`)
src/builtin/tags/unless.ts
+4 -4
View File
@@ -1,7 +1,7 @@
import { Emitter, Expression, isFalsy, ParseStream, Context, TagImplOptions, Token, TagToken } from '../../types'
import { TopLevelToken, Template, Emitter, Expression, isFalsy, ParseStream, Context, TagImplOptions, Token, TagToken } from '../../types'
export default {
parse: function (tagToken: TagToken, remainTokens: Token[]) {
parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
this.templates = []
this.elseTemplates = []
let p
@@ -12,9 +12,9 @@ export default {
})
.on('tag:else', () => (p = this.elseTemplates))
.on('tag:endunless', () => stream.stop())
.on('template', tpl => p.push(tpl))
.on('template', (tpl: Template) => p.push(tpl))
.on('end', () => {
throw new Error(`tag ${tagToken.raw} not closed`)
throw new Error(`tag ${tagToken.getText()} not closed`)
})
stream.start()
src/context/context.ts
+3 -77
View File
@@ -1,6 +1,5 @@
import { Drop } from '../drop/drop'
import { __assign } from 'tslib'
import { assert } from '../util/assert'
import { NormalizedFullOptions, defaultOptions } from '../liquid-options'
import { Scope } from './scope'
import { isArray, isNil, isString, isFunction, toLiquid } from '../util/underscore'
@@ -34,13 +33,12 @@ export class Context {
return [this.globals, this.environments, ...this.scopes]
.reduce((ctx, val) => __assign(ctx, val), {})
}
public get (path: string) {
const paths = this.parseProp(path)
public get (paths: string[]) {
const scope = this.findScope(paths[0])
return this.getFromScope(scope, paths)
}
public getFromScope (scope: object, paths: string[] | string) {
if (!isArray(paths)) paths = this.parseProp(paths)
if (typeof paths === 'string') paths = paths.split('.')
return paths.reduce((scope, path) => {
scope = readProperty(scope, path)
if (isNil(scope) && this.opts.strictVariables) {
@@ -61,67 +59,11 @@ export class Context {
private findScope (key: string) {
for (let i = this.scopes.length - 1; i >= 0; i--) {
const candidate = this.scopes[i]
if (key in candidate) {
return candidate
}
if (key in candidate) return candidate
}
if (key in this.environments) return this.environments
return this.globals
}
/*
* Parse property access sequence from access string
* @example
* accessSeq("foo.bar") // ['foo', 'bar']
* accessSeq("foo['bar']") // ['foo', 'bar']
* accessSeq("foo['b]r']") // ['foo', 'b]r']
* accessSeq("foo[bar.coo]") // ['foo', 'bar'], for bar.coo == 'bar'
*/
private parseProp (str: string) {
str = String(str)
const seq: string[] = []
const push = () => name.length && (seq.push(name), (name = ''))
let name = ''
let j
let i = 0
while (i < str.length) {
switch (str[i]) {
case '[':
push()
const delemiter = str[i + 1]
if (/['"]/.test(delemiter)) { // foo["bar"]
j = str.indexOf(delemiter, i + 2)
assert(j !== -1, `unbalanced ${delemiter}: ${str}`)
name = str.slice(i + 2, j)
push()
i = j + 2
} else { // foo[bar.coo]
j = matchRightBracket(str, i + 1)
assert(j !== -1, `unbalanced []: ${str}`)
name = str.slice(i + 1, j)
if (!/^[+-]?\d+$/.test(name)) { // foo[bar] vs. foo[1]
name = String(this.get(name))
}
push()
i = j + 1
}
break
case '.':// foo.bar, foo[0].bar
push()
i++
break
default:// foo.bar
name += str[i++]
}
}
push()
if (!seq.length) {
throw new TypeError(`invalid path:"${str}"`)
}
return seq
}
}
export function readProperty (obj: Scope, key: string) {
@@ -152,19 +94,3 @@ function readSize (obj: Scope) {
if (isArray(obj) || isString(obj)) return obj.length
return obj['size']
}
function matchRightBracket (str: string, begin: number) {
let stack = 1 // count of '[' - count of ']'
for (let i = begin; i < str.length; i++) {
if (str[i] === '[') {
stack++
}
if (str[i] === ']') {
stack--
if (stack === 0) {
return i
}
}
}
return -1
}
src/liquid.ts
+2 -2
View File
@@ -38,8 +38,8 @@ export class Liquid {
_.forOwn(builtinFilters, (handler, name) => this.registerFilter(name, handler))
}
public parse (html: string, filepath?: string): Template[] {
const tokenizer = new Tokenizer(html, filepath, this.options)
const tokens = tokenizer.readTokens()
const tokenizer = new Tokenizer(html, filepath)
const tokens = tokenizer.readTopLevelTokens(this.options)
return this.parser.parse(tokens)
}
src/parser/filter-arg.ts
+4 -3
View File
@@ -1,9 +1,10 @@
import { isArray } from '../util/underscore'
import { ValueToken } from '../tokens/value-token'
type KeyValuePair = [string?, string?]
type KeyValuePair = [string?, ValueToken?]
export type FilterArg = string|KeyValuePair
export type FilterArg = ValueToken | KeyValuePair
export function isKeyValuePair (arr: FilterArg): arr is KeyValuePair {
export function isKeyValuePair (arr: FilterArg): arr is KeyValuePair { // TODO check
return isArray(arr)
}
src/parser/html-token.ts
-11
View File
@@ -1,11 +0,0 @@
import { Token } from './token'
export class HTMLToken extends Token {
public constructor (str: string, input: string, line: number, col: number, file?: string) {
super(str, input, line, col, file)
this.content = str
}
public static is (token: Token): token is HTMLToken {
return token instanceof HTMLToken
}
}
src/parser/lexical.ts
-29
View File
@@ -1,29 +0,0 @@
// quote related
const singleQuoted = /'[^']*'/
const doubleQuoted = /"[^"]*"/
export const quoted = new RegExp(`${singleQuoted.source}|${doubleQuoted.source}`)
export const quoteBalanced = new RegExp(`(?:${quoted.source}|[^'"])*`)
// basic types
export const number = /[+-]?(?:\d+\.?\d*|\.?\d+)/
export const bool = /true|false/
// property access
export const identifier = /[\w-]+[?]?/
export const subscript = new RegExp(`\\[(?:${quoted.source}|[\\w-\\.]+)\\]`)
export const literal = new RegExp(`(?:${quoted.source}|${bool.source}|${number.source})`)
export const variable = new RegExp(`${identifier.source}(?:\\.${identifier.source}|${subscript.source})*`)
// range related
export const rangeLimit = new RegExp(`(?:${variable.source}|${number.source})`)
export const range = new RegExp(`\\(${rangeLimit.source}\\.\\.${rangeLimit.source}\\)`)
export const rangeCapture = new RegExp(`\\((${rangeLimit.source})\\.\\.(${rangeLimit.source})\\)`)
export const value = new RegExp(`(?:${variable.source}|${literal.source}|${range.source})`)
// full match
export const tagLine = new RegExp(`^\\s*(${identifier.source})\\s*([\\s\\S]*?)\\s*$`)
export const numberLine = new RegExp(`^${number.source}$`)
export const boolLine = new RegExp(`^${bool.source}$`, 'i')
export const quotedLine = new RegExp(`^${quoted.source}$`)
export const rangeLine = new RegExp(`^${rangeCapture.source}$`)
src/parser/match-operator.ts
+24
View File
@@ -0,0 +1,24 @@
import { VARIABLE } from '../util/character'
const trie = {
a: { n: { d: { end: true, needBoundary: true } } },
o: { r: { end: true, needBoundary: true } },
c: { o: { n: { t: { a: { i: { n: { s: { end: true, needBoundary: true } } } } } } } },
'=': { '=': { end: true } },
'!': { '=': { end: true } },
'>': { end: true, '=': { end: true } },
'<': { end: true, '=': { end: true } }
}
export function matchOperator (str: string, begin: number, end = str.length) {
let node = trie
let i = begin
let info
while (node[str[i]] && i < end) {
node = node[str[i++]]
if (node['end']) info = node
}
if (!info) return -1
if (info['needBoundary'] && str.charCodeAt(i) & VARIABLE) return -1
return i
}
src/parser/output-token.ts
-20
View File
@@ -1,20 +0,0 @@
import { DelimitedToken } from './delimited-token'
import { Token } from './token'
import { NormalizedFullOptions } from '../liquid-options'
export class OutputToken extends DelimitedToken {
public constructor (
raw: string,
value: string,
input: string,
line: number,
pos: number,
options: NormalizedFullOptions,
file?: string
) {
super(raw, value, input, line, pos, options.trimOutputLeft, options.trimOutputRight, file)
}
public static is (token: Token): token is OutputToken {
return token instanceof OutputToken
}
}
src/parser/parse-stream.ts
+11 -10
View File
@@ -1,20 +1,21 @@
import { Token } from '../parser/token'
import { Token } from '../tokens/token'
import { Template } from '../template/template'
import { TagToken } from './tag-token'
import { isTagToken } from '../util/type-guards'
import { TopLevelToken } from '../tokens/toplevel-token'
type ParseToken = ((token: Token, remainTokens: Token[]) => Template)
type ParseToken<T extends Token> = ((token: T, remainTokens: T[]) => Template)
export class ParseStream {
private tokens: Token[]
export class ParseStream<T extends Token = TopLevelToken> {
private tokens: T[]
private handlers: {[key: string]: (arg: any) => void} = {}
private stopRequested = false
private parseToken: ParseToken
private parseToken: ParseToken<T>
public constructor (tokens: Token[], parseToken: ParseToken) {
public constructor (tokens: T[], parseToken: ParseToken<T>) {
this.tokens = tokens
this.parseToken = parseToken
}
public on<T extends Template | Token | undefined> (name: string, cb: (arg: T) => void): ParseStream {
public on<T2 extends Template | T | undefined> (name: string, cb: (arg: T2) => void): ParseStream<T> {
this.handlers[name] = cb
return this
}
@@ -24,10 +25,10 @@ export class ParseStream {
}
public start () {
this.trigger('start')
let token: Token | undefined
let token: T | undefined
while (!this.stopRequested && (token = this.tokens.shift())) {
if (this.trigger('token', token)) continue
if (TagToken.is(token) && this.trigger(`tag:${token.name}`, token)) {
if (isTagToken(token) && this.trigger(`tag:${token.name}`, token)) {
continue
}
const template = this.parseToken(token, this.tokens)
src/parser/literal.ts → src/parser/parse-string-literal.ts
+1 -19
View File
@@ -1,10 +1,3 @@
import { last } from '../util/underscore'
import { NullDrop } from '../drop/null-drop'
import { EmptyDrop } from '../drop/empty-drop'
import { BlankDrop } from '../drop/blank-drop'
type literal = true | false | NullDrop | EmptyDrop | BlankDrop | number | string
const rHex = /[\da-fA-F]/
const rOct = /[0-7]/
const escapeChar = {
@@ -23,18 +16,6 @@ function hexVal (c: string) {
return code - 48
}
export function parseLiteral (str: string): literal | undefined {
str = str.trim()
if (str === 'true') return true
if (str === 'false') return false
if (str === 'nil' || str === 'null') return new NullDrop()
if (str === 'empty') return new EmptyDrop()
if (str === 'blank') return new BlankDrop()
if (!isNaN(Number(str))) return Number(str)
if ((str[0] === '"' || str[0] === "'") && str[0] === last(str)) return parseStringLiteral(str)
}
export function parseStringLiteral (str: string): string {
let ret = ''
for (let i = 1; i < str.length - 1; i++) {
@@ -66,3 +47,4 @@ export function parseStringLiteral (str: string): string {
}
return ret
}
src/parser/parser.ts
+8 -8
View File
@@ -1,13 +1,13 @@
import { ParseError } from '../util/error'
import { Liquid } from '../liquid'
import { ParseStream } from './parse-stream'
import { Token } from './token'
import { TagToken } from './tag-token'
import { OutputToken } from './output-token'
import { isTagToken, isOutputToken } from '../util/type-guards'
import { OutputToken } from '../tokens/output-token'
import { Tag } from '../template/tag/tag'
import { Output } from '../template/output'
import { HTML } from '../template/html'
import { Template } from '../template/template'
import { TopLevelToken } from '../tokens/toplevel-token'
export default class Parser {
private liquid: Liquid
@@ -15,7 +15,7 @@ export default class Parser {
public constructor (liquid: Liquid) {
this.liquid = liquid
}
public parse (tokens: Token[]) {
public parse (tokens: TopLevelToken[]) {
let token
const templates: Template[] = []
while ((token = tokens.shift())) {
@@ -23,12 +23,12 @@ export default class Parser {
}
return templates
}
public parseToken (token: Token, remainTokens: Token[]) {
public parseToken (token: TopLevelToken, remainTokens: TopLevelToken[]) {
try {
if (TagToken.is(token)) {
if (isTagToken(token)) {
return new Tag(token, remainTokens, this.liquid)
}
if (OutputToken.is(token)) {
if (isOutputToken(token)) {
return new Output(token as OutputToken, this.liquid.filters)
}
return new HTML(token)
@@ -36,7 +36,7 @@ export default class Parser {
throw new ParseError(e, token)
}
}
public parseStream (tokens: Token[]) {
public parseStream (tokens: TopLevelToken[]) {
return new ParseStream(tokens, (token, tokens) => this.parseToken(token, tokens))
}
}
src/parser/substr.ts
-19
View File
@@ -1,19 +0,0 @@
export class Substr {
constructor (
public str: string,
public begin: number,
public end: number = begin
) {}
size () {
return this.end - this.begin
}
toString () {
return this.str.slice(this.begin, this.end)
}
first () {
return this.str[this.begin]
}
last () {
return this.str[this.end - 1]
}
}
src/parser/tag-token.ts
-30
View File
@@ -1,30 +0,0 @@
import { DelimitedToken } from './delimited-token'
import { Token } from './token'
import { TokenizationError } from '../util/error'
import * as lexical from './lexical'
import { NormalizedFullOptions } from '../liquid-options'
export class TagToken extends DelimitedToken {
public name: string
public args: string
public constructor (
raw: string,
value: string,
input: string,
line: number,
pos: number,
options: NormalizedFullOptions,
file?: string
) {
super(raw, value, input, line, pos, options.trimTagLeft, options.trimTagRight, file)
const match = this.content.match(lexical.tagLine)
if (!match) {
throw new TokenizationError(`illegal tag syntax`, this)
}
this.name = match[1]
this.args = match[2]
}
public static is (token: Token): token is TagToken {
return token instanceof TagToken
}
}
src/parser/token-kind.ts
+14
View File
@@ -0,0 +1,14 @@
export enum TokenKind {
Number,
Literal,
Tag,
Output,
HTML,
Filter,
Hash,
PropertyAccess,
Word,
Range,
Quoted,
Operator
}
src/parser/token.ts
-14
View File
@@ -1,14 +0,0 @@
export class Token {
public trimLeft = false
public trimRight = false
public content: string
public constructor (
public raw: string,
public input: string,
public line: number,
public col: number,
public file?: string
) {
this.content = raw
}
}
src/parser/tokenizer.ts
+195 -193
View File
@@ -1,158 +1,158 @@
import { whiteSpaceCtrl } from './whitespace-ctrl'
import { Substr } from './substr'
import { NumberToken } from '../tokens/number-token'
import { WordToken } from '../tokens/word-token'
import { literalValues } from '../util/literal'
import { LiteralToken } from '../tokens/literal-token'
import { OperatorToken } from '../tokens/operator-token'
import { PropertyAccessToken } from '../tokens/property-access-token'
import { assert } from '../util/assert'
import { TopLevelToken } from '../tokens/toplevel-token'
import { FilterArg } from './filter-arg'
import { FilterToken } from './filter-token'
import { FilterToken } from '../tokens/filter-token'
import { HashToken } from '../tokens/hash-token'
import { QuotedToken } from '../tokens/quoted-token'
import { ellipsis } from '../util/underscore'
import { HTMLToken } from './html-token'
import { TagToken } from './tag-token'
import { Token } from './token'
import { OutputToken } from './output-token'
import { HTMLToken } from '../tokens/html-token'
import { TagToken } from '../tokens/tag-token'
import { Token } from '../tokens/token'
import { RangeToken } from '../tokens/range-token'
import { ValueToken } from '../tokens/value-token'
import { OutputToken } from '../tokens/output-token'
import { TokenizationError } from '../util/error'
import { NormalizedFullOptions, defaultOptions } from '../liquid-options'
// bitmask character types to boost performance
// generated by bin/char-types.js
const TYPES = '00000000044004000000000000000000428000080000010011111111110022210111111111111111111111111110000101111111111111111111111111100000'
const VARIABLE = 1
const OPERATOR = 2
const BLANK = 4
const QUOTE = 8
import { TYPES, QUOTE, BLANK, VARIABLE } from '../util/character'
import { matchOperator } from './match-operator'
export class Tokenizer {
private p = 0
private N: number
private line = 1
private col = 1
p = 0
N: number
constructor (
private input: string,
private file: string = '',
private options: NormalizedFullOptions = defaultOptions
private file: string = ''
) {
this.N = input.length
}
* readExpression (): IterableIterator<string> {
* readExpression (): IterableIterator<Token> {
const operand = this.readValue()
if (!operand) return
yield operand
while (this.p < this.N) {
const operator = this.readOperator()
if (!operator) return
const operand = this.readValue()
if (operand.size()) {
yield operand.toString()
continue
}
this.readBlank()
const operator = new Substr(this.input, this.p)
while (OPERATOR & this.peekType()) operator.end = this.read()
if (operator.size()) {
yield operator.toString()
continue
}
this.read()
if (!operand) return
yield operator
yield operand
}
}
readFilterTokens (): FilterToken[] {
readOperator (): OperatorToken | undefined {
this.skipBlank()
const end = matchOperator(this.input, this.p, this.p + 8)
if (end === -1) return
return new OperatorToken(this.input, this.p, (this.p = end), this.file)
}
readFilters (): FilterToken[] {
const filters = []
while (true) {
const filter = this.readFilterToken()
const filter = this.readFilter()
if (!filter) return filters
filters.push(filter)
}
}
// | foo
// | foo: a
// | foo: a, b
// | foo: a, b: 1
readFilterToken (): FilterToken | null {
readFilter (): FilterToken | null {
this.readTo('|')
const begin = this.p
const name = this.readVariable().toString()
if (!name) return null
const name = this.readWord()
if (!name.size()) return null
const args = []
this.readBlank()
this.skipBlank()
if (this.peek() === ':') {
do {
this.read()
++this.p
const arg = this.readFilterArg()
arg && args.push(arg)
while (this.p < this.N && this.peek() !== ',' && this.peek() !== '|') this.read()
while (this.p < this.N && this.peek() !== ',' && this.peek() !== '|') ++this.p
} while (this.peek() === ',')
}
const raw = this.input.slice(begin, this.p)
return new FilterToken(name, args, raw, this.input, this.line, this.col, this.file)
return new FilterToken(name.getText(), args, this.input, begin, this.p, this.file)
}
readFilterArg (): FilterArg | null {
readFilterArg (): FilterArg | undefined {
const key = this.readValue()
if (!key.size()) return null
this.readBlank()
if (this.peek() === ':') {
this.read()
return [key.toString(), this.readValue().toString()]
}
return key.toString()
if (!key) return
this.skipBlank()
if (this.peek() !== ':') return key
++this.p
const value = this.readValue()
return [key.getText(), value]
}
readTokens (): Token[] {
const tokens: Token[] = []
readTopLevelTokens (options: NormalizedFullOptions = defaultOptions): TopLevelToken[] {
const tokens: TopLevelToken[] = []
while (this.p < this.N) {
const token = this.readToken()
const token = this.readTopLevelToken(options)
tokens.push(token)
}
whiteSpaceCtrl(tokens, this.options)
whiteSpaceCtrl(tokens, options)
return tokens
}
readToken (): Token {
const { tagDelimiterLeft, outputDelimiterLeft } = this.options
if (this.matchWord(tagDelimiterLeft)) return this.readTagToken()
if (this.matchWord(outputDelimiterLeft)) return this.readOutputToken()
return this.readHTMLToken()
readTopLevelToken (options: NormalizedFullOptions): TopLevelToken {
const { tagDelimiterLeft, outputDelimiterLeft } = options
if (this.matchWord(tagDelimiterLeft)) return this.readTagToken(options)
if (this.matchWord(outputDelimiterLeft)) return this.readOutputToken(options)
return this.readHTMLToken(options)
}
readHTMLToken (): HTMLToken {
const html = new Substr(this.input, this.p)
readHTMLToken (options: NormalizedFullOptions): HTMLToken {
const begin = this.p
while (this.p < this.N) {
const { tagDelimiterLeft, outputDelimiterLeft } = this.options
const { tagDelimiterLeft, outputDelimiterLeft } = options
if (this.matchWord(tagDelimiterLeft)) break
if (this.matchWord(outputDelimiterLeft)) break
html.end = this.read()
++this.p
}
return new HTMLToken(html.toString(), this.input, this.line, this.col, this.file)
return new HTMLToken(this.input, begin, this.p, this.file)
}
readTagToken (): TagToken {
const { line, col, file, input, options } = this
const { tagDelimiterLeft, tagDelimiterRight } = options
const buffer = this.readTo(tagDelimiterRight).toString()
if (!this.reverseMatchWord(tagDelimiterRight, buffer)) {
throw new TokenizationError(
`tag "${ellipsis(buffer, 16)}" not closed`,
new Token(buffer, input, line, col, file)
)
readTagToken (options: NormalizedFullOptions): TagToken {
const { file, input } = this
const { tagDelimiterRight } = options
const begin = this.p
if (this.readTo(tagDelimiterRight) === -1) {
this.mkError(`tag "${this.ellipsis(begin)}" not closed`, begin)
}
const value = buffer.slice(tagDelimiterLeft.length, -tagDelimiterRight.length)
return new TagToken(buffer, value, input, line, col, options, file)
return new TagToken(input, begin, this.p, options, file)
}
readOutputToken (): OutputToken {
const { line, col, file, input, options } = this
const { outputDelimiterLeft, outputDelimiterRight } = options
const buffer = this.readTo(outputDelimiterRight).toString()
if (!this.reverseMatchWord(outputDelimiterRight, buffer)) {
throw new TokenizationError(
`output "${ellipsis(buffer, 16)}" not closed`,
new Token(buffer, input, line, col, file)
)
readOutputToken (options: NormalizedFullOptions): OutputToken {
const { file, input } = this
const { outputDelimiterRight } = options
const begin = this.p
if (this.readTo(outputDelimiterRight) === -1) {
this.mkError(`output "${this.ellipsis(begin)}" not closed`, begin)
}
const value = buffer.slice(outputDelimiterLeft.length, -outputDelimiterRight.length)
return new OutputToken(buffer, value, input, line, col, options, file)
return new OutputToken(input, begin, this.p, options, file)
}
readVariable (): Substr {
this.readBlank()
const ans = new Substr(this.input, this.p)
while (this.peekType() & VARIABLE) ans.end = this.read()
return ans
mkError (msg: string, begin: number) {
throw new TokenizationError(msg, new WordToken(this.input, begin, this.N, this.file))
}
ellipsis (begin: number = this.p) {
return ellipsis(this.input.slice(begin), 16)
}
readWord (): WordToken { // rename to identifier
this.skipBlank()
const begin = this.p
while (this.peekType() & VARIABLE) ++this.p
return new WordToken(this.input, begin, this.p, this.file)
}
readHashes () {
@@ -164,133 +164,135 @@ export class Tokenizer {
}
}
readHash () {
this.readBlank()
if (this.peek() === ',') this.read()
const name = this.readVariable().toString()
if (!name) return null
readHash (): HashToken | undefined {
this.skipBlank()
if (this.peek() === ',') ++this.p
const begin = this.p
const name = this.readWord()
if (!name.size()) return
let value
this.readBlank()
let value = ''
this.skipBlank()
if (this.peek() === ':') {
this.read()
value = this.readValue().toString()
++this.p
value = this.readValue()
}
return [name, value]
return new HashToken(this.input, begin, this.p, name, value, this.file)
}
readPropertyAccess (): Substr {
this.readBlank()
const ans = new Substr(this.input, this.p)
let nested = 0
remaining () {
return this.input.slice(this.p)
}
advance (i = 1) {
this.p += i
}
end () {
return this.p >= this.N
}
readTo (end: string): number {
while (this.p < this.N) {
const c = this.peek()
const code = this.peekType()
if (c === '[') {
this.read()
ans.end = this.readValue().end
nested++
} else if (c === ']') {
if (!nested) break
ans.end = this.read()
nested--
} else if (c === '.') {
if (this.peekType(1) & VARIABLE) {
this.read()
ans.end = this.readVariable().end
} else break
} else if (code & VARIABLE) {
ans.end = this.read()
} else {
if (nested) this.read()
else break
}
++this.p
if (this.reverseMatchWord(end)) return this.p
}
return ans
return -1
}
readTo (end: string): Substr {
const ans = new Substr(this.input, this.p)
while (this.p < this.N) {
ans.end = this.read()
if (this.reverseMatchWord(end)) break
readValue (): ValueToken | undefined {
const value = this.readQuoted() || this.readRange()
if (value) return value
const variable = this.readWord()
if (!variable.size()) return
let isNumber = variable.isNumber(true)
const props: (QuotedToken | WordToken)[] = []
while (true) {
if (this.peek() === '[') {
isNumber = false
this.p++
const prop = this.readValue() || new WordToken(this.input, this.p, this.p, this.file)
this.readTo(']')
props.push(prop)
} else if (this.peek() === '.' && this.peek(1) !== '.') { // skip range syntax
this.p++
const prop = this.readWord()
if (!prop.size()) break
if (!prop.isNumber()) isNumber = false
props.push(prop)
} else break
}
return ans
if (!props.length && literalValues.hasOwnProperty(variable.content)) {
return new LiteralToken(this.input, variable.begin, variable.end, this.file)
}
if (isNumber) return new NumberToken(variable, props[0] as WordToken)
return new PropertyAccessToken(variable, props, this.p)
}
readValue (): Substr {
let val = this.readQuoted()
if (val.size()) return val
val = this.readBoolean()
if (val.size()) return val
val = this.readPropertyAccess()
if (val.size()) return val
return this.readRange()
readRange (): RangeToken | undefined {
this.skipBlank()
const begin = this.p
if (this.peek() !== '(') return
++this.p
const lhs = this.readValueOrThrow()
this.p += 2
const rhs = this.readValueOrThrow()
++this.p
return new RangeToken(this.input, begin, this.p, lhs, rhs, this.file)
}
readRange (): Substr {
this.readBlank()
const ans = new Substr(this.input, this.p)
if (this.peek() !== '(') return ans
this.read()
this.readValue()
this.read(2)
this.readValue()
ans.end = this.read()
return ans
readValueOrThrow (): ValueToken {
const value = this.readValue()
assert(value, () => `unexpected token "${this.ellipsis()}", value expected`)
return value!
}
readBoolean (): Substr {
this.readBlank()
const ans = new Substr(this.input, this.p)
if (this.matchWord('true') && !(this.peekType(4) & VARIABLE)) ans.end = this.read(4)
else if (this.matchWord('false') && !(this.peekType(5) & VARIABLE)) ans.end = this.read(5)
return ans
}
readQuoted (): Substr {
this.readBlank()
const ans = new Substr(this.input, this.p)
if (!(this.peekType() & QUOTE)) return ans
ans.end = this.read()
readQuoted (): QuotedToken | undefined {
this.skipBlank()
const begin = this.p
if (!(this.peekType() & QUOTE)) return
++this.p
let escaped = false
while (this.p < this.N) {
ans.end = this.read()
if (ans.last() === ans.first() && !escaped) break
++this.p
if (this.input[this.p - 1] === this.input[begin] && !escaped) break
if (escaped) escaped = false
else if (ans.last() === '\\') escaped = true
else if (this.input[this.p - 1] === '\\') escaped = true
}
return ans
return new QuotedToken(this.input, begin, this.p, this.file)
}
read (n = 1): number {
if (n > 1) this.read(n - 1)
const c = this.input[this.p++]
if (c === '\n') {
this.line++
this.col = 1
} else {
this.col++
}
return this.p
readFileName (): WordToken {
const begin = this.p
while (!(this.peekType() & BLANK) && this.peek() !== ',' && this.p < this.N) this.p++
return new WordToken(this.input, begin, this.p, this.file)
}
matchWord (word: string) {
for (let i = 0; i < word.length; i++) {
if (word[i] !== this.input[this.p + i]) return false
}
return true
}
reverseMatchWord (word: string, buffer?: string) {
const str = buffer || this.input
const end = buffer === undefined ? this.p : buffer.length
reverseMatchWord (word: string) {
for (let i = 0; i < word.length; i++) {
if (word[word.length - 1 - i] !== str[end - 1 - i]) return false
if (word[word.length - 1 - i] !== this.input[this.p - 1 - i]) return false
}
return true
}
peekType (n = 0) {
return +TYPES[this.input.charCodeAt(this.p + n)]
return TYPES[this.input.charCodeAt(this.p + n)]
}
peek (n = 0) {
return this.input[this.p + n]
}
readBlank () {
let ans = ''
while (this.peekType() & BLANK) ans += this.read()
return ans
skipBlank () {
while (this.peekType() & BLANK) ++this.p
}
}
src/parser/whitespace-ctrl.ts
+13 -10
View File
@@ -1,7 +1,8 @@
import { Token } from '../parser/token'
import { TagToken } from '../parser/tag-token'
import { HTMLToken } from '../parser/html-token'
import { Token } from '../tokens/token'
import { DelimitedToken } from '../tokens/delimited-token'
import { isTagToken, isHTMLToken } from '../util/type-guards'
import { NormalizedFullOptions } from '../liquid-options'
import { TYPES, INLINE_BLANK, BLANK } from '../util/character'
export function whiteSpaceCtrl (tokens: Token[], options: NormalizedFullOptions) {
options = { greedy: true, ...options }
@@ -9,11 +10,12 @@ export function whiteSpaceCtrl (tokens: Token[], options: NormalizedFullOptions)
for (let i = 0; i < tokens.length; i++) {
const token = tokens[i]
if (!(token instanceof DelimitedToken)) continue
if (!inRaw && token.trimLeft) {
trimLeft(tokens[i - 1], options.greedy)
}
if (TagToken.is(token)) {
if (isTagToken(token)) {
if (token.name === 'raw') inRaw = true
else if (token.name === 'endraw') inRaw = false
}
@@ -25,15 +27,16 @@ export function whiteSpaceCtrl (tokens: Token[], options: NormalizedFullOptions)
}
function trimLeft (token: Token, greedy: boolean) {
if (!token || !HTMLToken.is(token)) return
if (!token || !isHTMLToken(token)) return
const rLeft = greedy ? /\s+$/g : /[\t\r ]*$/g
token.content = token.content.replace(rLeft, '')
const mask = greedy ? BLANK : INLINE_BLANK
while (TYPES[token.input.charCodeAt(token.end - 1 - token.trimRight)] & mask) token.trimRight++
}
function trimRight (token: Token, greedy: boolean) {
if (!token || !HTMLToken.is(token)) return
if (!token || !isHTMLToken(token)) return
const rRight = greedy ? /^\s+/g : /^[\t\r ]*\n?/g
token.content = token.content.replace(rRight, '')
const mask = greedy ? BLANK : INLINE_BLANK
while (TYPES[token.input.charCodeAt(token.begin + token.trimLeft)] & mask) token.trimLeft++
if (token.input.charAt(token.begin + token.trimLeft) === '\n') token.trimLeft++
}
src/render/expression.ts
+60 -33
View File
@@ -1,47 +1,87 @@
import { QuotedToken } from '../tokens/quoted-token'
import { NumberToken } from '../tokens/number-token'
import { assert } from '../util/assert'
import { rangeLine } from '../parser/lexical'
import { parseLiteral } from '../parser/literal'
import { literalValues } from '../util/literal'
import { LiteralToken } from '../tokens/literal-token'
import * as TypeGuards from '../util/type-guards'
import { Token } from '../tokens/token'
import { OperatorToken } from '../tokens/operator-token'
import { RangeToken } from '../tokens/range-token'
import { parseStringLiteral } from '../parser/parse-string-literal'
import { Context } from '../context/context'
import { range, toValue } from '../util/underscore'
import { isOperator, precedence, operatorImpls } from './operator'
import { Tokenizer } from '../parser/tokenizer'
import { operatorImpls } from '../render/operator'
export class Expression {
private operands: any[] = []
private postfix: string[]
private postfix: IterableIterator<Token>
public constructor (str = '') {
public constructor (str: string) {
const tokenizer = new Tokenizer(str)
this.postfix = [...toPostfix(tokenizer.readExpression())]
this.postfix = toPostfix(tokenizer.readExpression())
}
public * evaluate (ctx: Context): any {
assert(ctx, 'unable to evaluate: context not defined')
public evaluate (ctx: Context): any {
for (const token of this.postfix) {
if (isOperator(token)) {
if (TypeGuards.isOperatorToken(token)) {
const r = this.operands.pop()
const l = this.operands.pop()
const result = operatorImpls[token](l, r)
const result = evalOperatorToken(token, l, r)
this.operands.push(result)
} else if (isRange(token)) {
this.operands.push(yield rangeValue(token, ctx))
} else {
const literal = parseLiteral(token)
this.operands.push(literal !== undefined ? literal : yield ctx.get(token))
this.operands.push(evalToken(token, ctx))
}
}
return this.operands[0]
}
public * value (ctx: Context) {
return toValue(yield this.evaluate(ctx))
return toValue(this.evaluate(ctx))
}
}
function * toPostfix (tokens: IterableIterator<string>): IterableIterator<string> {
const ops = []
export function evalToken (token: Token | undefined, ctx: Context): any {
assert(ctx, () => 'unable to evaluate: context not defined')
if (TypeGuards.isPropertyAccessToken(token)) {
const variable = token.variable.getText()
const props: string[] = token.props.map(prop => evalToken(prop, ctx))
return ctx.get([variable, ...props])
}
if (TypeGuards.isRangeToken(token)) return evalRangeToken(token, ctx)
if (TypeGuards.isLiteralToken(token)) return evalLiteralToken(token)
if (TypeGuards.isNumberToken(token)) return evalNumberToken(token)
if (TypeGuards.isWordToken(token)) return token.getText()
if (TypeGuards.isQuotedToken(token)) return evalQuotedToken(token)
}
function evalNumberToken (token: NumberToken) {
const str = token.whole.content + '.' + (token.decimal ? token.decimal.content : '')
return Number(str)
}
export function evalQuotedToken (token: QuotedToken) {
return parseStringLiteral(token.getText())
}
function evalOperatorToken (token: OperatorToken, lhs: any, rhs: any) {
const impl = operatorImpls[token.operator]
return impl(lhs, rhs)
}
function evalLiteralToken (token: LiteralToken) {
return literalValues[token.literal]
}
function evalRangeToken (token: RangeToken, ctx: Context) {
const low: number = evalToken(token.lhs, ctx)
const high: number = evalToken(token.rhs, ctx)
return range(+low, +high + 1)
}
function * toPostfix (tokens: IterableIterator<Token>): IterableIterator<Token> {
const ops: OperatorToken[] = []
for (const token of tokens) {
if (isOperator(token)) {
while (ops.length && precedence[ops[ops.length - 1]] > precedence[token]) {
if (TypeGuards.isOperatorToken(token)) {
while (ops.length && ops[ops.length - 1].getPrecedence() > token.getPrecedence()) {
yield ops.pop()!
}
ops.push(token)
@@ -51,16 +91,3 @@ function * toPostfix (tokens: IterableIterator<string>): IterableIterator<string
yield ops.pop()!
}
}
function * rangeValue (token: string, ctx: Context) {
let match
if ((match = token.match(rangeLine))) {
const low = yield new Expression(match[1]).value(ctx)
const high = yield new Expression(match[2]).value(ctx)
return range(+low, +high + 1)
}
}
function isRange (str: string) {
return rangeLine.test(str)
}
src/render/operator.ts
-16
View File
@@ -2,18 +2,6 @@ import { isComparable } from '../drop/icomparable'
import { isFunction } from '../util/underscore'
import { isTruthy } from '../render/boolean'
export const precedence = {
'==': 1,
'!=': 1,
'>': 1,
'<': 1,
'>=': 1,
'<=': 1,
'contains': 1,
'and': 0,
'or': 0
}
export const operatorImpls: {[key: string]: (lhs: any, rhs: any) => boolean} = {
'==': (l: any, r: any) => {
if (isComparable(l)) return l.equals(r)
@@ -51,7 +39,3 @@ export const operatorImpls: {[key: string]: (lhs: any, rhs: any) => boolean} = {
'and': (l: any, r: any) => isTruthy(l) && isTruthy(r),
'or': (l: any, r: any) => isTruthy(l) || isTruthy(r)
}
export function isOperator (token: string) {
return precedence.hasOwnProperty(token)
}
src/template/filter/filter-map.ts
+1 -1
View File
@@ -10,7 +10,7 @@ export class FilterMap {
get (name: string) {
const impl = this.impls[name]
assert(impl || !this.strictFilters, `undefined filter: ${name}`)
assert(impl || !this.strictFilters, () => `undefined filter: ${name}`)
return impl
}
src/template/filter/filter.ts
+4 -4
View File
@@ -1,4 +1,4 @@
import { Expression } from '../../render/expression'
import { evalToken } from '../../render/expression'
import { Context } from '../../context/context'
import { identify } from '../../util/underscore'
import { FilterImplOptions } from './filter-impl-options'
@@ -16,9 +16,9 @@ export class Filter {
}
public * render (value: any, context: Context) {
const argv: any[] = []
for (const arg of this.args) {
if (isKeyValuePair(arg)) argv.push([arg[0], yield new Expression(arg[1]).evaluate(context)])
else argv.push(yield new Expression(arg).evaluate(context))
for (const arg of this.args as FilterArg[]) {
if (isKeyValuePair(arg)) argv.push([arg[0], yield evalToken(arg[1], context)])
else argv.push(yield evalToken(arg, context))
}
return this.impl.apply({ context }, [value, ...argv])
}
src/template/html.ts
+2 -2
View File
@@ -1,6 +1,6 @@
import { TemplateImpl } from '../template/template-impl'
import { Template } from '../template/template'
import { HTMLToken } from '../parser/html-token'
import { HTMLToken } from '../tokens/html-token'
import { Context } from '../context/context'
import { Emitter } from '../render/emitter'
@@ -8,7 +8,7 @@ export class HTML extends TemplateImpl<HTMLToken> implements Template {
private str: string
public constructor (token: HTMLToken) {
super(token)
this.str = token.content
this.str = token.getContent()
}
public * render (ctx: Context, emitter: Emitter): IterableIterator<void> {
emitter.write(this.str)
src/template/output.ts
+1 -1
View File
@@ -5,7 +5,7 @@ import { TemplateImpl } from '../template/template-impl'
import { Template } from '../template/template'
import { Context } from '../context/context'
import { Emitter } from '../render/emitter'
import { OutputToken } from '../parser/output-token'
import { OutputToken } from '../tokens/output-token'
export class Output extends TemplateImpl<OutputToken> implements Template {
private value: Value
src/template/tag/hash.ts
+6 -6
View File
@@ -1,4 +1,4 @@
import { Expression } from '../../render/expression'
import { evalToken } from '../../render/expression'
import { Context } from '../../context/context'
import { Tokenizer } from '../../parser/tokenizer'
@@ -11,17 +11,17 @@ import { Tokenizer } from '../../parser/tokenizer'
* hash['reversed'] === undefined
*/
export class Hash {
[key: string]: any
hash: { [key: string]: any } = {}
constructor (markup: string) {
const tokenizer = new Tokenizer(markup)
for (const [name, value] of tokenizer.readHashes()) {
this[name] = value
for (const hash of tokenizer.readHashes()) {
this.hash[hash.name.content] = hash.value
}
}
* render (ctx: Context) {
const hash = {}
for (const key of Object.keys(this)) {
hash[key] = yield new Expression(this[key]).evaluate(ctx)
for (const key of Object.keys(this.hash)) {
hash[key] = evalToken(this.hash[key], ctx)
}
return hash
}
src/template/tag/tag-impl-options.ts
+3 -3
View File
@@ -1,11 +1,11 @@
import { Context } from '../../context/context'
import { TagToken } from '../../parser/tag-token'
import { Token } from '../../parser/token'
import { TagToken } from '../../tokens/tag-token'
import { TopLevelToken } from '../../tokens/toplevel-token'
import { TagImpl } from './tag-impl'
import { Hash } from '../../template/tag/hash'
import { Emitter } from '../../render/emitter'
export interface TagImplOptions {
parse?: (this: TagImpl, token: TagToken, remainingTokens: Token[]) => void;
parse?: (this: TagImpl, token: TagToken, remainingTokens: TopLevelToken[]) => void;
render: (this: TagImpl, ctx: Context, emitter: Emitter, hash: Hash) => any;
}
src/template/tag/tag-map.ts
+1 -1
View File
@@ -6,7 +6,7 @@ export class TagMap {
get (name: string) {
const impl = this.impls[name]
assert(impl, `tag "${name}" not found`)
assert(impl, () => `tag "${name}" not found`)
return impl
}
src/template/tag/tag.ts
+2 -2
View File
@@ -1,7 +1,7 @@
import { isFunction } from '../../util/underscore'
import { Liquid } from '../../liquid'
import { TemplateImpl } from '../../template/template-impl'
import { Emitter, Hash, Context, TagImplOptions, TagToken, Template, Token } from '../../types'
import { Emitter, Hash, Context, TagImplOptions, TagToken, Template, TopLevelToken } from '../../types'
import { TagImpl } from './tag-impl'
export class Tag extends TemplateImpl<TagToken> implements Template {
@@ -9,7 +9,7 @@ export class Tag extends TemplateImpl<TagToken> implements Template {
private impl: TagImpl
private static impls: { [key: string]: TagImplOptions } = {}
public constructor (token: TagToken, tokens: Token[], liquid: Liquid) {
public constructor (token: TagToken, tokens: TopLevelToken[], liquid: Liquid) {
super(token)
this.name = token.name
src/template/template.ts
+1 -1
View File
@@ -1,5 +1,5 @@
import { Context } from '../context/context'
import { Token } from '../parser/token'
import { Token } from '../tokens/token'
import { Emitter } from '../render/emitter'
export interface Template {
src/template/value.ts
+6 -5
View File
@@ -1,23 +1,24 @@
import { Expression } from '../render/expression'
import { evalToken } from '../render/expression'
import { Tokenizer } from '../parser/tokenizer'
import { FilterMap } from '../template/filter/filter-map'
import { Filter } from './filter/filter'
import { Context } from '../context/context'
import { ValueToken } from '../tokens/value-token'
export class Value {
public readonly filters: Filter[] = []
public readonly initial: string
public readonly initial?: ValueToken
/**
* @param str the value to be valuated, eg.: "foobar" | truncate: 3
*/
public constructor (str: string, private readonly filterMap: FilterMap) {
const tokenizer = new Tokenizer(str)
this.initial = tokenizer.readValue().toString()
this.filters = tokenizer.readFilterTokens().map(({ name, args }) => new Filter(name, this.filterMap.get(name), args))
this.initial = tokenizer.readValue()
this.filters = tokenizer.readFilters().map(({ name, args }) => new Filter(name, this.filterMap.get(name), args))
}
public * value (ctx: Context) {
let val = yield new Expression(this.initial).evaluate(ctx)
let val = yield evalToken(this.initial, ctx)
for (const filter of this.filters) {
val = yield filter.render(val, ctx)
}
src/parser/delimited-token.ts → src/tokens/delimited-token.ts
+10 -5
View File
@@ -1,18 +1,23 @@
import { Token } from './token'
import { TokenKind } from '../parser/token-kind'
import { last } from '../util/underscore'
export class DelimitedToken extends Token {
export abstract class DelimitedToken extends Token {
public trimLeft = false
public trimRight = false
public content: string
public constructor (
raw: string,
kind: TokenKind,
content: string,
input: string,
line: number,
pos: number,
begin: number,
end: number,
trimLeft: boolean,
trimRight: boolean,
file?: string
) {
super(raw, input, line, pos, file)
super(kind, input, begin, end, file)
this.content = this.getText()
const tl = content[0] === '-'
const tr = last(content) === '-'
this.content = content
src/parser/filter-token.ts → src/tokens/filter-token.ts
+5 -5
View File
@@ -1,16 +1,16 @@
import { Token } from './token'
import { FilterArg } from './filter-arg'
import { FilterArg } from '../parser/filter-arg'
import { TokenKind } from '../parser/token-kind'
export class FilterToken extends Token {
public constructor (
public name: string,
public args: FilterArg[],
raw: string,
input: string,
line: number,
col: number,
begin: number,
end: number,
file?: string
) {
super(raw, input, line, col, file)
super(TokenKind.Filter, input, begin, end, file)
}
}
src/tokens/hash-token.ts
+17
View File
@@ -0,0 +1,17 @@
import { Token } from './token'
import { ValueToken } from './value-token'
import { WordToken } from './word-token'
import { TokenKind } from '../parser/token-kind'
export class HashToken extends Token {
constructor (
public input: string,
public begin: number,
public end: number,
public name: WordToken,
public value?: ValueToken,
public file?: string
) {
super(TokenKind.Hash, input, begin, end, file)
}
}
src/tokens/html-token.ts
+18
View File
@@ -0,0 +1,18 @@
import { Token } from './token'
import { TokenKind } from '../parser/token-kind'
export class HTMLToken extends Token {
trimLeft = 0
trimRight = 0
constructor (
public input: string,
public begin: number,
public end: number,
public file?: string
) {
super(TokenKind.HTML, input, begin, end, file)
}
public getContent () {
return this.input.slice(this.begin + this.trimLeft, this.end - this.trimRight)
}
}
src/tokens/literal-token.ts
+15
View File
@@ -0,0 +1,15 @@
import { Token } from './token'
import { TokenKind } from '../parser/token-kind'
export class LiteralToken extends Token {
public literal: string
public constructor (
public input: string,
public begin: number,
public end: number,
public file?: string
) {
super(TokenKind.Literal, input, begin, end, file)
this.literal = this.getText()
}
}
src/tokens/number-token.ts
+12
View File
@@ -0,0 +1,12 @@
import { Token } from './token'
import { WordToken } from './word-token'
import { TokenKind } from '../parser/token-kind'
export class NumberToken extends Token {
constructor (
public whole: WordToken,
public decimal?: WordToken
) {
super(TokenKind.Number, whole.input, whole.begin, decimal ? decimal.end : whole.end, whole.file)
}
}
src/tokens/operator-token.ts
+30
View File
@@ -0,0 +1,30 @@
import { Token } from './token'
import { TokenKind } from '../parser/token-kind'
export const precedence = {
'==': 1,
'!=': 1,
'>': 1,
'<': 1,
'>=': 1,
'<=': 1,
'contains': 1,
'and': 0,
'or': 0
}
export class OperatorToken extends Token {
public operator: string
public constructor (
public input: string,
public begin: number,
public end: number,
public file?: string
) {
super(TokenKind.Operator, input, begin, end, file)
this.operator = this.getText()
}
getPrecedence () {
return precedence[this.getText()]
}
}
src/tokens/output-token.ts
+17
View File
@@ -0,0 +1,17 @@
import { DelimitedToken } from './delimited-token'
import { NormalizedFullOptions } from '../liquid-options'
import { TokenKind } from '../parser/token-kind'
export class OutputToken extends DelimitedToken {
public constructor (
input: string,
begin: number,
end: number,
options: NormalizedFullOptions,
file?: string
) {
const { trimOutputLeft, trimOutputRight, outputDelimiterLeft, outputDelimiterRight } = options
const value = input.slice(begin + outputDelimiterLeft.length, end - outputDelimiterRight.length)
super(TokenKind.Output, value, input, begin, end, trimOutputLeft, trimOutputRight, file)
}
}
src/tokens/property-access-token.ts
+14
View File
@@ -0,0 +1,14 @@
import { Token } from './token'
import { WordToken } from './word-token'
import { QuotedToken } from './quoted-token'
import { TokenKind } from '../parser/token-kind'
export class PropertyAccessToken extends Token {
constructor (
public variable: WordToken,
public props: (WordToken | QuotedToken | PropertyAccessToken)[],
end: number
) {
super(TokenKind.PropertyAccess, variable.input, variable.begin, end, variable.file)
}
}
src/tokens/quoted-token.ts
+13
View File
@@ -0,0 +1,13 @@
import { Token } from './token'
import { TokenKind } from '../parser/token-kind'
export class QuotedToken extends Token {
constructor (
public input: string,
public begin: number,
public end: number,
public file?: string
) {
super(TokenKind.Quoted, input, begin, end, file)
}
}
src/tokens/range-token.ts
+16
View File
@@ -0,0 +1,16 @@
import { Token } from './token'
import { ValueToken } from './value-token'
import { TokenKind } from '../parser/token-kind'
export class RangeToken extends Token {
constructor (
public input: string,
public begin: number,
public end: number,
public lhs: ValueToken,
public rhs: ValueToken,
public file?: string
) {
super(TokenKind.Range, input, begin, end, file)
}
}
src/tokens/tag-token.ts
+30
View File
@@ -0,0 +1,30 @@
import { DelimitedToken } from './delimited-token'
import { BLANK, TYPES, VARIABLE } from '../util/character'
import { TokenizationError } from '../util/error'
import { NormalizedFullOptions } from '../liquid-options'
import { TokenKind } from '../parser/token-kind'
export class TagToken extends DelimitedToken {
public name: string
public args: string
public constructor (
input: string,
begin: number,
end: number,
options: NormalizedFullOptions,
file?: string
) {
const { trimTagLeft, trimTagRight, tagDelimiterLeft, tagDelimiterRight } = options
const value = input.slice(begin + tagDelimiterLeft.length, end - tagDelimiterRight.length)
super(TokenKind.Tag, value, input, begin, end, trimTagLeft, trimTagRight, file)
let nameEnd = 0
while (TYPES[this.content.charCodeAt(nameEnd)] & VARIABLE) nameEnd++
this.name = this.content.slice(0, nameEnd)
if (!this.name) throw new TokenizationError(`illegal tag syntax`, this)
let argsBegin = nameEnd
while (TYPES[this.content.charCodeAt(argsBegin)] & BLANK) argsBegin++
this.args = this.content.slice(argsBegin)
}
}
src/tokens/token.ts
+27
View File
@@ -0,0 +1,27 @@
import { TokenKind } from '../parser/token-kind'
export abstract class Token {
public constructor (
public kind: TokenKind,
public input: string,
public begin: number,
public end: number,
public file?: string
) {}
public getText () {
return this.input.slice(this.begin, this.end)
}
public getPosition () {
let [row, col] = [1, 1]
for (let i = 0; i < this.begin; i++) {
if (this.input[i] === '\n') {
row++
col = 1
} else col++
}
return [row, col]
}
public size () {
return this.end - this.begin
}
}
src/tokens/toplevel-token.ts
+5
View File
@@ -0,0 +1,5 @@
import { TagToken } from './tag-token'
import { HTMLToken } from './html-token'
import { OutputToken } from './output-token'
export type TopLevelToken = TagToken | OutputToken | HTMLToken
src/tokens/value-token.ts
+6
View File
@@ -0,0 +1,6 @@
import { RangeToken } from './range-token'
import { LiteralToken } from './literal-token'
import { QuotedToken } from './quoted-token'
import { PropertyAccessToken } from './property-access-token'
export type ValueToken = RangeToken | LiteralToken | QuotedToken | PropertyAccessToken
src/tokens/word-token.ts
+26
View File
@@ -0,0 +1,26 @@
import { Token } from './token'
import { NUMBER, TYPES, SIGN } from '../util/character'
import { TokenKind } from '../parser/token-kind'
// a word can be an identifier, a number, a keyword or a single-word-literal
export class WordToken extends Token {
public content: string
constructor (
public input: string,
public begin: number,
public end: number,
public file?: string
) {
super(TokenKind.Word, input, begin, end, file)
this.content = this.getText()
}
isNumber (allowSign = false) {
const begin = allowSign && TYPES[this.input.charCodeAt(this.begin)] & SIGN
? this.begin + 1
: this.begin
for (let i = begin; i < this.end; i++) {
if (!(TYPES[this.input.charCodeAt(i)] & NUMBER)) return false
}
return true
}
}
src/types.ts
+7 -2
View File
@@ -1,13 +1,18 @@
import * as TypeGuards from './util/type-guards'
export { TypeGuards }
export { ParseError, TokenizationError, AssertionError } from './util/error'
export { assert } from './util/assert'
export { Drop } from './drop/drop'
export { Emitter } from './render/emitter'
export { Expression } from './render/expression'
export { isFalsy, isTruthy } from './render/boolean'
export { TagToken } from './parser/tag-token'
export { TagToken } from './tokens/tag-token'
export { Context } from './context/context'
export { Template } from './template/template'
export { TagImplOptions } from './template/tag/tag-impl-options'
export { ParseStream } from './parser/parse-stream'
export { Token } from './parser/token'
export { Token } from './tokens/token'
export { TopLevelToken } from './tokens/toplevel-token'
export { Tokenizer } from './parser/tokenizer'
export { Hash } from './template/tag/hash'
export { evalToken, evalQuotedToken } from './render/expression'
src/util/assert.ts
+3 -3
View File
@@ -1,8 +1,8 @@
import { AssertionError } from './error'
export function assert <T> (predicate: T | null | undefined, message?: string) {
export function assert <T> (predicate: T | null | undefined, message?: () => string) {
if (!predicate) {
message = message || `expect ${predicate} to be true`
throw new AssertionError(message)
const msg = message ? message() : `expect ${predicate} to be true`
throw new AssertionError(msg)
}
}
src/util/async.ts
+1 -1
View File
@@ -1,6 +1,6 @@
import { isFunction } from './underscore'
type resolver = (x?: any) => Thenable
type resolver = (x?: any) => any
interface Thenable {
then (resolve: resolver, reject?: resolver): Thenable;
src/util/character.ts
+10
View File
@@ -0,0 +1,10 @@
// bitmask character types to boost performance
// generated by bin/char-types.js
export const TYPES = [0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2, 8, 0, 0, 0, 0, 8, 0, 0, 0, 64, 0, 65, 0, 0, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 0, 0, 2, 2, 2, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]
export const VARIABLE = 1
export const OPERATOR = 2
export const BLANK = 4
export const QUOTE = 8
export const INLINE_BLANK = 16
export const NUMBER = 32
export const SIGN = 64
src/util/error.ts
+7 -5
View File
@@ -1,5 +1,5 @@
import * as _ from './underscore'
import { Token } from '../parser/token'
import { Token } from '../tokens/token'
import { Template } from '../template/template'
abstract class LiquidError extends Error {
@@ -57,14 +57,15 @@ export class AssertionError extends Error {
}
function mkContext (token: Token) {
const [line] = token.getPosition()
const lines = token.input.split('\n')
const begin = Math.max(token.line - 2, 1)
const end = Math.min(token.line + 3, lines.length)
const begin = Math.max(line - 2, 1)
const end = Math.min(line + 3, lines.length)
const context = _
.range(begin, end + 1)
.map(lineNumber => {
const indicator = (lineNumber === token.line) ? '>> ' : ' '
const indicator = (lineNumber === line) ? '>> ' : ' '
const num = _.padStart(String(lineNumber), String(end).length)
const text = lines[lineNumber - 1]
return `${indicator}${num}| ${text}`
@@ -76,6 +77,7 @@ function mkContext (token: Token) {
function mkMessage (msg: string, token: Token) {
if (token.file) msg += `, file:${token.file}`
msg += `, line:${token.line}, col:${token.col}`
const [line, col] = token.getPosition()
msg += `, line:${line}, col:${col}`
return msg
}
src/util/literal.ts
+12
View File
@@ -0,0 +1,12 @@
import { NullDrop } from '../drop/null-drop'
import { EmptyDrop } from '../drop/empty-drop'
import { BlankDrop } from '../drop/blank-drop'
export const literalValues = {
'true': true,
'false': false,
'nil': new NullDrop(),
'null': new NullDrop(),
'empty': new EmptyDrop(),
'blank': new BlankDrop()
}
src/util/type-guards.ts
+55
View File
@@ -0,0 +1,55 @@
import { OperatorToken } from '../tokens/operator-token'
import { WordToken } from '../tokens/word-token'
import { TagToken } from '../tokens/tag-token'
import { HTMLToken } from '../tokens/html-token'
import { OutputToken } from '../tokens/output-token'
import { PropertyAccessToken } from '../tokens/property-access-token'
import { LiteralToken } from '../tokens/literal-token'
import { QuotedToken } from '../tokens/quoted-token'
import { NumberToken } from '../tokens/number-token'
import { RangeToken } from '../tokens/range-token'
import { TokenKind } from '../parser/token-kind'
export function isOperatorToken (val: any): val is OperatorToken {
return getKind(val) === TokenKind.Operator
}
export function isHTMLToken (val: any): val is HTMLToken {
return getKind(val) === TokenKind.HTML
}
export function isOutputToken (val: any): val is OutputToken {
return getKind(val) === TokenKind.Output
}
export function isTagToken (val: any): val is TagToken {
return getKind(val) === TokenKind.Tag
}
export function isQuotedToken (val: any): val is QuotedToken {
return getKind(val) === TokenKind.Quoted
}
export function isLiteralToken (val: any): val is LiteralToken {
return getKind(val) === TokenKind.Literal
}
export function isNumberToken (val: any): val is NumberToken {
return getKind(val) === TokenKind.Number
}
export function isPropertyAccessToken (val: any): val is PropertyAccessToken {
return getKind(val) === TokenKind.PropertyAccess
}
export function isWordToken (val: any): val is WordToken {
return getKind(val) === TokenKind.Word
}
export function isRangeToken (val: any): val is RangeToken {
return getKind(val) === TokenKind.Range
}
function getKind (val: any) {
return val ? val.kind : -1
}