From d2d6a382356d17fbc5737f300edb6604bba391ba Mon Sep 17 00:00:00 2001 From: harttle Date: Wed, 11 Mar 2020 18:43:32 +0800 Subject: [PATCH] perf: introduce AST to avoid reparse --- bin/char-types.js | 43 -- bin/character-gen.js | 36 ++ src/builtin/filters/array.ts | 2 +- src/builtin/tags/assign.ts | 16 +- src/builtin/tags/block.ts | 6 +- src/builtin/tags/capture.ts | 16 +- src/builtin/tags/case.ts | 6 +- src/builtin/tags/comment.ts | 8 +- src/builtin/tags/cycle.ts | 36 +- src/builtin/tags/decrement.ts | 9 +- src/builtin/tags/for.ts | 32 +- src/builtin/tags/if.ts | 6 +- src/builtin/tags/include.ts | 42 +- src/builtin/tags/increment.ts | 9 +- src/builtin/tags/layout.ts | 30 +- src/builtin/tags/raw.ts | 8 +- src/builtin/tags/render.ts | 77 +-- src/builtin/tags/tablerow.ts | 30 +- src/builtin/tags/unless.ts | 8 +- src/context/context.ts | 80 +-- src/liquid.ts | 4 +- src/parser/filter-arg.ts | 7 +- src/parser/html-token.ts | 11 - src/parser/lexical.ts | 29 -- src/parser/match-operator.ts | 24 + src/parser/output-token.ts | 20 - src/parser/parse-stream.ts | 21 +- .../{literal.ts => parse-string-literal.ts} | 20 +- src/parser/parser.ts | 16 +- src/parser/substr.ts | 19 - src/parser/tag-token.ts | 30 -- src/parser/token-kind.ts | 14 + src/parser/token.ts | 14 - src/parser/tokenizer.ts | 388 +++++++------- src/parser/whitespace-ctrl.ts | 23 +- src/render/expression.ts | 93 ++-- src/render/operator.ts | 16 - src/template/filter/filter-map.ts | 2 +- src/template/filter/filter.ts | 8 +- src/template/html.ts | 4 +- src/template/output.ts | 2 +- src/template/tag/hash.ts | 12 +- src/template/tag/tag-impl-options.ts | 6 +- src/template/tag/tag-map.ts | 2 +- src/template/tag/tag.ts | 4 +- src/template/template.ts | 2 +- src/template/value.ts | 11 +- src/{parser => tokens}/delimited-token.ts | 15 +- src/{parser => tokens}/filter-token.ts | 10 +- src/tokens/hash-token.ts | 17 + src/tokens/html-token.ts | 18 + src/tokens/literal-token.ts | 15 + src/tokens/number-token.ts | 12 + src/tokens/operator-token.ts | 30 ++ src/tokens/output-token.ts | 17 + src/tokens/property-access-token.ts | 14 + src/tokens/quoted-token.ts | 13 + src/tokens/range-token.ts | 16 + src/tokens/tag-token.ts | 30 ++ src/tokens/token.ts | 27 + src/tokens/toplevel-token.ts | 5 + src/tokens/value-token.ts | 6 + src/tokens/word-token.ts | 26 + src/types.ts | 9 +- src/util/assert.ts | 6 +- src/util/async.ts | 2 +- src/util/character.ts | 10 + src/util/error.ts | 12 +- src/util/literal.ts | 12 + src/util/type-guards.ts | 55 ++ test/e2e/eval-value-sync.ts | 2 +- test/e2e/eval-value.ts | 4 +- test/integration/builtin/tags/decrement.ts | 5 - test/integration/builtin/tags/for.ts | 6 + test/integration/builtin/tags/include.ts | 16 + test/integration/builtin/tags/layout.ts | 14 +- test/integration/builtin/tags/render.ts | 26 +- test/integration/builtin/tags/tablerow.ts | 11 +- test/integration/liquid/liquid.ts | 2 +- test/integration/liquid/register-tags.ts | 2 +- .../liquid/{liquid-options.ts => root.ts} | 2 +- test/integration/liquid/trimming.ts | 2 +- .../liquid}/whitespace-ctrl.ts | 2 +- test/integration/util/error.ts | 23 +- test/unit/context/context.ts | 134 ++--- test/unit/parser/match-operator.ts | 26 + .../{literal.ts => parse-string-literal.ts} | 29 +- test/unit/parser/tokenizer.ts | 491 +++++++++++++----- test/unit/render/expression.ts | 140 +++-- test/unit/render/render.ts | 4 +- test/unit/template/filter/filter.ts | 21 +- test/unit/template/hash.ts | 3 +- test/unit/template/output.ts | 2 +- test/unit/template/tag.ts | 2 +- test/unit/template/value.ts | 69 +-- test/unit/util/assert.ts | 4 +- 96 files changed, 1553 insertions(+), 1168 deletions(-) delete mode 100644 bin/char-types.js create mode 100644 bin/character-gen.js delete mode 100644 src/parser/html-token.ts delete mode 100644 src/parser/lexical.ts create mode 100644 src/parser/match-operator.ts delete mode 100644 src/parser/output-token.ts rename src/parser/{literal.ts => parse-string-literal.ts} (60%) delete mode 100644 src/parser/substr.ts delete mode 100644 src/parser/tag-token.ts create mode 100644 src/parser/token-kind.ts delete mode 100644 src/parser/token.ts rename src/{parser => tokens}/delimited-token.ts (60%) rename src/{parser => tokens}/filter-token.ts (51%) create mode 100644 src/tokens/hash-token.ts create mode 100644 src/tokens/html-token.ts create mode 100644 src/tokens/literal-token.ts create mode 100644 src/tokens/number-token.ts create mode 100644 src/tokens/operator-token.ts create mode 100644 src/tokens/output-token.ts create mode 100644 src/tokens/property-access-token.ts create mode 100644 src/tokens/quoted-token.ts create mode 100644 src/tokens/range-token.ts create mode 100644 src/tokens/tag-token.ts create mode 100644 src/tokens/token.ts create mode 100644 src/tokens/toplevel-token.ts create mode 100644 src/tokens/value-token.ts create mode 100644 src/tokens/word-token.ts create mode 100644 src/util/character.ts create mode 100644 src/util/literal.ts create mode 100644 src/util/type-guards.ts rename test/integration/liquid/{liquid-options.ts => root.ts} (91%) rename test/{e2e => integration/liquid}/whitespace-ctrl.ts (99%) create mode 100644 test/unit/parser/match-operator.ts rename test/unit/parser/{literal.ts => parse-string-literal.ts} (57%) diff --git a/bin/char-types.js b/bin/char-types.js deleted file mode 100644 index 4ad77c785..000000000 --- a/bin/char-types.js +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env node - -function isQuote (c) { - return c === '"' || c === "'" -} - -function isOperator (c) { - return '!=<>'.includes(c) -} - -function isNumber (c) { - return c >= '0' && c <= '9' -} - -function isCharacter (c) { - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') -} - -function isVariable (c) { - return '_-?'.includes(c) || isCharacter(c) || isNumber(c) -} - -function isBlank (c) { - return c === '\n' || c === '\t' || c === ' ' || c === '\r' -} - -const types = [] -for (let i = 0; i < 128; i++) { - const c = String.fromCharCode(i) - let n = 0 - if (isVariable(c)) n |= 1 - if (isOperator(c)) n |= 2 - if (isBlank(c)) n |= 4 - if (isQuote(c)) n |= 8 - types.push(n) -} -console.log(` -const TYPES = '${types.join('')}' -const VARIABLE = 1 -const OPERATOR = 2 -const BLANK = 4 -const QUOTE = 8 -`.trim()) diff --git a/bin/character-gen.js b/bin/character-gen.js new file mode 100644 index 000000000..290f8c0e9 --- /dev/null +++ b/bin/character-gen.js @@ -0,0 +1,36 @@ +#!/usr/bin/env node + +const isQuote = c => c === '"' || c === "'" +const isOperator = c => '!=<>'.includes(c) +const isNumber = c => c >= '0' && c <= '9' +const isCharacter = c => (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') +const isVariable = c => '_-?'.includes(c) || isCharacter(c) || isNumber(c) +const isBlank = c => c === '\n' || c === '\t' || c === ' ' || c === '\r' +const isInlineBlank = c => c === '\t' || c === ' ' || c === '\r' +const isSign = c => c === '-' || c === '+' + +const types = [] +for (let i = 0; i < 128; i++) { + const c = String.fromCharCode(i) + let n = 0 + if (isVariable(c)) n |= 1 + if (isOperator(c)) n |= 2 + if (isBlank(c)) n |= 4 + if (isQuote(c)) n |= 8 + if (isInlineBlank(c)) n |= 16 + if (isNumber(c)) n |= 32 + if (isSign(c)) n |= 64 + types.push(n) +} +console.log(` +// bitmask character types to boost performance +// generated by bin/character-gen.js +export const TYPES = [${types.join(', ')}] +export const VARIABLE = 1 +export const OPERATOR = 2 +export const BLANK = 4 +export const QUOTE = 8 +export const INLINE_BLANK = 16 +export const NUMBER = 32 +export const SIGN = 64 +`.trim()) diff --git a/src/builtin/filters/array.ts b/src/builtin/filters/array.ts index 2e760948e..7ccb73284 100644 --- a/src/builtin/filters/array.ts +++ b/src/builtin/filters/array.ts @@ -31,7 +31,7 @@ function slice (v: T[], begin: number, length = 1): T[] { function where (this: FilterImpl, arr: T[], property: string, expected?: any): T[] { return arr.filter(obj => { - const value = this.context.getFromScope(obj, property) + const value = this.context.getFromScope(obj, property.split('.')) return expected === undefined ? isTruthy(value) : value === expected }) } diff --git a/src/builtin/tags/assign.ts b/src/builtin/tags/assign.ts index 5c60f8bc3..482d9bd0c 100644 --- a/src/builtin/tags/assign.ts +++ b/src/builtin/tags/assign.ts @@ -1,15 +1,13 @@ -import { assert } from '../../util/assert' -import { identifier } from '../../parser/lexical' -import { TagImplOptions, TagToken, Context } from '../../types' - -const re = new RegExp(`(${identifier.source})\\s*=([^]*)`) +import { Tokenizer, assert, TagImplOptions, TagToken, Context } from '../../types' export default { parse: function (token: TagToken) { - const match = token.args.match(re) as RegExpMatchArray - assert(match, `illegal token ${token.raw}`) - this.key = match[1] - this.value = match[2] + const tokenizer = new Tokenizer(token.args) + this.key = tokenizer.readWord().content + tokenizer.skipBlank() + assert(tokenizer.peek() === '=', () => `illegal token ${token.getText()}`) + tokenizer.advance() + this.value = tokenizer.remaining() }, render: function * (ctx: Context) { ctx.bottom()[this.key] = yield this.liquid._evalValue(this.value, ctx) diff --git a/src/builtin/tags/block.ts b/src/builtin/tags/block.ts index dad49b498..f8ad384bf 100644 --- a/src/builtin/tags/block.ts +++ b/src/builtin/tags/block.ts @@ -1,8 +1,8 @@ import BlockMode from '../../context/block-mode' -import { ParseStream, TagToken, Token, Template, Context, TagImplOptions, Emitter } from '../../types' +import { ParseStream, TagToken, TopLevelToken, Template, Context, TagImplOptions, Emitter } from '../../types' export default { - parse: function (token: TagToken, remainTokens: Token[]) { + parse: function (token: TagToken, remainTokens: TopLevelToken[]) { const match = /\w+/.exec(token.args) this.block = match ? match[0] : '' this.tpls = [] as Template[] @@ -10,7 +10,7 @@ export default { .on('tag:endblock', () => stream.stop()) .on('template', (tpl: Template) => this.tpls.push(tpl)) .on('end', () => { - throw new Error(`tag ${token.raw} not closed`) + throw new Error(`tag ${token.getText()} not closed`) }) stream.start() }, diff --git a/src/builtin/tags/capture.ts b/src/builtin/tags/capture.ts index 22d69ca1c..8fce59f02 100644 --- a/src/builtin/tags/capture.ts +++ b/src/builtin/tags/capture.ts @@ -1,22 +1,18 @@ -import { assert } from '../../util/assert' -import { identifier } from '../../parser/lexical' -import { Template, Context, TagImplOptions, TagToken, Token } from '../../types' - -const re = new RegExp(`(${identifier.source})`) +import { Tokenizer, assert, Template, Context, TagImplOptions, TagToken, TopLevelToken } from '../../types' export default { - parse: function (tagToken: TagToken, remainTokens: Token[]) { - const match = tagToken.args.match(re) as RegExpMatchArray - assert(match, `${tagToken.args} not valid identifier`) + parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) { + const tokenizer = new Tokenizer(tagToken.args) + this.variable = tokenizer.readWord().content + assert(this.variable, () => `${tagToken.args} not valid identifier`) - this.variable = match[1] this.templates = [] const stream = this.liquid.parser.parseStream(remainTokens) stream.on('tag:endcapture', () => stream.stop()) .on('template', (tpl: Template) => this.templates.push(tpl)) .on('end', () => { - throw new Error(`tag ${tagToken.raw} not closed`) + throw new Error(`tag ${tagToken.getText()} not closed`) }) stream.start() }, diff --git a/src/builtin/tags/case.ts b/src/builtin/tags/case.ts index 28ffbeaef..081abcfaf 100644 --- a/src/builtin/tags/case.ts +++ b/src/builtin/tags/case.ts @@ -1,7 +1,7 @@ -import { Expression, Emitter, TagToken, Token, Context, Template, TagImplOptions, ParseStream } from '../../types' +import { Expression, Emitter, TagToken, TopLevelToken, Context, Template, TagImplOptions, ParseStream } from '../../types' export default { - parse: function (tagToken: TagToken, remainTokens: Token[]) { + parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) { this.cond = tagToken.args this.cases = [] this.elseTemplates = [] @@ -18,7 +18,7 @@ export default { .on('tag:endcase', () => stream.stop()) .on('template', (tpl: Template) => p.push(tpl)) .on('end', () => { - throw new Error(`tag ${tagToken.raw} not closed`) + throw new Error(`tag ${tagToken.getText()} not closed`) }) stream.start() diff --git a/src/builtin/tags/comment.ts b/src/builtin/tags/comment.ts index 0d36aec07..dacb19fa9 100644 --- a/src/builtin/tags/comment.ts +++ b/src/builtin/tags/comment.ts @@ -1,16 +1,16 @@ -import { TagToken } from '../../parser/tag-token' -import { Token } from '../../parser/token' +import { TagToken } from '../../tokens/tag-token' +import { TopLevelToken } from '../../tokens/toplevel-token' import { TagImplOptions } from '../../template/tag/tag-impl-options' export default { - parse: function (tagToken: TagToken, remainTokens: Token[]) { + parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) { const stream = this.liquid.parser.parseStream(remainTokens) stream .on('token', (token: TagToken) => { if (token.name === 'endcomment') stream.stop() }) .on('end', () => { - throw new Error(`tag ${tagToken.raw} not closed`) + throw new Error(`tag ${tagToken.getText()} not closed`) }) stream.start() } diff --git a/src/builtin/tags/cycle.ts b/src/builtin/tags/cycle.ts index 970117656..2d71be048 100644 --- a/src/builtin/tags/cycle.ts +++ b/src/builtin/tags/cycle.ts @@ -1,28 +1,32 @@ import { assert } from '../../util/assert' -import { value as rValue } from '../../parser/lexical' -import { Emitter, Expression, TagToken, Context, TagImplOptions } from '../../types' - -const groupRE = new RegExp(`^(?:(${rValue.source})\\s*:\\s*)?(.*)$`) -const candidatesRE = new RegExp(rValue.source, 'g') +import { evalToken, Emitter, TagToken, Context, TagImplOptions } from '../../types' +import { Tokenizer } from '../../parser/tokenizer' export default { parse: function (tagToken: TagToken) { - let match: RegExpExecArray | null = groupRE.exec(tagToken.args) as RegExpExecArray - assert(match, `illegal tag: ${tagToken.raw}`) - - this.group = new Expression(match[1]) - const candidates = match[2] + const tokenizer = new Tokenizer(tagToken.args) + const group = tokenizer.readValue() + tokenizer.skipBlank() this.candidates = [] - while ((match = candidatesRE.exec(candidates))) { - this.candidates.push(match[0]) + if (group) { + if (tokenizer.peek() === ':') { + this.group = group + tokenizer.advance() + } else this.candidates.push(group) } - assert(this.candidates.length, `empty candidates: ${tagToken.raw}`) + + while (!tokenizer.end()) { + const value = tokenizer.readValue() + if (value) this.candidates.push(value) + tokenizer.readTo(',') + } + assert(this.candidates.length, () => `empty candidates: ${tagToken.getText()}`) }, - render: function * (ctx: Context, emitter: Emitter) { - const group = yield this.group.value(ctx) + render: function (ctx: Context, emitter: Emitter) { + const group = evalToken(this.group, ctx) const fingerprint = `cycle:${group}:` + this.candidates.join(',') const groups = ctx.getRegister('cycle') let idx = groups[fingerprint] @@ -34,7 +38,7 @@ export default { const candidate = this.candidates[idx] idx = (idx + 1) % this.candidates.length groups[fingerprint] = idx - const html = yield new Expression(candidate).value(ctx) + const html = evalToken(candidate, ctx) emitter.write(html) } } as TagImplOptions diff --git a/src/builtin/tags/decrement.ts b/src/builtin/tags/decrement.ts index b234ee7d7..e7e94be17 100644 --- a/src/builtin/tags/decrement.ts +++ b/src/builtin/tags/decrement.ts @@ -1,13 +1,10 @@ -import { assert } from '../../util/assert' -import { identifier } from '../../parser/lexical' -import { Emitter, TagToken, Context, TagImplOptions } from '../../types' +import { Tokenizer, Emitter, TagToken, Context, TagImplOptions } from '../../types' import { isNumber, stringify } from '../../util/underscore' export default { parse: function (token: TagToken) { - const match = token.args.match(identifier) as RegExpMatchArray - assert(match, `illegal identifier ${token.args}`) - this.variable = match[0] + const tokenizer = new Tokenizer(token.args) + this.variable = tokenizer.readWord().content }, render: function (context: Context, emitter: Emitter) { const scope = context.environments diff --git a/src/builtin/tags/for.ts b/src/builtin/tags/for.ts index 343e0f85e..c2fdcfda1 100644 --- a/src/builtin/tags/for.ts +++ b/src/builtin/tags/for.ts @@ -1,21 +1,24 @@ -import { Emitter, TagToken, Token, Context, Template, TagImplOptions, ParseStream } from '../../types' +import { assert, Tokenizer, evalToken, Emitter, TagToken, TopLevelToken, Context, Template, TagImplOptions, ParseStream } from '../../types' import { toCollection } from '../../util/collection' -import { Expression } from '../../render/expression' -import { assert } from '../../util/assert' -import { identifier, value } from '../../parser/lexical' import { ForloopDrop } from '../../drop/forloop-drop' import { Hash } from '../../template/tag/hash' -const re = new RegExp(`^(${identifier.source})\\s+in\\s+(${value.source})`) - export default { type: 'block', - parse: function (tagToken: TagToken, remainTokens: Token[]) { - const match = re.exec(tagToken.args) as RegExpExecArray - assert(match, `illegal tag: ${tagToken.raw}`) - this.variable = match[1] - this.collection = match[2] - this.hash = new Hash(tagToken.args.slice(match[0].length)) + parse: function (token: TagToken, remainTokens: TopLevelToken[]) { + const toknenizer = new Tokenizer(token.args) + + const variable = toknenizer.readWord() + const inStr = toknenizer.readWord() + const collection = toknenizer.readValue() + assert( + variable.size() && inStr.content === 'in' && collection, + () => `illegal tag: ${token.getText()}` + ) + + this.variable = variable.content + this.collection = collection + this.hash = new Hash(toknenizer.remaining()) this.templates = [] this.elseTemplates = [] @@ -26,15 +29,14 @@ export default { .on('tag:endfor', () => stream.stop()) .on('template', (tpl: Template) => p.push(tpl)) .on('end', () => { - throw new Error(`tag ${tagToken.raw} not closed`) + throw new Error(`tag ${token.getText()} not closed`) }) stream.start() }, render: function * (ctx: Context, emitter: Emitter) { const r = this.liquid.renderer - let collection = yield new Expression(this.collection).value(ctx) - collection = toCollection(collection) + let collection = toCollection(evalToken(this.collection, ctx)) if (!collection.length) { yield r.renderTemplates(this.elseTemplates, ctx, emitter) diff --git a/src/builtin/tags/if.ts b/src/builtin/tags/if.ts index 9801f10f8..2dc50167a 100644 --- a/src/builtin/tags/if.ts +++ b/src/builtin/tags/if.ts @@ -1,7 +1,7 @@ -import { Emitter, isTruthy, Expression, TagToken, Token, Context, Template, TagImplOptions, ParseStream } from '../../types' +import { Emitter, isTruthy, Expression, TagToken, TopLevelToken, Context, Template, TagImplOptions, ParseStream } from '../../types' export default { - parse: function (tagToken: TagToken, remainTokens: Token[]) { + parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) { this.branches = [] this.elseTemplates = [] @@ -21,7 +21,7 @@ export default { .on('tag:endif', () => stream.stop()) .on('template', (tpl: Template) => p.push(tpl)) .on('end', () => { - throw new Error(`tag ${tagToken.raw} not closed`) + throw new Error(`tag ${tagToken.getText()} not closed`) }) stream.start() diff --git a/src/builtin/tags/include.ts b/src/builtin/tags/include.ts index e71fd218f..daef7d4b6 100644 --- a/src/builtin/tags/include.ts +++ b/src/builtin/tags/include.ts @@ -1,35 +1,41 @@ -import { assert } from '../../util/assert' -import { Expression, Hash, Emitter, TagToken, Context, TagImplOptions } from '../../types' -import { quoted, value, quotedLine } from '../../parser/lexical' +import { assert, evalQuotedToken, TypeGuards, Tokenizer, evalToken, Hash, Emitter, TagToken, Context, TagImplOptions } from '../../types' import BlockMode from '../../context/block-mode' -const rFile = new RegExp(`^(${quoted.source}|[^\\s,]+)(?:\\s+with\\s+(${value.source}))?`) - export default { parse: function (token: TagToken) { - const match = rFile.exec(token.args) - if (!match) { - throw new Error(`illegal argument "${token.args}"`) - } - this.file = match[1] - this.hash = new Hash(token.args.slice(match[0].length)) - this.withVar = match[2] + const args = token.args + const tokenizer = new Tokenizer(args) + this.file = this.liquid.options.dynamicPartials + ? tokenizer.readValue() + : tokenizer.readFileName() + assert(this.file, () => `illegal argument "${token.args}"`) + + const begin = tokenizer.p + const withStr = tokenizer.readWord() + if (withStr.content === 'with') { + tokenizer.skipBlank() + if (tokenizer.peek() !== ':') { + this.withVar = tokenizer.readValue() + } else tokenizer.p = begin + } else tokenizer.p = begin + + this.hash = new Hash(tokenizer.remaining()) }, render: function * (ctx: Context, emitter: Emitter) { const { liquid, hash, withVar, file } = this const { renderer } = liquid const filepath = ctx.opts.dynamicPartials - ? (quotedLine.exec(file) - ? yield renderer.renderTemplates(liquid.parse(file.slice(1, -1)), ctx) - : yield new Expression(file).value(ctx)) - : file - assert(filepath, `illegal filename "${file}":"${filepath}"`) + ? (TypeGuards.isQuotedToken(file) + ? yield renderer.renderTemplates(liquid.parse(evalQuotedToken(file)), ctx) + : yield evalToken(file, ctx)) + : file.getText() + assert(filepath, () => `illegal filename "${file}":"${filepath}"`) const saved = ctx.saveRegister('blocks', 'blockMode') ctx.setRegister('blocks', {}) ctx.setRegister('blockMode', BlockMode.OUTPUT) const scope = yield hash.render(ctx) - if (withVar) scope[filepath] = yield new Expression(withVar).evaluate(ctx) + if (withVar) scope[filepath] = evalToken(withVar, ctx) const templates = yield liquid._parseFile(filepath, ctx.opts, ctx.sync) ctx.push(scope) yield renderer.renderTemplates(templates, ctx, emitter) diff --git a/src/builtin/tags/increment.ts b/src/builtin/tags/increment.ts index 02ec03f32..39d2655eb 100644 --- a/src/builtin/tags/increment.ts +++ b/src/builtin/tags/increment.ts @@ -1,13 +1,10 @@ -import { assert } from '../../util/assert' -import { identifier } from '../../parser/lexical' import { isNumber, stringify } from '../../util/underscore' -import { Emitter, TagToken, Context, TagImplOptions } from '../../types' +import { Tokenizer, Emitter, TagToken, Context, TagImplOptions } from '../../types' export default { parse: function (token: TagToken) { - const match = token.args.match(identifier) - assert(match, `illegal identifier ${token.args}`) - this.variable = match![0] + const tokenizer = new Tokenizer(token.args) + this.variable = tokenizer.readWord().content }, render: function (context: Context, emitter: Emitter) { const scope = context.environments diff --git a/src/builtin/tags/layout.ts b/src/builtin/tags/layout.ts index 21029c3b7..a27cf5a13 100644 --- a/src/builtin/tags/layout.ts +++ b/src/builtin/tags/layout.ts @@ -1,29 +1,25 @@ -import { assert } from '../../util/assert' -import { quotedLine, quoted } from '../../parser/lexical' -import { Emitter, Hash, Expression, TagToken, Token, Context, TagImplOptions } from '../../types' +import { assert, evalQuotedToken, TypeGuards, evalToken, Tokenizer, Emitter, Hash, TagToken, TopLevelToken, Context, TagImplOptions } from '../../types' import BlockMode from '../../context/block-mode' -const rFile = new RegExp(`^(${quoted.source}|[^\\s,]+)`) - export default { - parse: function (token: TagToken, remainTokens: Token[]) { - const match = rFile.exec(token.args) - if (!match) { - throw new Error(`illegal argument "${token.args}"`) - } - this.file = match[1] - this.hash = new Hash(token.args.slice(match[0].length)) + parse: function (token: TagToken, remainTokens: TopLevelToken[]) { + const tokenizer = new Tokenizer(token.args) + const file = this.liquid.options.dynamicPartials ? tokenizer.readValue() : tokenizer.readFileName() + assert(file, () => `illegal argument "${token.args}"`) + + this.file = file + this.hash = new Hash(tokenizer.remaining()) this.tpls = this.liquid.parser.parse(remainTokens) }, render: function * (ctx: Context, emitter: Emitter) { const { liquid, hash, file } = this const { renderer } = liquid const filepath = ctx.opts.dynamicPartials - ? (quotedLine.exec(file) - ? yield renderer.renderTemplates(liquid.parse(file.slice(1, -1)), ctx) - : yield new Expression(file).value(ctx)) - : this.file - assert(filepath, `illegal filename "${file}":"${filepath}"`) + ? (TypeGuards.isQuotedToken(file) + ? yield renderer.renderTemplates(liquid.parse(evalQuotedToken(file)), ctx) + : evalToken(this.file, ctx)) + : file.getText() + assert(filepath, () => `illegal filename "${file.getText()}":"${filepath}"`) // render the remaining tokens immediately ctx.setRegister('blockMode', BlockMode.STORE) diff --git a/src/builtin/tags/raw.ts b/src/builtin/tags/raw.ts index cd55a51f9..1aa5d1bf6 100644 --- a/src/builtin/tags/raw.ts +++ b/src/builtin/tags/raw.ts @@ -1,7 +1,7 @@ -import { TagToken, Token, TagImplOptions } from '../../types' +import { TagToken, TopLevelToken, TagImplOptions } from '../../types' export default { - parse: function (tagToken: TagToken, remainTokens: Token[]) { + parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) { this.tokens = [] const stream = this.liquid.parser.parseStream(remainTokens) @@ -11,11 +11,11 @@ export default { else this.tokens.push(token) }) .on('end', () => { - throw new Error(`tag ${tagToken.raw} not closed`) + throw new Error(`tag ${tagToken.getText()} not closed`) }) stream.start() }, render: function () { - return this.tokens.map((token: Token) => token.raw).join('') + return this.tokens.map((token: TopLevelToken) => token.getText()).join('') } } as TagImplOptions diff --git a/src/builtin/tags/render.ts b/src/builtin/tags/render.ts index 6b470c5db..86eda1ba6 100644 --- a/src/builtin/tags/render.ts +++ b/src/builtin/tags/render.ts @@ -1,56 +1,69 @@ import { assert } from '../../util/assert' import { ForloopDrop } from '../../drop/forloop-drop' import { toCollection } from '../../util/collection' -import { Expression, Hash, Emitter, TagToken, Context, TagImplOptions } from '../../types' -import { identifier, value, quoted, quotedLine } from '../../parser/lexical' - -const rFile = new RegExp(`^(${quoted.source}|[^\\s,]+)`) -const rWith = new RegExp(`^\\s+with\\s+(${value.source})(?:\\s+as\\s+(${identifier.source}))?`) -const rFor = new RegExp(`^\\s+for\\s+(${value.source})\\s+as\\s+(${identifier.source})`) +import { evalQuotedToken, TypeGuards, Tokenizer, evalToken, Hash, Emitter, TagToken, Context, TagImplOptions } from '../../types' export default { parse: function (token: TagToken) { - let args = token.args - let match = rFile.exec(args) + const args = token.args + const tokenizer = new Tokenizer(args) + this.file = this.liquid.options.dynamicPartials + ? tokenizer.readValue() + : tokenizer.readFileName() + assert(this.file, () => `illegal argument "${token.args}"`) - assert(match, `illegal argument "${token.args}"`) - this.file = match![1] - args = args.substr(match![0].length) + while (!tokenizer.end()) { + tokenizer.skipBlank() + const begin = tokenizer.p + const keyword = tokenizer.readWord() + if (keyword.content === 'with' || keyword.content === 'for') { + tokenizer.skipBlank() + if (tokenizer.peek() !== ':') { + const value = tokenizer.readValue() + if (value) { + const beforeAs = tokenizer.p + const asStr = tokenizer.readWord() + let alias + if (asStr.content === 'as') alias = tokenizer.readWord() + else tokenizer.p = beforeAs - while (true) { - if ((match = rWith.exec(args))) { - this.withVar = match[1] - this.withAs = match[2] - args = args.substr(match[0].length) - } else if ((match = rFor.exec(args))) { - this.forVar = match[1] - this.forAs = match[2] - args = args.substr(match[0].length) - } else break + this[keyword.content] = { value, alias: alias && alias.content } + tokenizer.skipBlank() + if (tokenizer.peek() === ',') tokenizer.advance() + continue + } + } + } + tokenizer.p = begin + break } - this.hash = new Hash(args) + this.hash = new Hash(tokenizer.remaining()) }, render: function * (ctx: Context, emitter: Emitter) { - const { liquid, withVar, withAs, forVar, forAs, file, hash } = this + const { liquid, file, hash } = this const { renderer } = liquid const filepath = ctx.opts.dynamicPartials - ? (quotedLine.exec(file) - ? yield renderer.renderTemplates(liquid.parse(file.slice(1, -1)), ctx) - : yield new Expression(file).value(ctx)) - : this.file - assert(filepath, `illegal filename "${file}":"${filepath}"`) + ? (TypeGuards.isQuotedToken(file) + ? yield renderer.renderTemplates(liquid.parse(evalQuotedToken(file)), ctx) + : evalToken(file, ctx)) + : file.getText() + assert(filepath, () => `illegal filename "${file.getText()}":"${filepath}"`) const childCtx = new Context({}, ctx.opts, ctx.sync) const scope = yield hash.render(ctx) - if (withVar) scope[withAs || filepath] = yield new Expression(withVar).evaluate(ctx) + if (this['with']) { + const { value, alias } = this['with'] + scope[alias || filepath] = evalToken(value, ctx) + } childCtx.push(scope) - if (forVar) { - let collection = yield new Expression(forVar).value(ctx) + if (this['for']) { + const { value, alias } = this['for'] + let collection = evalToken(value, ctx) collection = toCollection(collection) scope['forloop'] = new ForloopDrop(collection.length) for (const item of collection) { - scope[forAs] = item + scope[alias] = item const templates = yield liquid._parseFile(filepath, childCtx.opts, childCtx.sync) yield renderer.renderTemplates(templates, childCtx, emitter) scope.forloop.next() diff --git a/src/builtin/tags/tablerow.ts b/src/builtin/tags/tablerow.ts index f6a346318..48bc08817 100644 --- a/src/builtin/tags/tablerow.ts +++ b/src/builtin/tags/tablerow.ts @@ -1,21 +1,21 @@ -import { assert } from '../../util/assert' import { toCollection } from '../../util/collection' -import { Expression, Emitter, Hash, TagToken, Token, Context, Template, TagImplOptions, ParseStream } from '../../types' -import { identifier, value } from '../../parser/lexical' +import { assert, evalToken, Emitter, Hash, TagToken, TopLevelToken, Context, Template, TagImplOptions, ParseStream } from '../../types' import { TablerowloopDrop } from '../../drop/tablerowloop-drop' - -const re = new RegExp(`^(${identifier.source})\\s+in\\s+` + - `(${value.source})`) +import { Tokenizer } from '../../parser/tokenizer' export default { - parse: function (tagToken: TagToken, remainTokens: Token[]) { - const match = re.exec(tagToken.args) as RegExpExecArray - assert(match, `illegal tag: ${tagToken.raw}`) + parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) { + const tokenizer = new Tokenizer(tagToken.args) - this.variable = match[1] - this.collection = match[2] + this.variable = tokenizer.readWord() + tokenizer.skipBlank() + + const tmp = tokenizer.readWord() + assert(tmp && tmp.content === 'in', () => `illegal tag: ${tagToken.getText()}`) + + this.collection = tokenizer.readValue() + this.hash = new Hash(tokenizer.remaining()) this.templates = [] - this.hash = new Hash(tagToken.args.slice(match[0].length)) let p const stream: ParseStream = this.liquid.parser.parseStream(remainTokens) @@ -23,14 +23,14 @@ export default { .on('tag:endtablerow', () => stream.stop()) .on('template', (tpl: Template) => p.push(tpl)) .on('end', () => { - throw new Error(`tag ${tagToken.raw} not closed`) + throw new Error(`tag ${tagToken.getText()} not closed`) }) stream.start() }, render: function * (ctx: Context, emitter: Emitter) { - let collection = toCollection(yield new Expression(this.collection).value(ctx)) + let collection = toCollection(evalToken(this.collection, ctx)) const hash = yield this.hash.render(ctx) const offset = hash.offset || 0 const limit = (hash.limit === undefined) ? collection.length : hash.limit @@ -44,7 +44,7 @@ export default { ctx.push(scope) for (let idx = 0; idx < collection.length; idx++, tablerowloop.next()) { - scope[this.variable] = collection[idx] + scope[this.variable.content] = collection[idx] if (tablerowloop.col0() === 0) { if (tablerowloop.row() !== 1) emitter.write('') emitter.write(``) diff --git a/src/builtin/tags/unless.ts b/src/builtin/tags/unless.ts index 206d53ae3..643e48b19 100644 --- a/src/builtin/tags/unless.ts +++ b/src/builtin/tags/unless.ts @@ -1,7 +1,7 @@ -import { Emitter, Expression, isFalsy, ParseStream, Context, TagImplOptions, Token, TagToken } from '../../types' +import { TopLevelToken, Template, Emitter, Expression, isFalsy, ParseStream, Context, TagImplOptions, Token, TagToken } from '../../types' export default { - parse: function (tagToken: TagToken, remainTokens: Token[]) { + parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) { this.templates = [] this.elseTemplates = [] let p @@ -12,9 +12,9 @@ export default { }) .on('tag:else', () => (p = this.elseTemplates)) .on('tag:endunless', () => stream.stop()) - .on('template', tpl => p.push(tpl)) + .on('template', (tpl: Template) => p.push(tpl)) .on('end', () => { - throw new Error(`tag ${tagToken.raw} not closed`) + throw new Error(`tag ${tagToken.getText()} not closed`) }) stream.start() diff --git a/src/context/context.ts b/src/context/context.ts index 3263eb307..a0f729b59 100644 --- a/src/context/context.ts +++ b/src/context/context.ts @@ -1,6 +1,5 @@ import { Drop } from '../drop/drop' import { __assign } from 'tslib' -import { assert } from '../util/assert' import { NormalizedFullOptions, defaultOptions } from '../liquid-options' import { Scope } from './scope' import { isArray, isNil, isString, isFunction, toLiquid } from '../util/underscore' @@ -34,13 +33,12 @@ export class Context { return [this.globals, this.environments, ...this.scopes] .reduce((ctx, val) => __assign(ctx, val), {}) } - public get (path: string) { - const paths = this.parseProp(path) + public get (paths: string[]) { const scope = this.findScope(paths[0]) return this.getFromScope(scope, paths) } public getFromScope (scope: object, paths: string[] | string) { - if (!isArray(paths)) paths = this.parseProp(paths) + if (typeof paths === 'string') paths = paths.split('.') return paths.reduce((scope, path) => { scope = readProperty(scope, path) if (isNil(scope) && this.opts.strictVariables) { @@ -61,67 +59,11 @@ export class Context { private findScope (key: string) { for (let i = this.scopes.length - 1; i >= 0; i--) { const candidate = this.scopes[i] - if (key in candidate) { - return candidate - } + if (key in candidate) return candidate } if (key in this.environments) return this.environments return this.globals } - - /* - * Parse property access sequence from access string - * @example - * accessSeq("foo.bar") // ['foo', 'bar'] - * accessSeq("foo['bar']") // ['foo', 'bar'] - * accessSeq("foo['b]r']") // ['foo', 'b]r'] - * accessSeq("foo[bar.coo]") // ['foo', 'bar'], for bar.coo == 'bar' - */ - private parseProp (str: string) { - str = String(str) - const seq: string[] = [] - const push = () => name.length && (seq.push(name), (name = '')) - let name = '' - let j - let i = 0 - while (i < str.length) { - switch (str[i]) { - case '[': - push() - - const delemiter = str[i + 1] - if (/['"]/.test(delemiter)) { // foo["bar"] - j = str.indexOf(delemiter, i + 2) - assert(j !== -1, `unbalanced ${delemiter}: ${str}`) - name = str.slice(i + 2, j) - push() - i = j + 2 - } else { // foo[bar.coo] - j = matchRightBracket(str, i + 1) - assert(j !== -1, `unbalanced []: ${str}`) - name = str.slice(i + 1, j) - if (!/^[+-]?\d+$/.test(name)) { // foo[bar] vs. foo[1] - name = String(this.get(name)) - } - push() - i = j + 1 - } - break - case '.':// foo.bar, foo[0].bar - push() - i++ - break - default:// foo.bar - name += str[i++] - } - } - push() - - if (!seq.length) { - throw new TypeError(`invalid path:"${str}"`) - } - return seq - } } export function readProperty (obj: Scope, key: string) { @@ -152,19 +94,3 @@ function readSize (obj: Scope) { if (isArray(obj) || isString(obj)) return obj.length return obj['size'] } - -function matchRightBracket (str: string, begin: number) { - let stack = 1 // count of '[' - count of ']' - for (let i = begin; i < str.length; i++) { - if (str[i] === '[') { - stack++ - } - if (str[i] === ']') { - stack-- - if (stack === 0) { - return i - } - } - } - return -1 -} diff --git a/src/liquid.ts b/src/liquid.ts index 4c36cc8c5..d68d4c57c 100644 --- a/src/liquid.ts +++ b/src/liquid.ts @@ -38,8 +38,8 @@ export class Liquid { _.forOwn(builtinFilters, (handler, name) => this.registerFilter(name, handler)) } public parse (html: string, filepath?: string): Template[] { - const tokenizer = new Tokenizer(html, filepath, this.options) - const tokens = tokenizer.readTokens() + const tokenizer = new Tokenizer(html, filepath) + const tokens = tokenizer.readTopLevelTokens(this.options) return this.parser.parse(tokens) } diff --git a/src/parser/filter-arg.ts b/src/parser/filter-arg.ts index 43190ed62..b3e499068 100644 --- a/src/parser/filter-arg.ts +++ b/src/parser/filter-arg.ts @@ -1,9 +1,10 @@ import { isArray } from '../util/underscore' +import { ValueToken } from '../tokens/value-token' -type KeyValuePair = [string?, string?] +type KeyValuePair = [string?, ValueToken?] -export type FilterArg = string|KeyValuePair +export type FilterArg = ValueToken | KeyValuePair -export function isKeyValuePair (arr: FilterArg): arr is KeyValuePair { +export function isKeyValuePair (arr: FilterArg): arr is KeyValuePair { // TODO check return isArray(arr) } diff --git a/src/parser/html-token.ts b/src/parser/html-token.ts deleted file mode 100644 index 89ccf6eea..000000000 --- a/src/parser/html-token.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Token } from './token' - -export class HTMLToken extends Token { - public constructor (str: string, input: string, line: number, col: number, file?: string) { - super(str, input, line, col, file) - this.content = str - } - public static is (token: Token): token is HTMLToken { - return token instanceof HTMLToken - } -} diff --git a/src/parser/lexical.ts b/src/parser/lexical.ts deleted file mode 100644 index ca8a26a85..000000000 --- a/src/parser/lexical.ts +++ /dev/null @@ -1,29 +0,0 @@ -// quote related -const singleQuoted = /'[^']*'/ -const doubleQuoted = /"[^"]*"/ -export const quoted = new RegExp(`${singleQuoted.source}|${doubleQuoted.source}`) -export const quoteBalanced = new RegExp(`(?:${quoted.source}|[^'"])*`) - -// basic types -export const number = /[+-]?(?:\d+\.?\d*|\.?\d+)/ -export const bool = /true|false/ - -// property access -export const identifier = /[\w-]+[?]?/ -export const subscript = new RegExp(`\\[(?:${quoted.source}|[\\w-\\.]+)\\]`) -export const literal = new RegExp(`(?:${quoted.source}|${bool.source}|${number.source})`) -export const variable = new RegExp(`${identifier.source}(?:\\.${identifier.source}|${subscript.source})*`) - -// range related -export const rangeLimit = new RegExp(`(?:${variable.source}|${number.source})`) -export const range = new RegExp(`\\(${rangeLimit.source}\\.\\.${rangeLimit.source}\\)`) -export const rangeCapture = new RegExp(`\\((${rangeLimit.source})\\.\\.(${rangeLimit.source})\\)`) - -export const value = new RegExp(`(?:${variable.source}|${literal.source}|${range.source})`) - -// full match -export const tagLine = new RegExp(`^\\s*(${identifier.source})\\s*([\\s\\S]*?)\\s*$`) -export const numberLine = new RegExp(`^${number.source}$`) -export const boolLine = new RegExp(`^${bool.source}$`, 'i') -export const quotedLine = new RegExp(`^${quoted.source}$`) -export const rangeLine = new RegExp(`^${rangeCapture.source}$`) diff --git a/src/parser/match-operator.ts b/src/parser/match-operator.ts new file mode 100644 index 000000000..e1883b66d --- /dev/null +++ b/src/parser/match-operator.ts @@ -0,0 +1,24 @@ +import { VARIABLE } from '../util/character' + +const trie = { + a: { n: { d: { end: true, needBoundary: true } } }, + o: { r: { end: true, needBoundary: true } }, + c: { o: { n: { t: { a: { i: { n: { s: { end: true, needBoundary: true } } } } } } } }, + '=': { '=': { end: true } }, + '!': { '=': { end: true } }, + '>': { end: true, '=': { end: true } }, + '<': { end: true, '=': { end: true } } +} + +export function matchOperator (str: string, begin: number, end = str.length) { + let node = trie + let i = begin + let info + while (node[str[i]] && i < end) { + node = node[str[i++]] + if (node['end']) info = node + } + if (!info) return -1 + if (info['needBoundary'] && str.charCodeAt(i) & VARIABLE) return -1 + return i +} diff --git a/src/parser/output-token.ts b/src/parser/output-token.ts deleted file mode 100644 index 6145f235f..000000000 --- a/src/parser/output-token.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { DelimitedToken } from './delimited-token' -import { Token } from './token' -import { NormalizedFullOptions } from '../liquid-options' - -export class OutputToken extends DelimitedToken { - public constructor ( - raw: string, - value: string, - input: string, - line: number, - pos: number, - options: NormalizedFullOptions, - file?: string - ) { - super(raw, value, input, line, pos, options.trimOutputLeft, options.trimOutputRight, file) - } - public static is (token: Token): token is OutputToken { - return token instanceof OutputToken - } -} diff --git a/src/parser/parse-stream.ts b/src/parser/parse-stream.ts index f8f022567..5c7833907 100644 --- a/src/parser/parse-stream.ts +++ b/src/parser/parse-stream.ts @@ -1,20 +1,21 @@ -import { Token } from '../parser/token' +import { Token } from '../tokens/token' import { Template } from '../template/template' -import { TagToken } from './tag-token' +import { isTagToken } from '../util/type-guards' +import { TopLevelToken } from '../tokens/toplevel-token' -type ParseToken = ((token: Token, remainTokens: Token[]) => Template) +type ParseToken = ((token: T, remainTokens: T[]) => Template) -export class ParseStream { - private tokens: Token[] +export class ParseStream { + private tokens: T[] private handlers: {[key: string]: (arg: any) => void} = {} private stopRequested = false - private parseToken: ParseToken + private parseToken: ParseToken - public constructor (tokens: Token[], parseToken: ParseToken) { + public constructor (tokens: T[], parseToken: ParseToken) { this.tokens = tokens this.parseToken = parseToken } - public on (name: string, cb: (arg: T) => void): ParseStream { + public on (name: string, cb: (arg: T2) => void): ParseStream { this.handlers[name] = cb return this } @@ -24,10 +25,10 @@ export class ParseStream { } public start () { this.trigger('start') - let token: Token | undefined + let token: T | undefined while (!this.stopRequested && (token = this.tokens.shift())) { if (this.trigger('token', token)) continue - if (TagToken.is(token) && this.trigger(`tag:${token.name}`, token)) { + if (isTagToken(token) && this.trigger(`tag:${token.name}`, token)) { continue } const template = this.parseToken(token, this.tokens) diff --git a/src/parser/literal.ts b/src/parser/parse-string-literal.ts similarity index 60% rename from src/parser/literal.ts rename to src/parser/parse-string-literal.ts index 2262c4044..78e86eaea 100644 --- a/src/parser/literal.ts +++ b/src/parser/parse-string-literal.ts @@ -1,10 +1,3 @@ -import { last } from '../util/underscore' -import { NullDrop } from '../drop/null-drop' -import { EmptyDrop } from '../drop/empty-drop' -import { BlankDrop } from '../drop/blank-drop' - -type literal = true | false | NullDrop | EmptyDrop | BlankDrop | number | string - const rHex = /[\da-fA-F]/ const rOct = /[0-7]/ const escapeChar = { @@ -23,18 +16,6 @@ function hexVal (c: string) { return code - 48 } -export function parseLiteral (str: string): literal | undefined { - str = str.trim() - - if (str === 'true') return true - if (str === 'false') return false - if (str === 'nil' || str === 'null') return new NullDrop() - if (str === 'empty') return new EmptyDrop() - if (str === 'blank') return new BlankDrop() - if (!isNaN(Number(str))) return Number(str) - if ((str[0] === '"' || str[0] === "'") && str[0] === last(str)) return parseStringLiteral(str) -} - export function parseStringLiteral (str: string): string { let ret = '' for (let i = 1; i < str.length - 1; i++) { @@ -66,3 +47,4 @@ export function parseStringLiteral (str: string): string { } return ret } + diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 8cb613a1c..006eede24 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -1,13 +1,13 @@ import { ParseError } from '../util/error' import { Liquid } from '../liquid' import { ParseStream } from './parse-stream' -import { Token } from './token' -import { TagToken } from './tag-token' -import { OutputToken } from './output-token' +import { isTagToken, isOutputToken } from '../util/type-guards' +import { OutputToken } from '../tokens/output-token' import { Tag } from '../template/tag/tag' import { Output } from '../template/output' import { HTML } from '../template/html' import { Template } from '../template/template' +import { TopLevelToken } from '../tokens/toplevel-token' export default class Parser { private liquid: Liquid @@ -15,7 +15,7 @@ export default class Parser { public constructor (liquid: Liquid) { this.liquid = liquid } - public parse (tokens: Token[]) { + public parse (tokens: TopLevelToken[]) { let token const templates: Template[] = [] while ((token = tokens.shift())) { @@ -23,12 +23,12 @@ export default class Parser { } return templates } - public parseToken (token: Token, remainTokens: Token[]) { + public parseToken (token: TopLevelToken, remainTokens: TopLevelToken[]) { try { - if (TagToken.is(token)) { + if (isTagToken(token)) { return new Tag(token, remainTokens, this.liquid) } - if (OutputToken.is(token)) { + if (isOutputToken(token)) { return new Output(token as OutputToken, this.liquid.filters) } return new HTML(token) @@ -36,7 +36,7 @@ export default class Parser { throw new ParseError(e, token) } } - public parseStream (tokens: Token[]) { + public parseStream (tokens: TopLevelToken[]) { return new ParseStream(tokens, (token, tokens) => this.parseToken(token, tokens)) } } diff --git a/src/parser/substr.ts b/src/parser/substr.ts deleted file mode 100644 index b56c0c144..000000000 --- a/src/parser/substr.ts +++ /dev/null @@ -1,19 +0,0 @@ -export class Substr { - constructor ( - public str: string, - public begin: number, - public end: number = begin - ) {} - size () { - return this.end - this.begin - } - toString () { - return this.str.slice(this.begin, this.end) - } - first () { - return this.str[this.begin] - } - last () { - return this.str[this.end - 1] - } -} diff --git a/src/parser/tag-token.ts b/src/parser/tag-token.ts deleted file mode 100644 index b1811dd1f..000000000 --- a/src/parser/tag-token.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { DelimitedToken } from './delimited-token' -import { Token } from './token' -import { TokenizationError } from '../util/error' -import * as lexical from './lexical' -import { NormalizedFullOptions } from '../liquid-options' - -export class TagToken extends DelimitedToken { - public name: string - public args: string - public constructor ( - raw: string, - value: string, - input: string, - line: number, - pos: number, - options: NormalizedFullOptions, - file?: string - ) { - super(raw, value, input, line, pos, options.trimTagLeft, options.trimTagRight, file) - const match = this.content.match(lexical.tagLine) - if (!match) { - throw new TokenizationError(`illegal tag syntax`, this) - } - this.name = match[1] - this.args = match[2] - } - public static is (token: Token): token is TagToken { - return token instanceof TagToken - } -} diff --git a/src/parser/token-kind.ts b/src/parser/token-kind.ts new file mode 100644 index 000000000..a66a5330f --- /dev/null +++ b/src/parser/token-kind.ts @@ -0,0 +1,14 @@ +export enum TokenKind { + Number, + Literal, + Tag, + Output, + HTML, + Filter, + Hash, + PropertyAccess, + Word, + Range, + Quoted, + Operator +} diff --git a/src/parser/token.ts b/src/parser/token.ts deleted file mode 100644 index 6e05fd463..000000000 --- a/src/parser/token.ts +++ /dev/null @@ -1,14 +0,0 @@ -export class Token { - public trimLeft = false - public trimRight = false - public content: string - public constructor ( - public raw: string, - public input: string, - public line: number, - public col: number, - public file?: string - ) { - this.content = raw - } -} diff --git a/src/parser/tokenizer.ts b/src/parser/tokenizer.ts index e73458056..320a41e77 100644 --- a/src/parser/tokenizer.ts +++ b/src/parser/tokenizer.ts @@ -1,158 +1,158 @@ import { whiteSpaceCtrl } from './whitespace-ctrl' -import { Substr } from './substr' +import { NumberToken } from '../tokens/number-token' +import { WordToken } from '../tokens/word-token' +import { literalValues } from '../util/literal' +import { LiteralToken } from '../tokens/literal-token' +import { OperatorToken } from '../tokens/operator-token' +import { PropertyAccessToken } from '../tokens/property-access-token' +import { assert } from '../util/assert' +import { TopLevelToken } from '../tokens/toplevel-token' import { FilterArg } from './filter-arg' -import { FilterToken } from './filter-token' +import { FilterToken } from '../tokens/filter-token' +import { HashToken } from '../tokens/hash-token' +import { QuotedToken } from '../tokens/quoted-token' import { ellipsis } from '../util/underscore' -import { HTMLToken } from './html-token' -import { TagToken } from './tag-token' -import { Token } from './token' -import { OutputToken } from './output-token' +import { HTMLToken } from '../tokens/html-token' +import { TagToken } from '../tokens/tag-token' +import { Token } from '../tokens/token' +import { RangeToken } from '../tokens/range-token' +import { ValueToken } from '../tokens/value-token' +import { OutputToken } from '../tokens/output-token' import { TokenizationError } from '../util/error' import { NormalizedFullOptions, defaultOptions } from '../liquid-options' - -// bitmask character types to boost performance -// generated by bin/char-types.js -const TYPES = '00000000044004000000000000000000428000080000010011111111110022210111111111111111111111111110000101111111111111111111111111100000' -const VARIABLE = 1 -const OPERATOR = 2 -const BLANK = 4 -const QUOTE = 8 +import { TYPES, QUOTE, BLANK, VARIABLE } from '../util/character' +import { matchOperator } from './match-operator' export class Tokenizer { - private p = 0 - private N: number - private line = 1 - private col = 1 + p = 0 + N: number constructor ( private input: string, - private file: string = '', - private options: NormalizedFullOptions = defaultOptions + private file: string = '' ) { this.N = input.length } - * readExpression (): IterableIterator { + * readExpression (): IterableIterator { + const operand = this.readValue() + if (!operand) return + + yield operand + while (this.p < this.N) { + const operator = this.readOperator() + if (!operator) return + const operand = this.readValue() - if (operand.size()) { - yield operand.toString() - continue - } - this.readBlank() - const operator = new Substr(this.input, this.p) - while (OPERATOR & this.peekType()) operator.end = this.read() - if (operator.size()) { - yield operator.toString() - continue - } - this.read() + if (!operand) return + + yield operator + yield operand } } - - readFilterTokens (): FilterToken[] { + readOperator (): OperatorToken | undefined { + this.skipBlank() + const end = matchOperator(this.input, this.p, this.p + 8) + if (end === -1) return + return new OperatorToken(this.input, this.p, (this.p = end), this.file) + } + readFilters (): FilterToken[] { const filters = [] while (true) { - const filter = this.readFilterToken() + const filter = this.readFilter() if (!filter) return filters filters.push(filter) } } - - // | foo - // | foo: a - // | foo: a, b - // | foo: a, b: 1 - readFilterToken (): FilterToken | null { + readFilter (): FilterToken | null { this.readTo('|') const begin = this.p - const name = this.readVariable().toString() - if (!name) return null + const name = this.readWord() + if (!name.size()) return null const args = [] - this.readBlank() + this.skipBlank() if (this.peek() === ':') { do { - this.read() + ++this.p const arg = this.readFilterArg() arg && args.push(arg) - while (this.p < this.N && this.peek() !== ',' && this.peek() !== '|') this.read() + while (this.p < this.N && this.peek() !== ',' && this.peek() !== '|') ++this.p } while (this.peek() === ',') } - const raw = this.input.slice(begin, this.p) - return new FilterToken(name, args, raw, this.input, this.line, this.col, this.file) + return new FilterToken(name.getText(), args, this.input, begin, this.p, this.file) } - readFilterArg (): FilterArg | null { + readFilterArg (): FilterArg | undefined { const key = this.readValue() - if (!key.size()) return null - this.readBlank() - if (this.peek() === ':') { - this.read() - return [key.toString(), this.readValue().toString()] - } - return key.toString() + if (!key) return + this.skipBlank() + if (this.peek() !== ':') return key + ++this.p + const value = this.readValue() + return [key.getText(), value] } - readTokens (): Token[] { - const tokens: Token[] = [] + readTopLevelTokens (options: NormalizedFullOptions = defaultOptions): TopLevelToken[] { + const tokens: TopLevelToken[] = [] while (this.p < this.N) { - const token = this.readToken() + const token = this.readTopLevelToken(options) tokens.push(token) } - whiteSpaceCtrl(tokens, this.options) + whiteSpaceCtrl(tokens, options) return tokens } - readToken (): Token { - const { tagDelimiterLeft, outputDelimiterLeft } = this.options - if (this.matchWord(tagDelimiterLeft)) return this.readTagToken() - if (this.matchWord(outputDelimiterLeft)) return this.readOutputToken() - return this.readHTMLToken() + readTopLevelToken (options: NormalizedFullOptions): TopLevelToken { + const { tagDelimiterLeft, outputDelimiterLeft } = options + if (this.matchWord(tagDelimiterLeft)) return this.readTagToken(options) + if (this.matchWord(outputDelimiterLeft)) return this.readOutputToken(options) + return this.readHTMLToken(options) } - readHTMLToken (): HTMLToken { - const html = new Substr(this.input, this.p) + readHTMLToken (options: NormalizedFullOptions): HTMLToken { + const begin = this.p while (this.p < this.N) { - const { tagDelimiterLeft, outputDelimiterLeft } = this.options + const { tagDelimiterLeft, outputDelimiterLeft } = options if (this.matchWord(tagDelimiterLeft)) break if (this.matchWord(outputDelimiterLeft)) break - html.end = this.read() + ++this.p } - return new HTMLToken(html.toString(), this.input, this.line, this.col, this.file) + return new HTMLToken(this.input, begin, this.p, this.file) } - readTagToken (): TagToken { - const { line, col, file, input, options } = this - const { tagDelimiterLeft, tagDelimiterRight } = options - const buffer = this.readTo(tagDelimiterRight).toString() - if (!this.reverseMatchWord(tagDelimiterRight, buffer)) { - throw new TokenizationError( - `tag "${ellipsis(buffer, 16)}" not closed`, - new Token(buffer, input, line, col, file) - ) + readTagToken (options: NormalizedFullOptions): TagToken { + const { file, input } = this + const { tagDelimiterRight } = options + const begin = this.p + if (this.readTo(tagDelimiterRight) === -1) { + this.mkError(`tag "${this.ellipsis(begin)}" not closed`, begin) } - const value = buffer.slice(tagDelimiterLeft.length, -tagDelimiterRight.length) - return new TagToken(buffer, value, input, line, col, options, file) + return new TagToken(input, begin, this.p, options, file) } - readOutputToken (): OutputToken { - const { line, col, file, input, options } = this - const { outputDelimiterLeft, outputDelimiterRight } = options - const buffer = this.readTo(outputDelimiterRight).toString() - if (!this.reverseMatchWord(outputDelimiterRight, buffer)) { - throw new TokenizationError( - `output "${ellipsis(buffer, 16)}" not closed`, - new Token(buffer, input, line, col, file) - ) + readOutputToken (options: NormalizedFullOptions): OutputToken { + const { file, input } = this + const { outputDelimiterRight } = options + const begin = this.p + if (this.readTo(outputDelimiterRight) === -1) { + this.mkError(`output "${this.ellipsis(begin)}" not closed`, begin) } - const value = buffer.slice(outputDelimiterLeft.length, -outputDelimiterRight.length) - return new OutputToken(buffer, value, input, line, col, options, file) + return new OutputToken(input, begin, this.p, options, file) } - readVariable (): Substr { - this.readBlank() - const ans = new Substr(this.input, this.p) - while (this.peekType() & VARIABLE) ans.end = this.read() - return ans + mkError (msg: string, begin: number) { + throw new TokenizationError(msg, new WordToken(this.input, begin, this.N, this.file)) + } + + ellipsis (begin: number = this.p) { + return ellipsis(this.input.slice(begin), 16) + } + + readWord (): WordToken { // rename to identifier + this.skipBlank() + const begin = this.p + while (this.peekType() & VARIABLE) ++this.p + return new WordToken(this.input, begin, this.p, this.file) } readHashes () { @@ -164,133 +164,135 @@ export class Tokenizer { } } - readHash () { - this.readBlank() - if (this.peek() === ',') this.read() - const name = this.readVariable().toString() - if (!name) return null + readHash (): HashToken | undefined { + this.skipBlank() + if (this.peek() === ',') ++this.p + const begin = this.p + const name = this.readWord() + if (!name.size()) return + let value - this.readBlank() - let value = '' + this.skipBlank() if (this.peek() === ':') { - this.read() - value = this.readValue().toString() + ++this.p + value = this.readValue() } - return [name, value] + return new HashToken(this.input, begin, this.p, name, value, this.file) } - readPropertyAccess (): Substr { - this.readBlank() - const ans = new Substr(this.input, this.p) - let nested = 0 + remaining () { + return this.input.slice(this.p) + } + + advance (i = 1) { + this.p += i + } + + end () { + return this.p >= this.N + } + + readTo (end: string): number { while (this.p < this.N) { - const c = this.peek() - const code = this.peekType() - if (c === '[') { - this.read() - ans.end = this.readValue().end - nested++ - } else if (c === ']') { - if (!nested) break - ans.end = this.read() - nested-- - } else if (c === '.') { - if (this.peekType(1) & VARIABLE) { - this.read() - ans.end = this.readVariable().end - } else break - } else if (code & VARIABLE) { - ans.end = this.read() - } else { - if (nested) this.read() - else break - } + ++this.p + if (this.reverseMatchWord(end)) return this.p } - return ans + return -1 } - readTo (end: string): Substr { - const ans = new Substr(this.input, this.p) - while (this.p < this.N) { - ans.end = this.read() - if (this.reverseMatchWord(end)) break + + readValue (): ValueToken | undefined { + const value = this.readQuoted() || this.readRange() + if (value) return value + + const variable = this.readWord() + if (!variable.size()) return + + let isNumber = variable.isNumber(true) + const props: (QuotedToken | WordToken)[] = [] + while (true) { + if (this.peek() === '[') { + isNumber = false + this.p++ + const prop = this.readValue() || new WordToken(this.input, this.p, this.p, this.file) + this.readTo(']') + props.push(prop) + } else if (this.peek() === '.' && this.peek(1) !== '.') { // skip range syntax + this.p++ + const prop = this.readWord() + if (!prop.size()) break + if (!prop.isNumber()) isNumber = false + props.push(prop) + } else break } - return ans + if (!props.length && literalValues.hasOwnProperty(variable.content)) { + return new LiteralToken(this.input, variable.begin, variable.end, this.file) + } + if (isNumber) return new NumberToken(variable, props[0] as WordToken) + return new PropertyAccessToken(variable, props, this.p) } - readValue (): Substr { - let val = this.readQuoted() - if (val.size()) return val - val = this.readBoolean() - if (val.size()) return val - val = this.readPropertyAccess() - if (val.size()) return val - return this.readRange() + + readRange (): RangeToken | undefined { + this.skipBlank() + const begin = this.p + if (this.peek() !== '(') return + ++this.p + const lhs = this.readValueOrThrow() + this.p += 2 + const rhs = this.readValueOrThrow() + ++this.p + return new RangeToken(this.input, begin, this.p, lhs, rhs, this.file) } - readRange (): Substr { - this.readBlank() - const ans = new Substr(this.input, this.p) - if (this.peek() !== '(') return ans - this.read() - this.readValue() - this.read(2) - this.readValue() - ans.end = this.read() - return ans + + readValueOrThrow (): ValueToken { + const value = this.readValue() + assert(value, () => `unexpected token "${this.ellipsis()}", value expected`) + return value! } - readBoolean (): Substr { - this.readBlank() - const ans = new Substr(this.input, this.p) - if (this.matchWord('true') && !(this.peekType(4) & VARIABLE)) ans.end = this.read(4) - else if (this.matchWord('false') && !(this.peekType(5) & VARIABLE)) ans.end = this.read(5) - return ans - } - readQuoted (): Substr { - this.readBlank() - const ans = new Substr(this.input, this.p) - if (!(this.peekType() & QUOTE)) return ans - ans.end = this.read() + + readQuoted (): QuotedToken | undefined { + this.skipBlank() + const begin = this.p + if (!(this.peekType() & QUOTE)) return + ++this.p let escaped = false while (this.p < this.N) { - ans.end = this.read() - if (ans.last() === ans.first() && !escaped) break + ++this.p + if (this.input[this.p - 1] === this.input[begin] && !escaped) break if (escaped) escaped = false - else if (ans.last() === '\\') escaped = true + else if (this.input[this.p - 1] === '\\') escaped = true } - return ans + return new QuotedToken(this.input, begin, this.p, this.file) } - read (n = 1): number { - if (n > 1) this.read(n - 1) - const c = this.input[this.p++] - if (c === '\n') { - this.line++ - this.col = 1 - } else { - this.col++ - } - return this.p + + readFileName (): WordToken { + const begin = this.p + while (!(this.peekType() & BLANK) && this.peek() !== ',' && this.p < this.N) this.p++ + return new WordToken(this.input, begin, this.p, this.file) } + matchWord (word: string) { for (let i = 0; i < word.length; i++) { if (word[i] !== this.input[this.p + i]) return false } return true } - reverseMatchWord (word: string, buffer?: string) { - const str = buffer || this.input - const end = buffer === undefined ? this.p : buffer.length + + reverseMatchWord (word: string) { for (let i = 0; i < word.length; i++) { - if (word[word.length - 1 - i] !== str[end - 1 - i]) return false + if (word[word.length - 1 - i] !== this.input[this.p - 1 - i]) return false } return true } + peekType (n = 0) { - return +TYPES[this.input.charCodeAt(this.p + n)] + return TYPES[this.input.charCodeAt(this.p + n)] } + peek (n = 0) { return this.input[this.p + n] } - readBlank () { - let ans = '' - while (this.peekType() & BLANK) ans += this.read() - return ans + + skipBlank () { + while (this.peekType() & BLANK) ++this.p } } diff --git a/src/parser/whitespace-ctrl.ts b/src/parser/whitespace-ctrl.ts index 17e03e5e8..475314cd0 100644 --- a/src/parser/whitespace-ctrl.ts +++ b/src/parser/whitespace-ctrl.ts @@ -1,7 +1,8 @@ -import { Token } from '../parser/token' -import { TagToken } from '../parser/tag-token' -import { HTMLToken } from '../parser/html-token' +import { Token } from '../tokens/token' +import { DelimitedToken } from '../tokens/delimited-token' +import { isTagToken, isHTMLToken } from '../util/type-guards' import { NormalizedFullOptions } from '../liquid-options' +import { TYPES, INLINE_BLANK, BLANK } from '../util/character' export function whiteSpaceCtrl (tokens: Token[], options: NormalizedFullOptions) { options = { greedy: true, ...options } @@ -9,11 +10,12 @@ export function whiteSpaceCtrl (tokens: Token[], options: NormalizedFullOptions) for (let i = 0; i < tokens.length; i++) { const token = tokens[i] + if (!(token instanceof DelimitedToken)) continue if (!inRaw && token.trimLeft) { trimLeft(tokens[i - 1], options.greedy) } - if (TagToken.is(token)) { + if (isTagToken(token)) { if (token.name === 'raw') inRaw = true else if (token.name === 'endraw') inRaw = false } @@ -25,15 +27,16 @@ export function whiteSpaceCtrl (tokens: Token[], options: NormalizedFullOptions) } function trimLeft (token: Token, greedy: boolean) { - if (!token || !HTMLToken.is(token)) return + if (!token || !isHTMLToken(token)) return - const rLeft = greedy ? /\s+$/g : /[\t\r ]*$/g - token.content = token.content.replace(rLeft, '') + const mask = greedy ? BLANK : INLINE_BLANK + while (TYPES[token.input.charCodeAt(token.end - 1 - token.trimRight)] & mask) token.trimRight++ } function trimRight (token: Token, greedy: boolean) { - if (!token || !HTMLToken.is(token)) return + if (!token || !isHTMLToken(token)) return - const rRight = greedy ? /^\s+/g : /^[\t\r ]*\n?/g - token.content = token.content.replace(rRight, '') + const mask = greedy ? BLANK : INLINE_BLANK + while (TYPES[token.input.charCodeAt(token.begin + token.trimLeft)] & mask) token.trimLeft++ + if (token.input.charAt(token.begin + token.trimLeft) === '\n') token.trimLeft++ } diff --git a/src/render/expression.ts b/src/render/expression.ts index 817a5592d..26655dcda 100644 --- a/src/render/expression.ts +++ b/src/render/expression.ts @@ -1,47 +1,87 @@ +import { QuotedToken } from '../tokens/quoted-token' +import { NumberToken } from '../tokens/number-token' import { assert } from '../util/assert' -import { rangeLine } from '../parser/lexical' -import { parseLiteral } from '../parser/literal' +import { literalValues } from '../util/literal' +import { LiteralToken } from '../tokens/literal-token' +import * as TypeGuards from '../util/type-guards' +import { Token } from '../tokens/token' +import { OperatorToken } from '../tokens/operator-token' +import { RangeToken } from '../tokens/range-token' +import { parseStringLiteral } from '../parser/parse-string-literal' import { Context } from '../context/context' import { range, toValue } from '../util/underscore' -import { isOperator, precedence, operatorImpls } from './operator' import { Tokenizer } from '../parser/tokenizer' +import { operatorImpls } from '../render/operator' export class Expression { private operands: any[] = [] - private postfix: string[] + private postfix: IterableIterator - public constructor (str = '') { + public constructor (str: string) { const tokenizer = new Tokenizer(str) - this.postfix = [...toPostfix(tokenizer.readExpression())] + this.postfix = toPostfix(tokenizer.readExpression()) } - public * evaluate (ctx: Context): any { - assert(ctx, 'unable to evaluate: context not defined') - + public evaluate (ctx: Context): any { for (const token of this.postfix) { - if (isOperator(token)) { + if (TypeGuards.isOperatorToken(token)) { const r = this.operands.pop() const l = this.operands.pop() - const result = operatorImpls[token](l, r) + const result = evalOperatorToken(token, l, r) this.operands.push(result) - } else if (isRange(token)) { - this.operands.push(yield rangeValue(token, ctx)) } else { - const literal = parseLiteral(token) - this.operands.push(literal !== undefined ? literal : yield ctx.get(token)) + this.operands.push(evalToken(token, ctx)) } } return this.operands[0] } public * value (ctx: Context) { - return toValue(yield this.evaluate(ctx)) + return toValue(this.evaluate(ctx)) } } -function * toPostfix (tokens: IterableIterator): IterableIterator { - const ops = [] +export function evalToken (token: Token | undefined, ctx: Context): any { + assert(ctx, () => 'unable to evaluate: context not defined') + if (TypeGuards.isPropertyAccessToken(token)) { + const variable = token.variable.getText() + const props: string[] = token.props.map(prop => evalToken(prop, ctx)) + return ctx.get([variable, ...props]) + } + if (TypeGuards.isRangeToken(token)) return evalRangeToken(token, ctx) + if (TypeGuards.isLiteralToken(token)) return evalLiteralToken(token) + if (TypeGuards.isNumberToken(token)) return evalNumberToken(token) + if (TypeGuards.isWordToken(token)) return token.getText() + if (TypeGuards.isQuotedToken(token)) return evalQuotedToken(token) +} + +function evalNumberToken (token: NumberToken) { + const str = token.whole.content + '.' + (token.decimal ? token.decimal.content : '') + return Number(str) +} + +export function evalQuotedToken (token: QuotedToken) { + return parseStringLiteral(token.getText()) +} + +function evalOperatorToken (token: OperatorToken, lhs: any, rhs: any) { + const impl = operatorImpls[token.operator] + return impl(lhs, rhs) +} + +function evalLiteralToken (token: LiteralToken) { + return literalValues[token.literal] +} + +function evalRangeToken (token: RangeToken, ctx: Context) { + const low: number = evalToken(token.lhs, ctx) + const high: number = evalToken(token.rhs, ctx) + return range(+low, +high + 1) +} + +function * toPostfix (tokens: IterableIterator): IterableIterator { + const ops: OperatorToken[] = [] for (const token of tokens) { - if (isOperator(token)) { - while (ops.length && precedence[ops[ops.length - 1]] > precedence[token]) { + if (TypeGuards.isOperatorToken(token)) { + while (ops.length && ops[ops.length - 1].getPrecedence() > token.getPrecedence()) { yield ops.pop()! } ops.push(token) @@ -51,16 +91,3 @@ function * toPostfix (tokens: IterableIterator): IterableIterator': 1, - '<': 1, - '>=': 1, - '<=': 1, - 'contains': 1, - 'and': 0, - 'or': 0 -} - export const operatorImpls: {[key: string]: (lhs: any, rhs: any) => boolean} = { '==': (l: any, r: any) => { if (isComparable(l)) return l.equals(r) @@ -51,7 +39,3 @@ export const operatorImpls: {[key: string]: (lhs: any, rhs: any) => boolean} = { 'and': (l: any, r: any) => isTruthy(l) && isTruthy(r), 'or': (l: any, r: any) => isTruthy(l) || isTruthy(r) } - -export function isOperator (token: string) { - return precedence.hasOwnProperty(token) -} diff --git a/src/template/filter/filter-map.ts b/src/template/filter/filter-map.ts index af6b4264e..0724bae40 100644 --- a/src/template/filter/filter-map.ts +++ b/src/template/filter/filter-map.ts @@ -10,7 +10,7 @@ export class FilterMap { get (name: string) { const impl = this.impls[name] - assert(impl || !this.strictFilters, `undefined filter: ${name}`) + assert(impl || !this.strictFilters, () => `undefined filter: ${name}`) return impl } diff --git a/src/template/filter/filter.ts b/src/template/filter/filter.ts index 6cec08682..fd4de521b 100644 --- a/src/template/filter/filter.ts +++ b/src/template/filter/filter.ts @@ -1,4 +1,4 @@ -import { Expression } from '../../render/expression' +import { evalToken } from '../../render/expression' import { Context } from '../../context/context' import { identify } from '../../util/underscore' import { FilterImplOptions } from './filter-impl-options' @@ -16,9 +16,9 @@ export class Filter { } public * render (value: any, context: Context) { const argv: any[] = [] - for (const arg of this.args) { - if (isKeyValuePair(arg)) argv.push([arg[0], yield new Expression(arg[1]).evaluate(context)]) - else argv.push(yield new Expression(arg).evaluate(context)) + for (const arg of this.args as FilterArg[]) { + if (isKeyValuePair(arg)) argv.push([arg[0], yield evalToken(arg[1], context)]) + else argv.push(yield evalToken(arg, context)) } return this.impl.apply({ context }, [value, ...argv]) } diff --git a/src/template/html.ts b/src/template/html.ts index 0027fe280..faf0d2266 100644 --- a/src/template/html.ts +++ b/src/template/html.ts @@ -1,6 +1,6 @@ import { TemplateImpl } from '../template/template-impl' import { Template } from '../template/template' -import { HTMLToken } from '../parser/html-token' +import { HTMLToken } from '../tokens/html-token' import { Context } from '../context/context' import { Emitter } from '../render/emitter' @@ -8,7 +8,7 @@ export class HTML extends TemplateImpl implements Template { private str: string public constructor (token: HTMLToken) { super(token) - this.str = token.content + this.str = token.getContent() } public * render (ctx: Context, emitter: Emitter): IterableIterator { emitter.write(this.str) diff --git a/src/template/output.ts b/src/template/output.ts index 36281c931..29e147998 100644 --- a/src/template/output.ts +++ b/src/template/output.ts @@ -5,7 +5,7 @@ import { TemplateImpl } from '../template/template-impl' import { Template } from '../template/template' import { Context } from '../context/context' import { Emitter } from '../render/emitter' -import { OutputToken } from '../parser/output-token' +import { OutputToken } from '../tokens/output-token' export class Output extends TemplateImpl implements Template { private value: Value diff --git a/src/template/tag/hash.ts b/src/template/tag/hash.ts index 2d7825d3c..f48cbac00 100644 --- a/src/template/tag/hash.ts +++ b/src/template/tag/hash.ts @@ -1,4 +1,4 @@ -import { Expression } from '../../render/expression' +import { evalToken } from '../../render/expression' import { Context } from '../../context/context' import { Tokenizer } from '../../parser/tokenizer' @@ -11,17 +11,17 @@ import { Tokenizer } from '../../parser/tokenizer' * hash['reversed'] === undefined */ export class Hash { - [key: string]: any + hash: { [key: string]: any } = {} constructor (markup: string) { const tokenizer = new Tokenizer(markup) - for (const [name, value] of tokenizer.readHashes()) { - this[name] = value + for (const hash of tokenizer.readHashes()) { + this.hash[hash.name.content] = hash.value } } * render (ctx: Context) { const hash = {} - for (const key of Object.keys(this)) { - hash[key] = yield new Expression(this[key]).evaluate(ctx) + for (const key of Object.keys(this.hash)) { + hash[key] = evalToken(this.hash[key], ctx) } return hash } diff --git a/src/template/tag/tag-impl-options.ts b/src/template/tag/tag-impl-options.ts index a57e1ffde..432db1372 100644 --- a/src/template/tag/tag-impl-options.ts +++ b/src/template/tag/tag-impl-options.ts @@ -1,11 +1,11 @@ import { Context } from '../../context/context' -import { TagToken } from '../../parser/tag-token' -import { Token } from '../../parser/token' +import { TagToken } from '../../tokens/tag-token' +import { TopLevelToken } from '../../tokens/toplevel-token' import { TagImpl } from './tag-impl' import { Hash } from '../../template/tag/hash' import { Emitter } from '../../render/emitter' export interface TagImplOptions { - parse?: (this: TagImpl, token: TagToken, remainingTokens: Token[]) => void; + parse?: (this: TagImpl, token: TagToken, remainingTokens: TopLevelToken[]) => void; render: (this: TagImpl, ctx: Context, emitter: Emitter, hash: Hash) => any; } diff --git a/src/template/tag/tag-map.ts b/src/template/tag/tag-map.ts index fe0ef5ec4..f23ed2ba1 100644 --- a/src/template/tag/tag-map.ts +++ b/src/template/tag/tag-map.ts @@ -6,7 +6,7 @@ export class TagMap { get (name: string) { const impl = this.impls[name] - assert(impl, `tag "${name}" not found`) + assert(impl, () => `tag "${name}" not found`) return impl } diff --git a/src/template/tag/tag.ts b/src/template/tag/tag.ts index 13be5e753..1a38923ea 100644 --- a/src/template/tag/tag.ts +++ b/src/template/tag/tag.ts @@ -1,7 +1,7 @@ import { isFunction } from '../../util/underscore' import { Liquid } from '../../liquid' import { TemplateImpl } from '../../template/template-impl' -import { Emitter, Hash, Context, TagImplOptions, TagToken, Template, Token } from '../../types' +import { Emitter, Hash, Context, TagImplOptions, TagToken, Template, TopLevelToken } from '../../types' import { TagImpl } from './tag-impl' export class Tag extends TemplateImpl implements Template { @@ -9,7 +9,7 @@ export class Tag extends TemplateImpl implements Template { private impl: TagImpl private static impls: { [key: string]: TagImplOptions } = {} - public constructor (token: TagToken, tokens: Token[], liquid: Liquid) { + public constructor (token: TagToken, tokens: TopLevelToken[], liquid: Liquid) { super(token) this.name = token.name diff --git a/src/template/template.ts b/src/template/template.ts index 5ca052fdf..1764c5877 100644 --- a/src/template/template.ts +++ b/src/template/template.ts @@ -1,5 +1,5 @@ import { Context } from '../context/context' -import { Token } from '../parser/token' +import { Token } from '../tokens/token' import { Emitter } from '../render/emitter' export interface Template { diff --git a/src/template/value.ts b/src/template/value.ts index cd2b525d0..e8ef76563 100644 --- a/src/template/value.ts +++ b/src/template/value.ts @@ -1,23 +1,24 @@ -import { Expression } from '../render/expression' +import { evalToken } from '../render/expression' import { Tokenizer } from '../parser/tokenizer' import { FilterMap } from '../template/filter/filter-map' import { Filter } from './filter/filter' import { Context } from '../context/context' +import { ValueToken } from '../tokens/value-token' export class Value { public readonly filters: Filter[] = [] - public readonly initial: string + public readonly initial?: ValueToken /** * @param str the value to be valuated, eg.: "foobar" | truncate: 3 */ public constructor (str: string, private readonly filterMap: FilterMap) { const tokenizer = new Tokenizer(str) - this.initial = tokenizer.readValue().toString() - this.filters = tokenizer.readFilterTokens().map(({ name, args }) => new Filter(name, this.filterMap.get(name), args)) + this.initial = tokenizer.readValue() + this.filters = tokenizer.readFilters().map(({ name, args }) => new Filter(name, this.filterMap.get(name), args)) } public * value (ctx: Context) { - let val = yield new Expression(this.initial).evaluate(ctx) + let val = yield evalToken(this.initial, ctx) for (const filter of this.filters) { val = yield filter.render(val, ctx) } diff --git a/src/parser/delimited-token.ts b/src/tokens/delimited-token.ts similarity index 60% rename from src/parser/delimited-token.ts rename to src/tokens/delimited-token.ts index 764f6c970..1a9f2948f 100644 --- a/src/parser/delimited-token.ts +++ b/src/tokens/delimited-token.ts @@ -1,18 +1,23 @@ import { Token } from './token' +import { TokenKind } from '../parser/token-kind' import { last } from '../util/underscore' -export class DelimitedToken extends Token { +export abstract class DelimitedToken extends Token { + public trimLeft = false + public trimRight = false + public content: string public constructor ( - raw: string, + kind: TokenKind, content: string, input: string, - line: number, - pos: number, + begin: number, + end: number, trimLeft: boolean, trimRight: boolean, file?: string ) { - super(raw, input, line, pos, file) + super(kind, input, begin, end, file) + this.content = this.getText() const tl = content[0] === '-' const tr = last(content) === '-' this.content = content diff --git a/src/parser/filter-token.ts b/src/tokens/filter-token.ts similarity index 51% rename from src/parser/filter-token.ts rename to src/tokens/filter-token.ts index 316c83a4f..dbd8b3262 100644 --- a/src/parser/filter-token.ts +++ b/src/tokens/filter-token.ts @@ -1,16 +1,16 @@ import { Token } from './token' -import { FilterArg } from './filter-arg' +import { FilterArg } from '../parser/filter-arg' +import { TokenKind } from '../parser/token-kind' export class FilterToken extends Token { public constructor ( public name: string, public args: FilterArg[], - raw: string, input: string, - line: number, - col: number, + begin: number, + end: number, file?: string ) { - super(raw, input, line, col, file) + super(TokenKind.Filter, input, begin, end, file) } } diff --git a/src/tokens/hash-token.ts b/src/tokens/hash-token.ts new file mode 100644 index 000000000..288807c97 --- /dev/null +++ b/src/tokens/hash-token.ts @@ -0,0 +1,17 @@ +import { Token } from './token' +import { ValueToken } from './value-token' +import { WordToken } from './word-token' +import { TokenKind } from '../parser/token-kind' + +export class HashToken extends Token { + constructor ( + public input: string, + public begin: number, + public end: number, + public name: WordToken, + public value?: ValueToken, + public file?: string + ) { + super(TokenKind.Hash, input, begin, end, file) + } +} diff --git a/src/tokens/html-token.ts b/src/tokens/html-token.ts new file mode 100644 index 000000000..015467fdc --- /dev/null +++ b/src/tokens/html-token.ts @@ -0,0 +1,18 @@ +import { Token } from './token' +import { TokenKind } from '../parser/token-kind' + +export class HTMLToken extends Token { + trimLeft = 0 + trimRight = 0 + constructor ( + public input: string, + public begin: number, + public end: number, + public file?: string + ) { + super(TokenKind.HTML, input, begin, end, file) + } + public getContent () { + return this.input.slice(this.begin + this.trimLeft, this.end - this.trimRight) + } +} diff --git a/src/tokens/literal-token.ts b/src/tokens/literal-token.ts new file mode 100644 index 000000000..4e6a56be2 --- /dev/null +++ b/src/tokens/literal-token.ts @@ -0,0 +1,15 @@ +import { Token } from './token' +import { TokenKind } from '../parser/token-kind' + +export class LiteralToken extends Token { + public literal: string + public constructor ( + public input: string, + public begin: number, + public end: number, + public file?: string + ) { + super(TokenKind.Literal, input, begin, end, file) + this.literal = this.getText() + } +} diff --git a/src/tokens/number-token.ts b/src/tokens/number-token.ts new file mode 100644 index 000000000..b029d0a0d --- /dev/null +++ b/src/tokens/number-token.ts @@ -0,0 +1,12 @@ +import { Token } from './token' +import { WordToken } from './word-token' +import { TokenKind } from '../parser/token-kind' + +export class NumberToken extends Token { + constructor ( + public whole: WordToken, + public decimal?: WordToken + ) { + super(TokenKind.Number, whole.input, whole.begin, decimal ? decimal.end : whole.end, whole.file) + } +} diff --git a/src/tokens/operator-token.ts b/src/tokens/operator-token.ts new file mode 100644 index 000000000..31631f562 --- /dev/null +++ b/src/tokens/operator-token.ts @@ -0,0 +1,30 @@ +import { Token } from './token' +import { TokenKind } from '../parser/token-kind' + +export const precedence = { + '==': 1, + '!=': 1, + '>': 1, + '<': 1, + '>=': 1, + '<=': 1, + 'contains': 1, + 'and': 0, + 'or': 0 +} + +export class OperatorToken extends Token { + public operator: string + public constructor ( + public input: string, + public begin: number, + public end: number, + public file?: string + ) { + super(TokenKind.Operator, input, begin, end, file) + this.operator = this.getText() + } + getPrecedence () { + return precedence[this.getText()] + } +} diff --git a/src/tokens/output-token.ts b/src/tokens/output-token.ts new file mode 100644 index 000000000..d6f8b748f --- /dev/null +++ b/src/tokens/output-token.ts @@ -0,0 +1,17 @@ +import { DelimitedToken } from './delimited-token' +import { NormalizedFullOptions } from '../liquid-options' +import { TokenKind } from '../parser/token-kind' + +export class OutputToken extends DelimitedToken { + public constructor ( + input: string, + begin: number, + end: number, + options: NormalizedFullOptions, + file?: string + ) { + const { trimOutputLeft, trimOutputRight, outputDelimiterLeft, outputDelimiterRight } = options + const value = input.slice(begin + outputDelimiterLeft.length, end - outputDelimiterRight.length) + super(TokenKind.Output, value, input, begin, end, trimOutputLeft, trimOutputRight, file) + } +} diff --git a/src/tokens/property-access-token.ts b/src/tokens/property-access-token.ts new file mode 100644 index 000000000..79512c25e --- /dev/null +++ b/src/tokens/property-access-token.ts @@ -0,0 +1,14 @@ +import { Token } from './token' +import { WordToken } from './word-token' +import { QuotedToken } from './quoted-token' +import { TokenKind } from '../parser/token-kind' + +export class PropertyAccessToken extends Token { + constructor ( + public variable: WordToken, + public props: (WordToken | QuotedToken | PropertyAccessToken)[], + end: number + ) { + super(TokenKind.PropertyAccess, variable.input, variable.begin, end, variable.file) + } +} diff --git a/src/tokens/quoted-token.ts b/src/tokens/quoted-token.ts new file mode 100644 index 000000000..803023a38 --- /dev/null +++ b/src/tokens/quoted-token.ts @@ -0,0 +1,13 @@ +import { Token } from './token' +import { TokenKind } from '../parser/token-kind' + +export class QuotedToken extends Token { + constructor ( + public input: string, + public begin: number, + public end: number, + public file?: string + ) { + super(TokenKind.Quoted, input, begin, end, file) + } +} diff --git a/src/tokens/range-token.ts b/src/tokens/range-token.ts new file mode 100644 index 000000000..62ffa8727 --- /dev/null +++ b/src/tokens/range-token.ts @@ -0,0 +1,16 @@ +import { Token } from './token' +import { ValueToken } from './value-token' +import { TokenKind } from '../parser/token-kind' + +export class RangeToken extends Token { + constructor ( + public input: string, + public begin: number, + public end: number, + public lhs: ValueToken, + public rhs: ValueToken, + public file?: string + ) { + super(TokenKind.Range, input, begin, end, file) + } +} diff --git a/src/tokens/tag-token.ts b/src/tokens/tag-token.ts new file mode 100644 index 000000000..b7b3d68e7 --- /dev/null +++ b/src/tokens/tag-token.ts @@ -0,0 +1,30 @@ +import { DelimitedToken } from './delimited-token' +import { BLANK, TYPES, VARIABLE } from '../util/character' +import { TokenizationError } from '../util/error' +import { NormalizedFullOptions } from '../liquid-options' +import { TokenKind } from '../parser/token-kind' + +export class TagToken extends DelimitedToken { + public name: string + public args: string + public constructor ( + input: string, + begin: number, + end: number, + options: NormalizedFullOptions, + file?: string + ) { + const { trimTagLeft, trimTagRight, tagDelimiterLeft, tagDelimiterRight } = options + const value = input.slice(begin + tagDelimiterLeft.length, end - tagDelimiterRight.length) + super(TokenKind.Tag, value, input, begin, end, trimTagLeft, trimTagRight, file) + + let nameEnd = 0 + while (TYPES[this.content.charCodeAt(nameEnd)] & VARIABLE) nameEnd++ + this.name = this.content.slice(0, nameEnd) + if (!this.name) throw new TokenizationError(`illegal tag syntax`, this) + + let argsBegin = nameEnd + while (TYPES[this.content.charCodeAt(argsBegin)] & BLANK) argsBegin++ + this.args = this.content.slice(argsBegin) + } +} diff --git a/src/tokens/token.ts b/src/tokens/token.ts new file mode 100644 index 000000000..9cae30a6c --- /dev/null +++ b/src/tokens/token.ts @@ -0,0 +1,27 @@ +import { TokenKind } from '../parser/token-kind' + +export abstract class Token { + public constructor ( + public kind: TokenKind, + public input: string, + public begin: number, + public end: number, + public file?: string + ) {} + public getText () { + return this.input.slice(this.begin, this.end) + } + public getPosition () { + let [row, col] = [1, 1] + for (let i = 0; i < this.begin; i++) { + if (this.input[i] === '\n') { + row++ + col = 1 + } else col++ + } + return [row, col] + } + public size () { + return this.end - this.begin + } +} diff --git a/src/tokens/toplevel-token.ts b/src/tokens/toplevel-token.ts new file mode 100644 index 000000000..320d2df5f --- /dev/null +++ b/src/tokens/toplevel-token.ts @@ -0,0 +1,5 @@ +import { TagToken } from './tag-token' +import { HTMLToken } from './html-token' +import { OutputToken } from './output-token' + +export type TopLevelToken = TagToken | OutputToken | HTMLToken diff --git a/src/tokens/value-token.ts b/src/tokens/value-token.ts new file mode 100644 index 000000000..55650ce30 --- /dev/null +++ b/src/tokens/value-token.ts @@ -0,0 +1,6 @@ +import { RangeToken } from './range-token' +import { LiteralToken } from './literal-token' +import { QuotedToken } from './quoted-token' +import { PropertyAccessToken } from './property-access-token' + +export type ValueToken = RangeToken | LiteralToken | QuotedToken | PropertyAccessToken diff --git a/src/tokens/word-token.ts b/src/tokens/word-token.ts new file mode 100644 index 000000000..f19de6fbd --- /dev/null +++ b/src/tokens/word-token.ts @@ -0,0 +1,26 @@ +import { Token } from './token' +import { NUMBER, TYPES, SIGN } from '../util/character' +import { TokenKind } from '../parser/token-kind' + +// a word can be an identifier, a number, a keyword or a single-word-literal +export class WordToken extends Token { + public content: string + constructor ( + public input: string, + public begin: number, + public end: number, + public file?: string + ) { + super(TokenKind.Word, input, begin, end, file) + this.content = this.getText() + } + isNumber (allowSign = false) { + const begin = allowSign && TYPES[this.input.charCodeAt(this.begin)] & SIGN + ? this.begin + 1 + : this.begin + for (let i = begin; i < this.end; i++) { + if (!(TYPES[this.input.charCodeAt(i)] & NUMBER)) return false + } + return true + } +} diff --git a/src/types.ts b/src/types.ts index c29eea062..30c0acf9f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,13 +1,18 @@ +import * as TypeGuards from './util/type-guards' +export { TypeGuards } export { ParseError, TokenizationError, AssertionError } from './util/error' +export { assert } from './util/assert' export { Drop } from './drop/drop' export { Emitter } from './render/emitter' export { Expression } from './render/expression' export { isFalsy, isTruthy } from './render/boolean' -export { TagToken } from './parser/tag-token' +export { TagToken } from './tokens/tag-token' export { Context } from './context/context' export { Template } from './template/template' export { TagImplOptions } from './template/tag/tag-impl-options' export { ParseStream } from './parser/parse-stream' -export { Token } from './parser/token' +export { Token } from './tokens/token' +export { TopLevelToken } from './tokens/toplevel-token' export { Tokenizer } from './parser/tokenizer' export { Hash } from './template/tag/hash' +export { evalToken, evalQuotedToken } from './render/expression' diff --git a/src/util/assert.ts b/src/util/assert.ts index c67205b40..c2fc92f1b 100644 --- a/src/util/assert.ts +++ b/src/util/assert.ts @@ -1,8 +1,8 @@ import { AssertionError } from './error' -export function assert (predicate: T | null | undefined, message?: string) { +export function assert (predicate: T | null | undefined, message?: () => string) { if (!predicate) { - message = message || `expect ${predicate} to be true` - throw new AssertionError(message) + const msg = message ? message() : `expect ${predicate} to be true` + throw new AssertionError(msg) } } diff --git a/src/util/async.ts b/src/util/async.ts index 01a442316..a79a7aefc 100644 --- a/src/util/async.ts +++ b/src/util/async.ts @@ -1,6 +1,6 @@ import { isFunction } from './underscore' -type resolver = (x?: any) => Thenable +type resolver = (x?: any) => any interface Thenable { then (resolve: resolver, reject?: resolver): Thenable; diff --git a/src/util/character.ts b/src/util/character.ts new file mode 100644 index 000000000..5b37e02a6 --- /dev/null +++ b/src/util/character.ts @@ -0,0 +1,10 @@ +// bitmask character types to boost performance +// generated by bin/char-types.js +export const TYPES = [0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2, 8, 0, 0, 0, 0, 8, 0, 0, 0, 64, 0, 65, 0, 0, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 0, 0, 2, 2, 2, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0] +export const VARIABLE = 1 +export const OPERATOR = 2 +export const BLANK = 4 +export const QUOTE = 8 +export const INLINE_BLANK = 16 +export const NUMBER = 32 +export const SIGN = 64 diff --git a/src/util/error.ts b/src/util/error.ts index 860355b50..f0a95ccec 100644 --- a/src/util/error.ts +++ b/src/util/error.ts @@ -1,5 +1,5 @@ import * as _ from './underscore' -import { Token } from '../parser/token' +import { Token } from '../tokens/token' import { Template } from '../template/template' abstract class LiquidError extends Error { @@ -57,14 +57,15 @@ export class AssertionError extends Error { } function mkContext (token: Token) { + const [line] = token.getPosition() const lines = token.input.split('\n') - const begin = Math.max(token.line - 2, 1) - const end = Math.min(token.line + 3, lines.length) + const begin = Math.max(line - 2, 1) + const end = Math.min(line + 3, lines.length) const context = _ .range(begin, end + 1) .map(lineNumber => { - const indicator = (lineNumber === token.line) ? '>> ' : ' ' + const indicator = (lineNumber === line) ? '>> ' : ' ' const num = _.padStart(String(lineNumber), String(end).length) const text = lines[lineNumber - 1] return `${indicator}${num}| ${text}` @@ -76,6 +77,7 @@ function mkContext (token: Token) { function mkMessage (msg: string, token: Token) { if (token.file) msg += `, file:${token.file}` - msg += `, line:${token.line}, col:${token.col}` + const [line, col] = token.getPosition() + msg += `, line:${line}, col:${col}` return msg } diff --git a/src/util/literal.ts b/src/util/literal.ts new file mode 100644 index 000000000..c1091e5d5 --- /dev/null +++ b/src/util/literal.ts @@ -0,0 +1,12 @@ +import { NullDrop } from '../drop/null-drop' +import { EmptyDrop } from '../drop/empty-drop' +import { BlankDrop } from '../drop/blank-drop' + +export const literalValues = { + 'true': true, + 'false': false, + 'nil': new NullDrop(), + 'null': new NullDrop(), + 'empty': new EmptyDrop(), + 'blank': new BlankDrop() +} diff --git a/src/util/type-guards.ts b/src/util/type-guards.ts new file mode 100644 index 000000000..095a5e9c0 --- /dev/null +++ b/src/util/type-guards.ts @@ -0,0 +1,55 @@ +import { OperatorToken } from '../tokens/operator-token' +import { WordToken } from '../tokens/word-token' +import { TagToken } from '../tokens/tag-token' +import { HTMLToken } from '../tokens/html-token' +import { OutputToken } from '../tokens/output-token' +import { PropertyAccessToken } from '../tokens/property-access-token' +import { LiteralToken } from '../tokens/literal-token' +import { QuotedToken } from '../tokens/quoted-token' +import { NumberToken } from '../tokens/number-token' +import { RangeToken } from '../tokens/range-token' +import { TokenKind } from '../parser/token-kind' + +export function isOperatorToken (val: any): val is OperatorToken { + return getKind(val) === TokenKind.Operator +} + +export function isHTMLToken (val: any): val is HTMLToken { + return getKind(val) === TokenKind.HTML +} + +export function isOutputToken (val: any): val is OutputToken { + return getKind(val) === TokenKind.Output +} + +export function isTagToken (val: any): val is TagToken { + return getKind(val) === TokenKind.Tag +} + +export function isQuotedToken (val: any): val is QuotedToken { + return getKind(val) === TokenKind.Quoted +} + +export function isLiteralToken (val: any): val is LiteralToken { + return getKind(val) === TokenKind.Literal +} + +export function isNumberToken (val: any): val is NumberToken { + return getKind(val) === TokenKind.Number +} + +export function isPropertyAccessToken (val: any): val is PropertyAccessToken { + return getKind(val) === TokenKind.PropertyAccess +} + +export function isWordToken (val: any): val is WordToken { + return getKind(val) === TokenKind.Word +} + +export function isRangeToken (val: any): val is RangeToken { + return getKind(val) === TokenKind.Range +} + +function getKind (val: any) { + return val ? val.kind : -1 +} diff --git a/test/e2e/eval-value-sync.ts b/test/e2e/eval-value-sync.ts index 030117d9f..377a57c1c 100644 --- a/test/e2e/eval-value-sync.ts +++ b/test/e2e/eval-value-sync.ts @@ -6,6 +6,6 @@ describe('#evalValueSync()', function () { beforeEach(() => { engine = new Liquid() }) it('should throw when scope undefined', async function () { - return expect(() => engine.evalValueSync('{{"foo"}}', null as any)).to.throw(/context not defined/) + return expect(() => engine.evalValueSync('foo', null as any)).to.throw(/context not defined/) }) }) diff --git a/test/e2e/eval-value.ts b/test/e2e/eval-value.ts index c46e9d4a4..9cf84f135 100644 --- a/test/e2e/eval-value.ts +++ b/test/e2e/eval-value.ts @@ -1,11 +1,11 @@ import { expect } from 'chai' import { Liquid } from '../..' -describe('.evalValue()', function () { +describe('#evalValue()', function () { var engine: Liquid beforeEach(() => { engine = new Liquid() }) it('should throw when scope undefined', async function () { - return expect(engine.evalValue('{{"foo"}}', null as any)).to.be.rejectedWith(/context not defined/) + return expect(engine.evalValue('"foo"', null as any)).to.be.rejectedWith(/context not defined/) }) }) diff --git a/test/integration/builtin/tags/decrement.ts b/test/integration/builtin/tags/decrement.ts index 32f23c8cf..e5eab0923 100644 --- a/test/integration/builtin/tags/decrement.ts +++ b/test/integration/builtin/tags/decrement.ts @@ -6,11 +6,6 @@ use(chaiAsPromised) describe('tags/decrement', function () { const liquid = new Liquid() - it('should throw when variable expression illegal', function () { - const src = '{% decrement / %}{{var}}' - const ctx = {} - return expect(liquid.parseAndRender(src, ctx)).to.be.rejectedWith(/illegal/) - }) it('should decrement undefined variable', async function () { const src = '{% decrement var %}{% decrement var %}{% decrement var %}' diff --git a/test/integration/builtin/tags/for.ts b/test/integration/builtin/tags/for.ts index 7e22bdd10..0c1cb8e0b 100644 --- a/test/integration/builtin/tags/for.ts +++ b/test/integration/builtin/tags/for.ts @@ -46,6 +46,12 @@ describe('tags/for', function () { .to.be.rejectedWith(/tag .* not closed/) }) + it('should reject when for in not found', function () { + const src = '{%for c alpha%}{{c}}' + return expect(liquid.parseAndRender(src, scope)) + .to.be.rejectedWith('illegal tag: {%for c alpha%}, line:1, col:1') + }) + it('should reject when inner templates rejected', function () { const src = '{%for c in alpha%}{%throwingTag%}{%endfor%}' return expect(liquid.parseAndRender(src, scope)) diff --git a/test/integration/builtin/tags/include.ts b/test/integration/builtin/tags/include.ts index 6daf2729e..ba10602db 100644 --- a/test/integration/builtin/tags/include.ts +++ b/test/integration/builtin/tags/include.ts @@ -83,6 +83,22 @@ describe('tags/include', function () { const html = await liquid.renderFile('with.html') return expect(html).to.equal('color:red, shape:rect') }) + it('should ignore if with value not specified', async function () { + mock({ + '/with.html': '{% include "color" with, shape: "rect" %}', + '/color.html': 'color:{{color}}, shape:{{shape}}' + }) + const html = await liquid.renderFile('with.html') + return expect(html).to.equal('color:, shape:rect') + }) + it('should treat with as a valid key', async function () { + mock({ + '/with.html': '{% include "color" with: "foo" %}', + '/color.html': 'with:{{with}}' + }) + const html = await liquid.renderFile('with.html') + return expect(html).to.equal('with:foo') + }) it('should support include: with as Drop', async function () { class ColorDrop extends Drop { public valueOf (): string { diff --git a/test/integration/builtin/tags/layout.ts b/test/integration/builtin/tags/layout.ts index 4e9926f4d..1b84f38a6 100644 --- a/test/integration/builtin/tags/layout.ts +++ b/test/integration/builtin/tags/layout.ts @@ -1,6 +1,9 @@ import { Liquid } from '../../../../src/liquid' -import { expect } from 'chai' +import { expect, use } from 'chai' import { mock, restore } from '../../../stub/mockfs' +import * as chaiAsPromised from 'chai-as-promised' + +use(chaiAsPromised) describe('tags/layout', function () { let liquid: Liquid @@ -29,6 +32,15 @@ describe('tags/layout', function () { expect(e.message).to.match(/illegal argument ""/) }) }) + it('should throw when filename resolved to falsy', function () { + mock({ + '/parent.html': '{%layout foo%}' + }) + return liquid.renderFile('/parent.html').catch(function (e) { + expect(e.name).to.equal('RenderError') + expect(e.message).to.match(/illegal filename "foo":"undefined"/) + }) + }) describe('anonymous block', function () { it('should handle anonymous block', async function () { mock({ diff --git a/test/integration/builtin/tags/render.ts b/test/integration/builtin/tags/render.ts index 797a0e547..61a19ce50 100644 --- a/test/integration/builtin/tags/render.ts +++ b/test/integration/builtin/tags/render.ts @@ -121,9 +121,33 @@ describe('tags/render', function () { const html = await liquid.renderFile('index.html', { colors: ['red', 'green'] }) expect(html).to.equal('1: red\n2: green\n') }) + it('should support for as', async function () { + mock({ + '/index.html': '{% render "item" for "green" as color %}', + '/item.html': '{{forloop.index}}: {{color}}\n' + }) + const html = await liquid.renderFile('index.html') + expect(html).to.equal('1: green\n') + }) + it('should support for without as', async function () { + mock({ + '/index.html': '{% render "item" for colors %}', + '/item.html': '{{forloop.index}}: {{color}}\n' + }) + const html = await liquid.renderFile('index.html', { colors: ['red', 'green'] }) + expect(html).to.equal('1: \n2: \n') + }) it('should support for...as with other parameters', async function () { mock({ - '/index.html': '{% render "item" for colors as color with ".\n" as tail, sep: ". "%}', + '/index.html': '{% render "item" for colors as color with ".\n" as tail sep: ". "%}', + '/item.html': '{{forloop.index}}{{sep}}{{color}}{{tail}}' + }) + const html = await liquid.renderFile('index.html', { colors: ['red', 'green'] }) + expect(html).to.equal('1. red.\n2. green.\n') + }) + it('should support for...as with other parameters (comma separated)', async function () { + mock({ + '/index.html': '{% render "item" for colors as color, with ".\n" as tail, sep: ". "%}', '/item.html': '{{forloop.index}}{{sep}}{{color}}{{tail}}' }) const html = await liquid.renderFile('index.html', { colors: ['red', 'green'] }) diff --git a/test/integration/builtin/tags/tablerow.ts b/test/integration/builtin/tags/tablerow.ts index eff3538a3..a71ee2e4f 100644 --- a/test/integration/builtin/tags/tablerow.ts +++ b/test/integration/builtin/tags/tablerow.ts @@ -1,5 +1,8 @@ import { Liquid } from '../../../../src/liquid' -import { expect } from 'chai' +import { expect, use } from 'chai' +import * as chaiAsPromised from 'chai-as-promised' + +use(chaiAsPromised) describe('tags/tablerow', function () { const liquid = new Liquid() @@ -50,6 +53,12 @@ describe('tags/tablerow', function () { .to.be.rejectedWith(/tag .* not closed/) }) + it('should throw when x in y not found', function () { + const src = '{% tablerow i (1..3) %}{{ i }}' + return expect(liquid.parseAndRender(src)) + .to.be.rejectedWith('illegal tag: {% tablerow i (1..3) %}, line:1, col:1') + }) + it('should support tablerow with range', async function () { const src = '{% tablerow i in (1..5) cols:2 %}{{ i }}{% endtablerow %}' const dst = diff --git a/test/integration/liquid/liquid.ts b/test/integration/liquid/liquid.ts index f5bb3e0ad..00a245a17 100644 --- a/test/integration/liquid/liquid.ts +++ b/test/integration/liquid/liquid.ts @@ -93,7 +93,7 @@ describe('Liquid', function () { }) const tpls = await engine.getTemplate('mocha') expect(tpls.length).to.gte(1) - expect(tpls[0].token.raw).to.contain('module.exports') + expect(tpls[0].token.getText()).to.contain('module.exports') }) }) describe('#evalValue', function () { diff --git a/test/integration/liquid/register-tags.ts b/test/integration/liquid/register-tags.ts index 1b73e7a5d..d0cf36edd 100644 --- a/test/integration/liquid/register-tags.ts +++ b/test/integration/liquid/register-tags.ts @@ -21,7 +21,7 @@ describe('liquid#registerTag()', function () { it('should have access to ctx in render()', async () => { const liquid = new Liquid() liquid.registerTag('dynamic-string', { - render: async (ctx) => ctx.get('c') + render: async (ctx) => ctx.get(['c']) }) const html = await liquid.parseAndRender(`A{% dynamic-string %}C`, { c: 'B' diff --git a/test/integration/liquid/liquid-options.ts b/test/integration/liquid/root.ts similarity index 91% rename from test/integration/liquid/liquid-options.ts rename to test/integration/liquid/root.ts index 80e9a7b1e..d6caf3428 100644 --- a/test/integration/liquid/liquid-options.ts +++ b/test/integration/liquid/root.ts @@ -1,7 +1,7 @@ import { normalize } from '../../../src/liquid-options' import { expect } from 'chai' -describe('LiquidOptions', function () { +describe('LiquidOptions#root', function () { describe('#normalize ()', function () { it('should normalize string typed root array', function () { const options = normalize({ root: 'foo' }) diff --git a/test/integration/liquid/trimming.ts b/test/integration/liquid/trimming.ts index 1d620eace..6d6240844 100644 --- a/test/integration/liquid/trimming.ts +++ b/test/integration/liquid/trimming.ts @@ -45,7 +45,7 @@ describe('LiquidOptions#trimming', function () { const html = await engine.parseAndRender(src, ctx) return expect(html).to.equal('aharttle') }) - it('should respect to greedy:false by default', async function () { + it('should allow greedy:false', async function () { const engine = new Liquid({ greedy: false } as any) const html = await engine.parseAndRender(src, ctx) return expect(html).to.equal('\n a \nharttle ') diff --git a/test/e2e/whitespace-ctrl.ts b/test/integration/liquid/whitespace-ctrl.ts similarity index 99% rename from test/e2e/whitespace-ctrl.ts rename to test/integration/liquid/whitespace-ctrl.ts index 606d572a4..ce8e47cd2 100644 --- a/test/e2e/whitespace-ctrl.ts +++ b/test/integration/liquid/whitespace-ctrl.ts @@ -1,5 +1,5 @@ import { expect } from 'chai' -import { Liquid } from '../..' +import { Liquid } from '../../../src/liquid' const liquid = new Liquid() diff --git a/test/integration/util/error.ts b/test/integration/util/error.ts index eb6a85324..2e9165864 100644 --- a/test/integration/util/error.ts +++ b/test/integration/util/error.ts @@ -41,11 +41,6 @@ describe('error', function () { const err = await expect(engine.parseAndRender(html)).be.rejected expect(err.token.input).to.equal(html) }) - it('should contain line number in err.token.line', async function () { - const err = await expect(engine.parseAndRender('1\n2\n{% . a %}\n4')).be.rejected - expect(err.name).to.equal('TokenizationError') - expect(err.token.line).to.equal(3) - }) it('should contain stack in err.stack', async function () { const err = await expect(engine.parseAndRender('{% . a %}')).be.rejected expect(err.message).to.contain('illegal tag syntax') @@ -58,11 +53,11 @@ describe('error', function () { expect(err.stack).to.not.contain('at Object.parse') }) }) - it('should throw error with line and pos if tag unmatched', async function () { + it('should throw error with [line, col] if tag unmatched', async function () { const err = await expect(engine.parseAndRender('1\n2\nfoo{% assign a = 4 }\n4')).be.rejected + console.log(err.stack) expect(err.name).to.equal('TokenizationError') - expect(err.token.line).to.equal(3) - expect(err.token.col).to.equal(4) + expect(err.message).to.equal('tag "{% assign a =..." not closed, line:3, col:4') }) }) @@ -175,12 +170,6 @@ describe('error', function () { expect(err.stack).to.contain(message.join('\n')) expect(err.name).to.equal('RenderError') }) - it('should contain line number in err.token.line', async function () { - const src = '1\n2\n{{1|throwingFilter}}\n4' - const err = await expect(engine.parseAndRender(src)).be.rejected - expect(err.token.line).to.equal(3) - expect(err.name).to.equal('RenderError') - }) it('should contain stack in err.stack', async function () { const err = await expect(engine.parseAndRender('{%rejectingTag%}')).be.rejected expect(err.message).to.contain('intended render reject') @@ -256,12 +245,6 @@ describe('error', function () { expect(err.stack).to.contain(message.join('\n')) }) - it('should contain line number in err.token.line', async function () { - const html = '\n\n\n{% raw %}\n\n' - const err = await expect(engine.parseAndRender(html)).be.rejected - expect(err.token.line).to.equal(4) - }) - it('should contain stack in err.stack', async function () { const err = await expect(engine.parseAndRender('{% -a %}')).be.rejected expect(err.stack).to.contain('ParseError: tag "-a" not found') diff --git a/test/unit/context/context.ts b/test/unit/context/context.ts index 66441f3a7..fb95deb8d 100644 --- a/test/unit/context/context.ts +++ b/test/unit/context/context.ts @@ -24,123 +24,49 @@ describe('Context', function () { ctx = new Context(scope) }) - describe('#propertyAccessSeq()', function () { - it('should handle dot syntax', async function () { - expect(ctx.parseProp('foo.bar')) - .to.deep.equal(['foo', 'bar']) - }) - it('should handle [] syntax', async function () { - expect(ctx.parseProp('foo["bar"]')) - .to.deep.equal(['foo', 'bar']) - }) - it('should handle [] syntax', async function () { - expect(ctx.parseProp('foo[foo]')) - .to.deep.equal(['foo', 'zoo']) - }) - it('should handle nested access 1', async function () { - expect(ctx.parseProp('foo[bar.zoo]')) - .to.deep.equal(['foo', 'coo']) - }) - it('should handle nested access 2', async function () { - expect(ctx.parseProp('foo[bar["zoo"]]')) - .to.deep.equal(['foo', 'coo']) - }) - it('should handle nested access 3', async function () { - expect(ctx.parseProp('bar["foo"].zoo')) - .to.deep.equal(['bar', 'foo', 'zoo']) - }) - it('should handle nested access 4', async function () { - expect(ctx.parseProp('foo[0].bar')) - .to.deep.equal(['foo', '0', 'bar']) - }) - it('should handle nested access 5', async function () { - expect(ctx.parseProp('foo[one].bar')) - .to.deep.equal(['foo', '1', 'bar']) - }) - it('should handle nested access 6', async function () { - expect(ctx.parseProp('foo[two].bar')) - .to.deep.equal(['foo', 'undefined', 'bar']) - }) - }) - describe('#get()', function () { it('should get direct property', async function () { - expect(ctx.get('foo')).equal('zoo') + expect(ctx.get(['foo'])).equal('zoo') + }) + it('should read nested property', async function () { + expect(ctx.get(['obj', 'first'])).to.equal('f') + expect(ctx.get(['obj', 'last'])).to.equal('l') + expect(ctx.get(['obj', 'size'])).to.equal(undefined) }) - it('undefined property should yield undefined', async function () { - expect(ctx.get('notdefined')).to.equal(undefined) - expect(ctx.get(false as any)).to.equal(undefined) - }) - - it('should throw for invalid path', async function () { - expect(() => ctx.get('')).to.throw('invalid path:""') - }) - - it('should throw when [] unbalanced', async function () { - expect(() => ctx.get('foo[bar')).to.throw(/unbalanced \[\]/) - }) - - it('should throw when "" unbalanced', async function () { - expect(() => ctx.get('foo["bar]')).to.throw(/unbalanced "/) - }) - - it("should throw when '' unbalanced", async function () { - expect(() => ctx.get("foo['bar]")).to.throw(/unbalanced '/) + expect(ctx.get(['notdefined'])).to.equal(undefined) + expect(ctx.get([false as any])).to.equal(undefined) }) it('should respect to toLiquid', async function () { const scope = new Context({ foo: { toLiquid: () => ({ bar: 'BAR' }), bar: 'bar' } }) - expect(scope.get('foo.bar')).to.equal('BAR') + expect(scope.get(['foo', 'bar'])).to.equal('BAR') }) - - it('should access child property via dot syntax', async function () { - expect(ctx.get('bar.zoo')).to.equal('coo') - expect(ctx.get('bar.arr')).to.deep.equal(['a', 'b']) - }) - - it('should access child property via [] syntax', async function () { - expect(ctx.get('bar["zoo"]')).to.equal('coo') - }) - - it('should access child property via [] syntax', async function () { - expect(ctx.get('bar.arr[0]')).to.equal('a') - }) - - it('should access child property via [] syntax', async function () { - expect(ctx.get('bar[foo]')).to.equal('coo') - }) - it('should return undefined when not exist', async function () { - expect(ctx.get('foo.foo.foo')).to.be.undefined + expect(ctx.get(['foo', 'foo', 'foo'])).to.be.undefined }) it('should return string length as size', async function () { - expect(ctx.get('foo.size')).to.equal(3) + expect(ctx.get(['foo', 'size'])).to.equal(3) }) it('should return array length as size', async function () { - expect(ctx.get('bar.arr.size')).to.equal(2) - }) - it('should return size property if exists', async function () { - expect(ctx.get('zoo.size')).to.equal(4) - }) - it('should return undefined if do not have size and length', async function () { - expect(ctx.get('one.size')).to.equal(undefined) + expect(ctx.get(['bar', 'arr', 'size'])).to.equal(2) }) it('should read .first of array', async function () { - expect(ctx.get('bar.arr.first')).to.equal('a') - }) - it('should read .first of object', async function () { - expect(ctx.get('obj.first')).to.equal('f') + expect(ctx.get(['bar', 'arr', 'first'])).to.equal('a') }) it('should read .last of array', async function () { - expect(ctx.get('bar.arr.last')).to.equal('b') - }) - it('should read .last of object', async function () { - expect(ctx.get('obj.last')).to.equal('l') + expect(ctx.get(['bar', 'arr', 'last'])).to.equal('b') }) }) + + describe('#getFromScope()', function () { + it('should support string', () => { + expect(ctx.getFromScope({ obj: { foo: 'FOO' } }, 'obj.foo')).to.equal('FOO') + }) + }) + describe('strictVariables', async function () { let ctx: Context beforeEach(function () { @@ -149,22 +75,22 @@ describe('Context', function () { } as any) }) it('should throw when variable not defined', function () { - return expect(() => ctx.get('notdefined')).to.throw(/undefined variable: notdefined/) + return expect(() => ctx.get(['notdefined'])).to.throw(/undefined variable: notdefined/) }) it('should throw when deep variable not exist', async function () { ctx.push({ foo: 'FOO' }) - return expect(() => ctx.get('foo.bar.not.defined')).to.throw(/undefined variable: bar/) + return expect(() => ctx.get(['foo', 'bar', 'not', 'defined'])).to.throw(/undefined variable: bar/) }) it('should throw when itself not defined', async function () { ctx.push({ foo: 'FOO' }) - return expect(() => ctx.get('foo.BAR')).to.throw(/undefined variable: BAR/) + return expect(() => ctx.get(['foo', 'BAR'])).to.throw(/undefined variable: BAR/) }) it('should find variable in parent scope', async function () { ctx.push({ 'foo': 'foo' }) ctx.push({ 'bar': 'bar' }) - expect(ctx.get('foo')).to.equal('foo') + expect(ctx.get(['foo'])).to.equal('foo') }) }) @@ -180,14 +106,14 @@ describe('Context', function () { ctx.push({ foo: 'foo' }) - expect(ctx.get('foo')).to.equal('foo') - expect(ctx.get('bar')).to.equal('bar') + expect(ctx.get(['foo'])).to.equal('foo') + expect(ctx.get(['bar'])).to.equal('bar') }) it('should hide deep properties by push', async function () { ctx.push({ bar: { bar: 'bar' } }) ctx.push({ bar: { foo: 'foo' } }) - expect(ctx.get('bar.foo')).to.equal('foo') - expect(ctx.get('bar.bar')).to.equal(undefined) + expect(ctx.get(['bar', 'foo'])).to.equal('foo') + expect(ctx.get(['bar', 'bar'])).to.equal(undefined) }) }) describe('.pop()', function () { @@ -196,7 +122,7 @@ describe('Context', function () { foo: 'foo' }) ctx.pop() - expect(ctx.get('foo')).to.equal('zoo') + expect(ctx.get(['foo'])).to.equal('zoo') }) }) }) diff --git a/test/unit/parser/match-operator.ts b/test/unit/parser/match-operator.ts new file mode 100644 index 000000000..3f5c5058c --- /dev/null +++ b/test/unit/parser/match-operator.ts @@ -0,0 +1,26 @@ +import { expect } from 'chai' +import { matchOperator } from '../../../src/parser/match-operator' + +describe('parser/matchOperator()', function () { + it('should match contains', () => { + expect(matchOperator('contains', 0)).to.equal(8) + }) + it('should match comparision', () => { + expect(matchOperator('>', 0)).to.equal(1) + expect(matchOperator('>=', 0)).to.equal(2) + expect(matchOperator('<', 0)).to.equal(1) + expect(matchOperator('<=', 0)).to.equal(2) + }) + it('should match binary logic', () => { + expect(matchOperator('and', 0)).to.equal(3) + expect(matchOperator('or', 0)).to.equal(2) + }) + it('should not match if word not terminate', () => { + expect(matchOperator('true1', 0)).to.equal(-1) + expect(matchOperator('containsa', 0)).to.equal(-1) + }) + it('should match if word boundary found', () => { + expect(matchOperator('>=1', 0)).to.equal(2) + expect(matchOperator('contains b', 0)).to.equal(8) + }) +}) diff --git a/test/unit/parser/literal.ts b/test/unit/parser/parse-string-literal.ts similarity index 57% rename from test/unit/parser/literal.ts rename to test/unit/parser/parse-string-literal.ts index bf11b70f2..b305ebe9d 100644 --- a/test/unit/parser/literal.ts +++ b/test/unit/parser/parse-string-literal.ts @@ -1,30 +1,5 @@ import { expect } from 'chai' -import { parseLiteral, parseStringLiteral } from '../../../src/parser/literal' -import { NullDrop } from '../../../src/drop/null-drop' - -describe('parseLiteral()', function () { - it('should eval boolean literal', async function () { - expect(parseLiteral('true')).to.equal(true) - expect(parseLiteral('TrUE')).to.equal(undefined) - expect(parseLiteral('false')).to.equal(false) - }) - it('should eval number literal', async function () { - expect(parseLiteral('2.3')).to.equal(2.3) - expect(parseLiteral('.32')).to.equal(0.32) - expect(parseLiteral('-23.')).to.equal(-23) - expect(parseLiteral('23')).to.equal(23) - }) - it('should eval string literal', async function () { - expect(parseLiteral('"ab\'c"')).to.equal("ab'c") - expect(parseLiteral("'ab\"c'")).to.equal('ab"c') - }) - it('should eval nil literal', async function () { - expect(parseLiteral('nil')).to.be.instanceOf(NullDrop) - }) - it('should eval null literal', async function () { - expect(parseLiteral('null')).to.be.instanceOf(NullDrop) - }) -}) +import { parseStringLiteral } from '../../../src/parser/parse-string-literal' describe('parseStringLiteral()', function () { it('should parse octal escape', () => { @@ -36,7 +11,7 @@ describe('parseStringLiteral()', function () { it('should skip invalid octal escape', () => { expect(parseStringLiteral(String.raw`"\9"`)).to.equal('9') }) - it('should parse \n, \t, \r', () => { + it('should parse \\n, \\t, \\r', () => { expect(parseStringLiteral(String.raw`"fo\no"`)).to.equal('fo\no') expect(parseStringLiteral(String.raw`'fo\to'`)).to.equal('fo\to') expect(parseStringLiteral(String.raw`'fo\ro'`)).to.equal('fo\ro') diff --git a/test/unit/parser/tokenizer.ts b/test/unit/parser/tokenizer.ts index 261a9d898..ac4f7e2e3 100644 --- a/test/unit/parser/tokenizer.ts +++ b/test/unit/parser/tokenizer.ts @@ -1,210 +1,431 @@ import { expect } from 'chai' +import { WordToken } from '../../../src/tokens/word-token' +import { NumberToken } from '../../../src/tokens/number-token' +import { PropertyAccessToken } from '../../../src/tokens/property-access-token' +import { RangeToken } from '../../../src/tokens/range-token' +import { OperatorToken } from '../../../src/tokens/operator-token' import { Tokenizer } from '../../../src/parser/tokenizer' -import { TagToken } from '../../../src/parser/tag-token' -import { OutputToken } from '../../../src/parser/output-token' -import { HTMLToken } from '../../../src/parser/html-token' +import { TagToken } from '../../../src/tokens/tag-token' +import { QuotedToken } from '../../../src/tokens/quoted-token' +import { OutputToken } from '../../../src/tokens/output-token' +import { HTMLToken } from '../../../src/tokens/html-token' describe('Tokenize', function () { it('should read quoted', () => { - expect(new Tokenizer('"foo" ff').readQuoted().toString()).to.equal('"foo"') - expect(new Tokenizer(' "foo"ff').readQuoted().toString()).to.equal('"foo"') - }) - it('should read property access', () => { - expect(new Tokenizer('a[ b][ "c d" ]').readPropertyAccess().toString()).to.equal('a[ b][ "c d" ]') - expect(new Tokenizer('a.b[c[d.e]]').readPropertyAccess().toString()).to.equal('a.b[c[d.e]]') + expect(new Tokenizer('"foo" ff').readQuoted()!.getText()).to.equal('"foo"') + expect(new Tokenizer(' "foo"ff').readQuoted()!.getText()).to.equal('"foo"') }) it('should read value', () => { - expect(new Tokenizer('2.33.2').readValue().toString()).to.equal('2.33.2') - expect(new Tokenizer('"foo"a').readValue().toString()).to.equal('"foo"') - expect(new Tokenizer('a[b]["c d"]').readValue().toString()).to.equal('a[b]["c d"]') + expect(new Tokenizer('a[ b][ "c d" ]').readValueOrThrow().getText()).to.equal('a[ b][ "c d" ]') + expect(new Tokenizer('a.b[c[d.e]]').readValueOrThrow().getText()).to.equal('a.b[c[d.e]]') + }) + it('should read number value', () => { + const token: NumberToken = new Tokenizer('2.33.2').readValueOrThrow() as any + expect(token).to.be.instanceOf(NumberToken) + expect(token.whole.getText()).to.equal('2') + expect(token.decimal!.getText()).to.equal('33') + expect(token.getText()).to.equal('2.33') + }) + it('should read quoted value', () => { + const value = new Tokenizer('"foo"a').readValue() + expect(value).to.be.instanceOf(QuotedToken) + expect(value!.getText()).to.equal('"foo"') + }) + it('should read property access value', () => { + expect(new Tokenizer('a[b]["c d"]').readValueOrThrow().getText()).to.equal('a[b]["c d"]') }) it('should read hash', () => { - expect(new Tokenizer('foo: 3').readHash()).to.deep.equal(['foo', '3']) - expect(new Tokenizer(', foo: a[ "bar"]').readHash()).to.deep.equal(['foo', 'a[ "bar"]']) + const hash1 = new Tokenizer('foo: 3').readHash() + expect(hash1!.name.content).to.equal('foo') + expect(hash1!.value!.getText()).to.equal('3') + + const hash2 = new Tokenizer(', foo: a[ "bar"]').readHash() + expect(hash2!.name.content).to.equal('foo') + expect(hash2!.value!.getText()).to.equal('a[ "bar"]') }) - it('should read hashs', () => { - expect(new Tokenizer(', limit: 3 reverse offset:off').readHashes()) - .to.deep.equal([['limit', '3'], ['reverse', ''], ['offset', 'off']]) - expect(new Tokenizer('cols: 2, rows: data["rows"]').readHashes()) - .to.deep.equal([['cols', '2'], ['rows', 'data["rows"]']]) + it('should read multiple hashs', () => { + const hashes = new Tokenizer(', limit: 3 reverse offset:off').readHashes() + expect(hashes).to.have.lengthOf(3) + const [limit, reverse, offset] = hashes + expect(limit.name.content).to.equal('limit') + expect(limit.value!.getText()).to.equal('3') + + expect(reverse.name.content).to.equal('reverse') + expect(reverse.value).to.be.undefined + + expect(offset.name.content).to.equal('offset') + expect(offset.value!.getText()).to.equal('off') + }) + it('should read hash value with property access', () => { + const hashes = new Tokenizer('cols: 2, rows: data["rows"]').readHashes() + expect(hashes).to.have.lengthOf(2) + const [cols, rols] = hashes + + expect(cols.name.content).to.equal('cols') + expect(cols.value!.getText()).to.equal('2') + + expect(rols.name.content).to.equal('rows') + expect(rols.value!.getText()).to.equal('data["rows"]') }) it('should read HTML token', function () { const html = '

Lorem Ipsum

' const tokenizer = new Tokenizer(html) - const tokens = tokenizer.readTokens() + const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).to.equal(1) - expect(tokens[0].content).to.equal(html) expect(tokens[0]).instanceOf(HTMLToken) + expect((tokens[0] as HTMLToken).getContent()).to.equal(html) }) it('should read tag token', function () { const html = '

{% for p in a[1]%}

' const tokenizer = new Tokenizer(html) - const tokens = tokenizer.readTokens() + const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).to.equal(3) - expect(tokens[1]).instanceOf(TagToken) - expect(tokens[1].content).to.equal('for p in a[1]') + const tag = tokens[1] as TagToken + expect(tag).instanceOf(TagToken) + expect(tag.name).to.equal('for') + expect(tag.args).to.equal('p in a[1]') }) - it('should read value token', function () { + it('should read output token', function () { const html = '

{{foo | date: "%Y-%m-%d"}}

' const tokenizer = new Tokenizer(html) - const tokens = tokenizer.readTokens() + const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).to.equal(3) - expect(tokens[1]).instanceOf(OutputToken) - expect(tokens[1].content).to.equal('foo | date: "%Y-%m-%d"') + const output = tokens[1] as OutputToken + expect(output).instanceOf(OutputToken) + expect(output.content).to.equal('foo | date: "%Y-%m-%d"') }) it('should handle consecutive value and tags', function () { const html = '{{foo}}{{bar}}{%foo%}{%bar%}' const tokenizer = new Tokenizer(html) - const tokens = tokenizer.readTokens() + const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).to.equal(4) - expect(tokens[0]).instanceOf(OutputToken) - expect(tokens[2]).instanceOf(TagToken) + const o1 = tokens[0] as OutputToken + const o2 = tokens[1] as OutputToken + const t1 = tokens[2] as TagToken + const t2 = tokens[3] as TagToken + expect(o1).instanceOf(OutputToken) + expect(o2).instanceOf(OutputToken) + expect(t1).instanceOf(TagToken) + expect(t2).instanceOf(TagToken) - expect(tokens[1].content).to.equal('bar') - expect(tokens[2].content).to.equal('foo') + expect(o1.content).to.equal('foo') + expect(o2.content).to.equal('bar') + expect(t1.name).to.equal('foo') + expect(t1.args).to.equal('') + expect(t2.name).to.equal('bar') + expect(t2.args).to.equal('') }) it('should keep white spaces and newlines', function () { const html = '{%foo%}\n{%bar %} \n {%alice%}' const tokenizer = new Tokenizer(html) - const tokens = tokenizer.readTokens() + const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).to.equal(5) expect(tokens[1]).instanceOf(HTMLToken) - expect(tokens[1].raw).to.equal('\n') + expect(tokens[1].getText()).to.equal('\n') expect(tokens[3]).instanceOf(HTMLToken) - expect(tokens[3].raw).to.equal(' \n ') + expect(tokens[3].getText()).to.equal(' \n ') }) it('should handle multiple lines tag', function () { const html = '{%foo\na:a\nb:1.23\n%}' const tokenizer = new Tokenizer(html) - const tokens = tokenizer.readTokens() + const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).to.equal(1) expect(tokens[0]).instanceOf(TagToken) expect((tokens[0] as TagToken).args).to.equal('a:a\nb:1.23') - expect(tokens[0].raw).to.equal('{%foo\na:a\nb:1.23\n%}') + expect(tokens[0].getText()).to.equal('{%foo\na:a\nb:1.23\n%}') }) it('should handle multiple lines value', function () { const html = '{{foo\n|date:\n"%Y-%m-%d"\n}}' const tokenizer = new Tokenizer(html) - const tokens = tokenizer.readTokens() + const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).to.equal(1) expect(tokens[0]).instanceOf(OutputToken) - expect(tokens[0].raw).to.equal('{{foo\n|date:\n"%Y-%m-%d"\n}}') + expect(tokens[0].getText()).to.equal('{{foo\n|date:\n"%Y-%m-%d"\n}}') }) it('should handle complex object property access', function () { const html = '{{ obj["my:property with anything"] }}' const tokenizer = new Tokenizer(html) - const tokens = tokenizer.readTokens() + const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).to.equal(1) - expect(tokens[0]).instanceOf(OutputToken) - expect(tokens[0].content).to.equal('obj["my:property with anything"]') + const output = tokens[0] as OutputToken + expect(output).instanceOf(OutputToken) + expect(output.content).to.equal('obj["my:property with anything"]') }) it('should throw if tag not closed', function () { const html = '{% assign foo = bar {{foo}}' const tokenizer = new Tokenizer(html) - expect(() => tokenizer.readTokens()).to.throw(/tag "{% assign foo..." not closed/) + expect(() => tokenizer.readTopLevelTokens()).to.throw(/tag "{% assign foo..." not closed/) }) it('should throw if output not closed', function () { const tokenizer = new Tokenizer('{{name}') - expect(() => tokenizer.readTokens()).to.throw(/output "{{name}" not closed/) + expect(() => tokenizer.readTopLevelTokens()).to.throw(/output "{{name}" not closed/) }) - it('should read a simple filter', function () { - const tokenizer = new Tokenizer('| plus') - const token = tokenizer.readFilterToken() - expect(token).to.have.property('name', 'plus') - expect(token).to.have.property('args').to.deep.equal([]) + describe('#readRange()', () => { + it('should read `(1..3)`', () => { + const range = new Tokenizer('(1..3)').readRange() + expect(range).to.be.instanceOf(RangeToken) + expect(range!.getText()).to.deep.equal('(1..3)') + const { lhs, rhs } = range! + expect(lhs).to.be.instanceOf(NumberToken) + expect(lhs.getText()).to.equal('1') + expect(rhs).to.be.instanceOf(NumberToken) + expect(rhs.getText()).to.equal('3') + }) + it('should throw for `(..3)`', () => { + expect(() => new Tokenizer('(..3)').readRange()).to.throw('unexpected token "..3)", value expected') + }) + it('should read `(a.b..c["..d"])`', () => { + const range = new Tokenizer('(a.b..c["..d"])').readRange() + expect(range).to.be.instanceOf(RangeToken) + expect(range!.getText()).to.deep.equal('(a.b..c["..d"])') + }) }) - it('should read a filter with argument', function () { - const tokenizer = new Tokenizer(' | plus: 1') - const token = tokenizer.readFilterToken() - expect(token).to.have.property('name', 'plus') - expect(token).to.have.property('args').to.deep.equal(['1']) - }) - it('should read a filter with colon but no argument', function () { - const tokenizer = new Tokenizer('| plus:') - const token = tokenizer.readFilterToken() - expect(token).to.have.property('name', 'plus') - expect(token).to.have.property('args').to.deep.equal([]) - }) - it('should read a filter with k/v argument', function () { - const tokenizer = new Tokenizer(' | plus: a:1') - const token = tokenizer.readFilterToken() - expect(token).to.have.property('name', 'plus') - expect(token).to.have.property('args').to.deep.equal([['a', '1']]) - }) - it('should read a filter with "arr[0]" argument', function () { - const tokenizer = new Tokenizer('| plus: arr[0]') - const token = tokenizer.readFilterToken() - expect(token).to.have.property('name', 'plus') - expect(token).to.have.property('args').to.deep.equal(['arr[0]']) - }) - it('should read a filter with obj.foo argument', function () { - const tokenizer = new Tokenizer('| plus: obj.foo') - const token = tokenizer.readFilterToken() - expect(token).to.have.property('name', 'plus') - expect(token).to.have.property('args').to.deep.equal(['obj.foo']) - }) - it('should read a filter with obj["foo"] argument', function () { - const tokenizer = new Tokenizer('| plus: obj["good luck"]') - const token = tokenizer.readFilterToken() - expect(token).to.have.property('name', 'plus') - expect(token).to.have.property('args').to.deep.equal(['obj["good luck"]']) - }) - it('should read simple filters', function () { - const tokenizer = new Tokenizer('| plus: 3 | capitalize') - const tokens = tokenizer.readFilterTokens() + describe('#readFilter()', () => { + it('should read a simple filter', function () { + const tokenizer = new Tokenizer('| plus') + const token = tokenizer.readFilter() + expect(token).to.have.property('name', 'plus') + expect(token).to.have.property('args').to.deep.equal([]) + }) + it('should read a filter with argument', function () { + const tokenizer = new Tokenizer(' | plus: 1') + const token = tokenizer.readFilter() + expect(token).to.have.property('name', 'plus') + expect(token!.args).to.have.lengthOf(1) - expect(tokens).to.have.lengthOf(2) - expect(tokens[0]).to.have.property('name', 'plus') - expect(tokens[0]).to.have.property('args').to.deep.equal(['3']) - expect(tokens[1]).to.have.property('name', 'capitalize') - expect(tokens[1]).to.have.property('args').to.deep.equal([]) - }) - it('should read filters', function () { - const tokenizer = new Tokenizer('| plus: 3 | capitalize | append: foo[a.b["c d"]]') - const tokens = tokenizer.readFilterTokens() + const one: NumberToken = token!.args[0] as any + expect(one).to.be.instanceOf(NumberToken) + expect(one.getText()).to.equal('1') + }) + it('should read a filter with colon but no argument', function () { + const tokenizer = new Tokenizer('| plus:') + const token = tokenizer.readFilter() + expect(token).to.have.property('name', 'plus') + expect(token).to.have.property('args').to.deep.equal([]) + }) + it('should read a filter with k/v argument', function () { + const tokenizer = new Tokenizer(' | plus: a:1') + const token = tokenizer.readFilter() + expect(token).to.have.property('name', 'plus') + expect(token!.args).to.have.lengthOf(1) - expect(tokens).to.have.lengthOf(3) - expect(tokens[0]).to.have.property('name', 'plus') - expect(tokens[0]).to.have.property('args').to.deep.equal(['3']) - expect(tokens[1]).to.have.property('name', 'capitalize') - expect(tokens[1]).to.have.property('args').to.deep.equal([]) - expect(tokens[2]).to.have.property('name', 'append') - expect(tokens[2]).to.have.property('args').to.deep.equal(['foo[a.b["c d"]]']) + const [k, v]: [string, NumberToken] = token!.args[0] as any + expect(k).to.equal('a') + expect(v).to.be.instanceOf(NumberToken) + expect(v.getText()).to.equal('1') + }) + it('should read a filter with "arr[0]" argument', function () { + const tokenizer = new Tokenizer('| plus: arr[0]') + const token = tokenizer.readFilter() + expect(token).to.have.property('name', 'plus') + expect(token!.args).to.have.lengthOf(1) + + const pa: PropertyAccessToken = token!.args[0] as any + expect(token!.args[0]).to.be.instanceOf(PropertyAccessToken) + expect(pa.variable.content).to.equal('arr') + expect(pa.props).to.have.lengthOf(1) + expect(pa.props[0]).to.be.instanceOf(NumberToken) + expect(pa.props[0].getText()).to.equal('0') + }) + it('should read a filter with obj.foo argument', function () { + const tokenizer = new Tokenizer('| plus: obj.foo') + const token = tokenizer.readFilter() + expect(token).to.have.property('name', 'plus') + expect(token!.args).to.have.lengthOf(1) + + const pa: PropertyAccessToken = token!.args[0] as any + expect(token!.args[0]).to.be.instanceOf(PropertyAccessToken) + expect(pa.variable.content).to.equal('obj') + expect(pa.props).to.have.lengthOf(1) + expect(pa.props[0]).to.be.instanceOf(WordToken) + expect(pa.props[0].getText()).to.equal('foo') + }) + it('should read a filter with obj["foo"] argument', function () { + const tokenizer = new Tokenizer('| plus: obj["good luck"]') + const token = tokenizer.readFilter() + expect(token).to.have.property('name', 'plus') + expect(token!.args).to.have.lengthOf(1) + + const pa: PropertyAccessToken = token!.args[0] as any + expect(token!.args[0]).to.be.instanceOf(PropertyAccessToken) + expect(pa.getText()).to.equal('obj["good luck"]') + expect(pa.variable.content).to.equal('obj') + expect(pa.props[0].getText()).to.equal('"good luck"') + }) }) - it('should read expression `a==b`', () => { - const exp = new Tokenizer('a==b').readExpression() - expect([...exp]).to.deep.equal(['a', '==', 'b']) + describe('#readFilters()', () => { + it('should read simple filters', function () { + const tokenizer = new Tokenizer('| plus: 3 | capitalize') + const tokens = tokenizer.readFilters() + + expect(tokens).to.have.lengthOf(2) + expect(tokens[0]).to.have.property('name', 'plus') + expect(tokens[0].args).to.have.lengthOf(1) + expect(tokens[0].args[0]).to.be.instanceOf(NumberToken) + expect((tokens[0].args[0] as any).getText()).to.equal('3') + + expect(tokens[1]).to.have.property('name', 'capitalize') + expect(tokens[1].args).to.have.lengthOf(0) + }) + it('should read filters', function () { + const tokenizer = new Tokenizer('| plus: a:3 | capitalize | append: foo[a.b["c d"]]') + const tokens = tokenizer.readFilters() + + expect(tokens).to.have.lengthOf(3) + expect(tokens[0]).to.have.property('name', 'plus') + expect(tokens[0].args).to.have.lengthOf(1) + const [k, v]: [string, NumberToken] = tokens[0].args[0] as any + expect(k).to.equal('a') + expect(v).to.be.instanceOf(NumberToken) + expect(v.getText()).to.equal('3') + + expect(tokens[1]).to.have.property('name', 'capitalize') + expect(tokens[1].args).to.have.lengthOf(0) + + expect(tokens[2]).to.have.property('name', 'append') + expect(tokens[2].args).to.have.lengthOf(1) + expect(tokens[2].args[0]).to.be.instanceOf(PropertyAccessToken) + expect((tokens[2].args[0] as any).getText()).to.equal('foo[a.b["c d"]]') + expect((tokens[2].args[0] as any).props[0].getText()).to.equal('a.b["c d"]') + }) }) - it('should read expression `^`', () => { - const exp = new Tokenizer('^').readExpression() - expect([...exp]).to.deep.equal([]) - }) - it('should read expression `a == b`', () => { - const exp = new Tokenizer('a == b').readExpression() - expect([...exp]).to.deep.equal(['a', '==', 'b']) - }) - it('should read expression `(1..3) contains 3`', () => { - const exp = new Tokenizer('(1..3) contains 3').readExpression() - expect([...exp]).to.deep.equal(['(1..3)', 'contains', '3']) - }) - it('should read expression `a[b] = c`', () => { - const exp = new Tokenizer('a[b] = c').readExpression() - expect([...exp]).to.deep.equal(['a[b]', '=', 'c']) - }) - it('should read expression `c[a["b"]] >= c`', () => { - const exp = new Tokenizer('c[a["b"]] >= c').readExpression() - expect([...exp]).to.deep.equal(['c[a["b"]]', '>=', 'c']) - }) - it('should read expression `"][" == var`', () => { - const exp = new Tokenizer('"][" == var').readExpression() - expect([...exp]).to.deep.equal(['"]["', '==', 'var']) - }) - it('should read expression `"\\\'" == "\\""`', () => { - const exp = new Tokenizer('"\\\'" == "\\""').readExpression() - expect([...exp]).to.deep.equal(['"\\\'"', '==', '"\\""']) + describe('#readExpression()', () => { + it('should read expression `a `', () => { + const exp = [...new Tokenizer('a ').readExpression()] + + expect(exp).to.have.lengthOf(1) + expect(exp[0]).to.be.instanceOf(PropertyAccessToken) + expect(exp[0].getText()).to.deep.equal('a') + }) + it('should read expression `a[][b]`', () => { + const exp = [...new Tokenizer('a[][b]').readExpression()] + + expect(exp).to.have.lengthOf(1) + const pa = exp[0] as PropertyAccessToken + expect(pa).to.be.instanceOf(PropertyAccessToken) + expect(pa.variable.content).to.deep.equal('a') + expect(pa.props).to.have.lengthOf(2) + + const [p1, p2] = pa.props + expect(p1).to.be.instanceOf(WordToken) + expect(p1.getText()).to.equal('') + expect(p2).to.be.instanceOf(PropertyAccessToken) + expect(p2.getText()).to.equal('b') + }) + it('should read expression `a.`', () => { + const exp = [...new Tokenizer('a.').readExpression()] + + expect(exp).to.have.lengthOf(1) + const pa = exp[0] as PropertyAccessToken + expect(pa).to.be.instanceOf(PropertyAccessToken) + expect(pa.variable.content).to.deep.equal('a') + expect(pa.props).to.have.lengthOf(0) + }) + it('should read expression `a ==`', () => { + const exp = [...new Tokenizer('a ==').readExpression()] + + expect(exp).to.have.lengthOf(1) + expect(exp[0]).to.be.instanceOf(PropertyAccessToken) + expect(exp[0].getText()).to.deep.equal('a') + }) + it('should read expression `a==b`', () => { + const exp = new Tokenizer('a==b').readExpression() + const [a, equals, b] = exp + + expect(a).to.be.instanceOf(PropertyAccessToken) + expect(a.getText()).to.deep.equal('a') + + expect(equals).to.be.instanceOf(OperatorToken) + expect(equals.getText()).to.equal('==') + + expect(b).to.be.instanceOf(PropertyAccessToken) + expect(b.getText()).to.deep.equal('b') + }) + it('should read expression `^`', () => { + const exp = new Tokenizer('^').readExpression() + expect([...exp]).to.deep.equal([]) + }) + it('should read expression `a == b`', () => { + const exp = new Tokenizer('a == b').readExpression() + const [a, equals, b] = exp + + expect(a).to.be.instanceOf(PropertyAccessToken) + expect(a.getText()).to.deep.equal('a') + + expect(equals).to.be.instanceOf(OperatorToken) + expect(equals.getText()).to.equal('==') + + expect(b).to.be.instanceOf(PropertyAccessToken) + expect(b.getText()).to.deep.equal('b') + }) + it('should read expression `(1..3) contains 3`', () => { + const exp = new Tokenizer('(1..3) contains 3').readExpression() + const [range, contains, rhs] = exp + + expect(range).to.be.instanceOf(RangeToken) + expect(range.getText()).to.deep.equal('(1..3)') + + expect(contains).to.be.instanceOf(OperatorToken) + expect(contains.getText()).to.equal('contains') + + expect(rhs).to.be.instanceOf(NumberToken) + expect(rhs.getText()).to.deep.equal('3') + }) + it('should read expression `a[b] == c`', () => { + const exp = new Tokenizer('a[b] == c').readExpression() + const [lhs, contains, rhs] = exp + + expect(lhs).to.be.instanceOf(PropertyAccessToken) + expect(lhs.getText()).to.deep.equal('a[b]') + + expect(contains).to.be.instanceOf(OperatorToken) + expect(contains.getText()).to.equal('==') + + expect(rhs).to.be.instanceOf(PropertyAccessToken) + expect(rhs.getText()).to.deep.equal('c') + }) + it('should read expression `c[a["b"]] >= c`', () => { + const exp = new Tokenizer('c[a["b"]] >= c').readExpression() + const [lhs, op, rhs] = exp + + expect(lhs).to.be.instanceOf(PropertyAccessToken) + expect(lhs.getText()).to.deep.equal('c[a["b"]]') + + expect(op).to.be.instanceOf(OperatorToken) + expect(op.getText()).to.equal('>=') + + expect(rhs).to.be.instanceOf(PropertyAccessToken) + expect(rhs.getText()).to.deep.equal('c') + }) + it('should read expression `"][" == var`', () => { + const exp = new Tokenizer('"][" == var').readExpression() + const [lhs, equals, rhs] = exp + + expect(lhs).to.be.instanceOf(QuotedToken) + expect(lhs.getText()).to.deep.equal('"]["') + + expect(equals).to.be.instanceOf(OperatorToken) + expect(equals.getText()).to.equal('==') + + expect(rhs).to.be.instanceOf(PropertyAccessToken) + expect(rhs.getText()).to.deep.equal('var') + }) + it('should read expression `"\\\'" == "\\""`', () => { + const exp = new Tokenizer('"\\\'" == "\\""').readExpression() + const [lhs, equals, rhs] = exp + + expect(lhs).to.be.instanceOf(QuotedToken) + expect(lhs.getText()).to.deep.equal('"\\\'"') + + expect(equals).to.be.instanceOf(OperatorToken) + expect(equals.getText()).to.equal('==') + + expect(rhs).to.be.instanceOf(QuotedToken) + expect(rhs.getText()).to.deep.equal('"\\""') + }) }) }) diff --git a/test/unit/render/expression.ts b/test/unit/render/expression.ts index ebddef6e0..e972f2c43 100644 --- a/test/unit/render/expression.ts +++ b/test/unit/render/expression.ts @@ -4,59 +4,115 @@ import { Context } from '../../../src/context/context' import { toThenable } from '../../../src/util/async' describe('Expression', function () { - let ctx: Context - - beforeEach(function () { - ctx = new Context({ - one: 1, - two: 2, - empty: '', - quote: '"', - space: ' ', - x: 'XXX', - y: undefined, - z: null, - obj: { - ']': 'right bracket' - } - }) - }) + const ctx = new Context({}) it('should throw when context not defined', done => { - toThenable(new Expression().value(undefined!)).catch(err => { - expect(err.message).to.match(/context not defined/) - done() - return 0 as any + toThenable(new Expression('foo').value(undefined!)) + .then(() => done(new Error('should not resolved'))) + .catch(err => { + expect(err.message).to.match(/context not defined/) + done() + }) + }) + + describe('single value', function () { + it('should eval literal', async function () { + expect(await toThenable(new Expression('2.4').value(ctx))).to.equal(2.4) + expect(await toThenable(new Expression('"foo"').value(ctx))).to.equal('foo') + expect(await toThenable(new Expression('false').value(ctx))).to.equal(false) + }) + it('should eval range expression', async function () { + const ctx = new Context({ two: 2 }) + expect(await toThenable(new Expression('(2..4)').value(ctx))).to.deep.equal([2, 3, 4]) + expect(await toThenable(new Expression('(two..4)').value(ctx))).to.deep.equal([2, 3, 4]) + }) + it('should eval literal', async function () { + expect(await toThenable(new Expression('2.4').value(ctx))).to.equal(2.4) + expect(await toThenable(new Expression('"foo"').value(ctx))).to.equal('foo') + expect(await toThenable(new Expression('false').value(ctx))).to.equal(false) + }) + + it('should eval property access', async function () { + const ctx = new Context({ + foo: { bar: 'BAR' }, + coo: 'bar', + doo: { foo: 'bar', bar: { foo: 'bar' } } + }) + expect(await toThenable(new Expression('foo.bar').value(ctx))).to.equal('BAR') + expect(await toThenable(new Expression('foo["bar"]').value(ctx))).to.equal('BAR') + expect(await toThenable(new Expression('foo[coo]').value(ctx))).to.equal('BAR') + expect(await toThenable(new Expression('foo[doo.foo]').value(ctx))).to.equal('BAR') + expect(await toThenable(new Expression('foo[doo["foo"]]').value(ctx))).to.equal('BAR') + expect(await toThenable(new Expression('doo[coo].foo').value(ctx))).to.equal('bar') }) }) - it('should eval simple expression', async function () { - expect(await toThenable(new Expression('1==2').value(ctx))).to.equal(false) - expect(await toThenable(new Expression('1<2').value(ctx))).to.equal(true) - expect(await toThenable(new Expression('1 < 2').value(ctx))).to.equal(true) - expect(await toThenable(new Expression('1 < 2').value(ctx))).to.equal(true) - expect(await toThenable(new Expression('2 <= 2').value(ctx))).to.equal(true) - expect(await toThenable(new Expression('one <= two').value(ctx))).to.equal(true) - expect(await toThenable(new Expression('x contains "x"').value(ctx))).to.equal(false) - expect(await toThenable(new Expression('x contains "X"').value(ctx))).to.equal(true) - expect(await toThenable(new Expression('1 contains "x"').value(ctx))).to.equal(false) - expect(await toThenable(new Expression('y contains "x"').value(ctx))).to.equal(false) - expect(await toThenable(new Expression('z contains "x"').value(ctx))).to.equal(false) - expect(await toThenable(new Expression('(1..5) contains 3').value(ctx))).to.equal(true) - expect(await toThenable(new Expression('(1..5) contains 6').value(ctx))).to.equal(false) - expect(await toThenable(new Expression('"<=" == "<="').value(ctx))).to.equal(true) + describe('simple expression', function () { + it('should return false for "1==2"', async () => { + expect(await toThenable(new Expression('1==2').value(ctx))).to.equal(false) + }) + it('should return true for "1<2"', async () => { + expect(await toThenable(new Expression('1<2').value(ctx))).to.equal(true) + }) + it('should return true for "1 < 2"', async () => { + expect(await toThenable(new Expression('1 < 2').value(ctx))).to.equal(true) + }) + it('should return true for "1 < 2"', async () => { + expect(await toThenable(new Expression('1 < 2').value(ctx))).to.equal(true) + }) + it('should return true for "2 <= 2"', async () => { + expect(await toThenable(new Expression('2 <= 2').value(ctx))).to.equal(true) + }) + it('should return true for "one <= two"', async () => { + const ctx = new Context({ one: 1, two: 2 }) + expect(await toThenable(new Expression('one <= two').value(ctx))).to.equal(true) + }) + it('should return false for "x contains "x""', async () => { + const ctx = new Context({ x: 'XXX' }) + expect(await toThenable(new Expression('x contains "x"').value(ctx))).to.equal(false) + }) + it('should return true for "x contains "X""', async () => { + const ctx = new Context({ x: 'XXX' }) + expect(await toThenable(new Expression('x contains "X"').value(ctx))).to.equal(true) + }) + it('should return false for "1 contains "x""', async () => { + const ctx = new Context({ x: 'XXX' }) + expect(await toThenable(new Expression('1 contains "x"').value(ctx))).to.equal(false) + }) + it('should return false for "y contains "x""', async () => { + const ctx = new Context({ x: 'XXX' }) + expect(await toThenable(new Expression('y contains "x"').value(ctx))).to.equal(false) + }) + it('should return false for "z contains "x""', async () => { + const ctx = new Context({ x: 'XXX' }) + expect(await toThenable(new Expression('z contains "x"').value(ctx))).to.equal(false) + }) + it('should return true for "(1..5) contains 3"', async () => { + const ctx = new Context({ x: 'XXX' }) + expect(await toThenable(new Expression('(1..5) contains 3').value(ctx))).to.equal(true) + }) + it('should return false for "(1..5) contains 6"', async () => { + const ctx = new Context({ x: 'XXX' }) + expect(await toThenable(new Expression('(1..5) contains 6').value(ctx))).to.equal(false) + }) + it('should return true for ""<=" == "<=""', async () => { + expect(await toThenable(new Expression('"<=" == "<="').value(ctx))).to.equal(true) + }) }) it('should allow space in quoted value', async function () { + const ctx = new Context({ space: ' ' }) expect(await toThenable(new Expression('" " == space').value(ctx))).to.equal(true) }) describe('escape', () => { it('should escape quote', async function () { + const ctx = new Context({ quote: '"' }) expect(await toThenable(new Expression('"\\"" == quote').value(ctx))).to.equal(true) }) it('should escape square bracket', async function () { - expect(await toThenable(new Expression('obj["]"] == "right bracket"').value(ctx))).to.equal(true) + const ctx = new Context({ obj: { ']': 'bracket' } }) + expect(await toThenable(new Expression('obj["]"] == "bracket"').value(ctx))).to.equal(true) }) }) @@ -71,6 +127,7 @@ describe('Expression', function () { expect(await toThenable(new Expression('1 < 2 or x contains "x"').value(ctx))).to.equal(true) }) it('should support value and !=', async function () { + const ctx = new Context({ empty: '' }) expect(await toThenable(new Expression('empty and empty != ""').value(ctx))).to.equal(false) }) it('should recognize quoted value', async function () { @@ -84,10 +141,9 @@ describe('Expression', function () { const ctx = new Context({ obj: { foo: true } }) expect(await toThenable(new Expression('obj["foo"] and true').value(ctx))).to.equal(true) }) - }) - - it('should eval range expression', async function () { - expect(await toThenable(new Expression('(2..4)').value(ctx))).to.deep.equal([2, 3, 4]) - expect(await toThenable(new Expression('(two..4)').value(ctx))).to.deep.equal([2, 3, 4]) + it('should allow nested property access', async function () { + const ctx = new Context({ obj: { foo: 'FOO' }, keys: { "what's this": 'foo' } }) + expect(await toThenable(new Expression('obj[keys["what\'s this"]]').value(ctx))).to.equal('FOO') + }) }) }) diff --git a/test/unit/render/render.ts b/test/unit/render/render.ts index b51efcfe4..0f45f3615 100644 --- a/test/unit/render/render.ts +++ b/test/unit/render/render.ts @@ -1,6 +1,6 @@ import { expect } from 'chai' import { Context } from '../../../src/context/context' -import { Token } from '../../../src/parser/token' +import { HTMLToken } from '../../../src/tokens/html-token' import { Render } from '../../../src/render/render' import { HTML } from '../../../src/template/html' import { toThenable } from '../../../src/util/async' @@ -14,7 +14,7 @@ describe('render', function () { describe('.renderTemplates()', function () { it('should render html', async function () { const scope = new Context() - const token = { content: '

' } as Token + const token = { getContent: () => '

' } as HTMLToken const html = await toThenable(render.renderTemplates([new HTML(token)], scope)) return expect(html).to.equal('

') }) diff --git a/test/unit/template/filter/filter.ts b/test/unit/template/filter/filter.ts index 385fffee5..93061a310 100644 --- a/test/unit/template/filter/filter.ts +++ b/test/unit/template/filter/filter.ts @@ -3,6 +3,9 @@ import * as sinon from 'sinon' import * as sinonChai from 'sinon-chai' import { Context } from '../../../../src/context/context' import { toThenable } from '../../../../src/util/async' +import { NumberToken } from '../../../../src/tokens/number-token' +import { QuotedToken } from '../../../../src/tokens/quoted-token' +import { WordToken } from '../../../../src/tokens/word-token' import { FilterMap } from '../../../../src/template/filter/filter-map' chai.use(sinonChai) @@ -27,13 +30,15 @@ describe('filter', function () { it('should call filter impl with correct arguments', async function () { const spy = sinon.spy() filters.set('foo', spy) - await toThenable(filters.create('foo', ['33']).render('foo', ctx)) - expect(spy).to.have.been.calledWith('foo', 33) + const thirty = new NumberToken(new WordToken('30', 0, 2), undefined) + await toThenable(filters.create('foo', [thirty]).render('foo', ctx)) + expect(spy).to.have.been.calledWith('foo', 30) }) it('should call filter impl with correct this arg', async function () { const spy = sinon.spy() filters.set('foo', spy) - await toThenable(filters.create('foo', ['33']).render('foo', ctx)) + const thirty = new NumberToken(new WordToken('33', 0, 2), undefined) + await toThenable(filters.create('foo', [thirty]).render('foo', ctx)) expect(spy).to.have.been.calledOn(sinon.match.has('context', ctx)) }) it('should render a simple filter', async function () { @@ -43,12 +48,15 @@ describe('filter', function () { it('should render filters with argument', async function () { filters.set('add', (a, b) => a + b) - expect(await toThenable(filters.create('add', ['2']).render(3, ctx))).to.equal(5) + const two = new NumberToken(new WordToken('2', 0, 1), undefined) + expect(await toThenable(filters.create('add', [two]).render(3, ctx))).to.equal(5) }) it('should render filters with multiple arguments', async function () { filters.set('add', (a, b, c) => a + b + c) - expect(await toThenable(filters.create('add', ['2', '"c"']).render(3, ctx))).to.equal('5c') + const two = new NumberToken(new WordToken('2', 0, 1), undefined) + const c = new QuotedToken('"c"', 0, 3) + expect(await toThenable(filters.create('add', [two, c]).render(3, ctx))).to.equal('5c') }) it('should pass Objects/Drops as it is', async function () { @@ -65,6 +73,7 @@ describe('filter', function () { it('should support key value pairs', async function () { filters.set('add', (a, b) => b[0] + ':' + (a + b[1])) - expect(await toThenable((filters.create('add', [['num', '2']]).render(3, ctx)))).to.equal('num:5') + const two = new NumberToken(new WordToken('2', 0, 1), undefined) + expect(await toThenable((filters.create('add', [['num', two]]).render(3, ctx)))).to.equal('num:5') }) }) diff --git a/test/unit/template/hash.ts b/test/unit/template/hash.ts index 7cdeefddf..196694377 100644 --- a/test/unit/template/hash.ts +++ b/test/unit/template/hash.ts @@ -28,7 +28,8 @@ describe('Hash', function () { expect(hash.num).to.equal(2.3) }) it('should parse "num:bar.coo"', async function () { - const hash = await toThenable(new Hash('num:bar.coo').render(new Context({ bar: { coo: 3 } }))) + const pending = new Hash('num:bar.coo').render(new Context({ bar: { coo: 3 } })) + const hash = await toThenable(pending) expect(hash.num).to.equal(3) }) it('should parse "num1:2.3 reverse,num2:bar.coo\n num3: arr[0]"', async function () { diff --git a/test/unit/template/output.ts b/test/unit/template/output.ts index a47abca2d..e55da0edd 100644 --- a/test/unit/template/output.ts +++ b/test/unit/template/output.ts @@ -2,7 +2,7 @@ import * as chai from 'chai' import { toThenable } from '../../../src/util/async' import { Context } from '../../../src/context/context' import { Output } from '../../../src/template/output' -import { OutputToken } from '../../../src/parser/output-token' +import { OutputToken } from '../../../src/tokens/output-token' import { FilterMap } from '../../../src/template/filter/filter-map' const expect = chai.expect diff --git a/test/unit/template/tag.ts b/test/unit/template/tag.ts index 195fe3c07..1821ceb9f 100644 --- a/test/unit/template/tag.ts +++ b/test/unit/template/tag.ts @@ -3,7 +3,7 @@ import { Tag } from '../../../src/template/tag/tag' import { Context } from '../../../src/context/context' import * as sinon from 'sinon' import * as sinonChai from 'sinon-chai' -import { TagToken } from '../../../src/parser/tag-token' +import { TagToken } from '../../../src/tokens/tag-token' import { toThenable } from '../../../src/util/async' chai.use(sinonChai) diff --git a/test/unit/template/value.ts b/test/unit/template/value.ts index 62be991e3..29d343b90 100644 --- a/test/unit/template/value.ts +++ b/test/unit/template/value.ts @@ -1,4 +1,5 @@ import * as chai from 'chai' +import { QuotedToken } from '../../../src/tokens/quoted-token' import { toThenable } from '../../../src/util/async' import { FilterMap } from '../../../src/template/filter/filter-map' import * as sinonChai from 'sinon-chai' @@ -15,71 +16,17 @@ describe('Value', function () { const filterMap = new FilterMap(false) it('should parse "foo', function () { const tpl = new Value('foo', filterMap) - expect(tpl.initial).to.equal('foo') + expect(tpl.initial!.getText()).to.equal('foo') expect(tpl.filters).to.deep.equal([]) }) - - it('should parse "foo | add"', function () { - const tpl = new Value('foo | add', filterMap) - expect(tpl.initial).to.equal('foo') - expect(tpl.filters.length).to.equal(1) - expect(tpl.filters[0].args).to.eql([]) - }) - it('should parse "foo,foo | add"', function () { - const tpl = new Value('foo,foo | add', filterMap) - expect(tpl.initial).to.equal('foo') - expect(tpl.filters.length).to.equal(1) - expect(tpl.filters[0].args).to.eql([]) - }) - it('should parse "foo | add: 3, false"', function () { - const tpl = new Value('foo | add: 3, "foo"', filterMap) - expect(tpl.initial).to.equal('foo') - expect(tpl.filters.length).to.equal(1) - expect(tpl.filters[0].args).to.eql(['3', '"foo"']) - }) - it('should parse "foo | add: "foo" bar, 3"', function () { - const tpl = new Value('foo | add: "foo" bar, 3', filterMap) - expect(tpl.initial).to.equal('foo') - expect(tpl.filters.length).to.equal(1) - expect(tpl.filters[0].name).to.eql('add') - expect(tpl.filters[0].args).to.eql(['"foo"', '3']) - }) - it('should parse "foo | add: "|", 3', function () { - const tpl = new Value('foo | add: "|", 3', filterMap) - expect(tpl.initial).to.equal('foo') - expect(tpl.filters.length).to.equal(1) - expect(tpl.filters[0].args).to.eql(['"|"', '3']) - }) - it('should parse "foo | add: "|", 3', function () { - const tpl = new Value('foo | add: "|", 3', filterMap) - expect(tpl.initial).to.equal('foo') - expect(tpl.filters.length).to.equal(1) - expect(tpl.filters[0].args).to.eql(['"|"', '3']) - }) - it('should support arguments as named key/values', function () { - const f = new Value('o | foo: key1: "literal1", key2: value2', filterMap) - expect(f.filters[0].name).to.equal('foo') - expect(f.filters[0].args).to.eql([['key1', '"literal1"'], ['key2', 'value2']]) - }) - it('should support arguments as named key/values with inline literals', function () { - const f = new Value('o | foo: "test0", key1: "literal1", key2: value2', filterMap) - expect(f.filters[0].name).to.equal('foo') - expect(f.filters[0].args).to.deep.equal(['"test0"', ['key1', '"literal1"'], ['key2', 'value2']]) - }) - it('should support arguments as named key/values with inline values', function () { - const f = new Value('o | foo: test0, key1: "literal1", key2: value2', filterMap) - expect(f.filters[0].name).to.equal('foo') - expect(f.filters[0].args).to.deep.equal(['test0', ['key1', '"literal1"'], ['key2', 'value2']]) - }) - it('should support argument values named same as keys', function () { - const f = new Value('o | foo: a: a', filterMap) - expect(f.filters[0].name).to.equal('foo') - expect(f.filters[0].args).to.deep.equal([['a', 'a']]) - }) - it('should support argument literals named same as keys', function () { + it('should parse filters in value content', function () { const f = new Value('o | foo: a: "a"', filterMap) expect(f.filters[0].name).to.equal('foo') - expect(f.filters[0].args).to.deep.equal([['a', '"a"']]) + expect(f.filters[0].args).to.have.lengthOf(1) + const [k, v] = f.filters[0].args[0] as any + expect(k).to.equal('a') + expect(v).to.be.instanceOf(QuotedToken) + expect((v as QuotedToken).getText()).to.equal('"a"') }) }) diff --git a/test/unit/util/assert.ts b/test/unit/util/assert.ts index 1388d12b2..45fe73764 100644 --- a/test/unit/util/assert.ts +++ b/test/unit/util/assert.ts @@ -5,11 +5,11 @@ const expect = chai.expect describe('assert', function () { it('should not throw if predicate is truthy', function () { - const fn = () => assert('foo', 'bar') + const fn = () => assert('foo', () => 'bar') expect(fn).to.not.throw() }) it('should not throw if predicate is truthy', function () { - const fn = () => assert('', 'bar') + const fn = () => assert('', () => 'bar') expect(fn).to.throw(/bar/) }) it('should populate default message', function () {