feat: create trie programmatically in options

This commit is contained in:
Jason Etcovitch
2021-02-04 09:11:28 +08:00
committed by Jun Yang
parent 8734e2e6ce
commit befc33c4eb
28 changed files with 196 additions and 146 deletions
+1 -1
View File
@@ -2,7 +2,7 @@ import { Tokenizer, assert, TagImplOptions, TagToken, Context } from '../../type
export default { export default {
parse: function (token: TagToken) { parse: function (token: TagToken) {
const tokenizer = new Tokenizer(token.args) const tokenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
this.key = tokenizer.readIdentifier().content this.key = tokenizer.readIdentifier().content
tokenizer.skipBlank() tokenizer.skipBlank()
assert(tokenizer.peek() === '=', () => `illegal token ${token.getText()}`) assert(tokenizer.peek() === '=', () => `illegal token ${token.getText()}`)
+1 -1
View File
@@ -3,7 +3,7 @@ import { evalQuotedToken } from '../../render/expression'
export default { export default {
parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) { parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
const tokenizer = new Tokenizer(tagToken.args) const tokenizer = new Tokenizer(tagToken.args, this.liquid.options.operatorsTrie)
this.variable = readVariableName(tokenizer) this.variable = readVariableName(tokenizer)
assert(this.variable, () => `${tagToken.args} not valid identifier`) assert(this.variable, () => `${tagToken.args} not valid identifier`)
+3 -2
View File
@@ -26,10 +26,11 @@ export default {
render: function * (ctx: Context, emitter: Emitter) { render: function * (ctx: Context, emitter: Emitter) {
const r = this.liquid.renderer const r = this.liquid.renderer
const cond = yield new Expression(this.cond, this.liquid.options.operators).value(ctx) const { operators, operatorsTrie } = this.liquid.options
const cond = yield new Expression(this.cond, operators, operatorsTrie).value(ctx)
for (let i = 0; i < this.cases.length; i++) { for (let i = 0; i < this.cases.length; i++) {
const branch = this.cases[i] const branch = this.cases[i]
const val = yield new Expression(branch.val, this.liquid.options.operators).value(ctx) const val = yield new Expression(branch.val, operators, operatorsTrie).value(ctx)
if (val === cond) { if (val === cond) {
yield r.renderTemplates(branch.templates, ctx, emitter) yield r.renderTemplates(branch.templates, ctx, emitter)
return return
+1 -1
View File
@@ -4,7 +4,7 @@ import { Tokenizer } from '../../parser/tokenizer'
export default { export default {
parse: function (tagToken: TagToken) { parse: function (tagToken: TagToken) {
const tokenizer = new Tokenizer(tagToken.args) const tokenizer = new Tokenizer(tagToken.args, this.liquid.options.operatorsTrie)
const group = tokenizer.readValue() const group = tokenizer.readValue()
tokenizer.skipBlank() tokenizer.skipBlank()
+1 -1
View File
@@ -3,7 +3,7 @@ import { isNumber, stringify } from '../../util/underscore'
export default { export default {
parse: function (token: TagToken) { parse: function (token: TagToken) {
const tokenizer = new Tokenizer(token.args) const tokenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
this.variable = tokenizer.readIdentifier().content this.variable = tokenizer.readIdentifier().content
}, },
render: function (context: Context, emitter: Emitter) { render: function (context: Context, emitter: Emitter) {
+1 -1
View File
@@ -6,7 +6,7 @@ import { Hash } from '../../template/tag/hash'
export default { export default {
type: 'block', type: 'block',
parse: function (token: TagToken, remainTokens: TopLevelToken[]) { parse: function (token: TagToken, remainTokens: TopLevelToken[]) {
const toknenizer = new Tokenizer(token.args) const toknenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
const variable = toknenizer.readIdentifier() const variable = toknenizer.readIdentifier()
const inStr = toknenizer.readIdentifier() const inStr = toknenizer.readIdentifier()
+2 -1
View File
@@ -29,9 +29,10 @@ export default {
render: function * (ctx: Context, emitter: Emitter) { render: function * (ctx: Context, emitter: Emitter) {
const r = this.liquid.renderer const r = this.liquid.renderer
const { operators, operatorsTrie } = this.liquid.options
for (const branch of this.branches) { for (const branch of this.branches) {
const cond = yield new Expression(branch.cond, this.liquid.options.operators, ctx.opts.lenientIf).value(ctx) const cond = yield new Expression(branch.cond, operators, operatorsTrie, ctx.opts.lenientIf).value(ctx)
if (isTruthy(cond, ctx)) { if (isTruthy(cond, ctx)) {
yield r.renderTemplates(branch.templates, ctx, emitter) yield r.renderTemplates(branch.templates, ctx, emitter)
return return
+1 -1
View File
@@ -4,7 +4,7 @@ import BlockMode from '../../context/block-mode'
export default { export default {
parse: function (token: TagToken) { parse: function (token: TagToken) {
const args = token.args const args = token.args
const tokenizer = new Tokenizer(args) const tokenizer = new Tokenizer(args, this.liquid.options.operatorsTrie)
this.file = this.liquid.options.dynamicPartials this.file = this.liquid.options.dynamicPartials
? tokenizer.readValue() ? tokenizer.readValue()
: tokenizer.readFileName() : tokenizer.readFileName()
+1 -1
View File
@@ -3,7 +3,7 @@ import { Tokenizer, Emitter, TagToken, Context, TagImplOptions } from '../../typ
export default { export default {
parse: function (token: TagToken) { parse: function (token: TagToken) {
const tokenizer = new Tokenizer(token.args) const tokenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
this.variable = tokenizer.readIdentifier().content this.variable = tokenizer.readIdentifier().content
}, },
render: function (context: Context, emitter: Emitter) { render: function (context: Context, emitter: Emitter) {
+1 -1
View File
@@ -3,7 +3,7 @@ import BlockMode from '../../context/block-mode'
export default { export default {
parse: function (token: TagToken, remainTokens: TopLevelToken[]) { parse: function (token: TagToken, remainTokens: TopLevelToken[]) {
const tokenizer = new Tokenizer(token.args) const tokenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
const file = this.liquid.options.dynamicPartials ? tokenizer.readValue() : tokenizer.readFileName() const file = this.liquid.options.dynamicPartials ? tokenizer.readValue() : tokenizer.readFileName()
assert(file, () => `illegal argument "${token.args}"`) assert(file, () => `illegal argument "${token.args}"`)
+1 -1
View File
@@ -6,7 +6,7 @@ import { evalQuotedToken, TypeGuards, Tokenizer, evalToken, Hash, Emitter, TagTo
export default { export default {
parse: function (token: TagToken) { parse: function (token: TagToken) {
const args = token.args const args = token.args
const tokenizer = new Tokenizer(args) const tokenizer = new Tokenizer(args, this.liquid.options.operatorsTrie)
this.file = this.liquid.options.dynamicPartials this.file = this.liquid.options.dynamicPartials
? tokenizer.readValue() ? tokenizer.readValue()
: tokenizer.readFileName() : tokenizer.readFileName()
+1 -1
View File
@@ -5,7 +5,7 @@ import { Tokenizer } from '../../parser/tokenizer'
export default { export default {
parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) { parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
const tokenizer = new Tokenizer(tagToken.args) const tokenizer = new Tokenizer(tagToken.args, this.liquid.options.operatorsTrie)
this.variable = tokenizer.readIdentifier() this.variable = tokenizer.readIdentifier()
tokenizer.skipBlank() tokenizer.skipBlank()
+3 -2
View File
@@ -29,7 +29,8 @@ export default {
render: function * (ctx: Context, emitter: Emitter) { render: function * (ctx: Context, emitter: Emitter) {
const r = this.liquid.renderer const r = this.liquid.renderer
const cond = yield new Expression(this.cond, this.liquid.options.operators, ctx.opts.lenientIf).value(ctx) const { operators, operatorsTrie } = this.liquid.options
const cond = yield new Expression(this.cond, operators, operatorsTrie, ctx.opts.lenientIf).value(ctx)
if (isFalsy(cond, ctx)) { if (isFalsy(cond, ctx)) {
yield r.renderTemplates(this.templates, ctx, emitter) yield r.renderTemplates(this.templates, ctx, emitter)
@@ -37,7 +38,7 @@ export default {
} }
for (const branch of this.branches) { for (const branch of this.branches) {
const cond = yield new Expression(branch.cond, this.liquid.options.operators, ctx.opts.lenientIf).value(ctx) const cond = yield new Expression(branch.cond, operators, operatorsTrie, ctx.opts.lenientIf).value(ctx)
if (isTruthy(cond, ctx)) { if (isTruthy(cond, ctx)) {
yield r.renderTemplates(branch.templates, ctx, emitter) yield r.renderTemplates(branch.templates, ctx, emitter)
return return
+8 -1
View File
@@ -4,6 +4,7 @@ import { Cache } from './cache/cache'
import { LRU } from './cache/lru' import { LRU } from './cache/lru'
import { FS } from './fs/fs' import { FS } from './fs/fs'
import { defaultOperators, Operators } from './render/operator' import { defaultOperators, Operators } from './render/operator'
import { createTrie, Trie } from './util/operator-trie'
export interface LiquidOptions { export interface LiquidOptions {
/** A directory or an array of directories from where to resolve layout and include templates, and the filename passed to `.renderFile()`. If it's an array, the files are looked up in the order they occur in the array. Defaults to `["."]` */ /** A directory or an array of directories from where to resolve layout and include templates, and the filename passed to `.renderFile()`. If it's an array, the files are looked up in the order they occur in the array. Defaults to `["."]` */
@@ -55,6 +56,7 @@ export interface LiquidOptions {
interface NormalizedOptions extends LiquidOptions { interface NormalizedOptions extends LiquidOptions {
root?: string[]; root?: string[];
cache?: Cache<Template[]>; cache?: Cache<Template[]>;
operatorsTrie?: Trie;
} }
export interface NormalizedFullOptions extends NormalizedOptions { export interface NormalizedFullOptions extends NormalizedOptions {
@@ -79,6 +81,7 @@ export interface NormalizedFullOptions extends NormalizedOptions {
globals: object; globals: object;
keepOutputType: boolean; keepOutputType: boolean;
operators: Operators; operators: Operators;
operatorsTrie: Trie;
} }
export const defaultOptions: NormalizedFullOptions = { export const defaultOptions: NormalizedFullOptions = {
@@ -102,7 +105,8 @@ export const defaultOptions: NormalizedFullOptions = {
lenientIf: false, lenientIf: false,
globals: {}, globals: {},
keepOutputType: false, keepOutputType: false,
operators: defaultOperators operators: defaultOperators,
operatorsTrie: createTrie(defaultOperators)
} }
export function normalize (options?: LiquidOptions): NormalizedOptions { export function normalize (options?: LiquidOptions): NormalizedOptions {
@@ -117,6 +121,9 @@ export function normalize (options?: LiquidOptions): NormalizedOptions {
else cache = options.cache ? new LRU<Template[]>(1024) : undefined else cache = options.cache ? new LRU<Template[]>(1024) : undefined
options.cache = cache options.cache = cache
} }
if (options.hasOwnProperty('operators')) {
(options as NormalizedOptions).operatorsTrie = createTrie(options.operators!)
}
return options as NormalizedOptions return options as NormalizedOptions
} }
+1 -1
View File
@@ -39,7 +39,7 @@ export class Liquid {
forOwn(builtinFilters, (handler: FilterImplOptions, name: string) => this.registerFilter(snakeCase(name), handler)) forOwn(builtinFilters, (handler: FilterImplOptions, name: string) => this.registerFilter(snakeCase(name), handler))
} }
public parse (html: string, filepath?: string): Template[] { public parse (html: string, filepath?: string): Template[] {
const tokenizer = new Tokenizer(html, filepath) const tokenizer = new Tokenizer(html, this.options.operatorsTrie, filepath)
const tokens = tokenizer.readTopLevelTokens(this.options) const tokens = tokenizer.readTopLevelTokens(this.options)
return this.parser.parse(tokens) return this.parser.parse(tokens)
} }
+2 -11
View File
@@ -1,16 +1,7 @@
import { IDENTIFIER } from '../util/character' import { IDENTIFIER } from '../util/character'
import { Trie } from '../util/operator-trie'
const trie = { export function matchOperator (str: string, begin: number, trie: Trie, end = str.length) {
a: { n: { d: { end: true, needBoundary: true } } },
o: { r: { end: true, needBoundary: true } },
c: { o: { n: { t: { a: { i: { n: { s: { end: true, needBoundary: true } } } } } } } },
'=': { '=': { end: true } },
'!': { '=': { end: true } },
'>': { end: true, '=': { end: true } },
'<': { end: true, '=': { end: true } }
}
export function matchOperator (str: string, begin: number, end = str.length) {
let node = trie let node = trie
let i = begin let i = begin
let info let info
+3 -1
View File
@@ -22,6 +22,7 @@ import { TokenizationError } from '../util/error'
import { NormalizedFullOptions, defaultOptions } from '../liquid-options' import { NormalizedFullOptions, defaultOptions } from '../liquid-options'
import { TYPES, QUOTE, BLANK, IDENTIFIER } from '../util/character' import { TYPES, QUOTE, BLANK, IDENTIFIER } from '../util/character'
import { matchOperator } from './match-operator' import { matchOperator } from './match-operator'
import { Trie } from '../util/operator-trie'
export class Tokenizer { export class Tokenizer {
p = 0 p = 0
@@ -30,6 +31,7 @@ export class Tokenizer {
constructor ( constructor (
private input: string, private input: string,
private trie: Trie,
private file: string = '' private file: string = ''
) { ) {
this.N = input.length this.N = input.length
@@ -54,7 +56,7 @@ export class Tokenizer {
} }
readOperator (): OperatorToken | undefined { readOperator (): OperatorToken | undefined {
this.skipBlank() this.skipBlank()
const end = matchOperator(this.input, this.p, this.p + 8) const end = matchOperator(this.input, this.p, this.trie, this.p + 8)
if (end === -1) return if (end === -1) return
return new OperatorToken(this.input, this.p, (this.p = end), this.file) return new OperatorToken(this.input, this.p, (this.p = end), this.file)
} }
+3 -2
View File
@@ -13,6 +13,7 @@ import { range, toValue } from '../util/underscore'
import { Tokenizer } from '../parser/tokenizer' import { Tokenizer } from '../parser/tokenizer'
import { Operators } from '../render/operator' import { Operators } from '../render/operator'
import { UndefinedVariableError, InternalUndefinedVariableError } from '../util/error' import { UndefinedVariableError, InternalUndefinedVariableError } from '../util/error'
import { Trie } from '../util/operator-trie'
export class Expression { export class Expression {
private operands: any[] = [] private operands: any[] = []
@@ -20,8 +21,8 @@ export class Expression {
private lenient: boolean private lenient: boolean
private operators: Operators private operators: Operators
public constructor (str: string, operators: Operators, lenient = false) { public constructor (str: string, operators: Operators, operatorsTrie: Trie, lenient = false) {
const tokenizer = new Tokenizer(str) const tokenizer = new Tokenizer(str, operatorsTrie)
this.postfix = [...toPostfix(tokenizer.readExpression())] this.postfix = [...toPostfix(tokenizer.readExpression())]
this.lenient = lenient this.lenient = lenient
this.operators = operators this.operators = operators
+1 -1
View File
@@ -13,7 +13,7 @@ import { Tokenizer } from '../../parser/tokenizer'
export class Hash { export class Hash {
hash: { [key: string]: any } = {} hash: { [key: string]: any } = {}
constructor (markup: string) { constructor (markup: string) {
const tokenizer = new Tokenizer(markup) const tokenizer = new Tokenizer(markup, {})
for (const hash of tokenizer.readHashes()) { for (const hash of tokenizer.readHashes()) {
this.hash[hash.name.content] = hash.value this.hash[hash.name.content] = hash.value
} }
+1 -1
View File
@@ -14,7 +14,7 @@ export class Value {
* @param str the value to be valuated, eg.: "foobar" | truncate: 3 * @param str the value to be valuated, eg.: "foobar" | truncate: 3
*/ */
public constructor (str: string, private readonly filterMap: FilterMap, liquid: Liquid) { public constructor (str: string, private readonly filterMap: FilterMap, liquid: Liquid) {
const tokenizer = new Tokenizer(str) const tokenizer = new Tokenizer(str, liquid.options.operatorsTrie)
this.initial = tokenizer.readValue() this.initial = tokenizer.readValue()
this.filters = tokenizer.readFilters().map(({ name, args }) => new Filter(name, this.filterMap.get(name), args, liquid)) this.filters = tokenizer.readFilters().map(({ name, args }) => new Filter(name, this.filterMap.get(name), args, liquid))
} }
+1 -1
View File
@@ -18,7 +18,7 @@ export class TagToken extends DelimitedToken {
const value = input.slice(begin + tagDelimiterLeft.length, end - tagDelimiterRight.length) const value = input.slice(begin + tagDelimiterLeft.length, end - tagDelimiterRight.length)
super(TokenKind.Tag, value, input, begin, end, trimTagLeft, trimTagRight, file) super(TokenKind.Tag, value, input, begin, end, trimTagLeft, trimTagRight, file)
const tokenizer = new Tokenizer(this.content) const tokenizer = new Tokenizer(this.content, options.operatorsTrie)
this.name = tokenizer.readIdentifier().getText() this.name = tokenizer.readIdentifier().getText()
if (!this.name) throw new TokenizationError(`illegal tag syntax`, this) if (!this.name) throw new TokenizationError(`illegal tag syntax`, this)
+27
View File
@@ -0,0 +1,27 @@
import { Operators } from '../render/operator'
export interface Trie {
[key: string]: any;
}
export function createTrie (operators: Operators): Trie {
const trie: Trie = {}
for (const [name, handler] of Object.entries(operators)) {
let node = trie
for (let i = 0; i < name.length; i++) {
const c = name[i]
node[c] = node[c] || {}
if (i === name.length - 1 && c !== '=') {
node[c].needBoundary = true
}
node = node[c]
}
node.handler = handler
node.end = true
}
return trie
}
+4 -2
View File
@@ -19,7 +19,9 @@ describe('LiquidOptions#operators', function () {
}) })
it('should evaluate a custom operator', async function () { it('should evaluate a custom operator', async function () {
const result = await engine.parseAndRender('{% if "foo" isFooBar "bar" %}True{% endif %}') const first = await engine.parseAndRender('{% if "foo" isFooBar "bar" %}True{% else %}False{% endif %}')
expect(result).to.equal('True') expect(first).to.equal('True')
const second = await engine.parseAndRender('{% if "foo" isFooBar "foo" %}True{% else %}False{% endif %}')
expect(second).to.equal('False')
}) })
}) })
+14 -11
View File
@@ -1,26 +1,29 @@
import { expect } from 'chai' import { expect } from 'chai'
import { matchOperator } from '../../../src/parser/match-operator' import { matchOperator } from '../../../src/parser/match-operator'
import { defaultOperators } from '../../../src/types'
import { createTrie } from '../../../src/util/operator-trie'
describe('parser/matchOperator()', function () { describe('parser/matchOperator()', function () {
const trie = createTrie(defaultOperators)
it('should match contains', () => { it('should match contains', () => {
expect(matchOperator('contains', 0)).to.equal(8) expect(matchOperator('contains', 0, trie)).to.equal(8)
}) })
it('should match comparision', () => { it('should match comparision', () => {
expect(matchOperator('>', 0)).to.equal(1) expect(matchOperator('>', 0, trie)).to.equal(1)
expect(matchOperator('>=', 0)).to.equal(2) expect(matchOperator('>=', 0, trie)).to.equal(2)
expect(matchOperator('<', 0)).to.equal(1) expect(matchOperator('<', 0, trie)).to.equal(1)
expect(matchOperator('<=', 0)).to.equal(2) expect(matchOperator('<=', 0, trie)).to.equal(2)
}) })
it('should match binary logic', () => { it('should match binary logic', () => {
expect(matchOperator('and', 0)).to.equal(3) expect(matchOperator('and', 0, trie)).to.equal(3)
expect(matchOperator('or', 0)).to.equal(2) expect(matchOperator('or', 0, trie)).to.equal(2)
}) })
it('should not match if word not terminate', () => { it('should not match if word not terminate', () => {
expect(matchOperator('true1', 0)).to.equal(-1) expect(matchOperator('true1', 0, trie)).to.equal(-1)
expect(matchOperator('containsa', 0)).to.equal(-1) expect(matchOperator('containsa', 0, trie)).to.equal(-1)
}) })
it('should match if word boundary found', () => { it('should match if word boundary found', () => {
expect(matchOperator('>=1', 0)).to.equal(2) expect(matchOperator('>=1', 0, trie)).to.equal(2)
expect(matchOperator('contains b', 0)).to.equal(8) expect(matchOperator('contains b', 0, trie)).to.equal(8)
}) })
}) })
+60 -56
View File
@@ -9,59 +9,63 @@ import { TagToken } from '../../../src/tokens/tag-token'
import { QuotedToken } from '../../../src/tokens/quoted-token' import { QuotedToken } from '../../../src/tokens/quoted-token'
import { OutputToken } from '../../../src/tokens/output-token' import { OutputToken } from '../../../src/tokens/output-token'
import { HTMLToken } from '../../../src/tokens/html-token' import { HTMLToken } from '../../../src/tokens/html-token'
import { createTrie } from '../../../src/util/operator-trie'
import { defaultOperators } from '../../../src/types'
describe('Tokenizer', function () { describe('Tokenizer', function () {
const trie = createTrie(defaultOperators)
it('should read quoted', () => { it('should read quoted', () => {
expect(new Tokenizer('"foo" ff').readQuoted()!.getText()).to.equal('"foo"') expect(new Tokenizer('"foo" ff', trie).readQuoted()!.getText()).to.equal('"foo"')
expect(new Tokenizer(' "foo"ff').readQuoted()!.getText()).to.equal('"foo"') expect(new Tokenizer(' "foo"ff', trie).readQuoted()!.getText()).to.equal('"foo"')
}) })
it('should read value', () => { it('should read value', () => {
expect(new Tokenizer('a[ b][ "c d" ]').readValueOrThrow().getText()).to.equal('a[ b][ "c d" ]') expect(new Tokenizer('a[ b][ "c d" ]', trie).readValueOrThrow().getText()).to.equal('a[ b][ "c d" ]')
expect(new Tokenizer('a.b[c[d.e]]').readValueOrThrow().getText()).to.equal('a.b[c[d.e]]') expect(new Tokenizer('a.b[c[d.e]]', trie).readValueOrThrow().getText()).to.equal('a.b[c[d.e]]')
}) })
it('should read identifier', () => { it('should read identifier', () => {
expect(new Tokenizer('foo bar').readIdentifier()).to.haveOwnProperty('content', 'foo') expect(new Tokenizer('foo bar', trie).readIdentifier()).to.haveOwnProperty('content', 'foo')
expect(new Tokenizer('foo bar').readWord()).to.haveOwnProperty('content', 'foo') expect(new Tokenizer('foo bar', trie).readWord()).to.haveOwnProperty('content', 'foo')
}) })
it('should read number value', () => { it('should read number value', () => {
const token: NumberToken = new Tokenizer('2.33.2').readValueOrThrow() as any const token: NumberToken = new Tokenizer('2.33.2', trie).readValueOrThrow() as any
expect(token).to.be.instanceOf(NumberToken) expect(token).to.be.instanceOf(NumberToken)
expect(token.whole.getText()).to.equal('2') expect(token.whole.getText()).to.equal('2')
expect(token.decimal!.getText()).to.equal('33') expect(token.decimal!.getText()).to.equal('33')
expect(token.getText()).to.equal('2.33') expect(token.getText()).to.equal('2.33')
}) })
it('should read quoted value', () => { it('should read quoted value', () => {
const value = new Tokenizer('"foo"a').readValue() const value = new Tokenizer('"foo"a', trie).readValue()
expect(value).to.be.instanceOf(QuotedToken) expect(value).to.be.instanceOf(QuotedToken)
expect(value!.getText()).to.equal('"foo"') expect(value!.getText()).to.equal('"foo"')
}) })
it('should read property access value', () => { it('should read property access value', () => {
expect(new Tokenizer('a[b]["c d"]').readValueOrThrow().getText()).to.equal('a[b]["c d"]') expect(new Tokenizer('a[b]["c d"]', trie).readValueOrThrow().getText()).to.equal('a[b]["c d"]')
}) })
it('should read quoted property access value', () => { it('should read quoted property access value', () => {
const value = new Tokenizer('["a prop"]').readValue() const value = new Tokenizer('["a prop"]', trie).readValue()
expect(value).to.be.instanceOf(PropertyAccessToken) expect(value).to.be.instanceOf(PropertyAccessToken)
expect((value as PropertyAccessToken).variable.getText()).to.equal('"a prop"') expect((value as PropertyAccessToken).variable.getText()).to.equal('"a prop"')
}) })
it('should throw for broken quoted property access', () => { it('should throw for broken quoted property access', () => {
const tokenizer = new Tokenizer('[5]') const tokenizer = new Tokenizer('[5]', trie)
expect(() => tokenizer.readValueOrThrow()).to.throw() expect(() => tokenizer.readValueOrThrow()).to.throw()
}) })
it('should throw for incomplete quoted property access', () => { it('should throw for incomplete quoted property access', () => {
const tokenizer = new Tokenizer('["a prop"') const tokenizer = new Tokenizer('["a prop"', trie)
expect(() => tokenizer.readValueOrThrow()).to.throw() expect(() => tokenizer.readValueOrThrow()).to.throw()
}) })
it('should read hash', () => { it('should read hash', () => {
const hash1 = new Tokenizer('foo: 3').readHash() const hash1 = new Tokenizer('foo: 3', trie).readHash()
expect(hash1!.name.content).to.equal('foo') expect(hash1!.name.content).to.equal('foo')
expect(hash1!.value!.getText()).to.equal('3') expect(hash1!.value!.getText()).to.equal('3')
const hash2 = new Tokenizer(', foo: a[ "bar"]').readHash() const hash2 = new Tokenizer(', foo: a[ "bar"]', trie).readHash()
expect(hash2!.name.content).to.equal('foo') expect(hash2!.name.content).to.equal('foo')
expect(hash2!.value!.getText()).to.equal('a[ "bar"]') expect(hash2!.value!.getText()).to.equal('a[ "bar"]')
}) })
it('should read multiple hashs', () => { it('should read multiple hashs', () => {
const hashes = new Tokenizer(', limit: 3 reverse offset:off').readHashes() const hashes = new Tokenizer(', limit: 3 reverse offset:off', trie).readHashes()
expect(hashes).to.have.lengthOf(3) expect(hashes).to.have.lengthOf(3)
const [limit, reverse, offset] = hashes const [limit, reverse, offset] = hashes
expect(limit.name.content).to.equal('limit') expect(limit.name.content).to.equal('limit')
@@ -74,7 +78,7 @@ describe('Tokenizer', function () {
expect(offset.value!.getText()).to.equal('off') expect(offset.value!.getText()).to.equal('off')
}) })
it('should read hash value with property access', () => { it('should read hash value with property access', () => {
const hashes = new Tokenizer('cols: 2, rows: data["rows"]').readHashes() const hashes = new Tokenizer('cols: 2, rows: data["rows"]', trie).readHashes()
expect(hashes).to.have.lengthOf(2) expect(hashes).to.have.lengthOf(2)
const [cols, rols] = hashes const [cols, rols] = hashes
@@ -87,7 +91,7 @@ describe('Tokenizer', function () {
describe('#readTopLevelTokens()', () => { describe('#readTopLevelTokens()', () => {
it('should read HTML token', function () { it('should read HTML token', function () {
const html = '<html><body><p>Lorem Ipsum</p></body></html>' const html = '<html><body><p>Lorem Ipsum</p></body></html>'
const tokenizer = new Tokenizer(html) const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens() const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(1) expect(tokens.length).to.equal(1)
@@ -96,7 +100,7 @@ describe('Tokenizer', function () {
}) })
it('should read tag token', function () { it('should read tag token', function () {
const html = '<p>{% for p in a[1]%}</p>' const html = '<p>{% for p in a[1]%}</p>'
const tokenizer = new Tokenizer(html) const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens() const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(3) expect(tokens.length).to.equal(3)
@@ -107,7 +111,7 @@ describe('Tokenizer', function () {
}) })
it('should allow unclosed tag inside {% raw %}', function () { it('should allow unclosed tag inside {% raw %}', function () {
const html = '{%raw%} {%if%} {%else {%endraw%}' const html = '{%raw%} {%if%} {%else {%endraw%}'
const tokenizer = new Tokenizer(html) const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens() const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(3) expect(tokens.length).to.equal(3)
@@ -116,7 +120,7 @@ describe('Tokenizer', function () {
}) })
it('should allow unclosed endraw tag inside {% raw %}', function () { it('should allow unclosed endraw tag inside {% raw %}', function () {
const html = '{%raw%} {%endraw {%raw%} {%endraw%}' const html = '{%raw%} {%endraw {%raw%} {%endraw%}'
const tokenizer = new Tokenizer(html) const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens() const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(3) expect(tokens.length).to.equal(3)
@@ -125,12 +129,12 @@ describe('Tokenizer', function () {
}) })
it('should throw when {% raw %} not closed', function () { it('should throw when {% raw %} not closed', function () {
const html = '{%raw%} {%endraw {%raw%}' const html = '{%raw%} {%endraw {%raw%}'
const tokenizer = new Tokenizer(html) const tokenizer = new Tokenizer(html, trie)
expect(() => tokenizer.readTopLevelTokens()).to.throw('raw "{%raw%} {%end..." not closed, line:1, col:8') expect(() => tokenizer.readTopLevelTokens()).to.throw('raw "{%raw%} {%end..." not closed, line:1, col:8')
}) })
it('should read output token', function () { it('should read output token', function () {
const html = '<p>{{foo | date: "%Y-%m-%d"}}</p>' const html = '<p>{{foo | date: "%Y-%m-%d"}}</p>'
const tokenizer = new Tokenizer(html) const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens() const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(3) expect(tokens.length).to.equal(3)
@@ -140,7 +144,7 @@ describe('Tokenizer', function () {
}) })
it('should handle consecutive value and tags', function () { it('should handle consecutive value and tags', function () {
const html = '{{foo}}{{bar}}{%foo%}{%bar%}' const html = '{{foo}}{{bar}}{%foo%}{%bar%}'
const tokenizer = new Tokenizer(html) const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens() const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(4) expect(tokens.length).to.equal(4)
@@ -162,7 +166,7 @@ describe('Tokenizer', function () {
}) })
it('should keep white spaces and newlines', function () { it('should keep white spaces and newlines', function () {
const html = '{%foo%}\n{%bar %} \n {%alice%}' const html = '{%foo%}\n{%bar %} \n {%alice%}'
const tokenizer = new Tokenizer(html) const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens() const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(5) expect(tokens.length).to.equal(5)
expect(tokens[1]).instanceOf(HTMLToken) expect(tokens[1]).instanceOf(HTMLToken)
@@ -172,7 +176,7 @@ describe('Tokenizer', function () {
}) })
it('should handle multiple lines tag', function () { it('should handle multiple lines tag', function () {
const html = '{%foo\na:a\nb:1.23\n%}' const html = '{%foo\na:a\nb:1.23\n%}'
const tokenizer = new Tokenizer(html) const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens() const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(1) expect(tokens.length).to.equal(1)
expect(tokens[0]).instanceOf(TagToken) expect(tokens[0]).instanceOf(TagToken)
@@ -181,7 +185,7 @@ describe('Tokenizer', function () {
}) })
it('should handle multiple lines value', function () { it('should handle multiple lines value', function () {
const html = '{{foo\n|date:\n"%Y-%m-%d"\n}}' const html = '{{foo\n|date:\n"%Y-%m-%d"\n}}'
const tokenizer = new Tokenizer(html) const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens() const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(1) expect(tokens.length).to.equal(1)
expect(tokens[0]).instanceOf(OutputToken) expect(tokens[0]).instanceOf(OutputToken)
@@ -189,7 +193,7 @@ describe('Tokenizer', function () {
}) })
it('should handle complex object property access', function () { it('should handle complex object property access', function () {
const html = '{{ obj["my:property with anything"] }}' const html = '{{ obj["my:property with anything"] }}'
const tokenizer = new Tokenizer(html) const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens() const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(1) expect(tokens.length).to.equal(1)
const output = tokens[0] as OutputToken const output = tokens[0] as OutputToken
@@ -198,18 +202,18 @@ describe('Tokenizer', function () {
}) })
it('should throw if tag not closed', function () { it('should throw if tag not closed', function () {
const html = '{% assign foo = bar {{foo}}' const html = '{% assign foo = bar {{foo}}'
const tokenizer = new Tokenizer(html) const tokenizer = new Tokenizer(html, trie)
expect(() => tokenizer.readTopLevelTokens()).to.throw(/tag "{% assign foo..." not closed/) expect(() => tokenizer.readTopLevelTokens()).to.throw(/tag "{% assign foo..." not closed/)
}) })
it('should throw if output not closed', function () { it('should throw if output not closed', function () {
const tokenizer = new Tokenizer('{{name}') const tokenizer = new Tokenizer('{{name}', trie)
expect(() => tokenizer.readTopLevelTokens()).to.throw(/output "{{name}" not closed/) expect(() => tokenizer.readTopLevelTokens()).to.throw(/output "{{name}" not closed/)
}) })
}) })
describe('#readTagToken()', () => { describe('#readTagToken()', () => {
it('should skip quoted delimiters', function () { it('should skip quoted delimiters', function () {
const html = '{% assign a = "%} {% }} {{" %}' const html = '{% assign a = "%} {% }} {{" %}'
const tokenizer = new Tokenizer(html) const tokenizer = new Tokenizer(html, trie)
const token = tokenizer.readTagToken() const token = tokenizer.readTagToken()
expect(token).instanceOf(TagToken) expect(token).instanceOf(TagToken)
@@ -220,7 +224,7 @@ describe('Tokenizer', function () {
describe('#readOutputToken()', () => { describe('#readOutputToken()', () => {
it('should skip quoted delimiters', function () { it('should skip quoted delimiters', function () {
const html = '{{ "%} {%" | append: "}} {{" }}' const html = '{{ "%} {%" | append: "}} {{" }}'
const tokenizer = new Tokenizer(html) const tokenizer = new Tokenizer(html, trie)
const token = tokenizer.readOutputToken() const token = tokenizer.readOutputToken()
console.log(token) console.log(token)
@@ -230,7 +234,7 @@ describe('Tokenizer', function () {
}) })
describe('#readRange()', () => { describe('#readRange()', () => {
it('should read `(1..3)`', () => { it('should read `(1..3)`', () => {
const range = new Tokenizer('(1..3)').readRange() const range = new Tokenizer('(1..3)', trie).readRange()
expect(range).to.be.instanceOf(RangeToken) expect(range).to.be.instanceOf(RangeToken)
expect(range!.getText()).to.deep.equal('(1..3)') expect(range!.getText()).to.deep.equal('(1..3)')
const { lhs, rhs } = range! const { lhs, rhs } = range!
@@ -240,23 +244,23 @@ describe('Tokenizer', function () {
expect(rhs.getText()).to.equal('3') expect(rhs.getText()).to.equal('3')
}) })
it('should throw for `(..3)`', () => { it('should throw for `(..3)`', () => {
expect(() => new Tokenizer('(..3)').readRange()).to.throw('unexpected token "..3)", value expected') expect(() => new Tokenizer('(..3)', trie).readRange()).to.throw('unexpected token "..3)", value expected')
}) })
it('should read `(a.b..c["..d"])`', () => { it('should read `(a.b..c["..d"])`', () => {
const range = new Tokenizer('(a.b..c["..d"])').readRange() const range = new Tokenizer('(a.b..c["..d"])', trie).readRange()
expect(range).to.be.instanceOf(RangeToken) expect(range).to.be.instanceOf(RangeToken)
expect(range!.getText()).to.deep.equal('(a.b..c["..d"])') expect(range!.getText()).to.deep.equal('(a.b..c["..d"])')
}) })
}) })
describe('#readFilter()', () => { describe('#readFilter()', () => {
it('should read a simple filter', function () { it('should read a simple filter', function () {
const tokenizer = new Tokenizer('| plus') const tokenizer = new Tokenizer('| plus', trie)
const token = tokenizer.readFilter() const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus') expect(token).to.have.property('name', 'plus')
expect(token).to.have.property('args').to.deep.equal([]) expect(token).to.have.property('args').to.deep.equal([])
}) })
it('should read a filter with argument', function () { it('should read a filter with argument', function () {
const tokenizer = new Tokenizer(' | plus: 1') const tokenizer = new Tokenizer(' | plus: 1', trie)
const token = tokenizer.readFilter() const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus') expect(token).to.have.property('name', 'plus')
expect(token!.args).to.have.lengthOf(1) expect(token!.args).to.have.lengthOf(1)
@@ -266,18 +270,18 @@ describe('Tokenizer', function () {
expect(one.getText()).to.equal('1') expect(one.getText()).to.equal('1')
}) })
it('should read a filter with colon but no argument', function () { it('should read a filter with colon but no argument', function () {
const tokenizer = new Tokenizer('| plus:') const tokenizer = new Tokenizer('| plus:', trie)
const token = tokenizer.readFilter() const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus') expect(token).to.have.property('name', 'plus')
expect(token).to.have.property('args').to.deep.equal([]) expect(token).to.have.property('args').to.deep.equal([])
}) })
it('should read null if name not found', function () { it('should read null if name not found', function () {
const tokenizer = new Tokenizer('|') const tokenizer = new Tokenizer('|', trie)
const token = tokenizer.readFilter() const token = tokenizer.readFilter()
expect(token).to.be.null expect(token).to.be.null
}) })
it('should read a filter with k/v argument', function () { it('should read a filter with k/v argument', function () {
const tokenizer = new Tokenizer(' | plus: a:1') const tokenizer = new Tokenizer(' | plus: a:1', trie)
const token = tokenizer.readFilter() const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus') expect(token).to.have.property('name', 'plus')
expect(token!.args).to.have.lengthOf(1) expect(token!.args).to.have.lengthOf(1)
@@ -288,7 +292,7 @@ describe('Tokenizer', function () {
expect(v.getText()).to.equal('1') expect(v.getText()).to.equal('1')
}) })
it('should read a filter with "arr[0]" argument', function () { it('should read a filter with "arr[0]" argument', function () {
const tokenizer = new Tokenizer('| plus: arr[0]') const tokenizer = new Tokenizer('| plus: arr[0]', trie)
const token = tokenizer.readFilter() const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus') expect(token).to.have.property('name', 'plus')
expect(token!.args).to.have.lengthOf(1) expect(token!.args).to.have.lengthOf(1)
@@ -301,7 +305,7 @@ describe('Tokenizer', function () {
expect(pa.props[0].getText()).to.equal('0') expect(pa.props[0].getText()).to.equal('0')
}) })
it('should read a filter with obj.foo argument', function () { it('should read a filter with obj.foo argument', function () {
const tokenizer = new Tokenizer('| plus: obj.foo') const tokenizer = new Tokenizer('| plus: obj.foo', trie)
const token = tokenizer.readFilter() const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus') expect(token).to.have.property('name', 'plus')
expect(token!.args).to.have.lengthOf(1) expect(token!.args).to.have.lengthOf(1)
@@ -314,7 +318,7 @@ describe('Tokenizer', function () {
expect(pa.props[0].getText()).to.equal('foo') expect(pa.props[0].getText()).to.equal('foo')
}) })
it('should read a filter with obj["foo"] argument', function () { it('should read a filter with obj["foo"] argument', function () {
const tokenizer = new Tokenizer('| plus: obj["good luck"]') const tokenizer = new Tokenizer('| plus: obj["good luck"]', trie)
const token = tokenizer.readFilter() const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus') expect(token).to.have.property('name', 'plus')
expect(token!.args).to.have.lengthOf(1) expect(token!.args).to.have.lengthOf(1)
@@ -328,7 +332,7 @@ describe('Tokenizer', function () {
}) })
describe('#readFilters()', () => { describe('#readFilters()', () => {
it('should read simple filters', function () { it('should read simple filters', function () {
const tokenizer = new Tokenizer('| plus: 3 | capitalize') const tokenizer = new Tokenizer('| plus: 3 | capitalize', trie)
const tokens = tokenizer.readFilters() const tokens = tokenizer.readFilters()
expect(tokens).to.have.lengthOf(2) expect(tokens).to.have.lengthOf(2)
@@ -341,7 +345,7 @@ describe('Tokenizer', function () {
expect(tokens[1].args).to.have.lengthOf(0) expect(tokens[1].args).to.have.lengthOf(0)
}) })
it('should read filters', function () { it('should read filters', function () {
const tokenizer = new Tokenizer('| plus: a:3 | capitalize | append: foo[a.b["c d"]]') const tokenizer = new Tokenizer('| plus: a:3 | capitalize | append: foo[a.b["c d"]]', trie)
const tokens = tokenizer.readFilters() const tokens = tokenizer.readFilters()
expect(tokens).to.have.lengthOf(3) expect(tokens).to.have.lengthOf(3)
@@ -364,14 +368,14 @@ describe('Tokenizer', function () {
}) })
describe('#readExpression()', () => { describe('#readExpression()', () => {
it('should read expression `a `', () => { it('should read expression `a `', () => {
const exp = [...new Tokenizer('a ').readExpression()] const exp = [...new Tokenizer('a ', trie).readExpression()]
expect(exp).to.have.lengthOf(1) expect(exp).to.have.lengthOf(1)
expect(exp[0]).to.be.instanceOf(PropertyAccessToken) expect(exp[0]).to.be.instanceOf(PropertyAccessToken)
expect(exp[0].getText()).to.deep.equal('a') expect(exp[0].getText()).to.deep.equal('a')
}) })
it('should read expression `a[][b]`', () => { it('should read expression `a[][b]`', () => {
const exp = [...new Tokenizer('a[][b]').readExpression()] const exp = [...new Tokenizer('a[][b]', trie).readExpression()]
expect(exp).to.have.lengthOf(1) expect(exp).to.have.lengthOf(1)
const pa = exp[0] as PropertyAccessToken const pa = exp[0] as PropertyAccessToken
@@ -386,7 +390,7 @@ describe('Tokenizer', function () {
expect(p2.getText()).to.equal('b') expect(p2.getText()).to.equal('b')
}) })
it('should read expression `a.`', () => { it('should read expression `a.`', () => {
const exp = [...new Tokenizer('a.').readExpression()] const exp = [...new Tokenizer('a.', trie).readExpression()]
expect(exp).to.have.lengthOf(1) expect(exp).to.have.lengthOf(1)
const pa = exp[0] as PropertyAccessToken const pa = exp[0] as PropertyAccessToken
@@ -395,14 +399,14 @@ describe('Tokenizer', function () {
expect(pa.props).to.have.lengthOf(0) expect(pa.props).to.have.lengthOf(0)
}) })
it('should read expression `a ==`', () => { it('should read expression `a ==`', () => {
const exp = [...new Tokenizer('a ==').readExpression()] const exp = [...new Tokenizer('a ==', trie).readExpression()]
expect(exp).to.have.lengthOf(1) expect(exp).to.have.lengthOf(1)
expect(exp[0]).to.be.instanceOf(PropertyAccessToken) expect(exp[0]).to.be.instanceOf(PropertyAccessToken)
expect(exp[0].getText()).to.deep.equal('a') expect(exp[0].getText()).to.deep.equal('a')
}) })
it('should read expression `a==b`', () => { it('should read expression `a==b`', () => {
const exp = new Tokenizer('a==b').readExpression() const exp = new Tokenizer('a==b', trie).readExpression()
const [a, equals, b] = exp const [a, equals, b] = exp
expect(a).to.be.instanceOf(PropertyAccessToken) expect(a).to.be.instanceOf(PropertyAccessToken)
@@ -415,11 +419,11 @@ describe('Tokenizer', function () {
expect(b.getText()).to.deep.equal('b') expect(b.getText()).to.deep.equal('b')
}) })
it('should read expression `^`', () => { it('should read expression `^`', () => {
const exp = new Tokenizer('^').readExpression() const exp = new Tokenizer('^', trie).readExpression()
expect([...exp]).to.deep.equal([]) expect([...exp]).to.deep.equal([])
}) })
it('should read expression `a == b`', () => { it('should read expression `a == b`', () => {
const exp = new Tokenizer('a == b').readExpression() const exp = new Tokenizer('a == b', trie).readExpression()
const [a, equals, b] = exp const [a, equals, b] = exp
expect(a).to.be.instanceOf(PropertyAccessToken) expect(a).to.be.instanceOf(PropertyAccessToken)
@@ -432,7 +436,7 @@ describe('Tokenizer', function () {
expect(b.getText()).to.deep.equal('b') expect(b.getText()).to.deep.equal('b')
}) })
it('should read expression `(1..3) contains 3`', () => { it('should read expression `(1..3) contains 3`', () => {
const exp = new Tokenizer('(1..3) contains 3').readExpression() const exp = new Tokenizer('(1..3) contains 3', trie).readExpression()
const [range, contains, rhs] = exp const [range, contains, rhs] = exp
expect(range).to.be.instanceOf(RangeToken) expect(range).to.be.instanceOf(RangeToken)
@@ -445,7 +449,7 @@ describe('Tokenizer', function () {
expect(rhs.getText()).to.deep.equal('3') expect(rhs.getText()).to.deep.equal('3')
}) })
it('should read expression `a[b] == c`', () => { it('should read expression `a[b] == c`', () => {
const exp = new Tokenizer('a[b] == c').readExpression() const exp = new Tokenizer('a[b] == c', trie).readExpression()
const [lhs, contains, rhs] = exp const [lhs, contains, rhs] = exp
expect(lhs).to.be.instanceOf(PropertyAccessToken) expect(lhs).to.be.instanceOf(PropertyAccessToken)
@@ -458,7 +462,7 @@ describe('Tokenizer', function () {
expect(rhs.getText()).to.deep.equal('c') expect(rhs.getText()).to.deep.equal('c')
}) })
it('should read expression `c[a["b"]] >= c`', () => { it('should read expression `c[a["b"]] >= c`', () => {
const exp = new Tokenizer('c[a["b"]] >= c').readExpression() const exp = new Tokenizer('c[a["b"]] >= c', trie).readExpression()
const [lhs, op, rhs] = exp const [lhs, op, rhs] = exp
expect(lhs).to.be.instanceOf(PropertyAccessToken) expect(lhs).to.be.instanceOf(PropertyAccessToken)
@@ -471,7 +475,7 @@ describe('Tokenizer', function () {
expect(rhs.getText()).to.deep.equal('c') expect(rhs.getText()).to.deep.equal('c')
}) })
it('should read expression `"][" == var`', () => { it('should read expression `"][" == var`', () => {
const exp = new Tokenizer('"][" == var').readExpression() const exp = new Tokenizer('"][" == var', trie).readExpression()
const [lhs, equals, rhs] = exp const [lhs, equals, rhs] = exp
expect(lhs).to.be.instanceOf(QuotedToken) expect(lhs).to.be.instanceOf(QuotedToken)
@@ -484,7 +488,7 @@ describe('Tokenizer', function () {
expect(rhs.getText()).to.deep.equal('var') expect(rhs.getText()).to.deep.equal('var')
}) })
it('should read expression `"\\\'" == "\\""`', () => { it('should read expression `"\\\'" == "\\""`', () => {
const exp = new Tokenizer('"\\\'" == "\\""').readExpression() const exp = new Tokenizer('"\\\'" == "\\""', trie).readExpression()
const [lhs, equals, rhs] = exp const [lhs, equals, rhs] = exp
expect(lhs).to.be.instanceOf(QuotedToken) expect(lhs).to.be.instanceOf(QuotedToken)
+43 -41
View File
@@ -3,12 +3,14 @@ import { expect } from 'chai'
import { Context } from '../../../src/context/context' import { Context } from '../../../src/context/context'
import { toThenable } from '../../../src/util/async' import { toThenable } from '../../../src/util/async'
import { defaultOperators } from '../../../src/render/operator' import { defaultOperators } from '../../../src/render/operator'
import { createTrie } from '../../../src/util/operator-trie'
describe('Expression', function () { describe('Expression', function () {
const ctx = new Context({}) const ctx = new Context({})
const trie = createTrie(defaultOperators)
it('should throw when context not defined', done => { it('should throw when context not defined', done => {
toThenable(new Expression('foo', defaultOperators).value(undefined!)) toThenable(new Expression('foo', defaultOperators, trie).value(undefined!))
.then(() => done(new Error('should not resolved'))) .then(() => done(new Error('should not resolved')))
.catch(err => { .catch(err => {
expect(err.message).to.match(/context not defined/) expect(err.message).to.match(/context not defined/)
@@ -18,19 +20,19 @@ describe('Expression', function () {
describe('single value', function () { describe('single value', function () {
it('should eval literal', async function () { it('should eval literal', async function () {
expect(await toThenable(new Expression('2.4', defaultOperators).value(ctx))).to.equal(2.4) expect(await toThenable(new Expression('2.4', defaultOperators, trie).value(ctx))).to.equal(2.4)
expect(await toThenable(new Expression('"foo"', defaultOperators).value(ctx))).to.equal('foo') expect(await toThenable(new Expression('"foo"', defaultOperators, trie).value(ctx))).to.equal('foo')
expect(await toThenable(new Expression('false', defaultOperators).value(ctx))).to.equal(false) expect(await toThenable(new Expression('false', defaultOperators, trie).value(ctx))).to.equal(false)
}) })
it('should eval range expression', async function () { it('should eval range expression', async function () {
const ctx = new Context({ two: 2 }) const ctx = new Context({ two: 2 })
expect(await toThenable(new Expression('(2..4)', defaultOperators).value(ctx))).to.deep.equal([2, 3, 4]) expect(await toThenable(new Expression('(2..4)', defaultOperators, trie).value(ctx))).to.deep.equal([2, 3, 4])
expect(await toThenable(new Expression('(two..4)', defaultOperators).value(ctx))).to.deep.equal([2, 3, 4]) expect(await toThenable(new Expression('(two..4)', defaultOperators, trie).value(ctx))).to.deep.equal([2, 3, 4])
}) })
it('should eval literal', async function () { it('should eval literal', async function () {
expect(await toThenable(new Expression('2.4', defaultOperators).value(ctx))).to.equal(2.4) expect(await toThenable(new Expression('2.4', defaultOperators, trie).value(ctx))).to.equal(2.4)
expect(await toThenable(new Expression('"foo"', defaultOperators).value(ctx))).to.equal('foo') expect(await toThenable(new Expression('"foo"', defaultOperators, trie).value(ctx))).to.equal('foo')
expect(await toThenable(new Expression('false', defaultOperators).value(ctx))).to.equal(false) expect(await toThenable(new Expression('false', defaultOperators, trie).value(ctx))).to.equal(false)
}) })
it('should eval property access', async function () { it('should eval property access', async function () {
@@ -39,112 +41,112 @@ describe('Expression', function () {
coo: 'bar', coo: 'bar',
doo: { foo: 'bar', bar: { foo: 'bar' } } doo: { foo: 'bar', bar: { foo: 'bar' } }
}) })
expect(await toThenable(new Expression('foo.bar', defaultOperators).value(ctx))).to.equal('BAR') expect(await toThenable(new Expression('foo.bar', defaultOperators, trie).value(ctx))).to.equal('BAR')
expect(await toThenable(new Expression('foo["bar"]', defaultOperators).value(ctx))).to.equal('BAR') expect(await toThenable(new Expression('foo["bar"]', defaultOperators, trie).value(ctx))).to.equal('BAR')
expect(await toThenable(new Expression('foo[coo]', defaultOperators).value(ctx))).to.equal('BAR') expect(await toThenable(new Expression('foo[coo]', defaultOperators, trie).value(ctx))).to.equal('BAR')
expect(await toThenable(new Expression('foo[doo.foo]', defaultOperators).value(ctx))).to.equal('BAR') expect(await toThenable(new Expression('foo[doo.foo]', defaultOperators, trie).value(ctx))).to.equal('BAR')
expect(await toThenable(new Expression('foo[doo["foo"]]', defaultOperators).value(ctx))).to.equal('BAR') expect(await toThenable(new Expression('foo[doo["foo"]]', defaultOperators, trie).value(ctx))).to.equal('BAR')
expect(await toThenable(new Expression('doo[coo].foo', defaultOperators).value(ctx))).to.equal('bar') expect(await toThenable(new Expression('doo[coo].foo', defaultOperators, trie).value(ctx))).to.equal('bar')
}) })
}) })
describe('simple expression', function () { describe('simple expression', function () {
it('should return false for "1==2"', async () => { it('should return false for "1==2"', async () => {
expect(await toThenable(new Expression('1==2', defaultOperators).value(ctx))).to.equal(false) expect(await toThenable(new Expression('1==2', defaultOperators, trie).value(ctx))).to.equal(false)
}) })
it('should return true for "1<2"', async () => { it('should return true for "1<2"', async () => {
expect(await toThenable(new Expression('1<2', defaultOperators).value(ctx))).to.equal(true) expect(await toThenable(new Expression('1<2', defaultOperators, trie).value(ctx))).to.equal(true)
}) })
it('should return true for "1 < 2"', async () => { it('should return true for "1 < 2"', async () => {
expect(await toThenable(new Expression('1 < 2', defaultOperators).value(ctx))).to.equal(true) expect(await toThenable(new Expression('1 < 2', defaultOperators, trie).value(ctx))).to.equal(true)
}) })
it('should return true for "1 < 2"', async () => { it('should return true for "1 < 2"', async () => {
expect(await toThenable(new Expression('1 < 2', defaultOperators).value(ctx))).to.equal(true) expect(await toThenable(new Expression('1 < 2', defaultOperators, trie).value(ctx))).to.equal(true)
}) })
it('should return true for "2 <= 2"', async () => { it('should return true for "2 <= 2"', async () => {
expect(await toThenable(new Expression('2 <= 2', defaultOperators).value(ctx))).to.equal(true) expect(await toThenable(new Expression('2 <= 2', defaultOperators, trie).value(ctx))).to.equal(true)
}) })
it('should return true for "one <= two"', async () => { it('should return true for "one <= two"', async () => {
const ctx = new Context({ one: 1, two: 2 }) const ctx = new Context({ one: 1, two: 2 })
expect(await toThenable(new Expression('one <= two', defaultOperators).value(ctx))).to.equal(true) expect(await toThenable(new Expression('one <= two', defaultOperators, trie).value(ctx))).to.equal(true)
}) })
it('should return false for "x contains "x""', async () => { it('should return false for "x contains "x""', async () => {
const ctx = new Context({ x: 'XXX' }) const ctx = new Context({ x: 'XXX' })
expect(await toThenable(new Expression('x contains "x"', defaultOperators).value(ctx))).to.equal(false) expect(await toThenable(new Expression('x contains "x"', defaultOperators, trie).value(ctx))).to.equal(false)
}) })
it('should return true for "x contains "X""', async () => { it('should return true for "x contains "X""', async () => {
const ctx = new Context({ x: 'XXX' }) const ctx = new Context({ x: 'XXX' })
expect(await toThenable(new Expression('x contains "X"', defaultOperators).value(ctx))).to.equal(true) expect(await toThenable(new Expression('x contains "X"', defaultOperators, trie).value(ctx))).to.equal(true)
}) })
it('should return false for "1 contains "x""', async () => { it('should return false for "1 contains "x""', async () => {
const ctx = new Context({ x: 'XXX' }) const ctx = new Context({ x: 'XXX' })
expect(await toThenable(new Expression('1 contains "x"', defaultOperators).value(ctx))).to.equal(false) expect(await toThenable(new Expression('1 contains "x"', defaultOperators, trie).value(ctx))).to.equal(false)
}) })
it('should return false for "y contains "x""', async () => { it('should return false for "y contains "x""', async () => {
const ctx = new Context({ x: 'XXX' }) const ctx = new Context({ x: 'XXX' })
expect(await toThenable(new Expression('y contains "x"', defaultOperators).value(ctx))).to.equal(false) expect(await toThenable(new Expression('y contains "x"', defaultOperators, trie).value(ctx))).to.equal(false)
}) })
it('should return false for "z contains "x""', async () => { it('should return false for "z contains "x""', async () => {
const ctx = new Context({ x: 'XXX' }) const ctx = new Context({ x: 'XXX' })
expect(await toThenable(new Expression('z contains "x"', defaultOperators).value(ctx))).to.equal(false) expect(await toThenable(new Expression('z contains "x"', defaultOperators, trie).value(ctx))).to.equal(false)
}) })
it('should return true for "(1..5) contains 3"', async () => { it('should return true for "(1..5) contains 3"', async () => {
const ctx = new Context({ x: 'XXX' }) const ctx = new Context({ x: 'XXX' })
expect(await toThenable(new Expression('(1..5) contains 3', defaultOperators).value(ctx))).to.equal(true) expect(await toThenable(new Expression('(1..5) contains 3', defaultOperators, trie).value(ctx))).to.equal(true)
}) })
it('should return false for "(1..5) contains 6"', async () => { it('should return false for "(1..5) contains 6"', async () => {
const ctx = new Context({ x: 'XXX' }) const ctx = new Context({ x: 'XXX' })
expect(await toThenable(new Expression('(1..5) contains 6', defaultOperators).value(ctx))).to.equal(false) expect(await toThenable(new Expression('(1..5) contains 6', defaultOperators, trie).value(ctx))).to.equal(false)
}) })
it('should return true for ""<=" == "<=""', async () => { it('should return true for ""<=" == "<=""', async () => {
expect(await toThenable(new Expression('"<=" == "<="', defaultOperators).value(ctx))).to.equal(true) expect(await toThenable(new Expression('"<=" == "<="', defaultOperators, trie).value(ctx))).to.equal(true)
}) })
}) })
it('should allow space in quoted value', async function () { it('should allow space in quoted value', async function () {
const ctx = new Context({ space: ' ' }) const ctx = new Context({ space: ' ' })
expect(await toThenable(new Expression('" " == space', defaultOperators).value(ctx))).to.equal(true) expect(await toThenable(new Expression('" " == space', defaultOperators, trie).value(ctx))).to.equal(true)
}) })
describe('escape', () => { describe('escape', () => {
it('should escape quote', async function () { it('should escape quote', async function () {
const ctx = new Context({ quote: '"' }) const ctx = new Context({ quote: '"' })
expect(await toThenable(new Expression('"\\"" == quote', defaultOperators).value(ctx))).to.equal(true) expect(await toThenable(new Expression('"\\"" == quote', defaultOperators, trie).value(ctx))).to.equal(true)
}) })
it('should escape square bracket', async function () { it('should escape square bracket', async function () {
const ctx = new Context({ obj: { ']': 'bracket' } }) const ctx = new Context({ obj: { ']': 'bracket' } })
expect(await toThenable(new Expression('obj["]"] == "bracket"', defaultOperators).value(ctx))).to.equal(true) expect(await toThenable(new Expression('obj["]"] == "bracket"', defaultOperators, trie).value(ctx))).to.equal(true)
}) })
}) })
describe('complex expression', function () { describe('complex expression', function () {
it('should support value or value', async function () { it('should support value or value', async function () {
expect(await toThenable(new Expression('false or true', defaultOperators).value(ctx))).to.equal(true) expect(await toThenable(new Expression('false or true', defaultOperators, trie).value(ctx))).to.equal(true)
}) })
it('should support < and contains', async function () { it('should support < and contains', async function () {
expect(await toThenable(new Expression('1 < 2 and x contains "x"', defaultOperators).value(ctx))).to.equal(false) expect(await toThenable(new Expression('1 < 2 and x contains "x"', defaultOperators, trie).value(ctx))).to.equal(false)
}) })
it('should support < or contains', async function () { it('should support < or contains', async function () {
expect(await toThenable(new Expression('1 < 2 or x contains "x"', defaultOperators).value(ctx))).to.equal(true) expect(await toThenable(new Expression('1 < 2 or x contains "x"', defaultOperators, trie).value(ctx))).to.equal(true)
}) })
it('should support value and !=', async function () { it('should support value and !=', async function () {
const ctx = new Context({ empty: '' }) const ctx = new Context({ empty: '' })
expect(await toThenable(new Expression('empty and empty != ""', defaultOperators).value(ctx))).to.equal(false) expect(await toThenable(new Expression('empty and empty != ""', defaultOperators, trie).value(ctx))).to.equal(false)
}) })
it('should recognize quoted value', async function () { it('should recognize quoted value', async function () {
expect(await toThenable(new Expression('">"', defaultOperators).value(ctx))).to.equal('>') expect(await toThenable(new Expression('">"', defaultOperators, trie).value(ctx))).to.equal('>')
}) })
it('should evaluate from right to left', async function () { it('should evaluate from right to left', async function () {
expect(await toThenable(new Expression('true or false and false', defaultOperators).value(ctx))).to.equal(true) expect(await toThenable(new Expression('true or false and false', defaultOperators, trie).value(ctx))).to.equal(true)
expect(await toThenable(new Expression('true and false and false or true', defaultOperators).value(ctx))).to.equal(false) expect(await toThenable(new Expression('true and false and false or true', defaultOperators, trie).value(ctx))).to.equal(false)
}) })
it('should recognize property access', async function () { it('should recognize property access', async function () {
const ctx = new Context({ obj: { foo: true } }) const ctx = new Context({ obj: { foo: true } })
expect(await toThenable(new Expression('obj["foo"] and true', defaultOperators).value(ctx))).to.equal(true) expect(await toThenable(new Expression('obj["foo"] and true', defaultOperators, trie).value(ctx))).to.equal(true)
}) })
it('should allow nested property access', async function () { it('should allow nested property access', async function () {
const ctx = new Context({ obj: { foo: 'FOO' }, keys: { "what's this": 'foo' } }) const ctx = new Context({ obj: { foo: 'FOO' }, keys: { "what's this": 'foo' } })
expect(await toThenable(new Expression('obj[keys["what\'s this"]]', defaultOperators).value(ctx))).to.equal('FOO') expect(await toThenable(new Expression('obj[keys["what\'s this"]]', defaultOperators, trie).value(ctx))).to.equal('FOO')
}) })
}) })
}) })
+5 -1
View File
@@ -5,12 +5,16 @@ import { Output } from '../../../src/template/output'
import { OutputToken } from '../../../src/tokens/output-token' import { OutputToken } from '../../../src/tokens/output-token'
import { FilterMap } from '../../../src/template/filter/filter-map' import { FilterMap } from '../../../src/template/filter/filter-map'
import { defaultOptions } from '../../../src/liquid-options' import { defaultOptions } from '../../../src/liquid-options'
import { createTrie } from '../../../src/util/operator-trie'
import { defaultOperators } from '../../../src/types'
const expect = chai.expect const expect = chai.expect
describe('Output', function () { describe('Output', function () {
const emitter: any = { write: (html: string) => (emitter.html += html), html: '' } const emitter: any = { write: (html: string) => (emitter.html += html), html: '' }
const liquid = {} as any const liquid = {
options: { operatorsTrie: createTrie(defaultOperators) }
} as any
let filters: FilterMap let filters: FilterMap
beforeEach(function () { beforeEach(function () {
filters = new FilterMap(false, liquid) filters = new FilterMap(false, liquid)
+5 -1
View File
@@ -6,13 +6,17 @@ import * as sinonChai from 'sinon-chai'
import * as sinon from 'sinon' import * as sinon from 'sinon'
import { Context } from '../../../src/context/context' import { Context } from '../../../src/context/context'
import { Value } from '../../../src/template/value' import { Value } from '../../../src/template/value'
import { createTrie } from '../../../src/util/operator-trie'
import { defaultOperators } from '../../../src/types'
chai.use(sinonChai) chai.use(sinonChai)
const expect = chai.expect const expect = chai.expect
describe('Value', function () { describe('Value', function () {
const liquid = {} as any const liquid = {
options: { operatorsTrie: createTrie(defaultOperators) }
} as any
describe('#constructor()', function () { describe('#constructor()', function () {
const filterMap = new FilterMap(false, liquid) const filterMap = new FilterMap(false, liquid)