Labyricorn/liquidjs
1
0
Fork 0
mirror of https://github.com/harttle/liquidjs.git synced 2026-09-15 04:10:40 -07:00
Code Issues Packages Projects Releases Wiki Activity

refactor: delay creation of operatorsTrie and hide this implementation

Browse Source 
BREAKING CHANGE: use `operators` instead of `operatorsTrie` as Tokenizer constructor argument, #500
This commit is contained in:
Jun Yang
2022-11-27 14:04:01 +08:00
parent ff112a4750
commit bb58d3e549
21 changed files with 86 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/unit/parser/tokenizer.ts
+61 -65
View File
@@ -9,64 +9,60 @@ import { TagToken } from '../../../src/tokens/tag-token'
import { QuotedToken } from '../../../src/tokens/quoted-token'
import { OutputToken } from '../../../src/tokens/output-token'
import { HTMLToken } from '../../../src/tokens/html-token'
import { createTrie } from '../../../src/util/operator-trie'
import { defaultOperators } from '../../../src/types'
import { LiquidTagToken } from '../../../src/tokens/liquid-tag-token'
describe('Tokenizer', function () {
const trie = createTrie(defaultOperators)
it('should read quoted', () => {
expect(new Tokenizer('"foo" ff').readQuoted()!.getText()).to.equal('"foo"')
expect(new Tokenizer(' "foo"ff', trie).readQuoted()!.getText()).to.equal('"foo"')
expect(new Tokenizer(' "foo"ff').readQuoted()!.getText()).to.equal('"foo"')
})
it('should read value', () => {
expect(new Tokenizer('a[ b][ "c d" ]', trie).readValueOrThrow().getText()).to.equal('a[ b][ "c d" ]')
expect(new Tokenizer('a.b[c[d.e]]', trie).readValueOrThrow().getText()).to.equal('a.b[c[d.e]]')
expect(new Tokenizer('a[ b][ "c d" ]').readValueOrThrow().getText()).to.equal('a[ b][ "c d" ]')
expect(new Tokenizer('a.b[c[d.e]]').readValueOrThrow().getText()).to.equal('a.b[c[d.e]]')
})
it('should read identifier', () => {
expect(new Tokenizer('foo bar', trie).readIdentifier()).to.haveOwnProperty('content', 'foo')
expect(new Tokenizer('foo bar', trie).readWord()).to.haveOwnProperty('content', 'foo')
expect(new Tokenizer('foo bar').readIdentifier()).to.haveOwnProperty('content', 'foo')
expect(new Tokenizer('foo bar').readWord()).to.haveOwnProperty('content', 'foo')
})
it('should read number value', () => {
const token: NumberToken = new Tokenizer('2.33.2', trie).readValueOrThrow() as any
const token: NumberToken = new Tokenizer('2.33.2').readValueOrThrow() as any
expect(token).to.be.instanceOf(NumberToken)
expect(token.whole.getText()).to.equal('2')
expect(token.decimal!.getText()).to.equal('33')
expect(token.getText()).to.equal('2.33')
})
it('should read quoted value', () => {
const value = new Tokenizer('"foo"a', trie).readValue()
const value = new Tokenizer('"foo"a').readValue()
expect(value).to.be.instanceOf(QuotedToken)
expect(value!.getText()).to.equal('"foo"')
})
it('should read property access value', () => {
expect(new Tokenizer('a[b]["c d"]', trie).readValueOrThrow().getText()).to.equal('a[b]["c d"]')
expect(new Tokenizer('a[b]["c d"]').readValueOrThrow().getText()).to.equal('a[b]["c d"]')
})
it('should read quoted property access value', () => {
const value = new Tokenizer('["a prop"]', trie).readValue()
const value = new Tokenizer('["a prop"]').readValue()
expect(value).to.be.instanceOf(PropertyAccessToken)
expect((value as PropertyAccessToken).variable.getText()).to.equal('"a prop"')
})
it('should throw for broken quoted property access', () => {
const tokenizer = new Tokenizer('[5]', trie)
const tokenizer = new Tokenizer('[5]')
expect(() => tokenizer.readValueOrThrow()).to.throw()
})
it('should throw for incomplete quoted property access', () => {
const tokenizer = new Tokenizer('["a prop"', trie)
const tokenizer = new Tokenizer('["a prop"')
expect(() => tokenizer.readValueOrThrow()).to.throw()
})
it('should read hash', () => {
const hash1 = new Tokenizer('foo: 3', trie).readHash()
const hash1 = new Tokenizer('foo: 3').readHash()
expect(hash1!.name.content).to.equal('foo')
expect(hash1!.value!.getText()).to.equal('3')
const hash2 = new Tokenizer(', foo: a[ "bar"]', trie).readHash()
const hash2 = new Tokenizer(', foo: a[ "bar"]').readHash()
expect(hash2!.name.content).to.equal('foo')
expect(hash2!.value!.getText()).to.equal('a[ "bar"]')
})
it('should read multiple hashs', () => {
const hashes = new Tokenizer(', limit: 3 reverse offset:off', trie).readHashes()
const hashes = new Tokenizer(', limit: 3 reverse offset:off').readHashes()
expect(hashes).to.have.lengthOf(3)
const [limit, reverse, offset] = hashes
expect(limit.name.content).to.equal('limit')
@@ -79,7 +75,7 @@ describe('Tokenizer', function () {
expect(offset.value!.getText()).to.equal('off')
})
it('should read hash value with property access', () => {
const hashes = new Tokenizer('cols: 2, rows: data["rows"]', trie).readHashes()
const hashes = new Tokenizer('cols: 2, rows: data["rows"]').readHashes()
expect(hashes).to.have.lengthOf(2)
const [cols, rols] = hashes
@@ -92,7 +88,7 @@ describe('Tokenizer', function () {
describe('#readTopLevelTokens()', () => {
it('should read HTML token', function () {
const html = '<html><body><p>Lorem Ipsum</p></body></html>'
const tokenizer = new Tokenizer(html, trie)
const tokenizer = new Tokenizer(html)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(1)
@@ -101,7 +97,7 @@ describe('Tokenizer', function () {
})
it('should read tag token', function () {
const html = '<p>{% for p in a[1]%}</p>'
const tokenizer = new Tokenizer(html, trie)
const tokenizer = new Tokenizer(html)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(3)
@@ -112,7 +108,7 @@ describe('Tokenizer', function () {
})
it('should allow unclosed tag inside {% raw %}', function () {
const html = '{%raw%} {%if%} {%else {%endraw%}'
const tokenizer = new Tokenizer(html, trie)
const tokenizer = new Tokenizer(html)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(3)
@@ -121,7 +117,7 @@ describe('Tokenizer', function () {
})
it('should allow unclosed endraw tag inside {% raw %}', function () {
const html = '{%raw%} {%endraw {%raw%} {%endraw%}'
const tokenizer = new Tokenizer(html, trie)
const tokenizer = new Tokenizer(html)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(3)
@@ -130,12 +126,12 @@ describe('Tokenizer', function () {
})
it('should throw when {% raw %} not closed', function () {
const html = '{%raw%} {%endraw {%raw%}'
const tokenizer = new Tokenizer(html, trie)
const tokenizer = new Tokenizer(html)
expect(() => tokenizer.readTopLevelTokens()).to.throw('raw "{%raw%} {%end..." not closed, line:1, col:8')
})
it('should read output token', function () {
const html = '<p>{{foo | date: "%Y-%m-%d"}}</p>'
const tokenizer = new Tokenizer(html, trie)
const tokenizer = new Tokenizer(html)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(3)
@@ -145,7 +141,7 @@ describe('Tokenizer', function () {
})
it('should handle consecutive value and tags', function () {
const html = '{{foo}}{{bar}}{%foo%}{%bar%}'
const tokenizer = new Tokenizer(html, trie)
const tokenizer = new Tokenizer(html)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(4)
@@ -167,7 +163,7 @@ describe('Tokenizer', function () {
})
it('should keep white spaces and newlines', function () {
const html = '{%foo%}\n{%bar %} \n {%alice%}'
const tokenizer = new Tokenizer(html, trie)
const tokenizer = new Tokenizer(html)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(5)
expect(tokens[1]).instanceOf(HTMLToken)
@@ -177,7 +173,7 @@ describe('Tokenizer', function () {
})
it('should handle multiple lines tag', function () {
const html = '{%foo\na:a\nb:1.23\n%}'
const tokenizer = new Tokenizer(html, trie)
const tokenizer = new Tokenizer(html)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(1)
expect(tokens[0]).instanceOf(TagToken)
@@ -186,7 +182,7 @@ describe('Tokenizer', function () {
})
it('should handle multiple lines value', function () {
const html = '{{foo\n|date:\n"%Y-%m-%d"\n}}'
const tokenizer = new Tokenizer(html, trie)
const tokenizer = new Tokenizer(html)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(1)
expect(tokens[0]).instanceOf(OutputToken)
@@ -194,7 +190,7 @@ describe('Tokenizer', function () {
})
it('should handle complex object property access', function () {
const html = '{{ obj["my:property with anything"] }}'
const tokenizer = new Tokenizer(html, trie)
const tokenizer = new Tokenizer(html)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(1)
const output = tokens[0] as OutputToken
@@ -203,18 +199,18 @@ describe('Tokenizer', function () {
})
it('should throw if tag not closed', function () {
const html = '{% assign foo = bar {{foo}}'
const tokenizer = new Tokenizer(html, trie)
const tokenizer = new Tokenizer(html)
expect(() => tokenizer.readTopLevelTokens()).to.throw(/tag "{% assign foo..." not closed/)
})
it('should throw if output not closed', function () {
const tokenizer = new Tokenizer('{{name}', trie)
const tokenizer = new Tokenizer('{{name}')
expect(() => tokenizer.readTopLevelTokens()).to.throw(/output "{{name}" not closed/)
})
})
describe('#readTagToken()', () => {
it('should skip quoted delimiters', function () {
const html = '{% assign a = "%} {% }} {{" %}'
const tokenizer = new Tokenizer(html, trie)
const tokenizer = new Tokenizer(html)
const token = tokenizer.readTagToken()
expect(token).instanceOf(TagToken)
@@ -225,7 +221,7 @@ describe('Tokenizer', function () {
describe('#readOutputToken()', () => {
it('should skip quoted delimiters', function () {
const html = '{{ "%} {%" | append: "}} {{" }}'
const tokenizer = new Tokenizer(html, trie)
const tokenizer = new Tokenizer(html)
const token = tokenizer.readOutputToken()
expect(token).instanceOf(OutputToken)
@@ -234,7 +230,7 @@ describe('Tokenizer', function () {
})
describe('#readRange()', () => {
it('should read `(1..3)`', () => {
const range = new Tokenizer('(1..3)', trie).readRange()
const range = new Tokenizer('(1..3)').readRange()
expect(range).to.be.instanceOf(RangeToken)
expect(range!.getText()).to.deep.equal('(1..3)')
const { lhs, rhs } = range!
@@ -244,23 +240,23 @@ describe('Tokenizer', function () {
expect(rhs.getText()).to.equal('3')
})
it('should throw for `(..3)`', () => {
expect(() => new Tokenizer('(..3)', trie).readRange()).to.throw('unexpected token "..3)", value expected')
expect(() => new Tokenizer('(..3)').readRange()).to.throw('unexpected token "..3)", value expected')
})
it('should read `(a.b..c["..d"])`', () => {
const range = new Tokenizer('(a.b..c["..d"])', trie).readRange()
const range = new Tokenizer('(a.b..c["..d"])').readRange()
expect(range).to.be.instanceOf(RangeToken)
expect(range!.getText()).to.deep.equal('(a.b..c["..d"])')
})
})
describe('#readFilter()', () => {
it('should read a simple filter', function () {
const tokenizer = new Tokenizer('| plus', trie)
const tokenizer = new Tokenizer('| plus')
const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus')
expect(token).to.have.property('args').to.deep.equal([])
})
it('should read a filter with argument', function () {
const tokenizer = new Tokenizer(' | plus: 1', trie)
const tokenizer = new Tokenizer(' | plus: 1')
const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus')
expect(token!.args).to.have.lengthOf(1)
@@ -270,18 +266,18 @@ describe('Tokenizer', function () {
expect(one.getText()).to.equal('1')
})
it('should read a filter with colon but no argument', function () {
const tokenizer = new Tokenizer('| plus:', trie)
const tokenizer = new Tokenizer('| plus:')
const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus')
expect(token).to.have.property('args').to.deep.equal([])
})
it('should read null if name not found', function () {
const tokenizer = new Tokenizer('|', trie)
const tokenizer = new Tokenizer('|')
const token = tokenizer.readFilter()
expect(token).to.be.null
})
it('should read a filter with k/v argument', function () {
const tokenizer = new Tokenizer(' | plus: a:1', trie)
const tokenizer = new Tokenizer(' | plus: a:1')
const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus')
expect(token!.args).to.have.lengthOf(1)
@@ -292,7 +288,7 @@ describe('Tokenizer', function () {
expect(v.getText()).to.equal('1')
})
it('should read a filter with "arr[0]" argument', function () {
const tokenizer = new Tokenizer('| plus: arr[0]', trie)
const tokenizer = new Tokenizer('| plus: arr[0]')
const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus')
expect(token!.args).to.have.lengthOf(1)
@@ -305,7 +301,7 @@ describe('Tokenizer', function () {
expect(pa.props[0].getText()).to.equal('0')
})
it('should read a filter with obj.foo argument', function () {
const tokenizer = new Tokenizer('| plus: obj.foo', trie)
const tokenizer = new Tokenizer('| plus: obj.foo')
const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus')
expect(token!.args).to.have.lengthOf(1)
@@ -318,7 +314,7 @@ describe('Tokenizer', function () {
expect(pa.props[0].getText()).to.equal('foo')
})
it('should read a filter with obj["foo"] argument', function () {
const tokenizer = new Tokenizer('| plus: obj["good luck"]', trie)
const tokenizer = new Tokenizer('| plus: obj["good luck"]')
const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus')
expect(token!.args).to.have.lengthOf(1)
@@ -332,7 +328,7 @@ describe('Tokenizer', function () {
})
describe('#readFilters()', () => {
it('should read simple filters', function () {
const tokenizer = new Tokenizer('| plus: 3 | capitalize', trie)
const tokenizer = new Tokenizer('| plus: 3 | capitalize')
const tokens = tokenizer.readFilters()
expect(tokens).to.have.lengthOf(2)
@@ -345,7 +341,7 @@ describe('Tokenizer', function () {
expect(tokens[1].args).to.have.lengthOf(0)
})
it('should read filters', function () {
const tokenizer = new Tokenizer('| plus: a:3 | capitalize | append: foo[a.b["c d"]]', trie)
const tokenizer = new Tokenizer('| plus: a:3 | capitalize | append: foo[a.b["c d"]]')
const tokens = tokenizer.readFilters()
expect(tokens).to.have.lengthOf(3)
@@ -368,14 +364,14 @@ describe('Tokenizer', function () {
})
describe('#readExpression()', () => {
it('should read expression `a `', () => {
const exp = [...new Tokenizer('a ', trie).readExpressionTokens()]
const exp = [...new Tokenizer('a ').readExpressionTokens()]
expect(exp).to.have.lengthOf(1)
expect(exp[0]).to.be.instanceOf(PropertyAccessToken)
expect(exp[0].getText()).to.deep.equal('a')
})
it('should read expression `a[][b]`', () => {
const exp = [...new Tokenizer('a[][b]', trie).readExpressionTokens()]
const exp = [...new Tokenizer('a[][b]').readExpressionTokens()]
expect(exp).to.have.lengthOf(1)
const pa = exp[0] as PropertyAccessToken
@@ -390,7 +386,7 @@ describe('Tokenizer', function () {
expect(p2.getText()).to.equal('b')
})
it('should read expression `a.`', () => {
const exp = [...new Tokenizer('a.', trie).readExpressionTokens()]
const exp = [...new Tokenizer('a.').readExpressionTokens()]
expect(exp).to.have.lengthOf(1)
const pa = exp[0] as PropertyAccessToken
@@ -399,14 +395,14 @@ describe('Tokenizer', function () {
expect(pa.props).to.have.lengthOf(0)
})
it('should read expression `a ==`', () => {
const exp = [...new Tokenizer('a ==', trie).readExpressionTokens()]
const exp = [...new Tokenizer('a ==').readExpressionTokens()]
expect(exp).to.have.lengthOf(1)
expect(exp[0]).to.be.instanceOf(PropertyAccessToken)
expect(exp[0].getText()).to.deep.equal('a')
})
it('should read expression `a==b`', () => {
const exp = new Tokenizer('a==b', trie).readExpressionTokens()
const exp = new Tokenizer('a==b').readExpressionTokens()
const [a, equals, b] = exp
expect(a).to.be.instanceOf(PropertyAccessToken)
@@ -419,11 +415,11 @@ describe('Tokenizer', function () {
expect(b.getText()).to.deep.equal('b')
})
it('should read expression `^`', () => {
const exp = new Tokenizer('^', trie).readExpressionTokens()
const exp = new Tokenizer('^').readExpressionTokens()
expect([...exp]).to.deep.equal([])
})
it('should read expression `a == b`', () => {
const exp = new Tokenizer('a == b', trie).readExpressionTokens()
const exp = new Tokenizer('a == b').readExpressionTokens()
const [a, equals, b] = exp
expect(a).to.be.instanceOf(PropertyAccessToken)
@@ -436,7 +432,7 @@ describe('Tokenizer', function () {
expect(b.getText()).to.deep.equal('b')
})
it('should read expression `(1..3) contains 3`', () => {
const exp = new Tokenizer('(1..3) contains 3', trie).readExpressionTokens()
const exp = new Tokenizer('(1..3) contains 3').readExpressionTokens()
const [range, contains, rhs] = exp
expect(range).to.be.instanceOf(RangeToken)
@@ -449,7 +445,7 @@ describe('Tokenizer', function () {
expect(rhs.getText()).to.deep.equal('3')
})
it('should read expression `a[b] == c`', () => {
const exp = new Tokenizer('a[b] == c', trie).readExpressionTokens()
const exp = new Tokenizer('a[b] == c').readExpressionTokens()
const [lhs, contains, rhs] = exp
expect(lhs).to.be.instanceOf(PropertyAccessToken)
@@ -462,7 +458,7 @@ describe('Tokenizer', function () {
expect(rhs.getText()).to.deep.equal('c')
})
it('should read expression `c[a["b"]] >= c`', () => {
const exp = new Tokenizer('c[a["b"]] >= c', trie).readExpressionTokens()
const exp = new Tokenizer('c[a["b"]] >= c').readExpressionTokens()
const [lhs, op, rhs] = exp
expect(lhs).to.be.instanceOf(PropertyAccessToken)
@@ -475,7 +471,7 @@ describe('Tokenizer', function () {
expect(rhs.getText()).to.deep.equal('c')
})
it('should read expression `"][" == var`', () => {
const exp = new Tokenizer('"][" == var', trie).readExpressionTokens()
const exp = new Tokenizer('"][" == var').readExpressionTokens()
const [lhs, equals, rhs] = exp
expect(lhs).to.be.instanceOf(QuotedToken)
@@ -488,7 +484,7 @@ describe('Tokenizer', function () {
expect(rhs.getText()).to.deep.equal('var')
})
it('should read expression `"\\\'" == "\\""`', () => {
const exp = new Tokenizer('"\\\'" == "\\""', trie).readExpressionTokens()
const exp = new Tokenizer('"\\\'" == "\\""').readExpressionTokens()
const [lhs, equals, rhs] = exp
expect(lhs).to.be.instanceOf(QuotedToken)
@@ -503,7 +499,7 @@ describe('Tokenizer', function () {
})
describe('#readLiquidTagTokens', () => {
it('should read newline terminated tokens', () => {
const tokenizer = new Tokenizer('echo \'hello\'', trie)
const tokenizer = new Tokenizer('echo \'hello\'')
const tokens = tokenizer.readLiquidTagTokens()
expect(tokens.length).to.equal(1)
const tag = tokens[0]
@@ -516,18 +512,18 @@ describe('Tokenizer', function () {
echo 'hello'
decrement foo
`, trie)
`)
const tokens = tokenizer.readLiquidTagTokens()
expect(tokens.length).to.equal(2)
})
it('should throw if line does not start with an identifier', () => {
const tokenizer = new Tokenizer('!', trie)
const tokenizer = new Tokenizer('!')
expect(() => tokenizer.readLiquidTagTokens()).to.throw(/illegal liquid tag syntax/)
})
})
describe('#read inline comment tags', () => {
it('should allow hash characters in tag names', () => {
const tokenizer = new Tokenizer('{% # some comment %}', trie)
const tokenizer = new Tokenizer('{% # some comment %}')
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(1)
const tag = tokens[0] as TagToken
@@ -536,7 +532,7 @@ describe('Tokenizer', function () {
expect(tag.args).to.equal('some comment')
})
it('should handle leading whitespace', () => {
const tokenizer = new Tokenizer('{%\n # some comment %}', trie)
const tokenizer = new Tokenizer('{%\n # some comment %}')
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(1)
const tag = tokens[0] as TagToken
@@ -545,7 +541,7 @@ describe('Tokenizer', function () {
expect(tag.args).to.equal('some comment')
})
it('should handle no trailing whitespace', () => {
const tokenizer = new Tokenizer('{%\n #some comment %}', trie)
const tokenizer = new Tokenizer('{%\n #some comment %}')
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(1)
const tag = tokens[0] as TagToken
test/unit/render/expression.ts
+1 -4
View File
@@ -3,13 +3,10 @@ import { expect } from 'chai'
import { Drop } from '../../../src/drop/drop'
import { Context } from '../../../src/context/context'
import { toPromise, toValueSync } from '../../../src/util/async'
import { defaultOperators } from '../../../src/render/operator'
import { createTrie } from '../../../src/util/operator-trie'
describe('Expression', function () {
const ctx = new Context({})
const trie = createTrie(defaultOperators)
const create = (str: string) => new Tokenizer(str, trie).readExpression()
const create = (str: string) => new Tokenizer(str).readExpression()
it('should throw when context not defined', done => {
toPromise(create('foo').evaluate(undefined!, false))