Labyricorn/liquidjs
1
0
Fork 0
mirror of https://github.com/harttle/liquidjs.git synced 2026-09-16 04:40:39 -07:00
Code Issues Packages Projects Releases Wiki Activity

feat: create trie programmatically in options

Browse Source
This commit is contained in:
Jason Etcovitch
2021-02-04 09:11:28 +08:00
committed by Jun Yang
parent 8734e2e6ce
commit befc33c4eb
28 changed files with 196 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/unit/parser/tokenizer.ts
+60 -56
View File
@@ -9,59 +9,63 @@ import { TagToken } from '../../../src/tokens/tag-token'
import { QuotedToken } from '../../../src/tokens/quoted-token'
import { OutputToken } from '../../../src/tokens/output-token'
import { HTMLToken } from '../../../src/tokens/html-token'
import { createTrie } from '../../../src/util/operator-trie'
import { defaultOperators } from '../../../src/types'
describe('Tokenizer', function () {
const trie = createTrie(defaultOperators)
it('should read quoted', () => {
expect(new Tokenizer('"foo" ff').readQuoted()!.getText()).to.equal('"foo"')
expect(new Tokenizer(' "foo"ff').readQuoted()!.getText()).to.equal('"foo"')
expect(new Tokenizer('"foo" ff', trie).readQuoted()!.getText()).to.equal('"foo"')
expect(new Tokenizer(' "foo"ff', trie).readQuoted()!.getText()).to.equal('"foo"')
})
it('should read value', () => {
expect(new Tokenizer('a[ b][ "c d" ]').readValueOrThrow().getText()).to.equal('a[ b][ "c d" ]')
expect(new Tokenizer('a.b[c[d.e]]').readValueOrThrow().getText()).to.equal('a.b[c[d.e]]')
expect(new Tokenizer('a[ b][ "c d" ]', trie).readValueOrThrow().getText()).to.equal('a[ b][ "c d" ]')
expect(new Tokenizer('a.b[c[d.e]]', trie).readValueOrThrow().getText()).to.equal('a.b[c[d.e]]')
})
it('should read identifier', () => {
expect(new Tokenizer('foo bar').readIdentifier()).to.haveOwnProperty('content', 'foo')
expect(new Tokenizer('foo bar').readWord()).to.haveOwnProperty('content', 'foo')
expect(new Tokenizer('foo bar', trie).readIdentifier()).to.haveOwnProperty('content', 'foo')
expect(new Tokenizer('foo bar', trie).readWord()).to.haveOwnProperty('content', 'foo')
})
it('should read number value', () => {
const token: NumberToken = new Tokenizer('2.33.2').readValueOrThrow() as any
const token: NumberToken = new Tokenizer('2.33.2', trie).readValueOrThrow() as any
expect(token).to.be.instanceOf(NumberToken)
expect(token.whole.getText()).to.equal('2')
expect(token.decimal!.getText()).to.equal('33')
expect(token.getText()).to.equal('2.33')
})
it('should read quoted value', () => {
const value = new Tokenizer('"foo"a').readValue()
const value = new Tokenizer('"foo"a', trie).readValue()
expect(value).to.be.instanceOf(QuotedToken)
expect(value!.getText()).to.equal('"foo"')
})
it('should read property access value', () => {
expect(new Tokenizer('a[b]["c d"]').readValueOrThrow().getText()).to.equal('a[b]["c d"]')
expect(new Tokenizer('a[b]["c d"]', trie).readValueOrThrow().getText()).to.equal('a[b]["c d"]')
})
it('should read quoted property access value', () => {
const value = new Tokenizer('["a prop"]').readValue()
const value = new Tokenizer('["a prop"]', trie).readValue()
expect(value).to.be.instanceOf(PropertyAccessToken)
expect((value as PropertyAccessToken).variable.getText()).to.equal('"a prop"')
})
it('should throw for broken quoted property access', () => {
const tokenizer = new Tokenizer('[5]')
const tokenizer = new Tokenizer('[5]', trie)
expect(() => tokenizer.readValueOrThrow()).to.throw()
})
it('should throw for incomplete quoted property access', () => {
const tokenizer = new Tokenizer('["a prop"')
const tokenizer = new Tokenizer('["a prop"', trie)
expect(() => tokenizer.readValueOrThrow()).to.throw()
})
it('should read hash', () => {
const hash1 = new Tokenizer('foo: 3').readHash()
const hash1 = new Tokenizer('foo: 3', trie).readHash()
expect(hash1!.name.content).to.equal('foo')
expect(hash1!.value!.getText()).to.equal('3')
const hash2 = new Tokenizer(', foo: a[ "bar"]').readHash()
const hash2 = new Tokenizer(', foo: a[ "bar"]', trie).readHash()
expect(hash2!.name.content).to.equal('foo')
expect(hash2!.value!.getText()).to.equal('a[ "bar"]')
})
it('should read multiple hashs', () => {
const hashes = new Tokenizer(', limit: 3 reverse offset:off').readHashes()
const hashes = new Tokenizer(', limit: 3 reverse offset:off', trie).readHashes()
expect(hashes).to.have.lengthOf(3)
const [limit, reverse, offset] = hashes
expect(limit.name.content).to.equal('limit')
@@ -74,7 +78,7 @@ describe('Tokenizer', function () {
expect(offset.value!.getText()).to.equal('off')
})
it('should read hash value with property access', () => {
const hashes = new Tokenizer('cols: 2, rows: data["rows"]').readHashes()
const hashes = new Tokenizer('cols: 2, rows: data["rows"]', trie).readHashes()
expect(hashes).to.have.lengthOf(2)
const [cols, rols] = hashes
@@ -87,7 +91,7 @@ describe('Tokenizer', function () {
describe('#readTopLevelTokens()', () => {
it('should read HTML token', function () {
const html = '<html><body><p>Lorem Ipsum</p></body></html>'
const tokenizer = new Tokenizer(html)
const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(1)
@@ -96,7 +100,7 @@ describe('Tokenizer', function () {
})
it('should read tag token', function () {
const html = '<p>{% for p in a[1]%}</p>'
const tokenizer = new Tokenizer(html)
const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(3)
@@ -107,7 +111,7 @@ describe('Tokenizer', function () {
})
it('should allow unclosed tag inside {% raw %}', function () {
const html = '{%raw%} {%if%} {%else {%endraw%}'
const tokenizer = new Tokenizer(html)
const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(3)
@@ -116,7 +120,7 @@ describe('Tokenizer', function () {
})
it('should allow unclosed endraw tag inside {% raw %}', function () {
const html = '{%raw%} {%endraw {%raw%} {%endraw%}'
const tokenizer = new Tokenizer(html)
const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(3)
@@ -125,12 +129,12 @@ describe('Tokenizer', function () {
})
it('should throw when {% raw %} not closed', function () {
const html = '{%raw%} {%endraw {%raw%}'
const tokenizer = new Tokenizer(html)
const tokenizer = new Tokenizer(html, trie)
expect(() => tokenizer.readTopLevelTokens()).to.throw('raw "{%raw%} {%end..." not closed, line:1, col:8')
})
it('should read output token', function () {
const html = '<p>{{foo | date: "%Y-%m-%d"}}</p>'
const tokenizer = new Tokenizer(html)
const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(3)
@@ -140,7 +144,7 @@ describe('Tokenizer', function () {
})
it('should handle consecutive value and tags', function () {
const html = '{{foo}}{{bar}}{%foo%}{%bar%}'
const tokenizer = new Tokenizer(html)
const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(4)
@@ -162,7 +166,7 @@ describe('Tokenizer', function () {
})
it('should keep white spaces and newlines', function () {
const html = '{%foo%}\n{%bar %} \n {%alice%}'
const tokenizer = new Tokenizer(html)
const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(5)
expect(tokens[1]).instanceOf(HTMLToken)
@@ -172,7 +176,7 @@ describe('Tokenizer', function () {
})
it('should handle multiple lines tag', function () {
const html = '{%foo\na:a\nb:1.23\n%}'
const tokenizer = new Tokenizer(html)
const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(1)
expect(tokens[0]).instanceOf(TagToken)
@@ -181,7 +185,7 @@ describe('Tokenizer', function () {
})
it('should handle multiple lines value', function () {
const html = '{{foo\n|date:\n"%Y-%m-%d"\n}}'
const tokenizer = new Tokenizer(html)
const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(1)
expect(tokens[0]).instanceOf(OutputToken)
@@ -189,7 +193,7 @@ describe('Tokenizer', function () {
})
it('should handle complex object property access', function () {
const html = '{{ obj["my:property with anything"] }}'
const tokenizer = new Tokenizer(html)
const tokenizer = new Tokenizer(html, trie)
const tokens = tokenizer.readTopLevelTokens()
expect(tokens.length).to.equal(1)
const output = tokens[0] as OutputToken
@@ -198,18 +202,18 @@ describe('Tokenizer', function () {
})
it('should throw if tag not closed', function () {
const html = '{% assign foo = bar {{foo}}'
const tokenizer = new Tokenizer(html)
const tokenizer = new Tokenizer(html, trie)
expect(() => tokenizer.readTopLevelTokens()).to.throw(/tag "{% assign foo..." not closed/)
})
it('should throw if output not closed', function () {
const tokenizer = new Tokenizer('{{name}')
const tokenizer = new Tokenizer('{{name}', trie)
expect(() => tokenizer.readTopLevelTokens()).to.throw(/output "{{name}" not closed/)
})
})
describe('#readTagToken()', () => {
it('should skip quoted delimiters', function () {
const html = '{% assign a = "%} {% }} {{" %}'
const tokenizer = new Tokenizer(html)
const tokenizer = new Tokenizer(html, trie)
const token = tokenizer.readTagToken()
expect(token).instanceOf(TagToken)
@@ -220,7 +224,7 @@ describe('Tokenizer', function () {
describe('#readOutputToken()', () => {
it('should skip quoted delimiters', function () {
const html = '{{ "%} {%" | append: "}} {{" }}'
const tokenizer = new Tokenizer(html)
const tokenizer = new Tokenizer(html, trie)
const token = tokenizer.readOutputToken()
console.log(token)
@@ -230,7 +234,7 @@ describe('Tokenizer', function () {
})
describe('#readRange()', () => {
it('should read `(1..3)`', () => {
const range = new Tokenizer('(1..3)').readRange()
const range = new Tokenizer('(1..3)', trie).readRange()
expect(range).to.be.instanceOf(RangeToken)
expect(range!.getText()).to.deep.equal('(1..3)')
const { lhs, rhs } = range!
@@ -240,23 +244,23 @@ describe('Tokenizer', function () {
expect(rhs.getText()).to.equal('3')
})
it('should throw for `(..3)`', () => {
expect(() => new Tokenizer('(..3)').readRange()).to.throw('unexpected token "..3)", value expected')
expect(() => new Tokenizer('(..3)', trie).readRange()).to.throw('unexpected token "..3)", value expected')
})
it('should read `(a.b..c["..d"])`', () => {
const range = new Tokenizer('(a.b..c["..d"])').readRange()
const range = new Tokenizer('(a.b..c["..d"])', trie).readRange()
expect(range).to.be.instanceOf(RangeToken)
expect(range!.getText()).to.deep.equal('(a.b..c["..d"])')
})
})
describe('#readFilter()', () => {
it('should read a simple filter', function () {
const tokenizer = new Tokenizer('| plus')
const tokenizer = new Tokenizer('| plus', trie)
const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus')
expect(token).to.have.property('args').to.deep.equal([])
})
it('should read a filter with argument', function () {
const tokenizer = new Tokenizer(' | plus: 1')
const tokenizer = new Tokenizer(' | plus: 1', trie)
const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus')
expect(token!.args).to.have.lengthOf(1)
@@ -266,18 +270,18 @@ describe('Tokenizer', function () {
expect(one.getText()).to.equal('1')
})
it('should read a filter with colon but no argument', function () {
const tokenizer = new Tokenizer('| plus:')
const tokenizer = new Tokenizer('| plus:', trie)
const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus')
expect(token).to.have.property('args').to.deep.equal([])
})
it('should read null if name not found', function () {
const tokenizer = new Tokenizer('|')
const tokenizer = new Tokenizer('|', trie)
const token = tokenizer.readFilter()
expect(token).to.be.null
})
it('should read a filter with k/v argument', function () {
const tokenizer = new Tokenizer(' | plus: a:1')
const tokenizer = new Tokenizer(' | plus: a:1', trie)
const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus')
expect(token!.args).to.have.lengthOf(1)
@@ -288,7 +292,7 @@ describe('Tokenizer', function () {
expect(v.getText()).to.equal('1')
})
it('should read a filter with "arr[0]" argument', function () {
const tokenizer = new Tokenizer('| plus: arr[0]')
const tokenizer = new Tokenizer('| plus: arr[0]', trie)
const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus')
expect(token!.args).to.have.lengthOf(1)
@@ -301,7 +305,7 @@ describe('Tokenizer', function () {
expect(pa.props[0].getText()).to.equal('0')
})
it('should read a filter with obj.foo argument', function () {
const tokenizer = new Tokenizer('| plus: obj.foo')
const tokenizer = new Tokenizer('| plus: obj.foo', trie)
const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus')
expect(token!.args).to.have.lengthOf(1)
@@ -314,7 +318,7 @@ describe('Tokenizer', function () {
expect(pa.props[0].getText()).to.equal('foo')
})
it('should read a filter with obj["foo"] argument', function () {
const tokenizer = new Tokenizer('| plus: obj["good luck"]')
const tokenizer = new Tokenizer('| plus: obj["good luck"]', trie)
const token = tokenizer.readFilter()
expect(token).to.have.property('name', 'plus')
expect(token!.args).to.have.lengthOf(1)
@@ -328,7 +332,7 @@ describe('Tokenizer', function () {
})
describe('#readFilters()', () => {
it('should read simple filters', function () {
const tokenizer = new Tokenizer('| plus: 3 | capitalize')
const tokenizer = new Tokenizer('| plus: 3 | capitalize', trie)
const tokens = tokenizer.readFilters()
expect(tokens).to.have.lengthOf(2)
@@ -341,7 +345,7 @@ describe('Tokenizer', function () {
expect(tokens[1].args).to.have.lengthOf(0)
})
it('should read filters', function () {
const tokenizer = new Tokenizer('| plus: a:3 | capitalize | append: foo[a.b["c d"]]')
const tokenizer = new Tokenizer('| plus: a:3 | capitalize | append: foo[a.b["c d"]]', trie)
const tokens = tokenizer.readFilters()
expect(tokens).to.have.lengthOf(3)
@@ -364,14 +368,14 @@ describe('Tokenizer', function () {
})
describe('#readExpression()', () => {
it('should read expression `a `', () => {
const exp = [...new Tokenizer('a ').readExpression()]
const exp = [...new Tokenizer('a ', trie).readExpression()]
expect(exp).to.have.lengthOf(1)
expect(exp[0]).to.be.instanceOf(PropertyAccessToken)
expect(exp[0].getText()).to.deep.equal('a')
})
it('should read expression `a[][b]`', () => {
const exp = [...new Tokenizer('a[][b]').readExpression()]
const exp = [...new Tokenizer('a[][b]', trie).readExpression()]
expect(exp).to.have.lengthOf(1)
const pa = exp[0] as PropertyAccessToken
@@ -386,7 +390,7 @@ describe('Tokenizer', function () {
expect(p2.getText()).to.equal('b')
})
it('should read expression `a.`', () => {
const exp = [...new Tokenizer('a.').readExpression()]
const exp = [...new Tokenizer('a.', trie).readExpression()]
expect(exp).to.have.lengthOf(1)
const pa = exp[0] as PropertyAccessToken
@@ -395,14 +399,14 @@ describe('Tokenizer', function () {
expect(pa.props).to.have.lengthOf(0)
})
it('should read expression `a ==`', () => {
const exp = [...new Tokenizer('a ==').readExpression()]
const exp = [...new Tokenizer('a ==', trie).readExpression()]
expect(exp).to.have.lengthOf(1)
expect(exp[0]).to.be.instanceOf(PropertyAccessToken)
expect(exp[0].getText()).to.deep.equal('a')
})
it('should read expression `a==b`', () => {
const exp = new Tokenizer('a==b').readExpression()
const exp = new Tokenizer('a==b', trie).readExpression()
const [a, equals, b] = exp
expect(a).to.be.instanceOf(PropertyAccessToken)
@@ -415,11 +419,11 @@ describe('Tokenizer', function () {
expect(b.getText()).to.deep.equal('b')
})
it('should read expression `^`', () => {
const exp = new Tokenizer('^').readExpression()
const exp = new Tokenizer('^', trie).readExpression()
expect([...exp]).to.deep.equal([])
})
it('should read expression `a == b`', () => {
const exp = new Tokenizer('a == b').readExpression()
const exp = new Tokenizer('a == b', trie).readExpression()
const [a, equals, b] = exp
expect(a).to.be.instanceOf(PropertyAccessToken)
@@ -432,7 +436,7 @@ describe('Tokenizer', function () {
expect(b.getText()).to.deep.equal('b')
})
it('should read expression `(1..3) contains 3`', () => {
const exp = new Tokenizer('(1..3) contains 3').readExpression()
const exp = new Tokenizer('(1..3) contains 3', trie).readExpression()
const [range, contains, rhs] = exp
expect(range).to.be.instanceOf(RangeToken)
@@ -445,7 +449,7 @@ describe('Tokenizer', function () {
expect(rhs.getText()).to.deep.equal('3')
})
it('should read expression `a[b] == c`', () => {
const exp = new Tokenizer('a[b] == c').readExpression()
const exp = new Tokenizer('a[b] == c', trie).readExpression()
const [lhs, contains, rhs] = exp
expect(lhs).to.be.instanceOf(PropertyAccessToken)
@@ -458,7 +462,7 @@ describe('Tokenizer', function () {
expect(rhs.getText()).to.deep.equal('c')
})
it('should read expression `c[a["b"]] >= c`', () => {
const exp = new Tokenizer('c[a["b"]] >= c').readExpression()
const exp = new Tokenizer('c[a["b"]] >= c', trie).readExpression()
const [lhs, op, rhs] = exp
expect(lhs).to.be.instanceOf(PropertyAccessToken)
@@ -471,7 +475,7 @@ describe('Tokenizer', function () {
expect(rhs.getText()).to.deep.equal('c')
})
it('should read expression `"][" == var`', () => {
const exp = new Tokenizer('"][" == var').readExpression()
const exp = new Tokenizer('"][" == var', trie).readExpression()
const [lhs, equals, rhs] = exp
expect(lhs).to.be.instanceOf(QuotedToken)
@@ -484,7 +488,7 @@ describe('Tokenizer', function () {
expect(rhs.getText()).to.deep.equal('var')
})
it('should read expression `"\\\'" == "\\""`', () => {
const exp = new Tokenizer('"\\\'" == "\\""').readExpression()
const exp = new Tokenizer('"\\\'" == "\\""', trie).readExpression()
const [lhs, equals, rhs] = exp
expect(lhs).to.be.instanceOf(QuotedToken)