fix: raw block not ignoring {% characters, fixes #263

test/unit/parser/tokenizer.ts

@@ -1,5 +1,5 @@
 import { expect } from 'chai'
-import { WordToken } from '../../../src/tokens/word-token'
+import { IdentifierToken } from '../../../src/tokens/identifier-token'
 import { NumberToken } from '../../../src/tokens/number-token'
 import { PropertyAccessToken } from '../../../src/tokens/property-access-token'
 import { RangeToken } from '../../../src/tokens/range-token'
@@ -19,6 +19,10 @@ describe('Tokenize', function () {
     expect(new Tokenizer('a[ b][ "c d" ]').readValueOrThrow().getText()).to.equal('a[ b][ "c d" ]')
     expect(new Tokenizer('a.b[c[d.e]]').readValueOrThrow().getText()).to.equal('a.b[c[d.e]]')
   })
+  it('should read identifier', () => {
+    expect(new Tokenizer('foo bar').readIdentifier()).to.haveOwnProperty('content', 'foo')
+    expect(new Tokenizer('foo bar').readWord()).to.haveOwnProperty('content', 'foo')
+  })
   it('should read number value', () => {
     const token: NumberToken = new Tokenizer('2.33.2').readValueOrThrow() as any
     expect(token).to.be.instanceOf(NumberToken)
@@ -100,6 +104,29 @@ describe('Tokenize', function () {
     expect(tag.name).to.equal('for')
     expect(tag.args).to.equal('p in a[1]')
   })
+  it('should allow unclosed tag inside {% raw %}', function () {
+    const html = '{%raw%} {%if%} {%else {%endraw%}'
+    const tokenizer = new Tokenizer(html)
+    const tokens = tokenizer.readTopLevelTokens()
+
+    expect(tokens.length).to.equal(3)
+    expect(tokens[0]).to.haveOwnProperty('name', 'raw')
+    expect((tokens[1] as any).getContent()).to.equal(' {%if%} {%else ')
+  })
+  it('should allow unclosed endraw tag inside {% raw %}', function () {
+    const html = '{%raw%} {%endraw {%raw%} {%endraw%}'
+    const tokenizer = new Tokenizer(html)
+    const tokens = tokenizer.readTopLevelTokens()
+
+    expect(tokens.length).to.equal(3)
+    expect(tokens[0]).to.haveOwnProperty('name', 'raw')
+    expect((tokens[1] as any).getContent()).to.equal(' {%endraw {%raw%} ')
+  })
+  it('should throw when {% raw %} not closed', function () {
+    const html = '{%raw%} {%endraw {%raw%}'
+    const tokenizer = new Tokenizer(html)
+    expect(() => tokenizer.readTopLevelTokens()).to.throw('raw "{%raw%} {%end..." not closed, line:1, col:8')
+  })
   it('should read output token', function () {
     const html = '<p>{{foo | date: "%Y-%m-%d"}}</p>'
     const tokenizer = new Tokenizer(html)
@@ -259,7 +286,7 @@ describe('Tokenize', function () {
       expect(token!.args[0]).to.be.instanceOf(PropertyAccessToken)
       expect(pa.variable.content).to.equal('obj')
       expect(pa.props).to.have.lengthOf(1)
-      expect(pa.props[0]).to.be.instanceOf(WordToken)
+      expect(pa.props[0]).to.be.instanceOf(IdentifierToken)
       expect(pa.props[0].getText()).to.equal('foo')
     })
     it('should read a filter with obj["foo"] argument', function () {
@@ -329,7 +356,7 @@ describe('Tokenize', function () {
       expect(pa.props).to.have.lengthOf(2)
 
       const [p1, p2] = pa.props
-      expect(p1).to.be.instanceOf(WordToken)
+      expect(p1).to.be.instanceOf(IdentifierToken)
       expect(p1.getText()).to.equal('')
       expect(p2).to.be.instanceOf(PropertyAccessToken)
       expect(p2.getText()).to.equal('b')

test/unit/template/filter/filter.ts

@@ -5,7 +5,7 @@ import { Context } from '../../../../src/context/context'
 import { toThenable } from '../../../../src/util/async'
 import { NumberToken } from '../../../../src/tokens/number-token'
 import { QuotedToken } from '../../../../src/tokens/quoted-token'
-import { WordToken } from '../../../../src/tokens/word-token'
+import { IdentifierToken } from '../../../../src/tokens/identifier-token'
 import { FilterMap } from '../../../../src/template/filter/filter-map'
 
 chai.use(sinonChai)
@@ -30,14 +30,14 @@ describe('filter', function () {
   it('should call filter impl with correct arguments', async function () {
     const spy = sinon.spy()
     filters.set('foo', spy)
-    const thirty = new NumberToken(new WordToken('30', 0, 2), undefined)
+    const thirty = new NumberToken(new IdentifierToken('30', 0, 2), undefined)
     await toThenable(filters.create('foo', [thirty]).render('foo', ctx))
     expect(spy).to.have.been.calledWith('foo', 30)
   })
   it('should call filter impl with correct this arg', async function () {
     const spy = sinon.spy()
     filters.set('foo', spy)
-    const thirty = new NumberToken(new WordToken('33', 0, 2), undefined)
+    const thirty = new NumberToken(new IdentifierToken('33', 0, 2), undefined)
     await toThenable(filters.create('foo', [thirty]).render('foo', ctx))
     expect(spy).to.have.been.calledOn(sinon.match.has('context', ctx))
   })
@@ -48,13 +48,13 @@ describe('filter', function () {
 
   it('should render filters with argument', async function () {
     filters.set('add', (a, b) => a + b)
-    const two = new NumberToken(new WordToken('2', 0, 1), undefined)
+    const two = new NumberToken(new IdentifierToken('2', 0, 1), undefined)
     expect(await toThenable(filters.create('add', [two]).render(3, ctx))).to.equal(5)
   })
 
   it('should render filters with multiple arguments', async function () {
     filters.set('add', (a, b, c) => a + b + c)
-    const two = new NumberToken(new WordToken('2', 0, 1), undefined)
+    const two = new NumberToken(new IdentifierToken('2', 0, 1), undefined)
     const c = new QuotedToken('"c"', 0, 3)
     expect(await toThenable(filters.create('add', [two, c]).render(3, ctx))).to.equal('5c')
   })
@@ -73,7 +73,7 @@ describe('filter', function () {
 
   it('should support key value pairs', async function () {
     filters.set('add', (a, b) => b[0] + ':' + (a + b[1]))
-    const two = new NumberToken(new WordToken('2', 0, 1), undefined)
+    const two = new NumberToken(new IdentifierToken('2', 0, 1), undefined)
     expect(await toThenable((filters.create('add', [['num', two]]).render(3, ctx)))).to.equal('num:5')
   })
 })