fix: raw block not ignoring {% characters, fixes #263

bin/character-gen.js

@@ -4,7 +4,7 @@ const isQuote = c => c === '"' || c === "'"
 const isOperator = c => '!=<>'.includes(c)
 const isNumber = c => c >= '0' && c <= '9'
 const isCharacter = c => (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
-const isVariable = c => '_-?'.includes(c) || isCharacter(c) || isNumber(c)
+const isIdentifier = c => '_-?'.includes(c) || isCharacter(c) || isNumber(c)
 const isBlank = c => c === '\n' || c === '\t' || c === ' ' || c === '\r' || c === '\v' || c === '\f'
 const isInlineBlank = c => c === '\t' || c === ' ' || c === '\r'
 const isSign = c => c === '-' || c === '+'
@@ -15,7 +15,7 @@ const types = []
 for (let i = 0; i < 128; i++) {
   const c = String.fromCharCode(i)
   let n = 0
-  if (isVariable(c)) n |= 1
+  if (isIdentifier(c)) n |= 1
   if (isOperator(c)) n |= 2
   if (isBlank(c)) n |= 4
   if (isQuote(c)) n |= 8
@@ -31,7 +31,7 @@ console.log(`
 // This file is generated by bin/character-gen.js
 // bitmask character types to boost performance
 export const TYPES = [${types.join(', ')}]
-export const VARIABLE = 1
+export const IDENTIFIER = 1
 export const OPERATOR = 2
 export const BLANK = 4
 export const QUOTE = 8

src/builtin/tags/assign.ts

@@ -3,7 +3,7 @@ import { Tokenizer, assert, TagImplOptions, TagToken, Context } from '../../type
 export default {
   parse: function (token: TagToken) {
     const tokenizer = new Tokenizer(token.args)
-    this.key = tokenizer.readWord().content
+    this.key = tokenizer.readIdentifier().content
     tokenizer.skipBlank()
     assert(tokenizer.peek() === '=', () => `illegal token ${token.getText()}`)
     tokenizer.advance()

src/builtin/tags/capture.ts

@@ -25,7 +25,7 @@ export default {
 } as TagImplOptions
 
 function readVariableName (tokenizer: Tokenizer) {
-  const word = tokenizer.readWord().content
+  const word = tokenizer.readIdentifier().content
   if (word) return word
   const quoted = tokenizer.readQuoted()
   if (quoted) return evalQuotedToken(quoted)

src/builtin/tags/decrement.ts

@@ -4,7 +4,7 @@ import { isNumber, stringify } from '../../util/underscore'
 export default {
   parse: function (token: TagToken) {
     const tokenizer = new Tokenizer(token.args)
-    this.variable = tokenizer.readWord().content
+    this.variable = tokenizer.readIdentifier().content
   },
   render: function (context: Context, emitter: Emitter) {
     const scope = context.environments

src/builtin/tags/for.ts

@@ -8,8 +8,8 @@ export default {
   parse: function (token: TagToken, remainTokens: TopLevelToken[]) {
     const toknenizer = new Tokenizer(token.args)
 
-    const variable = toknenizer.readWord()
+    const variable = toknenizer.readIdentifier()
-    const inStr = toknenizer.readWord()
+    const inStr = toknenizer.readIdentifier()
     const collection = toknenizer.readValue()
     assert(
       variable.size() && inStr.content === 'in' && collection,

src/builtin/tags/include.ts

@@ -11,7 +11,7 @@ export default {
     assert(this.file, () => `illegal argument "${token.args}"`)
 
     const begin = tokenizer.p
-    const withStr = tokenizer.readWord()
+    const withStr = tokenizer.readIdentifier()
     if (withStr.content === 'with') {
       tokenizer.skipBlank()
       if (tokenizer.peek() !== ':') {

src/builtin/tags/increment.ts

@@ -4,7 +4,7 @@ import { Tokenizer, Emitter, TagToken, Context, TagImplOptions } from '../../typ
 export default {
   parse: function (token: TagToken) {
     const tokenizer = new Tokenizer(token.args)
-    this.variable = tokenizer.readWord().content
+    this.variable = tokenizer.readIdentifier().content
   },
   render: function (context: Context, emitter: Emitter) {
     const scope = context.environments

src/builtin/tags/render.ts

@@ -15,16 +15,16 @@ export default {
     while (!tokenizer.end()) {
       tokenizer.skipBlank()
       const begin = tokenizer.p
-      const keyword = tokenizer.readWord()
+      const keyword = tokenizer.readIdentifier()
       if (keyword.content === 'with' || keyword.content === 'for') {
         tokenizer.skipBlank()
         if (tokenizer.peek() !== ':') {
           const value = tokenizer.readValue()
           if (value) {
             const beforeAs = tokenizer.p
-            const asStr = tokenizer.readWord()
+            const asStr = tokenizer.readIdentifier()
             let alias
-            if (asStr.content === 'as') alias = tokenizer.readWord()
+            if (asStr.content === 'as') alias = tokenizer.readIdentifier()
             else tokenizer.p = beforeAs
 
             this[keyword.content] = { value, alias: alias && alias.content }

src/builtin/tags/tablerow.ts

@@ -7,10 +7,10 @@ export default {
   parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
     const tokenizer = new Tokenizer(tagToken.args)
 
-    this.variable = tokenizer.readWord()
+    this.variable = tokenizer.readIdentifier()
     tokenizer.skipBlank()
 
-    const tmp = tokenizer.readWord()
+    const tmp = tokenizer.readIdentifier()
     assert(tmp && tmp.content === 'in', () => `illegal tag: ${tagToken.getText()}`)
 
     this.collection = tokenizer.readValue()

src/parser/match-operator.ts

@@ -1,4 +1,4 @@
-import { VARIABLE } from '../util/character'
+import { IDENTIFIER } from '../util/character'
 
 const trie = {
   a: { n: { d: { end: true, needBoundary: true } } },
@@ -19,6 +19,6 @@ export function matchOperator (str: string, begin: number, end = str.length) {
     if (node['end']) info = node
   }
   if (!info) return -1
-  if (info['needBoundary'] && str.charCodeAt(i) & VARIABLE) return -1
+  if (info['needBoundary'] && str.charCodeAt(i) & IDENTIFIER) return -1
   return i
 }

src/parser/tokenizer.ts

@@ -1,6 +1,6 @@
 import { whiteSpaceCtrl } from './whitespace-ctrl'
 import { NumberToken } from '../tokens/number-token'
-import { WordToken } from '../tokens/word-token'
+import { IdentifierToken } from '../tokens/identifier-token'
 import { literalValues } from '../util/literal'
 import { LiteralToken } from '../tokens/literal-token'
 import { OperatorToken } from '../tokens/operator-token'
@@ -20,12 +20,14 @@ import { ValueToken } from '../tokens/value-token'
 import { OutputToken } from '../tokens/output-token'
 import { TokenizationError } from '../util/error'
 import { NormalizedFullOptions, defaultOptions } from '../liquid-options'
-import { TYPES, QUOTE, BLANK, VARIABLE } from '../util/character'
+import { TYPES, QUOTE, BLANK, IDENTIFIER } from '../util/character'
 import { matchOperator } from './match-operator'
 
 export class Tokenizer {
   p = 0
   N: number
+  private rawBeginAt = -1
+
   constructor (
     private input: string,
     private file: string = ''
@@ -70,7 +72,7 @@ export class Tokenizer {
     assert(this.peek() === '|', () => `unexpected token at ${this.snapshot()}`)
     this.p++
     const begin = this.p
-    const name = this.readWord()
+    const name = this.readIdentifier()
     if (!name.size()) return null
     const args = []
     this.skipBlank()
@@ -107,8 +109,9 @@ export class Tokenizer {
 
   readTopLevelToken (options: NormalizedFullOptions): TopLevelToken {
     const { tagDelimiterLeft, outputDelimiterLeft } = options
-    if (this.matchWord(tagDelimiterLeft)) return this.readTagToken(options)
+    if (this.rawBeginAt > -1) return this.readEndrawOrRawContent(options)
-    if (this.matchWord(outputDelimiterLeft)) return this.readOutputToken(options)
+    if (this.match(tagDelimiterLeft)) return this.readTagToken(options)
+    if (this.match(outputDelimiterLeft)) return this.readOutputToken(options)
     return this.readHTMLToken(options)
   }
 
@@ -116,8 +119,8 @@ export class Tokenizer {
     const begin = this.p
     while (this.p < this.N) {
       const { tagDelimiterLeft, outputDelimiterLeft } = options
-      if (this.matchWord(tagDelimiterLeft)) break
+      if (this.match(tagDelimiterLeft)) break
-      if (this.matchWord(outputDelimiterLeft)) break
+      if (this.match(outputDelimiterLeft)) break
       ++this.p
     }
     return new HTMLToken(this.input, begin, this.p, this.file)
@@ -128,9 +131,11 @@ export class Tokenizer {
     const { tagDelimiterRight } = options
     const begin = this.p
     if (this.readTo(tagDelimiterRight) === -1) {
-      this.mkError(`tag ${this.snapshot(begin)} not closed`, begin)
+      throw this.mkError(`tag ${this.snapshot(begin)} not closed`, begin)
     }
-    return new TagToken(input, begin, this.p, options, file)
+    const token = new TagToken(input, begin, this.p, options, file)
+    if (token.name === 'raw') this.rawBeginAt = begin
+    return token
   }
 
   readOutputToken (options: NormalizedFullOptions): OutputToken {
@@ -138,24 +143,59 @@ export class Tokenizer {
     const { outputDelimiterRight } = options
     const begin = this.p
     if (this.readTo(outputDelimiterRight) === -1) {
-      this.mkError(`output ${this.snapshot(begin)} not closed`, begin)
+      throw this.mkError(`output ${this.snapshot(begin)} not closed`, begin)
     }
     return new OutputToken(input, begin, this.p, options, file)
   }
 
+  readEndrawOrRawContent (options: NormalizedFullOptions): HTMLToken | TagToken {
+    const { tagDelimiterLeft, tagDelimiterRight } = options
+    const begin = this.p
+    let leftPos = this.readTo(tagDelimiterLeft) - tagDelimiterLeft.length
+    while (this.p < this.N) {
+      if (this.readIdentifier().getText() !== 'endraw') {
+        leftPos = this.readTo(tagDelimiterLeft) - tagDelimiterLeft.length
+        continue
+      }
+      while (this.p <= this.N) {
+        if (this.rmatch(tagDelimiterRight)) {
+          const end = this.p
+          if (begin === leftPos) {
+            this.rawBeginAt = -1
+            return new TagToken(this.input, begin, end, options, this.file)
+          } else {
+            this.p = leftPos
+            return new HTMLToken(this.input, begin, leftPos, this.file)
+          }
+        }
+        if (this.rmatch(tagDelimiterLeft)) break
+        this.p++
+      }
+    }
+    throw this.mkError(`raw ${this.snapshot(this.rawBeginAt)} not closed`, begin)
+  }
+
   mkError (msg: string, begin: number) {
-    throw new TokenizationError(msg, new WordToken(this.input, begin, this.N, this.file))
+    return new TokenizationError(msg, new IdentifierToken(this.input, begin, this.N, this.file))
   }
 
   snapshot (begin: number = this.p) {
     return JSON.stringify(ellipsis(this.input.slice(begin), 16))
   }
 
-  readWord (): WordToken { // rename to identifier
+  /**
+   * @deprecated
+   */
+  readWord () {
+    console.warn('Tokenizer#readWord() will be removed, use #readIdentifier instead')
+    return this.readIdentifier()
+  }
+
+  readIdentifier (): IdentifierToken {
     this.skipBlank()
     const begin = this.p
-    while (this.peekType() & VARIABLE) ++this.p
+    while (this.peekType() & IDENTIFIER) ++this.p
-    return new WordToken(this.input, begin, this.p, this.file)
+    return new IdentifierToken(this.input, begin, this.p, this.file)
   }
 
   readHashes () {
@@ -171,7 +211,7 @@ export class Tokenizer {
     this.skipBlank()
     if (this.peek() === ',') ++this.p
     const begin = this.p
-    const name = this.readWord()
+    const name = this.readIdentifier()
     if (!name.size()) return
     let value
 
@@ -198,7 +238,7 @@ export class Tokenizer {
   readTo (end: string): number {
     while (this.p < this.N) {
       ++this.p
-      if (this.reverseMatchWord(end)) return this.p
+      if (this.rmatch(end)) return this.p
     }
     return -1
   }
@@ -216,21 +256,21 @@ export class Tokenizer {
       return new PropertyAccessToken(prop, [], this.p)
     }
 
-    const variable = this.readWord()
+    const variable = this.readIdentifier()
     if (!variable.size()) return
 
     let isNumber = variable.isNumber(true)
-    const props: (QuotedToken | WordToken)[] = []
+    const props: (QuotedToken | IdentifierToken)[] = []
     while (true) {
       if (this.peek() === '[') {
         isNumber = false
         this.p++
-        const prop = this.readValue() || new WordToken(this.input, this.p, this.p, this.file)
+        const prop = this.readValue() || new IdentifierToken(this.input, this.p, this.p, this.file)
         this.readTo(']')
         props.push(prop)
       } else if (this.peek() === '.' && this.peek(1) !== '.') { // skip range syntax
         this.p++
-        const prop = this.readWord()
+        const prop = this.readIdentifier()
         if (!prop.size()) break
         if (!prop.isNumber()) isNumber = false
         props.push(prop)
@@ -239,7 +279,7 @@ export class Tokenizer {
     if (!props.length && literalValues.hasOwnProperty(variable.content)) {
       return new LiteralToken(this.input, variable.begin, variable.end, this.file)
     }
-    if (isNumber) return new NumberToken(variable, props[0] as WordToken)
+    if (isNumber) return new NumberToken(variable, props[0] as IdentifierToken)
     return new PropertyAccessToken(variable, props, this.p)
   }
 
@@ -276,22 +316,22 @@ export class Tokenizer {
     return new QuotedToken(this.input, begin, this.p, this.file)
   }
 
-  readFileName (): WordToken {
+  readFileName (): IdentifierToken {
     const begin = this.p
     while (!(this.peekType() & BLANK) && this.peek() !== ',' && this.p < this.N) this.p++
-    return new WordToken(this.input, begin, this.p, this.file)
+    return new IdentifierToken(this.input, begin, this.p, this.file)
   }
 
-  matchWord (word: string) {
+  match (word: string) {
     for (let i = 0; i < word.length; i++) {
       if (word[i] !== this.input[this.p + i]) return false
     }
     return true
   }
 
-  reverseMatchWord (word: string) {
+  rmatch (pattern: string) {
-    for (let i = 0; i < word.length; i++) {
+    for (let i = 0; i < pattern.length; i++) {
-      if (word[word.length - 1 - i] !== this.input[this.p - 1 - i]) return false
+      if (pattern[pattern.length - 1 - i] !== this.input[this.p - 1 - i]) return false
     }
     return true
   }

src/parser/whitespace-ctrl.ts

@@ -4,7 +4,6 @@ import { NormalizedFullOptions } from '../liquid-options'
 import { TYPES, INLINE_BLANK, BLANK } from '../util/character'
 
 export function whiteSpaceCtrl (tokens: Token[], options: NormalizedFullOptions) {
-  options = { greedy: true, ...options }
   let inRaw = false
 
   for (let i = 0; i < tokens.length; i++) {

src/template/tag/tag.ts

@@ -1,13 +1,12 @@
 import { isFunction } from '../../util/underscore'
 import { Liquid } from '../../liquid'
 import { TemplateImpl } from '../../template/template-impl'
-import { Emitter, Hash, Context, TagImplOptions, TagToken, Template, TopLevelToken } from '../../types'
+import { Emitter, Hash, Context, TagToken, Template, TopLevelToken } from '../../types'
 import { TagImpl } from './tag-impl'
 
 export class Tag extends TemplateImpl<TagToken> implements Template {
   public name: string
   private impl: TagImpl
-  private static impls: { [key: string]: TagImplOptions } = {}
 
   public constructor (token: TagToken, tokens: TopLevelToken[], liquid: Liquid) {
     super(token)

src/tokens/hash-token.ts

@@ -1,6 +1,6 @@
 import { Token } from './token'
 import { ValueToken } from './value-token'
-import { WordToken } from './word-token'
+import { IdentifierToken } from './identifier-token'
 import { TokenKind } from '../parser/token-kind'
 
 export class HashToken extends Token {
@@ -8,7 +8,7 @@ export class HashToken extends Token {
     public input: string,
     public begin: number,
     public end: number,
-    public name: WordToken,
+    public name: IdentifierToken,
     public value?: ValueToken,
     public file?: string
   ) {

src/tokens/identifier-token.ts

@@ -2,8 +2,7 @@ import { Token } from './token'
 import { NUMBER, TYPES, SIGN } from '../util/character'
 import { TokenKind } from '../parser/token-kind'
 
-// a word can be an identifier, a number, a keyword or a single-word-literal
+export class IdentifierToken extends Token {
-export class WordToken extends Token {
   public content: string
   constructor (
     public input: string,

src/tokens/number-token.ts

@@ -1,11 +1,11 @@
 import { Token } from './token'
-import { WordToken } from './word-token'
+import { IdentifierToken } from './identifier-token'
 import { TokenKind } from '../parser/token-kind'
 
 export class NumberToken extends Token {
   constructor (
-    public whole: WordToken,
+    public whole: IdentifierToken,
-    public decimal?: WordToken
+    public decimal?: IdentifierToken
   ) {
     super(TokenKind.Number, whole.input, whole.begin, decimal ? decimal.end : whole.end, whole.file)
   }

src/tokens/property-access-token.ts

@@ -1,20 +1,20 @@
 import { Token } from './token'
-import { WordToken } from './word-token'
+import { IdentifierToken } from './identifier-token'
 import { QuotedToken } from './quoted-token'
 import { TokenKind } from '../parser/token-kind'
 import { parseStringLiteral } from '../parser/parse-string-literal'
 
 export class PropertyAccessToken extends Token {
   constructor (
-    public variable: WordToken | QuotedToken,
+    public variable: IdentifierToken | QuotedToken,
-    public props: (WordToken | QuotedToken | PropertyAccessToken)[],
+    public props: (IdentifierToken | QuotedToken | PropertyAccessToken)[],
     end: number
   ) {
     super(TokenKind.PropertyAccess, variable.input, variable.begin, end, variable.file)
   }
 
-  getVariableAsText() {
+  getVariableAsText () {
-    if (this.variable instanceof WordToken) {
+    if (this.variable instanceof IdentifierToken) {
       return this.variable.getText()
     } else {
       return parseStringLiteral(this.variable.getText())

src/tokens/tag-token.ts

@@ -1,8 +1,8 @@
 import { DelimitedToken } from './delimited-token'
-import { BLANK, TYPES, VARIABLE } from '../util/character'
 import { TokenizationError } from '../util/error'
 import { NormalizedFullOptions } from '../liquid-options'
 import { TokenKind } from '../parser/token-kind'
+import { Tokenizer } from '../parser/tokenizer'
 
 export class TagToken extends DelimitedToken {
   public name: string
@@ -18,13 +18,11 @@ export class TagToken extends DelimitedToken {
     const value = input.slice(begin + tagDelimiterLeft.length, end - tagDelimiterRight.length)
     super(TokenKind.Tag, value, input, begin, end, trimTagLeft, trimTagRight, file)
 
-    let nameEnd = 0
+    const tokenizer = new Tokenizer(this.content)
-    while (TYPES[this.content.charCodeAt(nameEnd)] & VARIABLE) nameEnd++
+    this.name = tokenizer.readIdentifier().getText()
-    this.name = this.content.slice(0, nameEnd)
     if (!this.name) throw new TokenizationError(`illegal tag syntax`, this)
 
-    let argsBegin = nameEnd
+    tokenizer.skipBlank()
-    while (TYPES[this.content.charCodeAt(argsBegin)] & BLANK) argsBegin++
+    this.args = tokenizer.remaining()
-    this.args = this.content.slice(argsBegin)
   }
 }

src/util/character.ts

@@ -3,7 +3,7 @@
 // This file is generated by bin/character-gen.js
 // bitmask character types to boost performance
 export const TYPES = [0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4, 4, 4, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2, 8, 0, 0, 0, 0, 8, 0, 0, 0, 64, 0, 65, 0, 0, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 0, 0, 2, 2, 2, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]
-export const VARIABLE = 1
+export const IDENTIFIER = 1
 export const OPERATOR = 2
 export const BLANK = 4
 export const QUOTE = 8

src/util/type-guards.ts

@@ -1,6 +1,6 @@
 import { OperatorToken } from '../tokens/operator-token'
 import { DelimitedToken } from '../tokens/delimited-token'
-import { WordToken } from '../tokens/word-token'
+import { IdentifierToken } from '../tokens/identifier-token'
 import { TagToken } from '../tokens/tag-token'
 import { HTMLToken } from '../tokens/html-token'
 import { OutputToken } from '../tokens/output-token'
@@ -47,7 +47,7 @@ export function isPropertyAccessToken (val: any): val is PropertyAccessToken {
   return getKind(val) === TokenKind.PropertyAccess
 }
 
-export function isWordToken (val: any): val is WordToken {
+export function isWordToken (val: any): val is IdentifierToken {
   return getKind(val) === TokenKind.Word
 }
 

test/e2e/issues.ts

@@ -36,4 +36,10 @@ describe('Issues', function () {
     // should stringify the regexp rather than execute it
     expect(html).to.equal(INPUT)
   })
+  it('#263 raw/endraw block not ignoring {% characters', () => {
+    const template = `{% raw %}This is a code snippet showing how {% breaks the raw block.{% endraw %}`
+    const engine = new Liquid()
+    const html = engine.parseAndRenderSync(template)
+    expect(html).to.equal('This is a code snippet showing how {% breaks the raw block.')
+  })
 })

test/integration/builtin/tags/raw.ts

@@ -1,11 +1,15 @@
 import { Liquid } from '../../../../src/liquid'
-import { expect } from 'chai'
+import * as chai from 'chai'
+import * as chaiAsPromised from 'chai-as-promised'
+
+const expect = chai.expect
+chai.use(chaiAsPromised)
 
 describe('tags/raw', function () {
   const liquid = new Liquid()
   it('should throw when not closed', async function () {
-    const p = liquid.parseAndRender('{% raw%}')
+    const p = liquid.parseAndRender('{% raw %}')
-    return expect(p).be.rejectedWith(/{% raw%} not closed/)
+    return expect(p).be.rejectedWith(/{% raw %} not closed/)
   })
   it('should output filters as it is', async function () {
     const src = '{% raw %}{{ 5 | plus: 6 }}{% endraw %} is equal to 11.'

test/unit/parser/tokenizer.ts

@@ -1,5 +1,5 @@
 import { expect } from 'chai'
-import { WordToken } from '../../../src/tokens/word-token'
+import { IdentifierToken } from '../../../src/tokens/identifier-token'
 import { NumberToken } from '../../../src/tokens/number-token'
 import { PropertyAccessToken } from '../../../src/tokens/property-access-token'
 import { RangeToken } from '../../../src/tokens/range-token'
@@ -19,6 +19,10 @@ describe('Tokenize', function () {
     expect(new Tokenizer('a[ b][ "c d" ]').readValueOrThrow().getText()).to.equal('a[ b][ "c d" ]')
     expect(new Tokenizer('a.b[c[d.e]]').readValueOrThrow().getText()).to.equal('a.b[c[d.e]]')
   })
+  it('should read identifier', () => {
+    expect(new Tokenizer('foo bar').readIdentifier()).to.haveOwnProperty('content', 'foo')
+    expect(new Tokenizer('foo bar').readWord()).to.haveOwnProperty('content', 'foo')
+  })
   it('should read number value', () => {
     const token: NumberToken = new Tokenizer('2.33.2').readValueOrThrow() as any
     expect(token).to.be.instanceOf(NumberToken)
@@ -100,6 +104,29 @@ describe('Tokenize', function () {
     expect(tag.name).to.equal('for')
     expect(tag.args).to.equal('p in a[1]')
   })
+  it('should allow unclosed tag inside {% raw %}', function () {
+    const html = '{%raw%} {%if%} {%else {%endraw%}'
+    const tokenizer = new Tokenizer(html)
+    const tokens = tokenizer.readTopLevelTokens()
+
+    expect(tokens.length).to.equal(3)
+    expect(tokens[0]).to.haveOwnProperty('name', 'raw')
+    expect((tokens[1] as any).getContent()).to.equal(' {%if%} {%else ')
+  })
+  it('should allow unclosed endraw tag inside {% raw %}', function () {
+    const html = '{%raw%} {%endraw {%raw%} {%endraw%}'
+    const tokenizer = new Tokenizer(html)
+    const tokens = tokenizer.readTopLevelTokens()
+
+    expect(tokens.length).to.equal(3)
+    expect(tokens[0]).to.haveOwnProperty('name', 'raw')
+    expect((tokens[1] as any).getContent()).to.equal(' {%endraw {%raw%} ')
+  })
+  it('should throw when {% raw %} not closed', function () {
+    const html = '{%raw%} {%endraw {%raw%}'
+    const tokenizer = new Tokenizer(html)
+    expect(() => tokenizer.readTopLevelTokens()).to.throw('raw "{%raw%} {%end..." not closed, line:1, col:8')
+  })
   it('should read output token', function () {
     const html = '<p>{{foo | date: "%Y-%m-%d"}}</p>'
     const tokenizer = new Tokenizer(html)
@@ -259,7 +286,7 @@ describe('Tokenize', function () {
       expect(token!.args[0]).to.be.instanceOf(PropertyAccessToken)
       expect(pa.variable.content).to.equal('obj')
       expect(pa.props).to.have.lengthOf(1)
-      expect(pa.props[0]).to.be.instanceOf(WordToken)
+      expect(pa.props[0]).to.be.instanceOf(IdentifierToken)
       expect(pa.props[0].getText()).to.equal('foo')
     })
     it('should read a filter with obj["foo"] argument', function () {
@@ -329,7 +356,7 @@ describe('Tokenize', function () {
       expect(pa.props).to.have.lengthOf(2)
 
       const [p1, p2] = pa.props
-      expect(p1).to.be.instanceOf(WordToken)
+      expect(p1).to.be.instanceOf(IdentifierToken)
       expect(p1.getText()).to.equal('')
       expect(p2).to.be.instanceOf(PropertyAccessToken)
       expect(p2.getText()).to.equal('b')

test/unit/template/filter/filter.ts

@@ -5,7 +5,7 @@ import { Context } from '../../../../src/context/context'
 import { toThenable } from '../../../../src/util/async'
 import { NumberToken } from '../../../../src/tokens/number-token'
 import { QuotedToken } from '../../../../src/tokens/quoted-token'
-import { WordToken } from '../../../../src/tokens/word-token'
+import { IdentifierToken } from '../../../../src/tokens/identifier-token'
 import { FilterMap } from '../../../../src/template/filter/filter-map'
 
 chai.use(sinonChai)
@@ -30,14 +30,14 @@ describe('filter', function () {
   it('should call filter impl with correct arguments', async function () {
     const spy = sinon.spy()
     filters.set('foo', spy)
-    const thirty = new NumberToken(new WordToken('30', 0, 2), undefined)
+    const thirty = new NumberToken(new IdentifierToken('30', 0, 2), undefined)
     await toThenable(filters.create('foo', [thirty]).render('foo', ctx))
     expect(spy).to.have.been.calledWith('foo', 30)
   })
   it('should call filter impl with correct this arg', async function () {
     const spy = sinon.spy()
     filters.set('foo', spy)
-    const thirty = new NumberToken(new WordToken('33', 0, 2), undefined)
+    const thirty = new NumberToken(new IdentifierToken('33', 0, 2), undefined)
     await toThenable(filters.create('foo', [thirty]).render('foo', ctx))
     expect(spy).to.have.been.calledOn(sinon.match.has('context', ctx))
   })
@@ -48,13 +48,13 @@ describe('filter', function () {
 
   it('should render filters with argument', async function () {
     filters.set('add', (a, b) => a + b)
-    const two = new NumberToken(new WordToken('2', 0, 1), undefined)
+    const two = new NumberToken(new IdentifierToken('2', 0, 1), undefined)
     expect(await toThenable(filters.create('add', [two]).render(3, ctx))).to.equal(5)
   })
 
   it('should render filters with multiple arguments', async function () {
     filters.set('add', (a, b, c) => a + b + c)
-    const two = new NumberToken(new WordToken('2', 0, 1), undefined)
+    const two = new NumberToken(new IdentifierToken('2', 0, 1), undefined)
     const c = new QuotedToken('"c"', 0, 3)
     expect(await toThenable(filters.create('add', [two, c]).render(3, ctx))).to.equal('5c')
   })
@@ -73,7 +73,7 @@ describe('filter', function () {
 
   it('should support key value pairs', async function () {
     filters.set('add', (a, b) => b[0] + ':' + (a + b[1]))
-    const two = new NumberToken(new WordToken('2', 0, 1), undefined)
+    const two = new NumberToken(new IdentifierToken('2', 0, 1), undefined)
     expect(await toThenable((filters.create('add', [['num', two]]).render(3, ctx)))).to.equal('num:5')
   })
 })