fix: raw block not ignoring {% characters, fixes #263

src/tokens/hash-token.ts

@@ -1,6 +1,6 @@
 import { Token } from './token'
 import { ValueToken } from './value-token'
-import { WordToken } from './word-token'
+import { IdentifierToken } from './identifier-token'
 import { TokenKind } from '../parser/token-kind'
 
 export class HashToken extends Token {
@@ -8,7 +8,7 @@ export class HashToken extends Token {
     public input: string,
     public begin: number,
     public end: number,
-    public name: WordToken,
+    public name: IdentifierToken,
     public value?: ValueToken,
     public file?: string
   ) {

src/tokens/identifier-token.ts

@@ -2,8 +2,7 @@ import { Token } from './token'
 import { NUMBER, TYPES, SIGN } from '../util/character'
 import { TokenKind } from '../parser/token-kind'
 
-// a word can be an identifier, a number, a keyword or a single-word-literal
-export class WordToken extends Token {
+export class IdentifierToken extends Token {
   public content: string
   constructor (
     public input: string,

src/tokens/number-token.ts

@@ -1,11 +1,11 @@
 import { Token } from './token'
-import { WordToken } from './word-token'
+import { IdentifierToken } from './identifier-token'
 import { TokenKind } from '../parser/token-kind'
 
 export class NumberToken extends Token {
   constructor (
-    public whole: WordToken,
-    public decimal?: WordToken
+    public whole: IdentifierToken,
+    public decimal?: IdentifierToken
   ) {
     super(TokenKind.Number, whole.input, whole.begin, decimal ? decimal.end : whole.end, whole.file)
   }

src/tokens/property-access-token.ts

@@ -1,20 +1,20 @@
 import { Token } from './token'
-import { WordToken } from './word-token'
+import { IdentifierToken } from './identifier-token'
 import { QuotedToken } from './quoted-token'
 import { TokenKind } from '../parser/token-kind'
 import { parseStringLiteral } from '../parser/parse-string-literal'
 
 export class PropertyAccessToken extends Token {
   constructor (
-    public variable: WordToken | QuotedToken,
-    public props: (WordToken | QuotedToken | PropertyAccessToken)[],
+    public variable: IdentifierToken | QuotedToken,
+    public props: (IdentifierToken | QuotedToken | PropertyAccessToken)[],
     end: number
   ) {
     super(TokenKind.PropertyAccess, variable.input, variable.begin, end, variable.file)
   }
 
-  getVariableAsText() {
-    if (this.variable instanceof WordToken) {
+  getVariableAsText () {
+    if (this.variable instanceof IdentifierToken) {
       return this.variable.getText()
     } else {
       return parseStringLiteral(this.variable.getText())

src/tokens/tag-token.ts

@@ -1,8 +1,8 @@
 import { DelimitedToken } from './delimited-token'
-import { BLANK, TYPES, VARIABLE } from '../util/character'
 import { TokenizationError } from '../util/error'
 import { NormalizedFullOptions } from '../liquid-options'
 import { TokenKind } from '../parser/token-kind'
+import { Tokenizer } from '../parser/tokenizer'
 
 export class TagToken extends DelimitedToken {
   public name: string
@@ -18,13 +18,11 @@ export class TagToken extends DelimitedToken {
     const value = input.slice(begin + tagDelimiterLeft.length, end - tagDelimiterRight.length)
     super(TokenKind.Tag, value, input, begin, end, trimTagLeft, trimTagRight, file)
 
-    let nameEnd = 0
-    while (TYPES[this.content.charCodeAt(nameEnd)] & VARIABLE) nameEnd++
-    this.name = this.content.slice(0, nameEnd)
+    const tokenizer = new Tokenizer(this.content)
+    this.name = tokenizer.readIdentifier().getText()
     if (!this.name) throw new TokenizationError(`illegal tag syntax`, this)
 
-    let argsBegin = nameEnd
-    while (TYPES[this.content.charCodeAt(argsBegin)] & BLANK) argsBegin++
-    this.args = this.content.slice(argsBegin)
+    tokenizer.skipBlank()
+    this.args = tokenizer.remaining()
   }
 }