fix: allow unicode to be identifiers, fixes #655

src/parser/tokenizer.ts

@@ -1,6 +1,6 @@
 import { FilteredValueToken, TagToken, HTMLToken, HashToken, QuotedToken, LiquidTagToken, OutputToken, ValueToken, Token, RangeToken, FilterToken, TopLevelToken, PropertyAccessToken, OperatorToken, LiteralToken, IdentifierToken, NumberToken } from '../tokens'
 import { OperatorHandler } from '../render/operator'
-import { TrieNode, LiteralValue, Trie, createTrie, ellipsis, literalValues, TokenizationError, TYPES, QUOTE, BLANK, IDENTIFIER, NUMBER, SIGN } from '../util'
+import { TrieNode, LiteralValue, Trie, createTrie, ellipsis, literalValues, TokenizationError, TYPES, QUOTE, BLANK, NUMBER, SIGN, isWord } from '../util'
 import { Operators, Expression } from '../render'
 import { NormalizedFullOptions, defaultOptions } from '../liquid-options'
 import { FilterArg } from './filter-arg'
@@ -59,7 +59,7 @@ export class Tokenizer {
       if (node['end']) info = node
     }
     if (!info) return -1
-    if (info['needBoundary'] && (this.peekType(i - this.p) & IDENTIFIER)) return -1
+    if (info['needBoundary'] && isWord(this.peek(i - this.p))) return -1
     return i
   }
   readFilteredValue (): FilteredValueToken {
@@ -245,7 +245,7 @@ export class Tokenizer {
   readIdentifier (): IdentifierToken {
     this.skipBlank()
     const begin = this.p
-    while (!this.end() && this.peekType() & IDENTIFIER) ++this.p
+    while (!this.end() && isWord(this.peek())) ++this.p
     return new IdentifierToken(this.input, begin, this.p, this.file)
   }
 
@@ -351,7 +351,7 @@ export class Tokenizer {
         n++
       } else break
     }
-    if (digitFound && !(this.peekType(n) & IDENTIFIER)) {
+    if (digitFound && !isWord(this.peek(n))) {
       const num = new NumberToken(this.input, this.p, this.p + n, this.file)
       this.advance(n)
       return num

src/util/character.ts

@@ -3,11 +3,18 @@
 // This file is generated by bin/character-gen.js
 // bitmask character types to boost performance
 export const TYPES = [0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4, 4, 4, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2, 8, 0, 0, 0, 0, 8, 0, 0, 0, 64, 0, 65, 0, 0, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 0, 0, 2, 2, 2, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]
-export const IDENTIFIER = 1
+export const WORD = 1
 export const OPERATOR = 2
 export const BLANK = 4
 export const QUOTE = 8
 export const INLINE_BLANK = 16
 export const NUMBER = 32
 export const SIGN = 64
+export const PUNCTUATION = 128
+
+export function isWord (char: string): boolean {
+  const code = char.charCodeAt(0)
+  return code >= 128 ? !TYPES[code] : !!(TYPES[code] & WORD)
+}
 TYPES[160] = TYPES[5760] = TYPES[6158] = TYPES[8192] = TYPES[8193] = TYPES[8194] = TYPES[8195] = TYPES[8196] = TYPES[8197] = TYPES[8198] = TYPES[8199] = TYPES[8200] = TYPES[8201] = TYPES[8202] = TYPES[8232] = TYPES[8233] = TYPES[8239] = TYPES[8287] = TYPES[12288] = BLANK
+TYPES[8220] = TYPES[8221] = PUNCTUATION

src/util/operator-trie.ts

@@ -1,4 +1,4 @@
-import { IDENTIFIER, TYPES } from '../util/character'
+import { isWord } from '../util/character'
 
 interface TrieInput<T> {
   [key: string]: T
@@ -25,7 +25,7 @@ export function createTrie<T = any> (input: TrieInput<T>): Trie<T> {
       const c = name[i]
       node[c] = node[c] || {}
 
-      if (i === name.length - 1 && (TYPES[name.charCodeAt(i)] & IDENTIFIER)) {
+      if (i === name.length - 1 && isWord(name[i])) {
         node[c].needBoundary = true
       }