Labyricorn/liquidjs
1
0
Fork 0
mirror of https://github.com/harttle/liquidjs.git synced 2026-09-16 04:40:39 -07:00
Code Issues Packages Projects Releases Wiki Activity

fix: allow unicode to be identifiers, fixes #655

Browse Source
This commit is contained in:
Harttle
2023-11-04 21:33:29 +08:00
parent d75ac93390
commit dd7616acb9
8 changed files with 41 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/parser/tokenizer.ts
+4 -4
View File
@@ -1,6 +1,6 @@
import { FilteredValueToken, TagToken, HTMLToken, HashToken, QuotedToken, LiquidTagToken, OutputToken, ValueToken, Token, RangeToken, FilterToken, TopLevelToken, PropertyAccessToken, OperatorToken, LiteralToken, IdentifierToken, NumberToken } from '../tokens'
import { OperatorHandler } from '../render/operator'
import { TrieNode, LiteralValue, Trie, createTrie, ellipsis, literalValues, TokenizationError, TYPES, QUOTE, BLANK, IDENTIFIER, NUMBER, SIGN } from '../util'
import { TrieNode, LiteralValue, Trie, createTrie, ellipsis, literalValues, TokenizationError, TYPES, QUOTE, BLANK, NUMBER, SIGN, isWord } from '../util'
import { Operators, Expression } from '../render'
import { NormalizedFullOptions, defaultOptions } from '../liquid-options'
import { FilterArg } from './filter-arg'
@@ -59,7 +59,7 @@ export class Tokenizer {
if (node['end']) info = node
}
if (!info) return -1
if (info['needBoundary'] && (this.peekType(i - this.p) & IDENTIFIER)) return -1
if (info['needBoundary'] && isWord(this.peek(i - this.p))) return -1
return i
}
readFilteredValue (): FilteredValueToken {
@@ -245,7 +245,7 @@ export class Tokenizer {
readIdentifier (): IdentifierToken {
this.skipBlank()
const begin = this.p
while (!this.end() && this.peekType() & IDENTIFIER) ++this.p
while (!this.end() && isWord(this.peek())) ++this.p
return new IdentifierToken(this.input, begin, this.p, this.file)
}
@@ -351,7 +351,7 @@ export class Tokenizer {
n++
} else break
}
if (digitFound && !(this.peekType(n) & IDENTIFIER)) {
if (digitFound && !isWord(this.peek(n))) {
const num = new NumberToken(this.input, this.p, this.p + n, this.file)
this.advance(n)
return num