Labyricorn/liquidjs
1
0
Fork 0
mirror of https://github.com/harttle/liquidjs.git synced 2026-09-15 04:10:40 -07:00
Code Issues Packages Projects Releases Wiki Activity

fix: allow unicode to be identifiers, fixes #655

Browse Source
This commit is contained in:
Harttle
2023-11-04 21:33:29 +08:00
parent d75ac93390
commit dd7616acb9
8 changed files with 41 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/character-gen.js
Regular → Executable
+12 -4
View File
@@ -4,18 +4,19 @@ const isQuote = c => c === '"' || c === "'"
const isOperator = c => '!=<>'.includes(c)
const isNumber = c => c >= '0' && c <= '9'
const isCharacter = c => (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
const isIdentifier = c => '_-?'.includes(c) || isCharacter(c) || isNumber(c)
const isBlank = c => c === '\n' || c === '\t' || c === ' ' || c === '\r' || c === '\v' || c === '\f'
const isWord = c => '_-?'.includes(c) || isCharacter(c) || isNumber(c)
const isBlank = c => '\n\t \r\v\f'.includes(c)
const isInlineBlank = c => c === '\t' || c === ' ' || c === '\r'
const isSign = c => c === '-' || c === '+'
// See https://developer.mozilla.org/zh-CN/docs/Web/JavaScript/Reference/Global_Objects/RegExp
const unicodeBlanks = '\u00a0\u1680\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000'
const unicodePunctuations = '“”'
const types = []
for (let i = 0; i < 128; i++) {
const c = String.fromCharCode(i)
let n = 0
if (isIdentifier(c)) n |= 1
if (isWord(c)) n |= 1
if (isOperator(c)) n |= 2
if (isBlank(c)) n |= 4
if (isQuote(c)) n |= 8
@@ -31,13 +32,20 @@ console.log(`
// This file is generated by bin/character-gen.js
// bitmask character types to boost performance
export const TYPES = [${types.join(', ')}]
export const IDENTIFIER = 1
export const WORD = 1
export const OPERATOR = 2
export const BLANK = 4
export const QUOTE = 8
export const INLINE_BLANK = 16
export const NUMBER = 32
export const SIGN = 64
export const PUNCTUATION = 128
export function isWord (char: string): boolean {
const code = char.charCodeAt(0)
return code >= 128 ? !TYPES[code] : !!(TYPES[code] & WORD)
}
`.trim())
console.log([...unicodeBlanks].map(char => `TYPES[${char.charCodeAt(0)}]`).join(' = ') + ' = BLANK')
console.log([...unicodePunctuations].map(char => `TYPES[${char.charCodeAt(0)}]`).join(' = ') + ' = PUNCTUATION')