fix: properly treat unicode blanks, fixes #221

bin/character-gen.js

@@ -5,9 +5,11 @@ const isOperator = c => '!=<>'.includes(c)
 const isNumber = c => c >= '0' && c <= '9'
 const isCharacter = c => (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
 const isVariable = c => '_-?'.includes(c) || isCharacter(c) || isNumber(c)
-const isBlank = c => c === '\n' || c === '\t' || c === ' ' || c === '\r'
+const isBlank = c => c === '\n' || c === '\t' || c === ' ' || c === '\r' || c === '\v' || c === '\f'
 const isInlineBlank = c => c === '\t' || c === ' ' || c === '\r'
 const isSign = c => c === '-' || c === '+'
+// See https://developer.mozilla.org/zh-CN/docs/Web/JavaScript/Reference/Global_Objects/RegExp
+const unicodeBlanks = '\u00a0\u1680\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000'
 
 const types = []
 for (let i = 0; i < 128; i++) {
@@ -22,9 +24,12 @@ for (let i = 0; i < 128; i++) {
   if (isSign(c)) n |= 64
   types.push(n)
 }
+
 console.log(`
+// **DO NOT CHANGE THIS FILE**
+//
+// This file is generated by bin/character-gen.js
 // bitmask character types to boost performance
-// generated by bin/character-gen.js
 export const TYPES = [${types.join(', ')}]
 export const VARIABLE = 1
 export const OPERATOR = 2
@@ -34,3 +39,5 @@ export const INLINE_BLANK = 16
 export const NUMBER = 32
 export const SIGN = 64
 `.trim())
+
+console.log([...unicodeBlanks].map(char => `TYPES[${char.charCodeAt(0)}]`).join(' = ') + ' = BLANK')

src/util/character.ts

@@ -1,6 +1,8 @@
+// **DO NOT CHANGE THIS FILE**
+//
+// This file is generated by bin/character-gen.js
 // bitmask character types to boost performance
-// generated by bin/char-types.js
+export const TYPES = [0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4, 4, 4, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2, 8, 0, 0, 0, 0, 8, 0, 0, 0, 64, 0, 65, 0, 0, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 0, 0, 2, 2, 2, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]
-export const TYPES = [0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2, 8, 0, 0, 0, 0, 8, 0, 0, 0, 64, 0, 65, 0, 0, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 0, 0, 2, 2, 2, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]
 export const VARIABLE = 1
 export const OPERATOR = 2
 export const BLANK = 4
@@ -8,3 +10,4 @@ export const QUOTE = 8
 export const INLINE_BLANK = 16
 export const NUMBER = 32
 export const SIGN = 64
+TYPES[160] = TYPES[5760] = TYPES[6158] = TYPES[8192] = TYPES[8193] = TYPES[8194] = TYPES[8195] = TYPES[8196] = TYPES[8197] = TYPES[8198] = TYPES[8199] = TYPES[8200] = TYPES[8201] = TYPES[8202] = TYPES[8232] = TYPES[8233] = TYPES[8239] = TYPES[8287] = TYPES[12288] = BLANK

test/e2e/issues.ts

@@ -0,0 +1,13 @@
+import { Liquid } from '../..'
+import { expect, use } from 'chai'
+import * as chaiAsPromised from 'chai-as-promised'
+
+use(chaiAsPromised)
+
+describe('Issues', function () {
+  it('#221 unicode blanks are not properly treated', async () => {
+    const engine = new Liquid({ strictVariables: true, strictFilters: true })
+    const html = engine.parseAndRenderSync('{{huh | truncate: 11}}', { huh: 'fdsafdsafdsafdsaaaaa' })
+    expect(html).to.equal('fdsafdsa...')
+  })
+})