fix: trim custom characters by Unicode code point (#955)

src/filters/string.ts

@@ -34,16 +34,32 @@ export function prepend (this: FilterImpl, v: string, arg: string) {
   return rhs + lhs
 }
 
+function getTrimStart (str: string, chars: Set<string>): number {
+  let start = 0
+  for (const char of str) {
+    if (!chars.has(char)) break
+    start += char.length
+  }
+  return start
+}
+
+function getTrimEnd (str: string, chars: Set<string>, start = 0): number {
+  let end = str.length
+  while (end > start) {
+    const size = end - start >= 2 && str.codePointAt(end - 2)! > 0xFFFF ? 2 : 1
+    if (!chars.has(str.slice(end - size, end))) break
+    end -= size
+  }
+  return end
+}
+
 export function lstrip (this: FilterImpl, v: string, chars?: string) {
   const str = stringify(v)
   this.context.memoryLimit.use(str.length)
   if (chars) {
     chars = stringify(chars)
     this.context.memoryLimit.use(chars.length)
-    for (let i = 0, set = new Set(chars); i < str.length; i++) {
+    return str.slice(getTrimStart(str, new Set(chars)))
-      if (!set.has(str[i])) return str.slice(i)
-    }
-    return ''
   }
   return str.trimStart()
 }
@@ -89,10 +105,7 @@ export function rstrip (this: FilterImpl, str: string, chars?: string) {
   if (chars) {
     chars = stringify(chars)
     this.context.memoryLimit.use(chars.length)
-    for (let i = str.length - 1, set = new Set(chars); i >= 0; i--) {
+    return str.slice(0, getTrimEnd(str, new Set(chars)))
-      if (!set.has(str[i])) return str.slice(0, i + 1)
-    }
-    return ''
   }
   return str.trimEnd()
 }
@@ -113,11 +126,8 @@ export function strip (this: FilterImpl, v: string, chars?: string) {
   if (chars) {
     const set = new Set(stringify(chars))
     this.context.memoryLimit.use(set.size)
-    let i = 0
+    const start = getTrimStart(str, set)
-    let j = str.length - 1
+    return str.slice(start, getTrimEnd(str, set, start))
-    while (set.has(str[i])) i++
-    while (j >= i && set.has(str[j])) j--
-    return str.slice(i, j + 1)
   }
   return str.trim()
 }

test/integration/filters/string.spec.ts

@@ -141,6 +141,23 @@ describe('filters/string', function () {
       'much room for activities!')
     await test('{{ "&[]{}" | strip: "&[]{}" }}', '')
   })
+  describe.each(['lstrip', 'rstrip', 'strip'])('%s with Unicode characters', filter => {
+    it.each([
+      ['𠮷𠮷text𠮷', '𠮷', ['text𠮷', '𠮷𠮷text', 'text']],
+      ['𠮷𠮶𠮷', '𠮷', ['𠮶𠮷', '𠮷𠮶', '𠮶']],
+      ['a𠮷text𠮷a', 'a𠮷', ['text𠮷a', 'a𠮷text', 'text']],
+      ['𠮷𠮷', '𠮷', ['', '', '']],
+      ['𠮶text𠮶', '𠮷', ['𠮶text𠮶', '𠮶text𠮶', '𠮶text𠮶']],
+      ['', '𠮷', ['', '', '']],
+      ['\uD842text\uD842', '\uD842', ['text\uD842', '\uD842text', 'text']],
+      ['\uD842𠮷x𠮷\uDFB7', '\uD842,\uDFB7', ['𠮷x𠮷\uDFB7', '\uD842𠮷x𠮷', '𠮷x𠮷']]
+    ])('should trim %s using %s', async (input, chars, outputs) => {
+      const expected = outputs[['lstrip', 'rstrip', 'strip'].indexOf(filter)]
+      const source = `{{ input | ${filter}: chars }}`
+      await test(source, { input, chars }, expected)
+      expect(liquid.parseAndRenderSync(source, { input, chars })).toBe(expected)
+    })
+  })
   it('should support strip_newlines', function () {
     return test('{% capture string_with_newlines %}\n' +
             'Hello\nthere\n{% endcapture %}' +