feat: array_to_sentence_string and number_of_words filters from Jekyll, #443

docs/source/_data/sidebar.yml

@@ -26,6 +26,7 @@ tutorials:
     contribution_guidelines: contribution-guidelines.html
 
 filters:
+  array_to_sentence_string: array_to_sentence_string.html
   overview: overview.html
   abs: abs.html
   append: append.html
@@ -33,6 +34,7 @@ filters:
   at_most: at_most.html
   capitalize: capitalize.html
   ceil: ceil.html
+  cgi_escape: cgi_escape.html
   compact: compact.html
   concat: concat.html
   date: date.html
@@ -62,6 +64,7 @@ filters:
   modulo: modulo.html
   newline_to_br: newline_to_br.html
   normalize_whitespace: normalize_whitespace.html
+  number_of_words: number_of_words.html
   plus: plus.html
   pop: pop.html
   push: push.html
@@ -93,10 +96,12 @@ filters:
   uniq: uniq.html
   unshift: unshift.html
   upcase: upcase.html
+  uri_escape: uri_escape.html
   url_decode: url_decode.html
   url_encode: url_encode.html
   where: where.html
   where_exp: where_exp.html
+  xml_escape: xml_escape.html
 
 tags:
   overview: overview.html

docs/source/filters/array_to_sentence_string.md

@@ -0,0 +1,27 @@
+---
+title: array_to_sentence_string
+---
+
+{% since %}v10.13.0{% endsince %}
+
+Convert an array into a sentence. Useful for listing tags. Optional argument for connector.
+
+Input
+```liquid
+{{ "foo,bar,baz" | split: "," | array_to_sentence_string }}
+```
+
+Output
+```text
+foo, bar, and baz
+```
+
+Input
+```liquid
+{{ "foo,bar,baz" | split: "," | array_to_sentence_string: "or" }}
+```
+
+Output
+```text
+foo, bar, or baz
+```

docs/source/filters/cgi_escape.md

@@ -0,0 +1,17 @@
+---
+title: cgi_escape
+---
+
+{% since %}v10.13.0{% endsince %}
+
+CGI escape a string for use in a URL. Replaces any special characters with appropriate `%XX` replacements. CGI escape normally replaces a space with a plus `+` sign.
+
+Input
+```liquid
+{{ "foo, bar; baz?" | cgi_escape }}
+```
+
+Output
+```text
+foo%2C+bar%3B+baz%3F
+```

docs/source/filters/number_of_words.md

@@ -0,0 +1,49 @@
+---
+title: number_of_words
+---
+
+{% since %}v10.13.0{% endsince %}
+
+Count the number of words in some text. This filter takes an optional argument to control the handling of Chinese-Japanese-Korean (CJK) characters in the input string:
+- Passing 'cjk' as the argument will count every CJK character detected as one word irrespective of being separated by whitespace.
+- Passing 'auto' (auto-detect) works similar to 'cjk' but is more performant if the filter is used on a variable string that may or may not contain CJK chars.
+
+Input
+```liquid
+{{ "Hello world!" | number_of_words }}
+```
+
+Output
+```text
+2
+```
+
+Input
+```liquid
+{{ "你好hello世界world" | number_of_words }}
+```
+
+Output
+```text
+1
+```
+
+Input
+```liquid
+{{ "你好hello世界world" | number_of_words: "cjk" }}
+```
+
+Output
+```text
+6
+```
+
+Input
+```liquid
+{{ "你好hello世界world" | number_of_words: "auto" }}
+```
+
+Output
+```text
+6
+```

docs/source/filters/overview.md

@@ -10,8 +10,8 @@ There's 40+ filters supported by LiquidJS. These filters can be categorized into
 Categories | Filters
 --- | ---
 Math | plus, minus, modulo, times, floor, ceil, round, divided_by, abs, at_least, at_most
-String | append, prepend, capitalize, upcase, downcase, strip, lstrip, rstrip, strip_newlines, split, replace, replace_first, replace_last,remove, remove_first, remove_last, truncate, truncatewords, normalize_whitespace
-HTML/URI | escape, escape_once, url_encode, url_decode, strip_html, newline_to_br
+String | append, prepend, capitalize, upcase, downcase, strip, lstrip, rstrip, strip_newlines, split, replace, replace_first, replace_last,remove, remove_first, remove_last, truncate, truncatewords, normalize_whitespace, number_of_words, array_to_sentence_string
+HTML/URI | escape, escape_once, url_encode, url_decode, strip_html, newline_to_br, xml_escape, cgi_escape, uri_escape
 Array | slice, map, sort, sort_natural, uniq, where, where_exp, group_by, group_by_exp, find, find_exp, first, last, join, reverse, concat, compact, size, push, pop, shift, unshift
 Date | date, date_to_xmlschema, date_to_rfc822, date_to_string, date_to_long_string
 Misc | default, json, jsonify, inspect, raw, to_integer

docs/source/filters/uri_escape.md

@@ -0,0 +1,19 @@
+---
+title: uri_escape
+---
+
+{% since %}v10.13.0{% endsince %}
+
+Percent encodes any special characters in a URI. URI escape normally replaces a space with `%20`. [Reserved characters][reserved] will not be escaped.
+
+Input
+```liquid
+{{ "http://foo.com/?q=foo, \bar?" | uri_escape }}
+```
+
+Output
+```text
+http://foo.com/?q=foo,%20%5Cbar?
+```
+
+[reserved]: https://en.wikipedia.org/wiki/Percent-encoding#Types_of_URI_characters

docs/source/filters/xml_escape.md

@@ -0,0 +1,17 @@
+---
+title: xml_escape
+---
+
+{% since %}v10.13.0{% endsince %}
+
+Escape some text for use in XML.
+
+Input
+```liquid
+{{ "Have you read \'James & the Giant Peach\'?" | xml_escape }}
+```
+
+Output
+```text
+Have you read 'James & the Giant Peach'?
+```

docs/source/zh-cn/filters/array_to_sentence_string.md

@@ -0,0 +1,27 @@
+---
+title: array_to_sentence_string
+---
+
+{% since %}v10.13.0{% endsince %}
+
+把数组转化为句子，用于做标签列表。有一个可选的连接词参数。
+
+输入
+```liquid
+{{ "foo,bar,baz" | split: "," | array_to_sentence_string }}
+```
+
+输出
+```text
+foo, bar, and baz
+```
+
+输入
+```liquid
+{{ "foo,bar,baz" | split: "," | array_to_sentence_string: "or" }}
+```
+
+输出
+```text
+foo, bar, or baz
+```

docs/source/zh-cn/filters/cgi_escape.md

@@ -0,0 +1,17 @@
+---
+title: cgi_escape
+---
+
+{% since %}v10.13.0{% endsince %}
+
+把字符串 CGI 转义，用于 URL。用对应的 `%XX` 替换特殊字符，空格会被转义为 `+` 号。
+
+输入
+```liquid
+{{ "foo, bar; baz?" | cgi_escape }}
+```
+
+输出
+```text
+foo%2C+bar%3B+baz%3F
+```

docs/source/zh-cn/filters/number_of_words.md

@@ -0,0 +1,49 @@
+---
+title: number_of_words
+---
+
+{% since %}v10.13.0{% endsince %}
+
+计算文本中的单词数。此过滤器接受一个可选参数，用于控制输入字符串中汉字-日语-韩语（CJK）字符的处理方式：
+- 将 'cjk' 作为参数传递将会将每个检测到的 CJK 字符计为一个单词，无论是否由空格分隔。
+- 将 'auto' （自动检测）作为参数传递与 'cjk' 类似，但如果过滤器用于可能包含或不包含 CJK 字符的字符串，则性能更好。
+
+Input
+```liquid
+{{ "Hello world!" | number_of_words }}
+```
+
+Output
+```text
+2
+```
+
+Input
+```liquid
+{{ "你好hello世界world" | number_of_words }}
+```
+
+Output
+```text
+1
+```
+
+Input
+```liquid
+{{ "你好hello世界world" | number_of_words: "cjk" }}
+```
+
+Output
+```text
+6
+```
+
+Input
+```liquid
+{{ "你好hello世界world" | number_of_words: "auto" }}
+```
+
+Output
+```text
+6
+```

docs/source/zh-cn/filters/overview.md

@@ -10,8 +10,8 @@ LiquidJS 共支持 40+ 个过滤器，可以分为如下几类：
 类别 | 过滤器
 --- | ---
 数学 | plus, minus, modulo, times, floor, ceil, round, divided_by, abs, at_least, at_most
-字符串 | append, prepend, capitalize, upcase, downcase, strip, lstrip, rstrip, strip_newlines, split, replace, replace_first, replace_last, remove, remove_first, remove_last, truncate, truncatewords, normalize_whitespace
-HTML/URI | escape, escape_once, url_encode, url_decode, strip_html, newline_to_br
+字符串 | append, prepend, capitalize, upcase, downcase, strip, lstrip, rstrip, strip_newlines, split, replace, replace_first, replace_last, remove, remove_first, remove_last, truncate, truncatewords, normalize_whitespace, number_of_words, array_to_sentence_string
+HTML/URI | escape, escape_once, url_encode, url_decode, strip_html, newline_to_br, xml_escape, cgi_escape, uri_escape
 数组 | slice, map, sort, sort_natural, uniq, where, where_exp, group_by, group_by_exp, find, find_exp, first, last, join, reverse, concat, compact, size, push, pop, shift, unshift
 日期 | date, date_to_xmlschema, date_to_rfc822, date_to_string, date_to_long_string
 其他 | default, json, jsonify, inspect, raw, to_integer

docs/source/zh-cn/filters/uri_escape.md

@@ -0,0 +1,19 @@
+---
+title: uri_escape
+---
+
+{% since %}v10.13.0{% endsince %}
+
+把 URI 中的特殊字符做百分号编码，空格会变成 `%20`。[保留字][reserved] 不会被转义。
+
+输入
+```liquid
+{{ "http://foo.com/?q=foo, \bar?" | uri_escape }}
+```
+
+输出
+```text
+http://foo.com/?q=foo,%20%5Cbar?
+```
+
+[reserved]: https://en.wikipedia.org/wiki/Percent-encoding#Types_of_URI_characters

docs/source/zh-cn/filters/xml_escape.md

@@ -0,0 +1,17 @@
+---
+title: xml_escape
+---
+
+{% since %}v10.13.0{% endsince %}
+
+把文本做 XML 转义。
+
+输入
+```liquid
+{{ "Have you read \'James & the Giant Peach\'?" | xml_escape }}
+```
+
+输出
+```text
+Have you read 'James & the Giant Peach'?
+```

src/filters/html.ts

@@ -19,6 +19,10 @@ export function escape (str: string) {
   return stringify(str).replace(/&|<|>|"|'/g, m => escapeMap[m])
 }
 
+export function xml_escape (str: string) {
+  return escape(str)
+}
+
 function unescape (str: string) {
   return stringify(str).replace(/&(amp|lt|gt|#34|#39);/g, m => unescapeMap[m])
 }

src/filters/string.ts

@@ -3,6 +3,18 @@
  *
  * * prefer stringify() to String() since `undefined`, `null` should eval ''
  */
+
+// Han (Chinese) characters: \u4E00-\u9FFF
+// Additional Han characters: \uF900-\uFAFF (CJK Compatibility Ideographs)
+// Additional Han characters: \u3400-\u4DBF (CJK Unified Ideographs Extension A)
+// Katakana (Japanese): \u30A0-\u30FF
+// Hiragana (Japanese): \u3040-\u309F
+// Hangul (Korean): \uAC00-\uD7AF
+const rCJKWord = /[\u4E00-\u9FFF\uF900-\uFAFF\u3400-\u4DBF\u3040-\u309F\u30A0-\u30FF\uAC00-\uD7AF]/gu;
+
+// Word boundary followed by word characters (for detecting words)
+const rNonCJKWord = /[^\u4E00-\u9FFF\uF900-\uFAFF\u3400-\u4DBF\u3040-\u309F\u30A0-\u30FF\uAC00-\uD7AF\s]+/gu;
+
 import { assert, escapeRegExp, stringify } from '../util'
 
 export function append (v: string, arg: string) {
@@ -32,16 +44,16 @@ export function upcase (str: string) {
 }
 
 export function remove (v: string, arg: string) {
-  return stringify(v).split(String(arg)).join('')
+  return stringify(v).split(stringify(arg)).join('')
 }
 
 export function remove_first (v: string, l: string) {
-  return stringify(v).replace(String(l), '')
+  return stringify(v).replace(stringify(l), '')
 }
 
 export function remove_last (v: string, l: string) {
   const str = stringify(v)
-  const pattern = String(l)
+  const pattern = stringify(l)
   const index = str.lastIndexOf(pattern)
   if (index === -1) return str
   return str.substring(0, index) + str.substring(index + pattern.length)
@@ -56,7 +68,7 @@ export function rstrip (str: string, chars?: string) {
 }
 
 export function split (v: string, arg: string) {
-  const arr = stringify(v).split(String(arg))
+  const arr = stringify(v).split(stringify(arg))
   // align to ruby split, which is the behavior of shopify/liquid
   // see: https://ruby-doc.org/core-2.4.0/String.html#method-i-split
   while (arr.length && arr[arr.length - 1] === '') arr.pop()
@@ -83,19 +95,19 @@ export function capitalize (str: string) {
 }
 
 export function replace (v: string, pattern: string, replacement: string) {
-  return stringify(v).split(String(pattern)).join(replacement)
+  return stringify(v).split(stringify(pattern)).join(replacement)
 }
 
 export function replace_first (v: string, arg1: string, arg2: string) {
-  return stringify(v).replace(String(arg1), arg2)
+  return stringify(v).replace(stringify(arg1), arg2)
 }
 
 export function replace_last (v: string, arg1: string, arg2: string) {
   const str = stringify(v)
-  const pattern = String(arg1)
+  const pattern = stringify(arg1)
   const index = str.lastIndexOf(pattern)
   if (index === -1) return str
-  const replacement = String(arg2)
+  const replacement = stringify(arg2)
   return str.substring(0, index) + replacement + str.substring(index + pattern.length)
 }
 
@@ -117,3 +129,34 @@ export function normalize_whitespace (v: string) {
   v = stringify(v)
   return v.replace(/\s+/g, ' ')
 }
+
+export function number_of_words(input: string, mode?: 'cjk' | 'auto') {
+  input = stringify(input).trim()
+  if (!input) return 0
+  switch (mode) {
+    case 'cjk':
+      // Count CJK characters and words
+      return (input.match(rCJKWord) || []).length + (input.match(rNonCJKWord) || []).length;
+    case 'auto':
+      // Count CJK characters, if none, count words
+      return rCJKWord.test(input)
+        ? input.match(rCJKWord)!.length + (input.match(rNonCJKWord) || []).length
+        : input.split(/\s+/).length
+    default:
+      // Count words only
+      return input.split(/\s+/).length;
+  }
+}
+
+export function array_to_sentence_string(array: unknown[], connector = "and") {
+  switch (array.length) {
+    case 0:
+      return ""
+    case 1:
+      return array[0]
+    case 2:
+      return `${array[0]} ${connector} ${array[1]}`;
+    default:
+      return `${array.slice(0, -1).join(", ")}, ${connector} ${array[array.length - 1]}`;
+  }
+}

src/filters/url.ts

@@ -1,4 +1,10 @@
 import { stringify } from '../util/underscore'
 
-export const url_decode = (x: string) => stringify(x).split('+').map(decodeURIComponent).join(' ')
-export const url_encode = (x: string) => stringify(x).split(' ').map(encodeURIComponent).join('+')
+export const url_decode = (x: string) => decodeURIComponent(stringify(x)).replace(/\+/g, ' ')
+export const url_encode = (x: string) => encodeURIComponent(stringify(x)).replace(/%20/g, '+')
+export const cgi_escape = (x: string) => encodeURIComponent(stringify(x))
+  .replace(/%20/g, '+')
+  .replace(/[!'()*]/g, c => '%' + c.charCodeAt(0).toString(16).toUpperCase())
+export const uri_escape = (x: string) => encodeURI(stringify(x))
+  .replace(/%5B/g, '[')
+  .replace(/%5D/g, ']')

test/integration/filters/html.spec.ts

@@ -26,6 +26,12 @@ describe('filters/html', function () {
     it('should escape nil value to empty string', () =>
       test('{{ undefinedValue | escape_once }}', ''))
   })
+  describe('xml_escape', function () {
+    it('should xml_escape \' and &', function () {
+      return test('{{ "Have you read \'James & the Giant Peach\'?" | xml_escape }}',
+        'Have you read 'James & the Giant Peach'?')
+    })
+  })
   describe('newline_to_br', function () {
     it('should support string_with_newlines', function () {
       const src = '{% capture string_with_newlines %}\n' +

test/integration/filters/string.spec.ts

@@ -238,4 +238,101 @@ describe('filters/string', function () {
       expect(liquid.parseAndRenderSync('{{ "a \n b  c" | normalize_whitespace }}')).toEqual('a b c')
     })
   })
+  describe('number_of_words', () => {
+    it('should count words of Latin sentence', async () => {
+      const html = await liquid.parseAndRender('{{ "I\'m not hungry" | number_of_words: "auto"}}')
+      expect(html).toEqual('3')
+    });
+
+    it('should count words of mixed sentence', async () => {
+      const html = await liquid.parseAndRender('{{ "Hello world!" | number_of_words }}');
+      expect(html).toEqual('2');
+    });
+
+    it('should count words of CJK sentence', async () => {
+      const html = await liquid.parseAndRender('{{ "你好hello世界world" | number_of_words }}');
+      expect(html).toEqual('1');
+    });
+
+    it('should count words of CJK sentence with mode "cjk"', async () => {
+      const html = await liquid.parseAndRender('{{ "你好hello世界world" | number_of_words: "cjk" }}');
+      expect(html).toEqual('6');
+    });
+
+    it('should count words of CJK sentence with mode "auto"', async () => {
+      const html = await liquid.parseAndRender('{{ "你好hello世界world" | number_of_words: "auto" }}');
+      expect(html).toEqual('6');
+    });
+    it('should handle empty input', async () => {
+      const html = await liquid.parseAndRender('{{ "" | number_of_words }}');
+      expect(html).toEqual('0');
+    });
+
+    it('should handle input with only whitespace', async () => {
+      const html = await liquid.parseAndRender('{{ "   " | number_of_words }}');
+      expect(html).toEqual('0');
+    });
+
+    it('should count words with punctuation marks', async () => {
+      const html = await liquid.parseAndRender('{{ "Hello! This is a test." | number_of_words }}');
+      expect(html).toEqual('5');
+    });
+
+    it('should count words with special characters', async () => {
+      const html = await liquid.parseAndRender('{{ "This is a test with special characters: !@#$%^&*()-_+=`~[]{};:\'\\"\\|<,>.?/" | number_of_words }}');
+      expect(html).toEqual('8');
+    });
+
+    it('should count words with multiple spaces between words', async () => {
+      const html = await liquid.parseAndRender('{{ "   Hello    world!    " | number_of_words }}');
+      expect(html).toEqual('2');
+    });
+
+    it('should count words with mixed CJK characters', async () => {
+      const html = await liquid.parseAndRender('{{ "你好こんにちは안녕하세요" | number_of_words: "cjk" }}');
+      expect(html).toEqual('12');
+    });
+  });
+  describe('array_to_sentence_string', () => {
+    it('should handle an empty array', async () => {
+      const html = await liquid.parseAndRender('{{ arr | array_to_sentence_string }}', { arr: [] })
+      expect(html).toEqual('')
+    })
+
+    it('should handle an array with one element', async () => {
+      const html = await liquid.parseAndRender('{{ arr | array_to_sentence_string }}', { arr: ["apple"] })
+      expect(html).toEqual('apple')
+    })
+
+    it('should handle an array with two elements', async () => {
+      const html = await liquid.parseAndRender('{{ arr | array_to_sentence_string }}', { arr: ["apple", "banana"] })
+      expect(html).toEqual('apple and banana')
+    })
+
+    it('should handle an array with more than two elements', async () => {
+      const html = await liquid.parseAndRender('{{ arr | array_to_sentence_string }}', { arr: ["apple", "banana", "orange"] })
+      expect(html).toEqual('apple, banana, and orange')
+    })
+
+    it('should handle an array with custom connector', async () => {
+      const html = await liquid.parseAndRender('{{ arr | array_to_sentence_string: "or" }}', { arr: ["apple", "banana", "orange"] })
+      expect(html).toEqual('apple, banana, or orange')
+    })
+
+    it('should handle an array of numbers', async () => {
+      const html = await liquid.parseAndRender('{{ arr | array_to_sentence_string }}', { arr: [1, 2, 3] })
+      expect(html).toEqual('1, 2, and 3')
+    })
+
+    it('should handle an array of mixed types', async () => {
+      const html = await liquid.parseAndRender('{{ arr | array_to_sentence_string }}', { arr: ["apple", 2, "orange"] })
+      expect(html).toEqual('apple, 2, and orange')
+    })
+
+    it('should handle an array of mixed types', async () => {
+      const html = await liquid.parseAndRender('{{ "foo,bar,baz" | split: "," | array_to_sentence_string }}')
+      expect(html).toEqual('foo, bar, and baz')
+    })
+    
+  })
 })

test/integration/filters/url.spec.ts

@@ -1,15 +1,45 @@
-import { test } from '../../stub/render'
+import { Liquid } from '../../../src'
 
-describe('filters/url', function () {
-  describe('url_decode', function () {
-    it('should decode %xx and +',
-      () => test('{{ "%27Stop%21%27+said+Fred" | url_decode }}', "'Stop!' said Fred"))
+describe('filters/url', () => {
+  const liquid = new Liquid()
+  describe('url_decode', () => {
+    it('should decode %xx and +', () => {
+      const html = liquid.parseAndRenderSync('{{ "%27Stop%21%27+said+Fred" | url_decode }}')
+      expect(html).toEqual("'Stop!' said Fred")
+    })
   })
 
-  describe('url_encode', function () {
-    it('should encode @',
-      () => test('{{ "[email protected]" | url_encode }}', 'john%40liquid.com'))
-    it('should encode <space>',
-      () => test('{{ "Tetsuro Takara" | url_encode }}', 'Tetsuro+Takara'))
+  describe('url_encode', () => {
+    it('should encode @', () => {
+      const html = liquid.parseAndRenderSync('{{ "[email protected]" | url_encode }}')
+      expect(html).toEqual('john%40liquid.com')
+    })
+    it('should encode <space>', () => {
+      const html = liquid.parseAndRenderSync('{{ "Tetsuro Takara" | url_encode }}')
+      expect(html).toEqual('Tetsuro+Takara')
+    })
+  })
+
+  describe('cgi_escape', () => {
+    it('should escape CGI chars', () => {
+      const html = liquid.parseAndRenderSync('{{ "!\',()*\\"!" | cgi_escape }}')
+      expect(html).toEqual('%21%27%2C%28%29%2A%22%21')
+    })
+    it('should escape space as +', () => {
+      const html = liquid.parseAndRenderSync('{{ "foo, bar; baz?" | cgi_escape }}')
+      expect(html).toEqual('foo%2C+bar%3B+baz%3F')
+    })
+  })
+
+  describe('uri_escape', () => {
+    it('should escape unsupported chars for uri', () => {
+      const html = liquid.parseAndRenderSync('{{ "http://foo.com/?q=foo, \\\\bar?" | uri_escape }}')
+      expect(html).toEqual('http://foo.com/?q=foo,%20%5Cbar?')
+    })
+    it('should not escape reserved characters', () => {
+      const reserved = "!#$&'()*+,/:;=?@[]"
+      const html = liquid.parseAndRenderSync('{{ reserved | uri_escape }}', { reserved })
+      expect(html).toEqual(reserved)
+    })
   })
 })