From 8daf281fca939b67fdb4155815519f038eae9b0e Mon Sep 17 00:00:00 2001
From: harttle <yangjvn@126.com>
Date: Tue, 31 Mar 2020 22:05:37 +0800
Subject: [PATCH] fix: throw an error if : omitted unintentionally, #212, #208

---
 README.md                                     | 14 ++-----------
 docs/source/_data/sidebar.yml                 |  1 +
 docs/source/tutorials/differences.md          | 21 +++++++++++++++++++
 docs/source/tutorials/intro-to-liquid.md      |  5 ++++-
 docs/source/tutorials/setup.md                |  6 ++++--
 docs/source/zh-cn/tutorials/differences.md    | 21 +++++++++++++++++++
 .../source/zh-cn/tutorials/intro-to-liquid.md |  5 ++++-
 docs/source/zh-cn/tutorials/setup.md          |  6 ++++--
 docs/themes/navy/languages/en.yml             |  1 +
 docs/themes/navy/languages/zh-cn.yml          |  1 +
 src/parser/tokenizer.ts                       | 15 +++++++------
 src/template/value.ts                         |  1 +
 test/integration/builtin/filters/array.ts     |  9 +++++++-
 test/unit/parser/tokenizer.ts                 |  5 +++++
 14 files changed, 86 insertions(+), 25 deletions(-)
 create mode 100644 docs/source/tutorials/differences.md
 create mode 100644 docs/source/zh-cn/tutorials/differences.md

diff --git a/README.md b/README.md
index 92bceadcf..0c13540bb 100644
--- a/README.md
+++ b/README.md
@@ -19,6 +19,8 @@ A simple, expressive, safe and [Shopify][shopify/liquid] compatible template eng
 * Please star [LiquidJS on GitHub][github]!
 * Support [LiquidJS on Open Collective][oc] or [Patreon][patreon]
 
+<p align="center"><a href="https://liquidjs.com"><img height="155px" width="155px" src="https://liquidjs.com/icon/mstile-310x310.png" alt="logo"></a></p>
+
 ## Installation
 
 Install from npm in Node.js:
@@ -35,18 +37,6 @@ Or use the UMD bundle from jsDelivr:
 
 More details, refer to [The Setup Guide][setup].
 
-## Differences with Shopify/liquid
-
-All features, filters and tags in [shopify/liquid](https://github.com/Shopify/liquid) are supposed to be built in LiquidJS, but not those business-logic specific filters/tags which are typically from Shopify themes (see [Plugins List][plugins] in case you're looking for them and feel free to add yours to the list). Though being compatible the Ruby version is one of our priorities, there are still some differences:
-
-* Dynamic file locating (enabled by default), that means layout/partial names are treated as variables in liquidjs. See [#51](https://github.com/harttle/liquidjs/issues/51).
-* Truthy and Falsy. All values except `undefined`, `null`, `false` are truthy, whereas in Ruby Liquid all except `nil` and `false` are truthy. See [#26](https://github.com/harttle/liquidjs/pull/26).
-* Number. In JavaScript we cannot distinguish or convert between `float` and `integer`, see [#59](https://github.com/harttle/liquidjs/issues/59). And when applied `size` filter, numbers always return 0, which is 8 for integer in ruby, cause they do not have a `length` property.
-* [.to_liquid()](https://github.com/Shopify/liquid/wiki/Introduction-to-Drops) is replaced by `.toLiquid()`
-* [.to_s()](https://www.rubydoc.info/gems/liquid/Liquid/Drop) is replaced by JavaScript `.toString()`
-* Iteration order for objects. The iteration order of JavaScript objects, and thus LiquidJS objects, is a combination of the insertion order for string keys, and ascending order for number-like keys, while the iteration order of Ruby Hash is simply the insertion order.
-* Sort stability. The [sort](https://shopify.github.io/liquid/filters/sort/) stability is also not defined in both shopify/liquid and LiquidJS, but it's [considered stable](https://v8.dev/features/stable-sort) for LiquidJS in Node.js 12+ and Google Chrome 70+.
-
 ## Related Packages
 
 * [gulp-liquidjs](https://www.npmjs.com/package/@tuanpham-dev/gulp-liquidjs): A shopify compatible Liquid template engine for Gulp using liquidjs.
diff --git a/docs/source/_data/sidebar.yml b/docs/source/_data/sidebar.yml
index 3418815fb..3fad6e946 100644
--- a/docs/source/_data/sidebar.yml
+++ b/docs/source/_data/sidebar.yml
@@ -16,6 +16,7 @@ tutorials:
   miscellaneous:
     migration9: migrate-to-9.html
     changelog: changelog.html
+    differences: differences.html
     contribution_guidelines: contribution-guidelines.html
 
 filters:
diff --git a/docs/source/tutorials/differences.md b/docs/source/tutorials/differences.md
new file mode 100644
index 000000000..421042ae9
--- /dev/null
+++ b/docs/source/tutorials/differences.md
@@ -0,0 +1,21 @@
+---
+title: Differences with Shopify/liquid
+---
+
+All filters and tags in [shopify/liquid](https://github.com/Shopify/liquid) are supposed to be built in LiquidJS, but not those business-logic specific tags/filters which are typically from Shopify themes (see [Plugins List][plugins] in case you're looking for them and feel free to add yours to the list). Though being compatible the Ruby version is one of our priorities, there are still some differences:
+
+* Truthy and Falsy. All values except `undefined`, `null`, `false` are truthy, whereas in Ruby Liquid all except `nil` and `false` are truthy. See [#26][#26].
+* Number. In JavaScript we cannot distinguish or convert between `float` and `integer`, see [#59][#59]. And when applied `size` filter, numbers always return 0, which is 8 for integer in ruby, cause they do not have a `length` property.
+* [.to_liquid()](https://github.com/Shopify/liquid/wiki/Introduction-to-Drops) is replaced by `.toLiquid()`
+* [.to_s()](https://www.rubydoc.info/gems/liquid/Liquid/Drop) is replaced by JavaScript `.toString()`
+* Iteration order for objects. The iteration order of JavaScript objects, and thus LiquidJS objects, is a combination of the insertion order for string keys, and ascending order for number-like keys, while the iteration order of Ruby Hash is simply the insertion order.
+* Sort stability. The [sort][sort] stability is also not defined in both shopify/liquid and LiquidJS, but it's [considered stable][stable-sort] for LiquidJS in Node.js 12+ and Google Chrome 70+.
+* Trailing unmatched characters inside filters are allowed in shopify/liquid but not in LiquidJS. It means filter arguments without a colon like `{%raw%}{{ "a b" | split " "}}{%endraw%}` will throw an error in LiquidJS. This is intended to improve Liquid usability, see [#208][#208] and [#212][#212].
+
+[#26]: https://github.com/harttle/liquidjs/pull/26
+[#59]: https://github.com/harttle/liquidjs/issues/59
+[#208]: https://github.com/harttle/liquidjs/issues/208
+[#212]: https://github.com/harttle/liquidjs/issues/212
+[sort]: https://liquidjs.com/filters/sort.html
+[stable-sort]: https://v8.dev/features/stable-sort
+[plugins]: ./plugins.html#Plugin-List
diff --git a/docs/source/tutorials/intro-to-liquid.md b/docs/source/tutorials/intro-to-liquid.md
index 16ff6fa4d..8e83cccb9 100644
--- a/docs/source/tutorials/intro-to-liquid.md
+++ b/docs/source/tutorials/intro-to-liquid.md
@@ -1,7 +1,9 @@
 ---
-title: Basic Syntax
+title: Introduction to Liquid Template Language
 ---
 
+LiquidJS is a simple, expressive, safe and shopify compatible template engine in pure JavaScript. The purpose of this repo is to provide a standard Liquid implementation for the JavaScript community. Liquid is originally implemented in Ruby and used by Github Pages, Jekyll and Shopify, see [Differences with Shopify/liquid][diff].
+
 LiquidJS syntax is relatively simple. There're 2 types of markups in LiquidJS:
 
 - **Tags**. A tag consists of a tag name and optional arguments wrapped between `{%raw%}{%{%endraw%}` and `%}`.
@@ -50,3 +52,4 @@ Typically tags appear in pairs with a start tag and a corresponding end tag. For
 A complete list of tags supported by LiquidJS can be found [here](../tags/overview.html).
 
 [shopify/liquid]: https://github.com/Shopify/liquid
+[diff]: ./differences.html
diff --git a/docs/source/tutorials/setup.md b/docs/source/tutorials/setup.md
index 26563f2a2..ae5e35aed 100644
--- a/docs/source/tutorials/setup.md
+++ b/docs/source/tutorials/setup.md
@@ -2,7 +2,7 @@
 title: Setup
 ---
 
-LiquidJS is a simple, expressive, safe and shopify compatible template engine in pure JavaScript. The purpose of this repo is to provide a standard Liquid implementation for the JavaScript community.
+In case you're not familiar with Liquid Template Language, see [Introduction to Liquid Template Language][intro].
 
 ## LiquidJS in Node.js
 
@@ -67,4 +67,6 @@ echo 'Hello, {{ name }}.' | npx liquidjs '{"name": "Snake"}'
 
 ## Miscellaneous
 
-A ReactJS demo is also added by [@stevenanthonyrevo](https://github.com/stevenanthonyrevo), see [liquidjs/demo/reactjs/](https://github.com/harttle/liquidjs/blob/master/demo/reactjs/).
\ No newline at end of file
+A ReactJS demo is also added by [@stevenanthonyrevo](https://github.com/stevenanthonyrevo), see [liquidjs/demo/reactjs/](https://github.com/harttle/liquidjs/blob/master/demo/reactjs/).
+
+[intro]: ./intro-to-liquid.html
\ No newline at end of file
diff --git a/docs/source/zh-cn/tutorials/differences.md b/docs/source/zh-cn/tutorials/differences.md
new file mode 100644
index 000000000..296b9c109
--- /dev/null
+++ b/docs/source/zh-cn/tutorials/differences.md
@@ -0,0 +1,21 @@
+---
+title: 和 Shopify/liquid 的区别
+---
+
+[Shopify/liquid](https://github.com/Shopify/liquid) 中的所有标签和过滤器 LiquidJS 都支持，但不包括 Shopify 主题中业务逻辑相关的标签和过滤器（如果你在找这些标签可以参考 [插件列表][plugins]，也欢迎把你的插件添加到列表中）。尽管原则上我们尽力兼容于 Shopify/liquid，但仍然存在一些区别：
+
+* 真和假。在 LiquidJS 中 `undefined`, `null`, `false` 是假，之外的都是真；在 Ruby 中 `nil` 和 `false` 是假，其他都是真。见 [#26][#26]。
+* 数字。JavaScript 不区分浮点数和整数，因此缺失一部分整数算术，见 [#59][#59]。此外 `size` 过滤器作用于数字时总是返回零，而不是 Ruby 中的浮点数或整数的内存大小。
+* Drop 中的 [.to_liquid()](https://github.com/Shopify/liquid/wiki/Introduction-to-Drops) 替换为 `.toLiquid()`。
+* 数据的 [.to_s()](https://www.rubydoc.info/gems/liquid/Liquid/Drop) 替换为 `.toString()`。
+* 对象的迭代顺序。JavaScript 对象的迭代顺序是插入顺序和数字键递增顺序的组合，但 Ruby Hash 中只是插入顺序（JavaScript 字面量 Object 和 Ruby 字面量 Hash 的插入顺序解释也不同）。
+* 排序稳定性。shopify/liquid 和 LiquidJS 都没有定义 [sort][sort] 过滤器的稳定性在，它取决于 Ruby/JavaScript 内置的排序算法，在 Node.js 12+ 和 Google Chrome 70+ LiquidJS 的排序是 [稳定的][stable-sort]。
+* shopify/liquid 允许过滤器尾部的未匹配字符，但 LiquidJS 不允许。这就是说如果过滤器参数前忘记写冒号比如 `{%raw%}{{ "a b" | split " "}}{%endraw%}` LiquidJS 会抛出异常。这是为了提升 Liquid 模板的易用性，参考 [#208][#208] 和 [#212][#212]。
+
+[#26]: https://github.com/harttle/liquidjs/pull/26
+[#59]: https://github.com/harttle/liquidjs/issues/59
+[#208]: https://github.com/harttle/liquidjs/issues/208
+[#212]: https://github.com/harttle/liquidjs/issues/212
+[sort]: https://liquidjs.com/filters/sort.html
+[stable-sort]: https://v8.dev/features/stable-sort
+[plugins]: ./plugins.html#插件列表
diff --git a/docs/source/zh-cn/tutorials/intro-to-liquid.md b/docs/source/zh-cn/tutorials/intro-to-liquid.md
index 956189454..025a97582 100644
--- a/docs/source/zh-cn/tutorials/intro-to-liquid.md
+++ b/docs/source/zh-cn/tutorials/intro-to-liquid.md
@@ -1,7 +1,9 @@
 ---
-title: 基本语法
+title: Liquid 模板语言简介
 ---
 
+LiquidJS 是一个简单的、安全的、兼容 Shopify 的、纯 JavaScript 编写的模板引擎。这个项目的目的是为 JavaScript 社区提供一个 Liquid 模板引擎的实现。Liquid 最初用 Ruby 实现并用于 Github Pages, Jekyll 和 Shopify，参考 [和 Shopify/liquid 的区别][diff]。
+
 LiquidJS 语法相对简单。LiquidJS 中有两种标记：
 
 - **标签**。标签由标签名和参数构成，由 `{%raw%}{%{%endraw%}` 和 `%}` 包裹。
@@ -50,3 +52,4 @@ LiquidJS 语法相对简单。LiquidJS 中有两种标记：
 [这里](../tags/overview.html) 是 LiquidJS 支持的完整的标签列表。
 
 [shopify/liquid]: https://github.com/Shopify/liquid
+[diff]: ./differences.html
diff --git a/docs/source/zh-cn/tutorials/setup.md b/docs/source/zh-cn/tutorials/setup.md
index 84b963f06..d1223c81c 100644
--- a/docs/source/zh-cn/tutorials/setup.md
+++ b/docs/source/zh-cn/tutorials/setup.md
@@ -2,7 +2,7 @@
 title: 安装和使用
 ---
 
-LiquidJS 是一个简单的、安全的、兼容 Shopify 的、纯 JavaScript 编写的模板引擎。这个项目的目的是为 JavaScript 社区提供一个 Liquid 模板引擎的实现。
+如果你还不了解 Liquid 模板语言，请参考 [Liquid 模板语言简介][intro]。
 
 ## 在 Node.js 里使用
 
@@ -65,4 +65,6 @@ echo 'Hello, {{ name }}.' | npx liquidjs '{"name": "Snake"}'
 
 ## 其他
 
-[@stevenanthonyrevo](https://github.com/stevenanthonyrevo) 还提供了一个 ReactJS demo，请参考 [liquidjs/demo/reactjs/](https://github.com/harttle/liquidjs/blob/master/demo/reactjs/)。
\ No newline at end of file
+[@stevenanthonyrevo](https://github.com/stevenanthonyrevo) 还提供了一个 ReactJS demo，请参考 [liquidjs/demo/reactjs/](https://github.com/harttle/liquidjs/blob/master/demo/reactjs/)。
+
+[intro]: ./intro-to-liquid.html
\ No newline at end of file
diff --git a/docs/themes/navy/languages/en.yml b/docs/themes/navy/languages/en.yml
index 520a5a005..dcc80780c 100644
--- a/docs/themes/navy/languages/en.yml
+++ b/docs/themes/navy/languages/en.yml
@@ -50,6 +50,7 @@ sidebar:
     migration9: 'Migrate to LiquidJS 9'
     contribution_guidelines: 'Contribution Guidelines'
     changelog: 'Changelog'
+    differences: Differences with Shopify/liquid
   filters:
     overview: Overview
   tags:
diff --git a/docs/themes/navy/languages/zh-cn.yml b/docs/themes/navy/languages/zh-cn.yml
index 7f7f0cfc1..e2f773c1e 100644
--- a/docs/themes/navy/languages/zh-cn.yml
+++ b/docs/themes/navy/languages/zh-cn.yml
@@ -49,6 +49,7 @@ sidebar:
     miscellaneous: 其他
     migration9: '迁移到 LiquidJS 9'
     contribution_guidelines: '贡献指南'
+    differences: 与 Shopify/liquid 的不同
     changelog: '更新日志'
   filters:
     overview: 概述
diff --git a/src/parser/tokenizer.ts b/src/parser/tokenizer.ts
index 320a41e77..cd9b866df 100644
--- a/src/parser/tokenizer.ts
+++ b/src/parser/tokenizer.ts
@@ -65,7 +65,10 @@ export class Tokenizer {
     }
   }
   readFilter (): FilterToken | null {
-    this.readTo('|')
+    this.skipBlank()
+    if (this.end()) return null
+    assert(this.peek() === '|', () => `unexpected token at ${this.snapshot()}`)
+    this.p++
     const begin = this.p
     const name = this.readWord()
     if (!name.size()) return null
@@ -125,7 +128,7 @@ export class Tokenizer {
     const { tagDelimiterRight } = options
     const begin = this.p
     if (this.readTo(tagDelimiterRight) === -1) {
-      this.mkError(`tag "${this.ellipsis(begin)}" not closed`, begin)
+      this.mkError(`tag ${this.snapshot(begin)} not closed`, begin)
     }
     return new TagToken(input, begin, this.p, options, file)
   }
@@ -135,7 +138,7 @@ export class Tokenizer {
     const { outputDelimiterRight } = options
     const begin = this.p
     if (this.readTo(outputDelimiterRight) === -1) {
-      this.mkError(`output "${this.ellipsis(begin)}" not closed`, begin)
+      this.mkError(`output ${this.snapshot(begin)} not closed`, begin)
     }
     return new OutputToken(input, begin, this.p, options, file)
   }
@@ -144,8 +147,8 @@ export class Tokenizer {
     throw new TokenizationError(msg, new WordToken(this.input, begin, this.N, this.file))
   }
 
-  ellipsis (begin: number = this.p) {
-    return ellipsis(this.input.slice(begin), 16)
+  snapshot (begin: number = this.p) {
+    return JSON.stringify(ellipsis(this.input.slice(begin), 16))
   }
 
   readWord (): WordToken { // rename to identifier
@@ -245,7 +248,7 @@ export class Tokenizer {
 
   readValueOrThrow (): ValueToken {
     const value = this.readValue()
-    assert(value, () => `unexpected token "${this.ellipsis()}", value expected`)
+    assert(value, () => `unexpected token ${this.snapshot()}, value expected`)
     return value!
   }
 
diff --git a/src/template/value.ts b/src/template/value.ts
index e8ef76563..36847ab8d 100644
--- a/src/template/value.ts
+++ b/src/template/value.ts
@@ -16,6 +16,7 @@ export class Value {
     const tokenizer = new Tokenizer(str)
     this.initial = tokenizer.readValue()
     this.filters = tokenizer.readFilters().map(({ name, args }) => new Filter(name, this.filterMap.get(name), args))
+    tokenizer.skipBlank()
   }
   public * value (ctx: Context) {
     let val = yield evalToken(this.initial, ctx)
diff --git a/test/integration/builtin/filters/array.ts b/test/integration/builtin/filters/array.ts
index 3d47dab87..d615082b8 100644
--- a/test/integration/builtin/filters/array.ts
+++ b/test/integration/builtin/filters/array.ts
@@ -1,6 +1,8 @@
 import { test } from '../../../stub/render'
 import { Liquid } from '../../../../src/liquid'
-import { expect } from 'chai'
+import { expect, use } from 'chai'
+import * as chaiAsPromised from 'chai-as-promised'
+use(chaiAsPromised)
 
 describe('filters/array', function () {
   let liquid: Liquid
@@ -25,6 +27,11 @@ describe('filters/array', function () {
         '{{ beatles | join }}'
       return test(src, 'John Paul George Ringo')
     })
+    it('should throw when comma missing', async () => {
+      const src = '{% assign beatles = "John, Paul, George, Ringo" | split: ", " %}' +
+        '{{ beatles | join " and " }}'
+      return expect(liquid.parseAndRender(src)).to.be.rejectedWith('unexpected token at "\\" and \\"", line:1, col:65')
+    })
   })
   it('should support split/last', function () {
     const src = '{% assign my_array = "zebra, octopus, giraffe, tiger" | split: ", " %}' +
diff --git a/test/unit/parser/tokenizer.ts b/test/unit/parser/tokenizer.ts
index ac4f7e2e3..97cd44cde 100644
--- a/test/unit/parser/tokenizer.ts
+++ b/test/unit/parser/tokenizer.ts
@@ -207,6 +207,11 @@ describe('Tokenize', function () {
       expect(token).to.have.property('name', 'plus')
       expect(token).to.have.property('args').to.deep.equal([])
     })
+    it('should read null if name not found', function () {
+      const tokenizer = new Tokenizer('|')
+      const token = tokenizer.readFilter()
+      expect(token).to.be.null
+    })
     it('should read a filter with k/v argument', function () {
       const tokenizer = new Tokenizer(' | plus: a:1')
       const token = tokenizer.readFilter()