perf: introduce AST to avoid reparse

bin/char-types.js

@@ -1,43 +0,0 @@
-#!/usr/bin/env node
-
-function isQuote (c) {
-  return c === '"' || c === "'"
-}
-
-function isOperator (c) {
-  return '!=<>'.includes(c)
-}
-
-function isNumber (c) {
-  return c >= '0' && c <= '9'
-}
-
-function isCharacter (c) {
-  return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
-}
-
-function isVariable (c) {
-  return '_-?'.includes(c) || isCharacter(c) || isNumber(c)
-}
-
-function isBlank (c) {
-  return c === '\n' || c === '\t' || c === ' ' || c === '\r'
-}
-
-const types = []
-for (let i = 0; i < 128; i++) {
-  const c = String.fromCharCode(i)
-  let n = 0
-  if (isVariable(c)) n |= 1
-  if (isOperator(c)) n |= 2
-  if (isBlank(c)) n |= 4
-  if (isQuote(c)) n |= 8
-  types.push(n)
-}
-console.log(`
-const TYPES = '${types.join('')}'
-const VARIABLE = 1
-const OPERATOR = 2
-const BLANK = 4
-const QUOTE = 8
-`.trim())

bin/character-gen.js

@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+
+const isQuote = c => c === '"' || c === "'"
+const isOperator = c => '!=<>'.includes(c)
+const isNumber = c => c >= '0' && c <= '9'
+const isCharacter = c => (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
+const isVariable = c => '_-?'.includes(c) || isCharacter(c) || isNumber(c)
+const isBlank = c => c === '\n' || c === '\t' || c === ' ' || c === '\r'
+const isInlineBlank = c => c === '\t' || c === ' ' || c === '\r'
+const isSign = c => c === '-' || c === '+'
+
+const types = []
+for (let i = 0; i < 128; i++) {
+  const c = String.fromCharCode(i)
+  let n = 0
+  if (isVariable(c)) n |= 1
+  if (isOperator(c)) n |= 2
+  if (isBlank(c)) n |= 4
+  if (isQuote(c)) n |= 8
+  if (isInlineBlank(c)) n |= 16
+  if (isNumber(c)) n |= 32
+  if (isSign(c)) n |= 64
+  types.push(n)
+}
+console.log(`
+// bitmask character types to boost performance
+// generated by bin/character-gen.js
+export const TYPES = [${types.join(', ')}]
+export const VARIABLE = 1
+export const OPERATOR = 2
+export const BLANK = 4
+export const QUOTE = 8
+export const INLINE_BLANK = 16
+export const NUMBER = 32
+export const SIGN = 64
+`.trim())

src/builtin/filters/array.ts

@@ -31,7 +31,7 @@ function slice<T> (v: T[], begin: number, length = 1): T[] {
 
 function where<T extends object> (this: FilterImpl, arr: T[], property: string, expected?: any): T[] {
   return arr.filter(obj => {
-    const value = this.context.getFromScope(obj, property)
+    const value = this.context.getFromScope(obj, property.split('.'))
     return expected === undefined ? isTruthy(value) : value === expected
   })
 }

src/builtin/tags/assign.ts

@@ -1,15 +1,13 @@
-import { assert } from '../../util/assert'
+import { Tokenizer, assert, TagImplOptions, TagToken, Context } from '../../types'
-import { identifier } from '../../parser/lexical'
-import { TagImplOptions, TagToken, Context } from '../../types'
-
-const re = new RegExp(`(${identifier.source})\\s*=([^]*)`)
 
 export default {
   parse: function (token: TagToken) {
-    const match = token.args.match(re) as RegExpMatchArray
+    const tokenizer = new Tokenizer(token.args)
-    assert(match, `illegal token ${token.raw}`)
+    this.key = tokenizer.readWord().content
-    this.key = match[1]
+    tokenizer.skipBlank()
-    this.value = match[2]
+    assert(tokenizer.peek() === '=', () => `illegal token ${token.getText()}`)
+    tokenizer.advance()
+    this.value = tokenizer.remaining()
   },
   render: function * (ctx: Context) {
     ctx.bottom()[this.key] = yield this.liquid._evalValue(this.value, ctx)

src/builtin/tags/block.ts

@@ -1,8 +1,8 @@
 import BlockMode from '../../context/block-mode'
-import { ParseStream, TagToken, Token, Template, Context, TagImplOptions, Emitter } from '../../types'
+import { ParseStream, TagToken, TopLevelToken, Template, Context, TagImplOptions, Emitter } from '../../types'
 
 export default {
-  parse: function (token: TagToken, remainTokens: Token[]) {
+  parse: function (token: TagToken, remainTokens: TopLevelToken[]) {
     const match = /\w+/.exec(token.args)
     this.block = match ? match[0] : ''
     this.tpls = [] as Template[]
@@ -10,7 +10,7 @@ export default {
       .on('tag:endblock', () => stream.stop())
       .on('template', (tpl: Template) => this.tpls.push(tpl))
       .on('end', () => {
-        throw new Error(`tag ${token.raw} not closed`)
+        throw new Error(`tag ${token.getText()} not closed`)
       })
     stream.start()
   },

src/builtin/tags/capture.ts

@@ -1,22 +1,18 @@
-import { assert } from '../../util/assert'
+import { Tokenizer, assert, Template, Context, TagImplOptions, TagToken, TopLevelToken } from '../../types'
-import { identifier } from '../../parser/lexical'
-import { Template, Context, TagImplOptions, TagToken, Token } from '../../types'
-
-const re = new RegExp(`(${identifier.source})`)
 
 export default {
-  parse: function (tagToken: TagToken, remainTokens: Token[]) {
+  parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
-    const match = tagToken.args.match(re) as RegExpMatchArray
+    const tokenizer = new Tokenizer(tagToken.args)
-    assert(match, `${tagToken.args} not valid identifier`)
+    this.variable = tokenizer.readWord().content
+    assert(this.variable, () => `${tagToken.args} not valid identifier`)
 
-    this.variable = match[1]
     this.templates = []
 
     const stream = this.liquid.parser.parseStream(remainTokens)
     stream.on('tag:endcapture', () => stream.stop())
       .on('template', (tpl: Template) => this.templates.push(tpl))
       .on('end', () => {
-        throw new Error(`tag ${tagToken.raw} not closed`)
+        throw new Error(`tag ${tagToken.getText()} not closed`)
       })
     stream.start()
   },

src/builtin/tags/case.ts

@@ -1,7 +1,7 @@
-import { Expression, Emitter, TagToken, Token, Context, Template, TagImplOptions, ParseStream } from '../../types'
+import { Expression, Emitter, TagToken, TopLevelToken, Context, Template, TagImplOptions, ParseStream } from '../../types'
 
 export default {
-  parse: function (tagToken: TagToken, remainTokens: Token[]) {
+  parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
     this.cond = tagToken.args
     this.cases = []
     this.elseTemplates = []
@@ -18,7 +18,7 @@ export default {
       .on('tag:endcase', () => stream.stop())
       .on('template', (tpl: Template) => p.push(tpl))
       .on('end', () => {
-        throw new Error(`tag ${tagToken.raw} not closed`)
+        throw new Error(`tag ${tagToken.getText()} not closed`)
       })
 
     stream.start()

src/builtin/tags/comment.ts

@@ -1,16 +1,16 @@
-import { TagToken } from '../../parser/tag-token'
+import { TagToken } from '../../tokens/tag-token'
-import { Token } from '../../parser/token'
+import { TopLevelToken } from '../../tokens/toplevel-token'
 import { TagImplOptions } from '../../template/tag/tag-impl-options'
 
 export default {
-  parse: function (tagToken: TagToken, remainTokens: Token[]) {
+  parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
     const stream = this.liquid.parser.parseStream(remainTokens)
     stream
       .on('token', (token: TagToken) => {
         if (token.name === 'endcomment') stream.stop()
       })
       .on('end', () => {
-        throw new Error(`tag ${tagToken.raw} not closed`)
+        throw new Error(`tag ${tagToken.getText()} not closed`)
       })
     stream.start()
   }

src/builtin/tags/cycle.ts

@@ -1,28 +1,32 @@
 import { assert } from '../../util/assert'
-import { value as rValue } from '../../parser/lexical'
+import { evalToken, Emitter, TagToken, Context, TagImplOptions } from '../../types'
-import { Emitter, Expression, TagToken, Context, TagImplOptions } from '../../types'
+import { Tokenizer } from '../../parser/tokenizer'
-
-const groupRE = new RegExp(`^(?:(${rValue.source})\\s*:\\s*)?(.*)$`)
-const candidatesRE = new RegExp(rValue.source, 'g')
 
 export default {
   parse: function (tagToken: TagToken) {
-    let match: RegExpExecArray | null = groupRE.exec(tagToken.args) as RegExpExecArray
+    const tokenizer = new Tokenizer(tagToken.args)
-    assert(match, `illegal tag: ${tagToken.raw}`)
+    const group = tokenizer.readValue()
-
+    tokenizer.skipBlank()
-    this.group = new Expression(match[1])
-    const candidates = match[2]
 
     this.candidates = []
 
-    while ((match = candidatesRE.exec(candidates))) {
+    if (group) {
-      this.candidates.push(match[0])
+      if (tokenizer.peek() === ':') {
+        this.group = group
+        tokenizer.advance()
+      } else this.candidates.push(group)
     }
-    assert(this.candidates.length, `empty candidates: ${tagToken.raw}`)
+
+    while (!tokenizer.end()) {
+      const value = tokenizer.readValue()
+      if (value) this.candidates.push(value)
+      tokenizer.readTo(',')
+    }
+    assert(this.candidates.length, () => `empty candidates: ${tagToken.getText()}`)
   },
 
-  render: function * (ctx: Context, emitter: Emitter) {
+  render: function (ctx: Context, emitter: Emitter) {
-    const group = yield this.group.value(ctx)
+    const group = evalToken(this.group, ctx)
     const fingerprint = `cycle:${group}:` + this.candidates.join(',')
     const groups = ctx.getRegister('cycle')
     let idx = groups[fingerprint]
@@ -34,7 +38,7 @@ export default {
     const candidate = this.candidates[idx]
     idx = (idx + 1) % this.candidates.length
     groups[fingerprint] = idx
-    const html = yield new Expression(candidate).value(ctx)
+    const html = evalToken(candidate, ctx)
     emitter.write(html)
   }
 } as TagImplOptions

src/builtin/tags/decrement.ts

@@ -1,13 +1,10 @@
-import { assert } from '../../util/assert'
+import { Tokenizer, Emitter, TagToken, Context, TagImplOptions } from '../../types'
-import { identifier } from '../../parser/lexical'
-import { Emitter, TagToken, Context, TagImplOptions } from '../../types'
 import { isNumber, stringify } from '../../util/underscore'
 
 export default {
   parse: function (token: TagToken) {
-    const match = token.args.match(identifier) as RegExpMatchArray
+    const tokenizer = new Tokenizer(token.args)
-    assert(match, `illegal identifier ${token.args}`)
+    this.variable = tokenizer.readWord().content
-    this.variable = match[0]
   },
   render: function (context: Context, emitter: Emitter) {
     const scope = context.environments

src/builtin/tags/for.ts

@@ -1,21 +1,24 @@
-import { Emitter, TagToken, Token, Context, Template, TagImplOptions, ParseStream } from '../../types'
+import { assert, Tokenizer, evalToken, Emitter, TagToken, TopLevelToken, Context, Template, TagImplOptions, ParseStream } from '../../types'
 import { toCollection } from '../../util/collection'
-import { Expression } from '../../render/expression'
-import { assert } from '../../util/assert'
-import { identifier, value } from '../../parser/lexical'
 import { ForloopDrop } from '../../drop/forloop-drop'
 import { Hash } from '../../template/tag/hash'
 
-const re = new RegExp(`^(${identifier.source})\\s+in\\s+(${value.source})`)
-
 export default {
   type: 'block',
-  parse: function (tagToken: TagToken, remainTokens: Token[]) {
+  parse: function (token: TagToken, remainTokens: TopLevelToken[]) {
-    const match = re.exec(tagToken.args) as RegExpExecArray
+    const toknenizer = new Tokenizer(token.args)
-    assert(match, `illegal tag: ${tagToken.raw}`)
+
-    this.variable = match[1]
+    const variable = toknenizer.readWord()
-    this.collection = match[2]
+    const inStr = toknenizer.readWord()
-    this.hash = new Hash(tagToken.args.slice(match[0].length))
+    const collection = toknenizer.readValue()
+    assert(
+      variable.size() && inStr.content === 'in' && collection,
+      () => `illegal tag: ${token.getText()}`
+    )
+
+    this.variable = variable.content
+    this.collection = collection
+    this.hash = new Hash(toknenizer.remaining())
     this.templates = []
     this.elseTemplates = []
 
@@ -26,15 +29,14 @@ export default {
       .on('tag:endfor', () => stream.stop())
       .on('template', (tpl: Template) => p.push(tpl))
       .on('end', () => {
-        throw new Error(`tag ${tagToken.raw} not closed`)
+        throw new Error(`tag ${token.getText()} not closed`)
       })
 
     stream.start()
   },
   render: function * (ctx: Context, emitter: Emitter) {
     const r = this.liquid.renderer
-    let collection = yield new Expression(this.collection).value(ctx)
+    let collection = toCollection(evalToken(this.collection, ctx))
-    collection = toCollection(collection)
 
     if (!collection.length) {
       yield r.renderTemplates(this.elseTemplates, ctx, emitter)

src/builtin/tags/if.ts

@@ -1,7 +1,7 @@
-import { Emitter, isTruthy, Expression, TagToken, Token, Context, Template, TagImplOptions, ParseStream } from '../../types'
+import { Emitter, isTruthy, Expression, TagToken, TopLevelToken, Context, Template, TagImplOptions, ParseStream } from '../../types'
 
 export default {
-  parse: function (tagToken: TagToken, remainTokens: Token[]) {
+  parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
     this.branches = []
     this.elseTemplates = []
 
@@ -21,7 +21,7 @@ export default {
       .on('tag:endif', () => stream.stop())
       .on('template', (tpl: Template) => p.push(tpl))
       .on('end', () => {
-        throw new Error(`tag ${tagToken.raw} not closed`)
+        throw new Error(`tag ${tagToken.getText()} not closed`)
       })
 
     stream.start()

src/builtin/tags/include.ts

@@ -1,35 +1,41 @@
-import { assert } from '../../util/assert'
+import { assert, evalQuotedToken, TypeGuards, Tokenizer, evalToken, Hash, Emitter, TagToken, Context, TagImplOptions } from '../../types'
-import { Expression, Hash, Emitter, TagToken, Context, TagImplOptions } from '../../types'
-import { quoted, value, quotedLine } from '../../parser/lexical'
 import BlockMode from '../../context/block-mode'
 
-const rFile = new RegExp(`^(${quoted.source}|[^\\s,]+)(?:\\s+with\\s+(${value.source}))?`)
-
 export default {
   parse: function (token: TagToken) {
-    const match = rFile.exec(token.args)
+    const args = token.args
-    if (!match) {
+    const tokenizer = new Tokenizer(args)
-      throw new Error(`illegal argument "${token.args}"`)
+    this.file = this.liquid.options.dynamicPartials
-    }
+      ? tokenizer.readValue()
-    this.file = match[1]
+      : tokenizer.readFileName()
-    this.hash = new Hash(token.args.slice(match[0].length))
+    assert(this.file, () => `illegal argument "${token.args}"`)
-    this.withVar = match[2]
+
+    const begin = tokenizer.p
+    const withStr = tokenizer.readWord()
+    if (withStr.content === 'with') {
+      tokenizer.skipBlank()
+      if (tokenizer.peek() !== ':') {
+        this.withVar = tokenizer.readValue()
+      } else tokenizer.p = begin
+    } else tokenizer.p = begin
+
+    this.hash = new Hash(tokenizer.remaining())
   },
   render: function * (ctx: Context, emitter: Emitter) {
     const { liquid, hash, withVar, file } = this
     const { renderer } = liquid
     const filepath = ctx.opts.dynamicPartials
-      ? (quotedLine.exec(file)
+      ? (TypeGuards.isQuotedToken(file)
-        ? yield renderer.renderTemplates(liquid.parse(file.slice(1, -1)), ctx)
+        ? yield renderer.renderTemplates(liquid.parse(evalQuotedToken(file)), ctx)
-        : yield new Expression(file).value(ctx))
+        : yield evalToken(file, ctx))
-      : file
+      : file.getText()
-    assert(filepath, `illegal filename "${file}":"${filepath}"`)
+    assert(filepath, () => `illegal filename "${file}":"${filepath}"`)
 
     const saved = ctx.saveRegister('blocks', 'blockMode')
     ctx.setRegister('blocks', {})
     ctx.setRegister('blockMode', BlockMode.OUTPUT)
     const scope = yield hash.render(ctx)
-    if (withVar) scope[filepath] = yield new Expression(withVar).evaluate(ctx)
+    if (withVar) scope[filepath] = evalToken(withVar, ctx)
     const templates = yield liquid._parseFile(filepath, ctx.opts, ctx.sync)
     ctx.push(scope)
     yield renderer.renderTemplates(templates, ctx, emitter)

src/builtin/tags/increment.ts

@@ -1,13 +1,10 @@
-import { assert } from '../../util/assert'
-import { identifier } from '../../parser/lexical'
 import { isNumber, stringify } from '../../util/underscore'
-import { Emitter, TagToken, Context, TagImplOptions } from '../../types'
+import { Tokenizer, Emitter, TagToken, Context, TagImplOptions } from '../../types'
 
 export default {
   parse: function (token: TagToken) {
-    const match = token.args.match(identifier)
+    const tokenizer = new Tokenizer(token.args)
-    assert(match, `illegal identifier ${token.args}`)
+    this.variable = tokenizer.readWord().content
-    this.variable = match![0]
   },
   render: function (context: Context, emitter: Emitter) {
     const scope = context.environments

src/builtin/tags/layout.ts

@@ -1,29 +1,25 @@
-import { assert } from '../../util/assert'
+import { assert, evalQuotedToken, TypeGuards, evalToken, Tokenizer, Emitter, Hash, TagToken, TopLevelToken, Context, TagImplOptions } from '../../types'
-import { quotedLine, quoted } from '../../parser/lexical'
-import { Emitter, Hash, Expression, TagToken, Token, Context, TagImplOptions } from '../../types'
 import BlockMode from '../../context/block-mode'
 
-const rFile = new RegExp(`^(${quoted.source}|[^\\s,]+)`)
-
 export default {
-  parse: function (token: TagToken, remainTokens: Token[]) {
+  parse: function (token: TagToken, remainTokens: TopLevelToken[]) {
-    const match = rFile.exec(token.args)
+    const tokenizer = new Tokenizer(token.args)
-    if (!match) {
+    const file = this.liquid.options.dynamicPartials ? tokenizer.readValue() : tokenizer.readFileName()
-      throw new Error(`illegal argument "${token.args}"`)
+    assert(file, () => `illegal argument "${token.args}"`)
-    }
+
-    this.file = match[1]
+    this.file = file
-    this.hash = new Hash(token.args.slice(match[0].length))
+    this.hash = new Hash(tokenizer.remaining())
     this.tpls = this.liquid.parser.parse(remainTokens)
   },
   render: function * (ctx: Context, emitter: Emitter) {
     const { liquid, hash, file } = this
     const { renderer } = liquid
     const filepath = ctx.opts.dynamicPartials
-      ? (quotedLine.exec(file)
+      ? (TypeGuards.isQuotedToken(file)
-        ? yield renderer.renderTemplates(liquid.parse(file.slice(1, -1)), ctx)
+        ? yield renderer.renderTemplates(liquid.parse(evalQuotedToken(file)), ctx)
-        : yield new Expression(file).value(ctx))
+        : evalToken(this.file, ctx))
-      : this.file
+      : file.getText()
-    assert(filepath, `illegal filename "${file}":"${filepath}"`)
+    assert(filepath, () => `illegal filename "${file.getText()}":"${filepath}"`)
 
     // render the remaining tokens immediately
     ctx.setRegister('blockMode', BlockMode.STORE)

src/builtin/tags/raw.ts

@@ -1,7 +1,7 @@
-import { TagToken, Token, TagImplOptions } from '../../types'
+import { TagToken, TopLevelToken, TagImplOptions } from '../../types'
 
 export default {
-  parse: function (tagToken: TagToken, remainTokens: Token[]) {
+  parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
     this.tokens = []
 
     const stream = this.liquid.parser.parseStream(remainTokens)
@@ -11,11 +11,11 @@ export default {
         else this.tokens.push(token)
       })
       .on('end', () => {
-        throw new Error(`tag ${tagToken.raw} not closed`)
+        throw new Error(`tag ${tagToken.getText()} not closed`)
       })
     stream.start()
   },
   render: function () {
-    return this.tokens.map((token: Token) => token.raw).join('')
+    return this.tokens.map((token: TopLevelToken) => token.getText()).join('')
   }
 } as TagImplOptions

src/builtin/tags/render.ts

@@ -1,56 +1,69 @@
 import { assert } from '../../util/assert'
 import { ForloopDrop } from '../../drop/forloop-drop'
 import { toCollection } from '../../util/collection'
-import { Expression, Hash, Emitter, TagToken, Context, TagImplOptions } from '../../types'
+import { evalQuotedToken, TypeGuards, Tokenizer, evalToken, Hash, Emitter, TagToken, Context, TagImplOptions } from '../../types'
-import { identifier, value, quoted, quotedLine } from '../../parser/lexical'
-
-const rFile = new RegExp(`^(${quoted.source}|[^\\s,]+)`)
-const rWith = new RegExp(`^\\s+with\\s+(${value.source})(?:\\s+as\\s+(${identifier.source}))?`)
-const rFor = new RegExp(`^\\s+for\\s+(${value.source})\\s+as\\s+(${identifier.source})`)
 
 export default {
   parse: function (token: TagToken) {
-    let args = token.args
+    const args = token.args
-    let match = rFile.exec(args)
+    const tokenizer = new Tokenizer(args)
+    this.file = this.liquid.options.dynamicPartials
+      ? tokenizer.readValue()
+      : tokenizer.readFileName()
+    assert(this.file, () => `illegal argument "${token.args}"`)
 
-    assert(match, `illegal argument "${token.args}"`)
+    while (!tokenizer.end()) {
-    this.file = match![1]
+      tokenizer.skipBlank()
-    args = args.substr(match![0].length)
+      const begin = tokenizer.p
+      const keyword = tokenizer.readWord()
+      if (keyword.content === 'with' || keyword.content === 'for') {
+        tokenizer.skipBlank()
+        if (tokenizer.peek() !== ':') {
+          const value = tokenizer.readValue()
+          if (value) {
+            const beforeAs = tokenizer.p
+            const asStr = tokenizer.readWord()
+            let alias
+            if (asStr.content === 'as') alias = tokenizer.readWord()
+            else tokenizer.p = beforeAs
 
-    while (true) {
+            this[keyword.content] = { value, alias: alias && alias.content }
-      if ((match = rWith.exec(args))) {
+            tokenizer.skipBlank()
-        this.withVar = match[1]
+            if (tokenizer.peek() === ',') tokenizer.advance()
-        this.withAs = match[2]
+            continue
-        args = args.substr(match[0].length)
+          }
-      } else if ((match = rFor.exec(args))) {
+        }
-        this.forVar = match[1]
+      }
-        this.forAs = match[2]
+      tokenizer.p = begin
-        args = args.substr(match[0].length)
+      break
-      } else break
     }
-    this.hash = new Hash(args)
+    this.hash = new Hash(tokenizer.remaining())
   },
   render: function * (ctx: Context, emitter: Emitter) {
-    const { liquid, withVar, withAs, forVar, forAs, file, hash } = this
+    const { liquid, file, hash } = this
     const { renderer } = liquid
     const filepath = ctx.opts.dynamicPartials
-      ? (quotedLine.exec(file)
+      ? (TypeGuards.isQuotedToken(file)
-        ? yield renderer.renderTemplates(liquid.parse(file.slice(1, -1)), ctx)
+        ? yield renderer.renderTemplates(liquid.parse(evalQuotedToken(file)), ctx)
-        : yield new Expression(file).value(ctx))
+        : evalToken(file, ctx))
-      : this.file
+      : file.getText()
-    assert(filepath, `illegal filename "${file}":"${filepath}"`)
+    assert(filepath, () => `illegal filename "${file.getText()}":"${filepath}"`)
 
     const childCtx = new Context({}, ctx.opts, ctx.sync)
     const scope = yield hash.render(ctx)
-    if (withVar) scope[withAs || filepath] = yield new Expression(withVar).evaluate(ctx)
+    if (this['with']) {
+      const { value, alias } = this['with']
+      scope[alias || filepath] = evalToken(value, ctx)
+    }
     childCtx.push(scope)
 
-    if (forVar) {
+    if (this['for']) {
-      let collection = yield new Expression(forVar).value(ctx)
+      const { value, alias } = this['for']
+      let collection = evalToken(value, ctx)
       collection = toCollection(collection)
       scope['forloop'] = new ForloopDrop(collection.length)
       for (const item of collection) {
-        scope[forAs] = item
+        scope[alias] = item
         const templates = yield liquid._parseFile(filepath, childCtx.opts, childCtx.sync)
         yield renderer.renderTemplates(templates, childCtx, emitter)
         scope.forloop.next()

src/builtin/tags/tablerow.ts

@@ -1,21 +1,21 @@
-import { assert } from '../../util/assert'
 import { toCollection } from '../../util/collection'
-import { Expression, Emitter, Hash, TagToken, Token, Context, Template, TagImplOptions, ParseStream } from '../../types'
+import { assert, evalToken, Emitter, Hash, TagToken, TopLevelToken, Context, Template, TagImplOptions, ParseStream } from '../../types'
-import { identifier, value } from '../../parser/lexical'
 import { TablerowloopDrop } from '../../drop/tablerowloop-drop'
-
+import { Tokenizer } from '../../parser/tokenizer'
-const re = new RegExp(`^(${identifier.source})\\s+in\\s+` +
-  `(${value.source})`)
 
 export default {
-  parse: function (tagToken: TagToken, remainTokens: Token[]) {
+  parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
-    const match = re.exec(tagToken.args) as RegExpExecArray
+    const tokenizer = new Tokenizer(tagToken.args)
-    assert(match, `illegal tag: ${tagToken.raw}`)
 
-    this.variable = match[1]
+    this.variable = tokenizer.readWord()
-    this.collection = match[2]
+    tokenizer.skipBlank()
+
+    const tmp = tokenizer.readWord()
+    assert(tmp && tmp.content === 'in', () => `illegal tag: ${tagToken.getText()}`)
+
+    this.collection = tokenizer.readValue()
+    this.hash = new Hash(tokenizer.remaining())
     this.templates = []
-    this.hash = new Hash(tagToken.args.slice(match[0].length))
 
     let p
     const stream: ParseStream = this.liquid.parser.parseStream(remainTokens)
@@ -23,14 +23,14 @@ export default {
       .on('tag:endtablerow', () => stream.stop())
       .on('template', (tpl: Template) => p.push(tpl))
       .on('end', () => {
-        throw new Error(`tag ${tagToken.raw} not closed`)
+        throw new Error(`tag ${tagToken.getText()} not closed`)
       })
 
     stream.start()
   },
 
   render: function * (ctx: Context, emitter: Emitter) {
-    let collection = toCollection(yield new Expression(this.collection).value(ctx))
+    let collection = toCollection(evalToken(this.collection, ctx))
     const hash = yield this.hash.render(ctx)
     const offset = hash.offset || 0
     const limit = (hash.limit === undefined) ? collection.length : hash.limit
@@ -44,7 +44,7 @@ export default {
     ctx.push(scope)
 
     for (let idx = 0; idx < collection.length; idx++, tablerowloop.next()) {
-      scope[this.variable] = collection[idx]
+      scope[this.variable.content] = collection[idx]
       if (tablerowloop.col0() === 0) {
         if (tablerowloop.row() !== 1) emitter.write('</tr>')
         emitter.write(`<tr class="row${tablerowloop.row()}">`)

src/builtin/tags/unless.ts

@@ -1,7 +1,7 @@
-import { Emitter, Expression, isFalsy, ParseStream, Context, TagImplOptions, Token, TagToken } from '../../types'
+import { TopLevelToken, Template, Emitter, Expression, isFalsy, ParseStream, Context, TagImplOptions, Token, TagToken } from '../../types'
 
 export default {
-  parse: function (tagToken: TagToken, remainTokens: Token[]) {
+  parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
     this.templates = []
     this.elseTemplates = []
     let p
@@ -12,9 +12,9 @@ export default {
       })
       .on('tag:else', () => (p = this.elseTemplates))
       .on('tag:endunless', () => stream.stop())
-      .on('template', tpl => p.push(tpl))
+      .on('template', (tpl: Template) => p.push(tpl))
       .on('end', () => {
-        throw new Error(`tag ${tagToken.raw} not closed`)
+        throw new Error(`tag ${tagToken.getText()} not closed`)
       })
 
     stream.start()

src/context/context.ts

@@ -1,6 +1,5 @@
 import { Drop } from '../drop/drop'
 import { __assign } from 'tslib'
-import { assert } from '../util/assert'
 import { NormalizedFullOptions, defaultOptions } from '../liquid-options'
 import { Scope } from './scope'
 import { isArray, isNil, isString, isFunction, toLiquid } from '../util/underscore'
@@ -34,13 +33,12 @@ export class Context {
     return [this.globals, this.environments, ...this.scopes]
       .reduce((ctx, val) => __assign(ctx, val), {})
   }
-  public get (path: string) {
+  public get (paths: string[]) {
-    const paths = this.parseProp(path)
     const scope = this.findScope(paths[0])
     return this.getFromScope(scope, paths)
   }
   public getFromScope (scope: object, paths: string[] | string) {
-    if (!isArray(paths)) paths = this.parseProp(paths)
+    if (typeof paths === 'string') paths = paths.split('.')
     return paths.reduce((scope, path) => {
       scope = readProperty(scope, path)
       if (isNil(scope) && this.opts.strictVariables) {
@@ -61,67 +59,11 @@ export class Context {
   private findScope (key: string) {
     for (let i = this.scopes.length - 1; i >= 0; i--) {
       const candidate = this.scopes[i]
-      if (key in candidate) {
+      if (key in candidate) return candidate
-        return candidate
-      }
     }
     if (key in this.environments) return this.environments
     return this.globals
   }
-
-  /*
-   * Parse property access sequence from access string
-   * @example
-   * accessSeq("foo.bar")         // ['foo', 'bar']
-   * accessSeq("foo['bar']")      // ['foo', 'bar']
-   * accessSeq("foo['b]r']")      // ['foo', 'b]r']
-   * accessSeq("foo[bar.coo]")    // ['foo', 'bar'], for bar.coo == 'bar'
-   */
-  private parseProp (str: string) {
-    str = String(str)
-    const seq: string[] = []
-    const push = () => name.length && (seq.push(name), (name = ''))
-    let name = ''
-    let j
-    let i = 0
-    while (i < str.length) {
-      switch (str[i]) {
-        case '[':
-          push()
-
-          const delemiter = str[i + 1]
-          if (/['"]/.test(delemiter)) { // foo["bar"]
-            j = str.indexOf(delemiter, i + 2)
-            assert(j !== -1, `unbalanced ${delemiter}: ${str}`)
-            name = str.slice(i + 2, j)
-            push()
-            i = j + 2
-          } else { // foo[bar.coo]
-            j = matchRightBracket(str, i + 1)
-            assert(j !== -1, `unbalanced []: ${str}`)
-            name = str.slice(i + 1, j)
-            if (!/^[+-]?\d+$/.test(name)) { // foo[bar] vs. foo[1]
-              name = String(this.get(name))
-            }
-            push()
-            i = j + 1
-          }
-          break
-        case '.':// foo.bar, foo[0].bar
-          push()
-          i++
-          break
-        default:// foo.bar
-          name += str[i++]
-      }
-    }
-    push()
-
-    if (!seq.length) {
-      throw new TypeError(`invalid path:"${str}"`)
-    }
-    return seq
-  }
 }
 
 export function readProperty (obj: Scope, key: string) {
@@ -152,19 +94,3 @@ function readSize (obj: Scope) {
   if (isArray(obj) || isString(obj)) return obj.length
   return obj['size']
 }
-
-function matchRightBracket (str: string, begin: number) {
-  let stack = 1 // count of '[' - count of ']'
-  for (let i = begin; i < str.length; i++) {
-    if (str[i] === '[') {
-      stack++
-    }
-    if (str[i] === ']') {
-      stack--
-      if (stack === 0) {
-        return i
-      }
-    }
-  }
-  return -1
-}

src/liquid.ts

@@ -38,8 +38,8 @@ export class Liquid {
     _.forOwn(builtinFilters, (handler, name) => this.registerFilter(name, handler))
   }
   public parse (html: string, filepath?: string): Template[] {
-    const tokenizer = new Tokenizer(html, filepath, this.options)
+    const tokenizer = new Tokenizer(html, filepath)
-    const tokens = tokenizer.readTokens()
+    const tokens = tokenizer.readTopLevelTokens(this.options)
     return this.parser.parse(tokens)
   }
 

src/parser/filter-arg.ts

@@ -1,9 +1,10 @@
 import { isArray } from '../util/underscore'
+import { ValueToken } from '../tokens/value-token'
 
-type KeyValuePair = [string?, string?]
+type KeyValuePair = [string?, ValueToken?]
 
-export type FilterArg = string|KeyValuePair
+export type FilterArg = ValueToken | KeyValuePair
 
-export function isKeyValuePair (arr: FilterArg): arr is KeyValuePair {
+export function isKeyValuePair (arr: FilterArg): arr is KeyValuePair { // TODO check
   return isArray(arr)
 }

src/parser/html-token.ts

@@ -1,11 +0,0 @@
-import { Token } from './token'
-
-export class HTMLToken extends Token {
-  public constructor (str: string, input: string, line: number, col: number, file?: string) {
-    super(str, input, line, col, file)
-    this.content = str
-  }
-  public static is (token: Token): token is HTMLToken {
-    return token instanceof HTMLToken
-  }
-}

src/parser/lexical.ts

@@ -1,29 +0,0 @@
-// quote related
-const singleQuoted = /'[^']*'/
-const doubleQuoted = /"[^"]*"/
-export const quoted = new RegExp(`${singleQuoted.source}|${doubleQuoted.source}`)
-export const quoteBalanced = new RegExp(`(?:${quoted.source}|[^'"])*`)
-
-// basic types
-export const number = /[+-]?(?:\d+\.?\d*|\.?\d+)/
-export const bool = /true|false/
-
-// property access
-export const identifier = /[\w-]+[?]?/
-export const subscript = new RegExp(`\\[(?:${quoted.source}|[\\w-\\.]+)\\]`)
-export const literal = new RegExp(`(?:${quoted.source}|${bool.source}|${number.source})`)
-export const variable = new RegExp(`${identifier.source}(?:\\.${identifier.source}|${subscript.source})*`)
-
-// range related
-export const rangeLimit = new RegExp(`(?:${variable.source}|${number.source})`)
-export const range = new RegExp(`\\(${rangeLimit.source}\\.\\.${rangeLimit.source}\\)`)
-export const rangeCapture = new RegExp(`\\((${rangeLimit.source})\\.\\.(${rangeLimit.source})\\)`)
-
-export const value = new RegExp(`(?:${variable.source}|${literal.source}|${range.source})`)
-
-// full match
-export const tagLine = new RegExp(`^\\s*(${identifier.source})\\s*([\\s\\S]*?)\\s*$`)
-export const numberLine = new RegExp(`^${number.source}$`)
-export const boolLine = new RegExp(`^${bool.source}$`, 'i')
-export const quotedLine = new RegExp(`^${quoted.source}$`)
-export const rangeLine = new RegExp(`^${rangeCapture.source}$`)

src/parser/match-operator.ts

@@ -0,0 +1,24 @@
+import { VARIABLE } from '../util/character'
+
+const trie = {
+  a: { n: { d: { end: true, needBoundary: true } } },
+  o: { r: { end: true, needBoundary: true } },
+  c: { o: { n: { t: { a: { i: { n: { s: { end: true, needBoundary: true } } } } } } } },
+  '=': { '=': { end: true } },
+  '!': { '=': { end: true } },
+  '>': { end: true, '=': { end: true } },
+  '<': { end: true, '=': { end: true } }
+}
+
+export function matchOperator (str: string, begin: number, end = str.length) {
+  let node = trie
+  let i = begin
+  let info
+  while (node[str[i]] && i < end) {
+    node = node[str[i++]]
+    if (node['end']) info = node
+  }
+  if (!info) return -1
+  if (info['needBoundary'] && str.charCodeAt(i) & VARIABLE) return -1
+  return i
+}

src/parser/output-token.ts

@@ -1,20 +0,0 @@
-import { DelimitedToken } from './delimited-token'
-import { Token } from './token'
-import { NormalizedFullOptions } from '../liquid-options'
-
-export class OutputToken extends DelimitedToken {
-  public constructor (
-    raw: string,
-    value: string,
-    input: string,
-    line: number,
-    pos: number,
-    options: NormalizedFullOptions,
-    file?: string
-  ) {
-    super(raw, value, input, line, pos, options.trimOutputLeft, options.trimOutputRight, file)
-  }
-  public static is (token: Token): token is OutputToken {
-    return token instanceof OutputToken
-  }
-}

src/parser/parse-stream.ts

@@ -1,20 +1,21 @@
-import { Token } from '../parser/token'
+import { Token } from '../tokens/token'
 import { Template } from '../template/template'
-import { TagToken } from './tag-token'
+import { isTagToken } from '../util/type-guards'
+import { TopLevelToken } from '../tokens/toplevel-token'
 
-type ParseToken = ((token: Token, remainTokens: Token[]) => Template)
+type ParseToken<T extends Token> = ((token: T, remainTokens: T[]) => Template)
 
-export class ParseStream {
+export class ParseStream<T extends Token = TopLevelToken> {
-  private tokens: Token[]
+  private tokens: T[]
   private handlers: {[key: string]: (arg: any) => void} = {}
   private stopRequested = false
-  private parseToken: ParseToken
+  private parseToken: ParseToken<T>
 
-  public constructor (tokens: Token[], parseToken: ParseToken) {
+  public constructor (tokens: T[], parseToken: ParseToken<T>) {
     this.tokens = tokens
     this.parseToken = parseToken
   }
-  public on<T extends Template | Token | undefined> (name: string, cb: (arg: T) => void): ParseStream {
+  public on<T2 extends Template | T | undefined> (name: string, cb: (arg: T2) => void): ParseStream<T> {
     this.handlers[name] = cb
     return this
   }
@@ -24,10 +25,10 @@ export class ParseStream {
   }
   public start () {
     this.trigger('start')
-    let token: Token | undefined
+    let token: T | undefined
     while (!this.stopRequested && (token = this.tokens.shift())) {
       if (this.trigger('token', token)) continue
-      if (TagToken.is(token) && this.trigger(`tag:${token.name}`, token)) {
+      if (isTagToken(token) && this.trigger(`tag:${token.name}`, token)) {
         continue
       }
       const template = this.parseToken(token, this.tokens)

src/parser/parse-string-literal.ts

@@ -1,10 +1,3 @@
-import { last } from '../util/underscore'
-import { NullDrop } from '../drop/null-drop'
-import { EmptyDrop } from '../drop/empty-drop'
-import { BlankDrop } from '../drop/blank-drop'
-
-type literal = true | false | NullDrop | EmptyDrop | BlankDrop | number | string
-
 const rHex = /[\da-fA-F]/
 const rOct = /[0-7]/
 const escapeChar = {
@@ -23,18 +16,6 @@ function hexVal (c: string) {
   return code - 48
 }
 
-export function parseLiteral (str: string): literal | undefined {
-  str = str.trim()
-
-  if (str === 'true') return true
-  if (str === 'false') return false
-  if (str === 'nil' || str === 'null') return new NullDrop()
-  if (str === 'empty') return new EmptyDrop()
-  if (str === 'blank') return new BlankDrop()
-  if (!isNaN(Number(str))) return Number(str)
-  if ((str[0] === '"' || str[0] === "'") && str[0] === last(str)) return parseStringLiteral(str)
-}
-
 export function parseStringLiteral (str: string): string {
   let ret = ''
   for (let i = 1; i < str.length - 1; i++) {
@@ -66,3 +47,4 @@ export function parseStringLiteral (str: string): string {
   }
   return ret
 }
+

src/parser/parser.ts

@@ -1,13 +1,13 @@
 import { ParseError } from '../util/error'
 import { Liquid } from '../liquid'
 import { ParseStream } from './parse-stream'
-import { Token } from './token'
+import { isTagToken, isOutputToken } from '../util/type-guards'
-import { TagToken } from './tag-token'
+import { OutputToken } from '../tokens/output-token'
-import { OutputToken } from './output-token'
 import { Tag } from '../template/tag/tag'
 import { Output } from '../template/output'
 import { HTML } from '../template/html'
 import { Template } from '../template/template'
+import { TopLevelToken } from '../tokens/toplevel-token'
 
 export default class Parser {
   private liquid: Liquid
@@ -15,7 +15,7 @@ export default class Parser {
   public constructor (liquid: Liquid) {
     this.liquid = liquid
   }
-  public parse (tokens: Token[]) {
+  public parse (tokens: TopLevelToken[]) {
     let token
     const templates: Template[] = []
     while ((token = tokens.shift())) {
@@ -23,12 +23,12 @@ export default class Parser {
     }
     return templates
   }
-  public parseToken (token: Token, remainTokens: Token[]) {
+  public parseToken (token: TopLevelToken, remainTokens: TopLevelToken[]) {
     try {
-      if (TagToken.is(token)) {
+      if (isTagToken(token)) {
         return new Tag(token, remainTokens, this.liquid)
       }
-      if (OutputToken.is(token)) {
+      if (isOutputToken(token)) {
         return new Output(token as OutputToken, this.liquid.filters)
       }
       return new HTML(token)
@@ -36,7 +36,7 @@ export default class Parser {
       throw new ParseError(e, token)
     }
   }
-  public parseStream (tokens: Token[]) {
+  public parseStream (tokens: TopLevelToken[]) {
     return new ParseStream(tokens, (token, tokens) => this.parseToken(token, tokens))
   }
 }

src/parser/substr.ts

@@ -1,19 +0,0 @@
-export class Substr {
-  constructor (
-    public str: string,
-    public begin: number,
-    public end: number = begin
-  ) {}
-  size () {
-    return this.end - this.begin
-  }
-  toString () {
-    return this.str.slice(this.begin, this.end)
-  }
-  first () {
-    return this.str[this.begin]
-  }
-  last () {
-    return this.str[this.end - 1]
-  }
-}

src/parser/tag-token.ts

@@ -1,30 +0,0 @@
-import { DelimitedToken } from './delimited-token'
-import { Token } from './token'
-import { TokenizationError } from '../util/error'
-import * as lexical from './lexical'
-import { NormalizedFullOptions } from '../liquid-options'
-
-export class TagToken extends DelimitedToken {
-  public name: string
-  public args: string
-  public constructor (
-    raw: string,
-    value: string,
-    input: string,
-    line: number,
-    pos: number,
-    options: NormalizedFullOptions,
-    file?: string
-  ) {
-    super(raw, value, input, line, pos, options.trimTagLeft, options.trimTagRight, file)
-    const match = this.content.match(lexical.tagLine)
-    if (!match) {
-      throw new TokenizationError(`illegal tag syntax`, this)
-    }
-    this.name = match[1]
-    this.args = match[2]
-  }
-  public static is (token: Token): token is TagToken {
-    return token instanceof TagToken
-  }
-}

src/parser/token-kind.ts

@@ -0,0 +1,14 @@
+export enum TokenKind {
+  Number,
+  Literal,
+  Tag,
+  Output,
+  HTML,
+  Filter,
+  Hash,
+  PropertyAccess,
+  Word,
+  Range,
+  Quoted,
+  Operator
+}

src/parser/token.ts

@@ -1,14 +0,0 @@
-export class Token {
-  public trimLeft = false
-  public trimRight = false
-  public content: string
-  public constructor (
-    public raw: string,
-    public input: string,
-    public line: number,
-    public col: number,
-    public file?: string
-  ) {
-    this.content = raw
-  }
-}

src/parser/tokenizer.ts

@@ -1,158 +1,158 @@
 import { whiteSpaceCtrl } from './whitespace-ctrl'
-import { Substr } from './substr'
+import { NumberToken } from '../tokens/number-token'
+import { WordToken } from '../tokens/word-token'
+import { literalValues } from '../util/literal'
+import { LiteralToken } from '../tokens/literal-token'
+import { OperatorToken } from '../tokens/operator-token'
+import { PropertyAccessToken } from '../tokens/property-access-token'
+import { assert } from '../util/assert'
+import { TopLevelToken } from '../tokens/toplevel-token'
 import { FilterArg } from './filter-arg'
-import { FilterToken } from './filter-token'
+import { FilterToken } from '../tokens/filter-token'
+import { HashToken } from '../tokens/hash-token'
+import { QuotedToken } from '../tokens/quoted-token'
 import { ellipsis } from '../util/underscore'
-import { HTMLToken } from './html-token'
+import { HTMLToken } from '../tokens/html-token'
-import { TagToken } from './tag-token'
+import { TagToken } from '../tokens/tag-token'
-import { Token } from './token'
+import { Token } from '../tokens/token'
-import { OutputToken } from './output-token'
+import { RangeToken } from '../tokens/range-token'
+import { ValueToken } from '../tokens/value-token'
+import { OutputToken } from '../tokens/output-token'
 import { TokenizationError } from '../util/error'
 import { NormalizedFullOptions, defaultOptions } from '../liquid-options'
-
+import { TYPES, QUOTE, BLANK, VARIABLE } from '../util/character'
-// bitmask character types to boost performance
+import { matchOperator } from './match-operator'
-// generated by bin/char-types.js
-const TYPES = '00000000044004000000000000000000428000080000010011111111110022210111111111111111111111111110000101111111111111111111111111100000'
-const VARIABLE = 1
-const OPERATOR = 2
-const BLANK = 4
-const QUOTE = 8
 
 export class Tokenizer {
-  private p = 0
+  p = 0
-  private N: number
+  N: number
-  private line = 1
-  private col = 1
   constructor (
     private input: string,
-    private file: string = '',
+    private file: string = ''
-    private options: NormalizedFullOptions = defaultOptions
   ) {
     this.N = input.length
   }
 
-  * readExpression (): IterableIterator<string> {
+  * readExpression (): IterableIterator<Token> {
+    const operand = this.readValue()
+    if (!operand) return
+
+    yield operand
+
     while (this.p < this.N) {
+      const operator = this.readOperator()
+      if (!operator) return
+
       const operand = this.readValue()
-      if (operand.size()) {
+      if (!operand) return
-        yield operand.toString()
+
-        continue
+      yield operator
-      }
+      yield operand
-      this.readBlank()
-      const operator = new Substr(this.input, this.p)
-      while (OPERATOR & this.peekType()) operator.end = this.read()
-      if (operator.size()) {
-        yield operator.toString()
-        continue
-      }
-      this.read()
     }
   }
-
+  readOperator (): OperatorToken | undefined {
-  readFilterTokens (): FilterToken[] {
+    this.skipBlank()
+    const end = matchOperator(this.input, this.p, this.p + 8)
+    if (end === -1) return
+    return new OperatorToken(this.input, this.p, (this.p = end), this.file)
+  }
+  readFilters (): FilterToken[] {
     const filters = []
     while (true) {
-      const filter = this.readFilterToken()
+      const filter = this.readFilter()
       if (!filter) return filters
       filters.push(filter)
     }
   }
-
+  readFilter (): FilterToken | null {
-  // | foo
-  // | foo: a
-  // | foo: a, b
-  // | foo: a, b: 1
-  readFilterToken (): FilterToken | null {
     this.readTo('|')
     const begin = this.p
-    const name = this.readVariable().toString()
+    const name = this.readWord()
-    if (!name) return null
+    if (!name.size()) return null
     const args = []
-    this.readBlank()
+    this.skipBlank()
     if (this.peek() === ':') {
       do {
-        this.read()
+        ++this.p
         const arg = this.readFilterArg()
         arg && args.push(arg)
-        while (this.p < this.N && this.peek() !== ',' && this.peek() !== '|') this.read()
+        while (this.p < this.N && this.peek() !== ',' && this.peek() !== '|') ++this.p
       } while (this.peek() === ',')
     }
-    const raw = this.input.slice(begin, this.p)
+    return new FilterToken(name.getText(), args, this.input, begin, this.p, this.file)
-    return new FilterToken(name, args, raw, this.input, this.line, this.col, this.file)
   }
 
-  readFilterArg (): FilterArg | null {
+  readFilterArg (): FilterArg | undefined {
     const key = this.readValue()
-    if (!key.size()) return null
+    if (!key) return
-    this.readBlank()
+    this.skipBlank()
-    if (this.peek() === ':') {
+    if (this.peek() !== ':') return key
-      this.read()
+    ++this.p
-      return [key.toString(), this.readValue().toString()]
+    const value = this.readValue()
-    }
+    return [key.getText(), value]
-    return key.toString()
   }
 
-  readTokens (): Token[] {
+  readTopLevelTokens (options: NormalizedFullOptions = defaultOptions): TopLevelToken[] {
-    const tokens: Token[] = []
+    const tokens: TopLevelToken[] = []
     while (this.p < this.N) {
-      const token = this.readToken()
+      const token = this.readTopLevelToken(options)
       tokens.push(token)
     }
-    whiteSpaceCtrl(tokens, this.options)
+    whiteSpaceCtrl(tokens, options)
     return tokens
   }
 
-  readToken (): Token {
+  readTopLevelToken (options: NormalizedFullOptions): TopLevelToken {
-    const { tagDelimiterLeft, outputDelimiterLeft } = this.options
+    const { tagDelimiterLeft, outputDelimiterLeft } = options
-    if (this.matchWord(tagDelimiterLeft)) return this.readTagToken()
+    if (this.matchWord(tagDelimiterLeft)) return this.readTagToken(options)
-    if (this.matchWord(outputDelimiterLeft)) return this.readOutputToken()
+    if (this.matchWord(outputDelimiterLeft)) return this.readOutputToken(options)
-    return this.readHTMLToken()
+    return this.readHTMLToken(options)
   }
 
-  readHTMLToken (): HTMLToken {
+  readHTMLToken (options: NormalizedFullOptions): HTMLToken {
-    const html = new Substr(this.input, this.p)
+    const begin = this.p
     while (this.p < this.N) {
-      const { tagDelimiterLeft, outputDelimiterLeft } = this.options
+      const { tagDelimiterLeft, outputDelimiterLeft } = options
       if (this.matchWord(tagDelimiterLeft)) break
       if (this.matchWord(outputDelimiterLeft)) break
-      html.end = this.read()
+      ++this.p
     }
-    return new HTMLToken(html.toString(), this.input, this.line, this.col, this.file)
+    return new HTMLToken(this.input, begin, this.p, this.file)
   }
 
-  readTagToken (): TagToken {
+  readTagToken (options: NormalizedFullOptions): TagToken {
-    const { line, col, file, input, options } = this
+    const { file, input } = this
-    const { tagDelimiterLeft, tagDelimiterRight } = options
+    const { tagDelimiterRight } = options
-    const buffer = this.readTo(tagDelimiterRight).toString()
+    const begin = this.p
-    if (!this.reverseMatchWord(tagDelimiterRight, buffer)) {
+    if (this.readTo(tagDelimiterRight) === -1) {
-      throw new TokenizationError(
+      this.mkError(`tag "${this.ellipsis(begin)}" not closed`, begin)
-        `tag "${ellipsis(buffer, 16)}" not closed`,
-        new Token(buffer, input, line, col, file)
-      )
     }
-    const value = buffer.slice(tagDelimiterLeft.length, -tagDelimiterRight.length)
+    return new TagToken(input, begin, this.p, options, file)
-    return new TagToken(buffer, value, input, line, col, options, file)
   }
 
-  readOutputToken (): OutputToken {
+  readOutputToken (options: NormalizedFullOptions): OutputToken {
-    const { line, col, file, input, options } = this
+    const { file, input } = this
-    const { outputDelimiterLeft, outputDelimiterRight } = options
+    const { outputDelimiterRight } = options
-    const buffer = this.readTo(outputDelimiterRight).toString()
+    const begin = this.p
-    if (!this.reverseMatchWord(outputDelimiterRight, buffer)) {
+    if (this.readTo(outputDelimiterRight) === -1) {
-      throw new TokenizationError(
+      this.mkError(`output "${this.ellipsis(begin)}" not closed`, begin)
-        `output "${ellipsis(buffer, 16)}" not closed`,
-        new Token(buffer, input, line, col, file)
-      )
     }
-    const value = buffer.slice(outputDelimiterLeft.length, -outputDelimiterRight.length)
+    return new OutputToken(input, begin, this.p, options, file)
-    return new OutputToken(buffer, value, input, line, col, options, file)
   }
 
-  readVariable (): Substr {
+  mkError (msg: string, begin: number) {
-    this.readBlank()
+    throw new TokenizationError(msg, new WordToken(this.input, begin, this.N, this.file))
-    const ans = new Substr(this.input, this.p)
+  }
-    while (this.peekType() & VARIABLE) ans.end = this.read()
+
-    return ans
+  ellipsis (begin: number = this.p) {
+    return ellipsis(this.input.slice(begin), 16)
+  }
+
+  readWord (): WordToken { // rename to identifier
+    this.skipBlank()
+    const begin = this.p
+    while (this.peekType() & VARIABLE) ++this.p
+    return new WordToken(this.input, begin, this.p, this.file)
   }
 
   readHashes () {
@@ -164,133 +164,135 @@ export class Tokenizer {
     }
   }
 
-  readHash () {
+  readHash (): HashToken | undefined {
-    this.readBlank()
+    this.skipBlank()
-    if (this.peek() === ',') this.read()
+    if (this.peek() === ',') ++this.p
-    const name = this.readVariable().toString()
+    const begin = this.p
-    if (!name) return null
+    const name = this.readWord()
+    if (!name.size()) return
+    let value
 
-    this.readBlank()
+    this.skipBlank()
-    let value = ''
     if (this.peek() === ':') {
-      this.read()
+      ++this.p
-      value = this.readValue().toString()
+      value = this.readValue()
     }
-    return [name, value]
+    return new HashToken(this.input, begin, this.p, name, value, this.file)
   }
 
-  readPropertyAccess (): Substr {
+  remaining () {
-    this.readBlank()
+    return this.input.slice(this.p)
-    const ans = new Substr(this.input, this.p)
+  }
-    let nested = 0
+
+  advance (i = 1) {
+    this.p += i
+  }
+
+  end () {
+    return this.p >= this.N
+  }
+
+  readTo (end: string): number {
     while (this.p < this.N) {
-      const c = this.peek()
+      ++this.p
-      const code = this.peekType()
+      if (this.reverseMatchWord(end)) return this.p
-      if (c === '[') {
-        this.read()
-        ans.end = this.readValue().end
-        nested++
-      } else if (c === ']') {
-        if (!nested) break
-        ans.end = this.read()
-        nested--
-      } else if (c === '.') {
-        if (this.peekType(1) & VARIABLE) {
-          this.read()
-          ans.end = this.readVariable().end
-        } else break
-      } else if (code & VARIABLE) {
-        ans.end = this.read()
-      } else {
-        if (nested) this.read()
-        else break
-      }
     }
-    return ans
+    return -1
   }
-  readTo (end: string): Substr {
+
-    const ans = new Substr(this.input, this.p)
+  readValue (): ValueToken | undefined {
-    while (this.p < this.N) {
+    const value = this.readQuoted() || this.readRange()
-      ans.end = this.read()
+    if (value) return value
-      if (this.reverseMatchWord(end)) break
+
+    const variable = this.readWord()
+    if (!variable.size()) return
+
+    let isNumber = variable.isNumber(true)
+    const props: (QuotedToken | WordToken)[] = []
+    while (true) {
+      if (this.peek() === '[') {
+        isNumber = false
+        this.p++
+        const prop = this.readValue() || new WordToken(this.input, this.p, this.p, this.file)
+        this.readTo(']')
+        props.push(prop)
+      } else if (this.peek() === '.' && this.peek(1) !== '.') { // skip range syntax
+        this.p++
+        const prop = this.readWord()
+        if (!prop.size()) break
+        if (!prop.isNumber()) isNumber = false
+        props.push(prop)
+      } else break
     }
-    return ans
+    if (!props.length && literalValues.hasOwnProperty(variable.content)) {
+      return new LiteralToken(this.input, variable.begin, variable.end, this.file)
+    }
+    if (isNumber) return new NumberToken(variable, props[0] as WordToken)
+    return new PropertyAccessToken(variable, props, this.p)
   }
-  readValue (): Substr {
+
-    let val = this.readQuoted()
+  readRange (): RangeToken | undefined {
-    if (val.size()) return val
+    this.skipBlank()
-    val = this.readBoolean()
+    const begin = this.p
-    if (val.size()) return val
+    if (this.peek() !== '(') return
-    val = this.readPropertyAccess()
+    ++this.p
-    if (val.size()) return val
+    const lhs = this.readValueOrThrow()
-    return this.readRange()
+    this.p += 2
+    const rhs = this.readValueOrThrow()
+    ++this.p
+    return new RangeToken(this.input, begin, this.p, lhs, rhs, this.file)
   }
-  readRange (): Substr {
+
-    this.readBlank()
+  readValueOrThrow (): ValueToken {
-    const ans = new Substr(this.input, this.p)
+    const value = this.readValue()
-    if (this.peek() !== '(') return ans
+    assert(value, () => `unexpected token "${this.ellipsis()}", value expected`)
-    this.read()
+    return value!
-    this.readValue()
-    this.read(2)
-    this.readValue()
-    ans.end = this.read()
-    return ans
   }
-  readBoolean (): Substr {
+
-    this.readBlank()
+  readQuoted (): QuotedToken | undefined {
-    const ans = new Substr(this.input, this.p)
+    this.skipBlank()
-    if (this.matchWord('true') && !(this.peekType(4) & VARIABLE)) ans.end = this.read(4)
+    const begin = this.p
-    else if (this.matchWord('false') && !(this.peekType(5) & VARIABLE)) ans.end = this.read(5)
+    if (!(this.peekType() & QUOTE)) return
-    return ans
+    ++this.p
-  }
-  readQuoted (): Substr {
-    this.readBlank()
-    const ans = new Substr(this.input, this.p)
-    if (!(this.peekType() & QUOTE)) return ans
-    ans.end = this.read()
     let escaped = false
     while (this.p < this.N) {
-      ans.end = this.read()
+      ++this.p
-      if (ans.last() === ans.first() && !escaped) break
+      if (this.input[this.p - 1] === this.input[begin] && !escaped) break
       if (escaped) escaped = false
-      else if (ans.last() === '\\') escaped = true
+      else if (this.input[this.p - 1] === '\\') escaped = true
     }
-    return ans
+    return new QuotedToken(this.input, begin, this.p, this.file)
   }
-  read (n = 1): number {
+
-    if (n > 1) this.read(n - 1)
+  readFileName (): WordToken {
-    const c = this.input[this.p++]
+    const begin = this.p
-    if (c === '\n') {
+    while (!(this.peekType() & BLANK) && this.peek() !== ',' && this.p < this.N) this.p++
-      this.line++
+    return new WordToken(this.input, begin, this.p, this.file)
-      this.col = 1
-    } else {
-      this.col++
-    }
-    return this.p
   }
+
   matchWord (word: string) {
     for (let i = 0; i < word.length; i++) {
       if (word[i] !== this.input[this.p + i]) return false
     }
     return true
   }
-  reverseMatchWord (word: string, buffer?: string) {
+
-    const str = buffer || this.input
+  reverseMatchWord (word: string) {
-    const end = buffer === undefined ? this.p : buffer.length
     for (let i = 0; i < word.length; i++) {
-      if (word[word.length - 1 - i] !== str[end - 1 - i]) return false
+      if (word[word.length - 1 - i] !== this.input[this.p - 1 - i]) return false
     }
     return true
   }
+
   peekType (n = 0) {
-    return +TYPES[this.input.charCodeAt(this.p + n)]
+    return TYPES[this.input.charCodeAt(this.p + n)]
   }
+
   peek (n = 0) {
     return this.input[this.p + n]
   }
-  readBlank () {
+
-    let ans = ''
+  skipBlank () {
-    while (this.peekType() & BLANK) ans += this.read()
+    while (this.peekType() & BLANK) ++this.p
-    return ans
   }
 }

src/parser/whitespace-ctrl.ts

@@ -1,7 +1,8 @@
-import { Token } from '../parser/token'
+import { Token } from '../tokens/token'
-import { TagToken } from '../parser/tag-token'
+import { DelimitedToken } from '../tokens/delimited-token'
-import { HTMLToken } from '../parser/html-token'
+import { isTagToken, isHTMLToken } from '../util/type-guards'
 import { NormalizedFullOptions } from '../liquid-options'
+import { TYPES, INLINE_BLANK, BLANK } from '../util/character'
 
 export function whiteSpaceCtrl (tokens: Token[], options: NormalizedFullOptions) {
   options = { greedy: true, ...options }
@@ -9,11 +10,12 @@ export function whiteSpaceCtrl (tokens: Token[], options: NormalizedFullOptions)
 
   for (let i = 0; i < tokens.length; i++) {
     const token = tokens[i]
+    if (!(token instanceof DelimitedToken)) continue
     if (!inRaw && token.trimLeft) {
       trimLeft(tokens[i - 1], options.greedy)
     }
 
-    if (TagToken.is(token)) {
+    if (isTagToken(token)) {
       if (token.name === 'raw') inRaw = true
       else if (token.name === 'endraw') inRaw = false
     }
@@ -25,15 +27,16 @@ export function whiteSpaceCtrl (tokens: Token[], options: NormalizedFullOptions)
 }
 
 function trimLeft (token: Token, greedy: boolean) {
-  if (!token || !HTMLToken.is(token)) return
+  if (!token || !isHTMLToken(token)) return
 
-  const rLeft = greedy ? /\s+$/g : /[\t\r ]*$/g
+  const mask = greedy ? BLANK : INLINE_BLANK
-  token.content = token.content.replace(rLeft, '')
+  while (TYPES[token.input.charCodeAt(token.end - 1 - token.trimRight)] & mask) token.trimRight++
 }
 
 function trimRight (token: Token, greedy: boolean) {
-  if (!token || !HTMLToken.is(token)) return
+  if (!token || !isHTMLToken(token)) return
 
-  const rRight = greedy ? /^\s+/g : /^[\t\r ]*\n?/g
+  const mask = greedy ? BLANK : INLINE_BLANK
-  token.content = token.content.replace(rRight, '')
+  while (TYPES[token.input.charCodeAt(token.begin + token.trimLeft)] & mask) token.trimLeft++
+  if (token.input.charAt(token.begin + token.trimLeft) === '\n') token.trimLeft++
 }

src/render/expression.ts

@@ -1,47 +1,87 @@
+import { QuotedToken } from '../tokens/quoted-token'
+import { NumberToken } from '../tokens/number-token'
 import { assert } from '../util/assert'
-import { rangeLine } from '../parser/lexical'
+import { literalValues } from '../util/literal'
-import { parseLiteral } from '../parser/literal'
+import { LiteralToken } from '../tokens/literal-token'
+import * as TypeGuards from '../util/type-guards'
+import { Token } from '../tokens/token'
+import { OperatorToken } from '../tokens/operator-token'
+import { RangeToken } from '../tokens/range-token'
+import { parseStringLiteral } from '../parser/parse-string-literal'
 import { Context } from '../context/context'
 import { range, toValue } from '../util/underscore'
-import { isOperator, precedence, operatorImpls } from './operator'
 import { Tokenizer } from '../parser/tokenizer'
+import { operatorImpls } from '../render/operator'
 
 export class Expression {
   private operands: any[] = []
-  private postfix: string[]
+  private postfix: IterableIterator<Token>
 
-  public constructor (str = '') {
+  public constructor (str: string) {
     const tokenizer = new Tokenizer(str)
-    this.postfix = [...toPostfix(tokenizer.readExpression())]
+    this.postfix = toPostfix(tokenizer.readExpression())
   }
-  public * evaluate (ctx: Context): any {
+  public evaluate (ctx: Context): any {
-    assert(ctx, 'unable to evaluate: context not defined')
-
     for (const token of this.postfix) {
-      if (isOperator(token)) {
+      if (TypeGuards.isOperatorToken(token)) {
         const r = this.operands.pop()
         const l = this.operands.pop()
-        const result = operatorImpls[token](l, r)
+        const result = evalOperatorToken(token, l, r)
         this.operands.push(result)
-      } else if (isRange(token)) {
-        this.operands.push(yield rangeValue(token, ctx))
       } else {
-        const literal = parseLiteral(token)
+        this.operands.push(evalToken(token, ctx))
-        this.operands.push(literal !== undefined ? literal : yield ctx.get(token))
       }
     }
     return this.operands[0]
   }
   public * value (ctx: Context) {
-    return toValue(yield this.evaluate(ctx))
+    return toValue(this.evaluate(ctx))
   }
 }
 
-function * toPostfix (tokens: IterableIterator<string>): IterableIterator<string> {
+export function evalToken (token: Token | undefined, ctx: Context): any {
-  const ops = []
+  assert(ctx, () => 'unable to evaluate: context not defined')
+  if (TypeGuards.isPropertyAccessToken(token)) {
+    const variable = token.variable.getText()
+    const props: string[] = token.props.map(prop => evalToken(prop, ctx))
+    return ctx.get([variable, ...props])
+  }
+  if (TypeGuards.isRangeToken(token)) return evalRangeToken(token, ctx)
+  if (TypeGuards.isLiteralToken(token)) return evalLiteralToken(token)
+  if (TypeGuards.isNumberToken(token)) return evalNumberToken(token)
+  if (TypeGuards.isWordToken(token)) return token.getText()
+  if (TypeGuards.isQuotedToken(token)) return evalQuotedToken(token)
+}
+
+function evalNumberToken (token: NumberToken) {
+  const str = token.whole.content + '.' + (token.decimal ? token.decimal.content : '')
+  return Number(str)
+}
+
+export function evalQuotedToken (token: QuotedToken) {
+  return parseStringLiteral(token.getText())
+}
+
+function evalOperatorToken (token: OperatorToken, lhs: any, rhs: any) {
+  const impl = operatorImpls[token.operator]
+  return impl(lhs, rhs)
+}
+
+function evalLiteralToken (token: LiteralToken) {
+  return literalValues[token.literal]
+}
+
+function evalRangeToken (token: RangeToken, ctx: Context) {
+  const low: number = evalToken(token.lhs, ctx)
+  const high: number = evalToken(token.rhs, ctx)
+  return range(+low, +high + 1)
+}
+
+function * toPostfix (tokens: IterableIterator<Token>): IterableIterator<Token> {
+  const ops: OperatorToken[] = []
   for (const token of tokens) {
-    if (isOperator(token)) {
+    if (TypeGuards.isOperatorToken(token)) {
-      while (ops.length && precedence[ops[ops.length - 1]] > precedence[token]) {
+      while (ops.length && ops[ops.length - 1].getPrecedence() > token.getPrecedence()) {
         yield ops.pop()!
       }
       ops.push(token)
@@ -51,16 +91,3 @@ function * toPostfix (tokens: IterableIterator<string>): IterableIterator<string
     yield ops.pop()!
   }
 }
-
-function * rangeValue (token: string, ctx: Context) {
-  let match
-  if ((match = token.match(rangeLine))) {
-    const low = yield new Expression(match[1]).value(ctx)
-    const high = yield new Expression(match[2]).value(ctx)
-    return range(+low, +high + 1)
-  }
-}
-
-function isRange (str: string) {
-  return rangeLine.test(str)
-}

src/render/operator.ts

@@ -2,18 +2,6 @@ import { isComparable } from '../drop/icomparable'
 import { isFunction } from '../util/underscore'
 import { isTruthy } from '../render/boolean'
 
-export const precedence = {
-  '==': 1,
-  '!=': 1,
-  '>': 1,
-  '<': 1,
-  '>=': 1,
-  '<=': 1,
-  'contains': 1,
-  'and': 0,
-  'or': 0
-}
-
 export const operatorImpls: {[key: string]: (lhs: any, rhs: any) => boolean} = {
   '==': (l: any, r: any) => {
     if (isComparable(l)) return l.equals(r)
@@ -51,7 +39,3 @@ export const operatorImpls: {[key: string]: (lhs: any, rhs: any) => boolean} = {
   'and': (l: any, r: any) => isTruthy(l) && isTruthy(r),
   'or': (l: any, r: any) => isTruthy(l) || isTruthy(r)
 }
-
-export function isOperator (token: string) {
-  return precedence.hasOwnProperty(token)
-}

src/template/filter/filter-map.ts

@@ -10,7 +10,7 @@ export class FilterMap {
 
   get (name: string) {
     const impl = this.impls[name]
-    assert(impl || !this.strictFilters, `undefined filter: ${name}`)
+    assert(impl || !this.strictFilters, () => `undefined filter: ${name}`)
     return impl
   }
 

src/template/filter/filter.ts

@@ -1,4 +1,4 @@
-import { Expression } from '../../render/expression'
+import { evalToken } from '../../render/expression'
 import { Context } from '../../context/context'
 import { identify } from '../../util/underscore'
 import { FilterImplOptions } from './filter-impl-options'
@@ -16,9 +16,9 @@ export class Filter {
   }
   public * render (value: any, context: Context) {
     const argv: any[] = []
-    for (const arg of this.args) {
+    for (const arg of this.args as FilterArg[]) {
-      if (isKeyValuePair(arg)) argv.push([arg[0], yield new Expression(arg[1]).evaluate(context)])
+      if (isKeyValuePair(arg)) argv.push([arg[0], yield evalToken(arg[1], context)])
-      else argv.push(yield new Expression(arg).evaluate(context))
+      else argv.push(yield evalToken(arg, context))
     }
     return this.impl.apply({ context }, [value, ...argv])
   }

src/template/html.ts

@@ -1,6 +1,6 @@
 import { TemplateImpl } from '../template/template-impl'
 import { Template } from '../template/template'
-import { HTMLToken } from '../parser/html-token'
+import { HTMLToken } from '../tokens/html-token'
 import { Context } from '../context/context'
 import { Emitter } from '../render/emitter'
 
@@ -8,7 +8,7 @@ export class HTML extends TemplateImpl<HTMLToken> implements Template {
   private str: string
   public constructor (token: HTMLToken) {
     super(token)
-    this.str = token.content
+    this.str = token.getContent()
   }
   public * render (ctx: Context, emitter: Emitter): IterableIterator<void> {
     emitter.write(this.str)

src/template/output.ts

@@ -5,7 +5,7 @@ import { TemplateImpl } from '../template/template-impl'
 import { Template } from '../template/template'
 import { Context } from '../context/context'
 import { Emitter } from '../render/emitter'
-import { OutputToken } from '../parser/output-token'
+import { OutputToken } from '../tokens/output-token'
 
 export class Output extends TemplateImpl<OutputToken> implements Template {
   private value: Value

src/template/tag/hash.ts

@@ -1,4 +1,4 @@
-import { Expression } from '../../render/expression'
+import { evalToken } from '../../render/expression'
 import { Context } from '../../context/context'
 import { Tokenizer } from '../../parser/tokenizer'
 
@@ -11,17 +11,17 @@ import { Tokenizer } from '../../parser/tokenizer'
  *    hash['reversed'] === undefined
  */
 export class Hash {
-  [key: string]: any
+  hash: { [key: string]: any } = {}
   constructor (markup: string) {
     const tokenizer = new Tokenizer(markup)
-    for (const [name, value] of tokenizer.readHashes()) {
+    for (const hash of tokenizer.readHashes()) {
-      this[name] = value
+      this.hash[hash.name.content] = hash.value
     }
   }
   * render (ctx: Context) {
     const hash = {}
-    for (const key of Object.keys(this)) {
+    for (const key of Object.keys(this.hash)) {
-      hash[key] = yield new Expression(this[key]).evaluate(ctx)
+      hash[key] = evalToken(this.hash[key], ctx)
     }
     return hash
   }

src/template/tag/tag-impl-options.ts

@@ -1,11 +1,11 @@
 import { Context } from '../../context/context'
-import { TagToken } from '../../parser/tag-token'
+import { TagToken } from '../../tokens/tag-token'
-import { Token } from '../../parser/token'
+import { TopLevelToken } from '../../tokens/toplevel-token'
 import { TagImpl } from './tag-impl'
 import { Hash } from '../../template/tag/hash'
 import { Emitter } from '../../render/emitter'
 
 export interface TagImplOptions {
-  parse?: (this: TagImpl, token: TagToken, remainingTokens: Token[]) => void;
+  parse?: (this: TagImpl, token: TagToken, remainingTokens: TopLevelToken[]) => void;
   render: (this: TagImpl, ctx: Context, emitter: Emitter, hash: Hash) => any;
 }

src/template/tag/tag-map.ts

@@ -6,7 +6,7 @@ export class TagMap {
 
   get (name: string) {
     const impl = this.impls[name]
-    assert(impl, `tag "${name}" not found`)
+    assert(impl, () => `tag "${name}" not found`)
     return impl
   }
 

src/template/tag/tag.ts

@@ -1,7 +1,7 @@
 import { isFunction } from '../../util/underscore'
 import { Liquid } from '../../liquid'
 import { TemplateImpl } from '../../template/template-impl'
-import { Emitter, Hash, Context, TagImplOptions, TagToken, Template, Token } from '../../types'
+import { Emitter, Hash, Context, TagImplOptions, TagToken, Template, TopLevelToken } from '../../types'
 import { TagImpl } from './tag-impl'
 
 export class Tag extends TemplateImpl<TagToken> implements Template {
@@ -9,7 +9,7 @@ export class Tag extends TemplateImpl<TagToken> implements Template {
   private impl: TagImpl
   private static impls: { [key: string]: TagImplOptions } = {}
 
-  public constructor (token: TagToken, tokens: Token[], liquid: Liquid) {
+  public constructor (token: TagToken, tokens: TopLevelToken[], liquid: Liquid) {
     super(token)
     this.name = token.name
 

src/template/template.ts

@@ -1,5 +1,5 @@
 import { Context } from '../context/context'
-import { Token } from '../parser/token'
+import { Token } from '../tokens/token'
 import { Emitter } from '../render/emitter'
 
 export interface Template {

src/template/value.ts

@@ -1,23 +1,24 @@
-import { Expression } from '../render/expression'
+import { evalToken } from '../render/expression'
 import { Tokenizer } from '../parser/tokenizer'
 import { FilterMap } from '../template/filter/filter-map'
 import { Filter } from './filter/filter'
 import { Context } from '../context/context'
+import { ValueToken } from '../tokens/value-token'
 
 export class Value {
   public readonly filters: Filter[] = []
-  public readonly initial: string
+  public readonly initial?: ValueToken
 
   /**
    * @param str the value to be valuated, eg.: "foobar" | truncate: 3
    */
   public constructor (str: string, private readonly filterMap: FilterMap) {
     const tokenizer = new Tokenizer(str)
-    this.initial = tokenizer.readValue().toString()
+    this.initial = tokenizer.readValue()
-    this.filters = tokenizer.readFilterTokens().map(({ name, args }) => new Filter(name, this.filterMap.get(name), args))
+    this.filters = tokenizer.readFilters().map(({ name, args }) => new Filter(name, this.filterMap.get(name), args))
   }
   public * value (ctx: Context) {
-    let val = yield new Expression(this.initial).evaluate(ctx)
+    let val = yield evalToken(this.initial, ctx)
     for (const filter of this.filters) {
       val = yield filter.render(val, ctx)
     }

src/tokens/delimited-token.ts

@@ -1,18 +1,23 @@
 import { Token } from './token'
+import { TokenKind } from '../parser/token-kind'
 import { last } from '../util/underscore'
 
-export class DelimitedToken extends Token {
+export abstract class DelimitedToken extends Token {
+  public trimLeft = false
+  public trimRight = false
+  public content: string
   public constructor (
-    raw: string,
+    kind: TokenKind,
     content: string,
     input: string,
-    line: number,
+    begin: number,
-    pos: number,
+    end: number,
     trimLeft: boolean,
     trimRight: boolean,
     file?: string
   ) {
-    super(raw, input, line, pos, file)
+    super(kind, input, begin, end, file)
+    this.content = this.getText()
     const tl = content[0] === '-'
     const tr = last(content) === '-'
     this.content = content

src/tokens/filter-token.ts

@@ -1,16 +1,16 @@
 import { Token } from './token'
-import { FilterArg } from './filter-arg'
+import { FilterArg } from '../parser/filter-arg'
+import { TokenKind } from '../parser/token-kind'
 
 export class FilterToken extends Token {
   public constructor (
     public name: string,
     public args: FilterArg[],
-    raw: string,
     input: string,
-    line: number,
+    begin: number,
-    col: number,
+    end: number,
     file?: string
   ) {
-    super(raw, input, line, col, file)
+    super(TokenKind.Filter, input, begin, end, file)
   }
 }

src/tokens/hash-token.ts

@@ -0,0 +1,17 @@
+import { Token } from './token'
+import { ValueToken } from './value-token'
+import { WordToken } from './word-token'
+import { TokenKind } from '../parser/token-kind'
+
+export class HashToken extends Token {
+  constructor (
+    public input: string,
+    public begin: number,
+    public end: number,
+    public name: WordToken,
+    public value?: ValueToken,
+    public file?: string
+  ) {
+    super(TokenKind.Hash, input, begin, end, file)
+  }
+}

src/tokens/html-token.ts

@@ -0,0 +1,18 @@
+import { Token } from './token'
+import { TokenKind } from '../parser/token-kind'
+
+export class HTMLToken extends Token {
+  trimLeft = 0
+  trimRight = 0
+  constructor (
+    public input: string,
+    public begin: number,
+    public end: number,
+    public file?: string
+  ) {
+    super(TokenKind.HTML, input, begin, end, file)
+  }
+  public getContent () {
+    return this.input.slice(this.begin + this.trimLeft, this.end - this.trimRight)
+  }
+}

src/tokens/literal-token.ts

@@ -0,0 +1,15 @@
+import { Token } from './token'
+import { TokenKind } from '../parser/token-kind'
+
+export class LiteralToken extends Token {
+  public literal: string
+  public constructor (
+    public input: string,
+    public begin: number,
+    public end: number,
+    public file?: string
+  ) {
+    super(TokenKind.Literal, input, begin, end, file)
+    this.literal = this.getText()
+  }
+}

src/tokens/number-token.ts

@@ -0,0 +1,12 @@
+import { Token } from './token'
+import { WordToken } from './word-token'
+import { TokenKind } from '../parser/token-kind'
+
+export class NumberToken extends Token {
+  constructor (
+    public whole: WordToken,
+    public decimal?: WordToken
+  ) {
+    super(TokenKind.Number, whole.input, whole.begin, decimal ? decimal.end : whole.end, whole.file)
+  }
+}

src/tokens/operator-token.ts

@@ -0,0 +1,30 @@
+import { Token } from './token'
+import { TokenKind } from '../parser/token-kind'
+
+export const precedence = {
+  '==': 1,
+  '!=': 1,
+  '>': 1,
+  '<': 1,
+  '>=': 1,
+  '<=': 1,
+  'contains': 1,
+  'and': 0,
+  'or': 0
+}
+
+export class OperatorToken extends Token {
+  public operator: string
+  public constructor (
+    public input: string,
+    public begin: number,
+    public end: number,
+    public file?: string
+  ) {
+    super(TokenKind.Operator, input, begin, end, file)
+    this.operator = this.getText()
+  }
+  getPrecedence () {
+    return precedence[this.getText()]
+  }
+}

src/tokens/output-token.ts

@@ -0,0 +1,17 @@
+import { DelimitedToken } from './delimited-token'
+import { NormalizedFullOptions } from '../liquid-options'
+import { TokenKind } from '../parser/token-kind'
+
+export class OutputToken extends DelimitedToken {
+  public constructor (
+    input: string,
+    begin: number,
+    end: number,
+    options: NormalizedFullOptions,
+    file?: string
+  ) {
+    const { trimOutputLeft, trimOutputRight, outputDelimiterLeft, outputDelimiterRight } = options
+    const value = input.slice(begin + outputDelimiterLeft.length, end - outputDelimiterRight.length)
+    super(TokenKind.Output, value, input, begin, end, trimOutputLeft, trimOutputRight, file)
+  }
+}

src/tokens/property-access-token.ts

@@ -0,0 +1,14 @@
+import { Token } from './token'
+import { WordToken } from './word-token'
+import { QuotedToken } from './quoted-token'
+import { TokenKind } from '../parser/token-kind'
+
+export class PropertyAccessToken extends Token {
+  constructor (
+    public variable: WordToken,
+    public props: (WordToken | QuotedToken | PropertyAccessToken)[],
+    end: number
+  ) {
+    super(TokenKind.PropertyAccess, variable.input, variable.begin, end, variable.file)
+  }
+}

src/tokens/quoted-token.ts

@@ -0,0 +1,13 @@
+import { Token } from './token'
+import { TokenKind } from '../parser/token-kind'
+
+export class QuotedToken extends Token {
+  constructor (
+    public input: string,
+    public begin: number,
+    public end: number,
+    public file?: string
+  ) {
+    super(TokenKind.Quoted, input, begin, end, file)
+  }
+}

src/tokens/range-token.ts

@@ -0,0 +1,16 @@
+import { Token } from './token'
+import { ValueToken } from './value-token'
+import { TokenKind } from '../parser/token-kind'
+
+export class RangeToken extends Token {
+  constructor (
+    public input: string,
+    public begin: number,
+    public end: number,
+    public lhs: ValueToken,
+    public rhs: ValueToken,
+    public file?: string
+  ) {
+    super(TokenKind.Range, input, begin, end, file)
+  }
+}

src/tokens/tag-token.ts

@@ -0,0 +1,30 @@
+import { DelimitedToken } from './delimited-token'
+import { BLANK, TYPES, VARIABLE } from '../util/character'
+import { TokenizationError } from '../util/error'
+import { NormalizedFullOptions } from '../liquid-options'
+import { TokenKind } from '../parser/token-kind'
+
+export class TagToken extends DelimitedToken {
+  public name: string
+  public args: string
+  public constructor (
+    input: string,
+    begin: number,
+    end: number,
+    options: NormalizedFullOptions,
+    file?: string
+  ) {
+    const { trimTagLeft, trimTagRight, tagDelimiterLeft, tagDelimiterRight } = options
+    const value = input.slice(begin + tagDelimiterLeft.length, end - tagDelimiterRight.length)
+    super(TokenKind.Tag, value, input, begin, end, trimTagLeft, trimTagRight, file)
+
+    let nameEnd = 0
+    while (TYPES[this.content.charCodeAt(nameEnd)] & VARIABLE) nameEnd++
+    this.name = this.content.slice(0, nameEnd)
+    if (!this.name) throw new TokenizationError(`illegal tag syntax`, this)
+
+    let argsBegin = nameEnd
+    while (TYPES[this.content.charCodeAt(argsBegin)] & BLANK) argsBegin++
+    this.args = this.content.slice(argsBegin)
+  }
+}

src/tokens/token.ts

@@ -0,0 +1,27 @@
+import { TokenKind } from '../parser/token-kind'
+
+export abstract class Token {
+  public constructor (
+    public kind: TokenKind,
+    public input: string,
+    public begin: number,
+    public end: number,
+    public file?: string
+  ) {}
+  public getText () {
+    return this.input.slice(this.begin, this.end)
+  }
+  public getPosition () {
+    let [row, col] = [1, 1]
+    for (let i = 0; i < this.begin; i++) {
+      if (this.input[i] === '\n') {
+        row++
+        col = 1
+      } else col++
+    }
+    return [row, col]
+  }
+  public size () {
+    return this.end - this.begin
+  }
+}

src/tokens/toplevel-token.ts

@@ -0,0 +1,5 @@
+import { TagToken } from './tag-token'
+import { HTMLToken } from './html-token'
+import { OutputToken } from './output-token'
+
+export type TopLevelToken = TagToken | OutputToken | HTMLToken

src/tokens/value-token.ts

@@ -0,0 +1,6 @@
+import { RangeToken } from './range-token'
+import { LiteralToken } from './literal-token'
+import { QuotedToken } from './quoted-token'
+import { PropertyAccessToken } from './property-access-token'
+
+export type ValueToken = RangeToken | LiteralToken | QuotedToken | PropertyAccessToken

src/tokens/word-token.ts

@@ -0,0 +1,26 @@
+import { Token } from './token'
+import { NUMBER, TYPES, SIGN } from '../util/character'
+import { TokenKind } from '../parser/token-kind'
+
+// a word can be an identifier, a number, a keyword or a single-word-literal
+export class WordToken extends Token {
+  public content: string
+  constructor (
+    public input: string,
+    public begin: number,
+    public end: number,
+    public file?: string
+  ) {
+    super(TokenKind.Word, input, begin, end, file)
+    this.content = this.getText()
+  }
+  isNumber (allowSign = false) {
+    const begin = allowSign && TYPES[this.input.charCodeAt(this.begin)] & SIGN
+      ? this.begin + 1
+      : this.begin
+    for (let i = begin; i < this.end; i++) {
+      if (!(TYPES[this.input.charCodeAt(i)] & NUMBER)) return false
+    }
+    return true
+  }
+}

src/types.ts

@@ -1,13 +1,18 @@
+import * as TypeGuards from './util/type-guards'
+export { TypeGuards }
 export { ParseError, TokenizationError, AssertionError } from './util/error'
+export { assert } from './util/assert'
 export { Drop } from './drop/drop'
 export { Emitter } from './render/emitter'
 export { Expression } from './render/expression'
 export { isFalsy, isTruthy } from './render/boolean'
-export { TagToken } from './parser/tag-token'
+export { TagToken } from './tokens/tag-token'
 export { Context } from './context/context'
 export { Template } from './template/template'
 export { TagImplOptions } from './template/tag/tag-impl-options'
 export { ParseStream } from './parser/parse-stream'
-export { Token } from './parser/token'
+export { Token } from './tokens/token'
+export { TopLevelToken } from './tokens/toplevel-token'
 export { Tokenizer } from './parser/tokenizer'
 export { Hash } from './template/tag/hash'
+export { evalToken, evalQuotedToken } from './render/expression'

src/util/assert.ts

@@ -1,8 +1,8 @@
 import { AssertionError } from './error'
 
-export function assert <T> (predicate: T | null | undefined, message?: string) {
+export function assert <T> (predicate: T | null | undefined, message?: () => string) {
   if (!predicate) {
-    message = message || `expect ${predicate} to be true`
+    const msg = message ? message() : `expect ${predicate} to be true`
-    throw new AssertionError(message)
+    throw new AssertionError(msg)
   }
 }

src/util/async.ts

@@ -1,6 +1,6 @@
 import { isFunction } from './underscore'
 
-type resolver = (x?: any) => Thenable
+type resolver = (x?: any) => any
 
 interface Thenable {
   then (resolve: resolver, reject?: resolver): Thenable;

src/util/character.ts

@@ -0,0 +1,10 @@
+// bitmask character types to boost performance
+// generated by bin/char-types.js
+export const TYPES = [0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2, 8, 0, 0, 0, 0, 8, 0, 0, 0, 64, 0, 65, 0, 0, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 0, 0, 2, 2, 2, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]
+export const VARIABLE = 1
+export const OPERATOR = 2
+export const BLANK = 4
+export const QUOTE = 8
+export const INLINE_BLANK = 16
+export const NUMBER = 32
+export const SIGN = 64

src/util/error.ts

@@ -1,5 +1,5 @@
 import * as _ from './underscore'
-import { Token } from '../parser/token'
+import { Token } from '../tokens/token'
 import { Template } from '../template/template'
 
 abstract class LiquidError extends Error {
@@ -57,14 +57,15 @@ export class AssertionError extends Error {
 }
 
 function mkContext (token: Token) {
+  const [line] = token.getPosition()
   const lines = token.input.split('\n')
-  const begin = Math.max(token.line - 2, 1)
+  const begin = Math.max(line - 2, 1)
-  const end = Math.min(token.line + 3, lines.length)
+  const end = Math.min(line + 3, lines.length)
 
   const context = _
     .range(begin, end + 1)
     .map(lineNumber => {
-      const indicator = (lineNumber === token.line) ? '>> ' : '   '
+      const indicator = (lineNumber === line) ? '>> ' : '   '
       const num = _.padStart(String(lineNumber), String(end).length)
       const text = lines[lineNumber - 1]
       return `${indicator}${num}| ${text}`
@@ -76,6 +77,7 @@ function mkContext (token: Token) {
 
 function mkMessage (msg: string, token: Token) {
   if (token.file) msg += `, file:${token.file}`
-  msg += `, line:${token.line}, col:${token.col}`
+  const [line, col] = token.getPosition()
+  msg += `, line:${line}, col:${col}`
   return msg
 }

src/util/literal.ts

@@ -0,0 +1,12 @@
+import { NullDrop } from '../drop/null-drop'
+import { EmptyDrop } from '../drop/empty-drop'
+import { BlankDrop } from '../drop/blank-drop'
+
+export const literalValues = {
+  'true': true,
+  'false': false,
+  'nil': new NullDrop(),
+  'null': new NullDrop(),
+  'empty': new EmptyDrop(),
+  'blank': new BlankDrop()
+}

src/util/type-guards.ts

@@ -0,0 +1,55 @@
+import { OperatorToken } from '../tokens/operator-token'
+import { WordToken } from '../tokens/word-token'
+import { TagToken } from '../tokens/tag-token'
+import { HTMLToken } from '../tokens/html-token'
+import { OutputToken } from '../tokens/output-token'
+import { PropertyAccessToken } from '../tokens/property-access-token'
+import { LiteralToken } from '../tokens/literal-token'
+import { QuotedToken } from '../tokens/quoted-token'
+import { NumberToken } from '../tokens/number-token'
+import { RangeToken } from '../tokens/range-token'
+import { TokenKind } from '../parser/token-kind'
+
+export function isOperatorToken (val: any): val is OperatorToken {
+  return getKind(val) === TokenKind.Operator
+}
+
+export function isHTMLToken (val: any): val is HTMLToken {
+  return getKind(val) === TokenKind.HTML
+}
+
+export function isOutputToken (val: any): val is OutputToken {
+  return getKind(val) === TokenKind.Output
+}
+
+export function isTagToken (val: any): val is TagToken {
+  return getKind(val) === TokenKind.Tag
+}
+
+export function isQuotedToken (val: any): val is QuotedToken {
+  return getKind(val) === TokenKind.Quoted
+}
+
+export function isLiteralToken (val: any): val is LiteralToken {
+  return getKind(val) === TokenKind.Literal
+}
+
+export function isNumberToken (val: any): val is NumberToken {
+  return getKind(val) === TokenKind.Number
+}
+
+export function isPropertyAccessToken (val: any): val is PropertyAccessToken {
+  return getKind(val) === TokenKind.PropertyAccess
+}
+
+export function isWordToken (val: any): val is WordToken {
+  return getKind(val) === TokenKind.Word
+}
+
+export function isRangeToken (val: any): val is RangeToken {
+  return getKind(val) === TokenKind.Range
+}
+
+function getKind (val: any) {
+  return val ? val.kind : -1
+}

test/e2e/eval-value-sync.ts

@@ -6,6 +6,6 @@ describe('#evalValueSync()', function () {
   beforeEach(() => { engine = new Liquid() })
 
   it('should throw when scope undefined', async function () {
-    return expect(() => engine.evalValueSync('{{"foo"}}', null as any)).to.throw(/context not defined/)
+    return expect(() => engine.evalValueSync('foo', null as any)).to.throw(/context not defined/)
   })
 })

test/e2e/eval-value.ts

@@ -1,11 +1,11 @@
 import { expect } from 'chai'
 import { Liquid } from '../..'
 
-describe('.evalValue()', function () {
+describe('#evalValue()', function () {
   var engine: Liquid
   beforeEach(() => { engine = new Liquid() })
 
   it('should throw when scope undefined', async function () {
-    return expect(engine.evalValue('{{"foo"}}', null as any)).to.be.rejectedWith(/context not defined/)
+    return expect(engine.evalValue('"foo"', null as any)).to.be.rejectedWith(/context not defined/)
   })
 })

test/integration/builtin/tags/decrement.ts

@@ -6,11 +6,6 @@ use(chaiAsPromised)
 
 describe('tags/decrement', function () {
   const liquid = new Liquid()
-  it('should throw when variable expression illegal', function () {
-    const src = '{% decrement / %}{{var}}'
-    const ctx = {}
-    return expect(liquid.parseAndRender(src, ctx)).to.be.rejectedWith(/illegal/)
-  })
 
   it('should decrement undefined variable', async function () {
     const src = '{% decrement var %}{% decrement var %}{% decrement var %}'

test/integration/builtin/tags/for.ts

@@ -46,6 +46,12 @@ describe('tags/for', function () {
         .to.be.rejectedWith(/tag .* not closed/)
     })
 
+    it('should reject when for in not found', function () {
+      const src = '{%for c alpha%}{{c}}'
+      return expect(liquid.parseAndRender(src, scope))
+        .to.be.rejectedWith('illegal tag: {%for c alpha%}, line:1, col:1')
+    })
+
     it('should reject when inner templates rejected', function () {
       const src = '{%for c in alpha%}{%throwingTag%}{%endfor%}'
       return expect(liquid.parseAndRender(src, scope))

test/integration/builtin/tags/include.ts

@@ -83,6 +83,22 @@ describe('tags/include', function () {
     const html = await liquid.renderFile('with.html')
     return expect(html).to.equal('color:red, shape:rect')
   })
+  it('should ignore if with value not specified', async function () {
+    mock({
+      '/with.html': '{% include "color" with, shape: "rect" %}',
+      '/color.html': 'color:{{color}}, shape:{{shape}}'
+    })
+    const html = await liquid.renderFile('with.html')
+    return expect(html).to.equal('color:, shape:rect')
+  })
+  it('should treat with as a valid key', async function () {
+    mock({
+      '/with.html': '{% include "color" with: "foo" %}',
+      '/color.html': 'with:{{with}}'
+    })
+    const html = await liquid.renderFile('with.html')
+    return expect(html).to.equal('with:foo')
+  })
   it('should support include: with as Drop', async function () {
     class ColorDrop extends Drop {
       public valueOf (): string {

test/integration/builtin/tags/layout.ts

@@ -1,6 +1,9 @@
 import { Liquid } from '../../../../src/liquid'
-import { expect } from 'chai'
+import { expect, use } from 'chai'
 import { mock, restore } from '../../../stub/mockfs'
+import * as chaiAsPromised from 'chai-as-promised'
+
+use(chaiAsPromised)
 
 describe('tags/layout', function () {
   let liquid: Liquid
@@ -29,6 +32,15 @@ describe('tags/layout', function () {
       expect(e.message).to.match(/illegal argument ""/)
     })
   })
+  it('should throw when filename resolved to falsy', function () {
+    mock({
+      '/parent.html': '{%layout foo%}'
+    })
+    return liquid.renderFile('/parent.html').catch(function (e) {
+      expect(e.name).to.equal('RenderError')
+      expect(e.message).to.match(/illegal filename "foo":"undefined"/)
+    })
+  })
   describe('anonymous block', function () {
     it('should handle anonymous block', async function () {
       mock({

test/integration/builtin/tags/render.ts

@@ -121,9 +121,33 @@ describe('tags/render', function () {
     const html = await liquid.renderFile('index.html', { colors: ['red', 'green'] })
     expect(html).to.equal('1: red\n2: green\n')
   })
+  it('should support for <non-array> as', async function () {
+    mock({
+      '/index.html': '{% render "item" for "green" as color %}',
+      '/item.html': '{{forloop.index}}: {{color}}\n'
+    })
+    const html = await liquid.renderFile('index.html')
+    expect(html).to.equal('1: green\n')
+  })
+  it('should support for without as', async function () {
+    mock({
+      '/index.html': '{% render "item" for colors %}',
+      '/item.html': '{{forloop.index}}: {{color}}\n'
+    })
+    const html = await liquid.renderFile('index.html', { colors: ['red', 'green'] })
+    expect(html).to.equal('1: \n2: \n')
+  })
   it('should support for...as with other parameters', async function () {
     mock({
-      '/index.html': '{% render "item" for colors as color with ".\n" as tail, sep: ". "%}',
+      '/index.html': '{% render "item" for colors as color with ".\n" as tail sep: ". "%}',
+      '/item.html': '{{forloop.index}}{{sep}}{{color}}{{tail}}'
+    })
+    const html = await liquid.renderFile('index.html', { colors: ['red', 'green'] })
+    expect(html).to.equal('1. red.\n2. green.\n')
+  })
+  it('should support for...as with other parameters (comma separated)', async function () {
+    mock({
+      '/index.html': '{% render "item" for colors as color, with ".\n" as tail, sep: ". "%}',
       '/item.html': '{{forloop.index}}{{sep}}{{color}}{{tail}}'
     })
     const html = await liquid.renderFile('index.html', { colors: ['red', 'green'] })

test/integration/builtin/tags/tablerow.ts

@@ -1,5 +1,8 @@
 import { Liquid } from '../../../../src/liquid'
-import { expect } from 'chai'
+import { expect, use } from 'chai'
+import * as chaiAsPromised from 'chai-as-promised'
+
+use(chaiAsPromised)
 
 describe('tags/tablerow', function () {
   const liquid = new Liquid()
@@ -50,6 +53,12 @@ describe('tags/tablerow', function () {
       .to.be.rejectedWith(/tag .* not closed/)
   })
 
+  it('should throw when x in y not found', function () {
+    const src = '{% tablerow i (1..3) %}{{ i }}'
+    return expect(liquid.parseAndRender(src))
+      .to.be.rejectedWith('illegal tag: {% tablerow i (1..3) %}, line:1, col:1')
+  })
+
   it('should support tablerow with range', async function () {
     const src = '{% tablerow i in (1..5) cols:2 %}{{ i }}{% endtablerow %}'
     const dst =

test/integration/liquid/liquid.ts

@@ -93,7 +93,7 @@ describe('Liquid', function () {
       })
       const tpls = await engine.getTemplate('mocha')
       expect(tpls.length).to.gte(1)
-      expect(tpls[0].token.raw).to.contain('module.exports')
+      expect(tpls[0].token.getText()).to.contain('module.exports')
     })
   })
   describe('#evalValue', function () {

test/integration/liquid/register-tags.ts

@@ -21,7 +21,7 @@ describe('liquid#registerTag()', function () {
   it('should have access to ctx in render()', async () => {
     const liquid = new Liquid()
     liquid.registerTag('dynamic-string', {
-      render: async (ctx) => ctx.get('c')
+      render: async (ctx) => ctx.get(['c'])
     })
     const html = await liquid.parseAndRender(`A{% dynamic-string %}C`, {
       c: 'B'

test/integration/liquid/root.ts

@@ -1,7 +1,7 @@
 import { normalize } from '../../../src/liquid-options'
 import { expect } from 'chai'
 
-describe('LiquidOptions', function () {
+describe('LiquidOptions#root', function () {
   describe('#normalize ()', function () {
     it('should normalize string typed root array', function () {
       const options = normalize({ root: 'foo' })

test/integration/liquid/trimming.ts

@@ -45,7 +45,7 @@ describe('LiquidOptions#trimming', function () {
       const html = await engine.parseAndRender(src, ctx)
       return expect(html).to.equal('aharttle')
     })
-    it('should respect to greedy:false by default', async function () {
+    it('should allow greedy:false', async function () {
       const engine = new Liquid({ greedy: false } as any)
       const html = await engine.parseAndRender(src, ctx)
       return expect(html).to.equal('\n a \nharttle ')

test/integration/liquid/whitespace-ctrl.ts

@@ -1,5 +1,5 @@
 import { expect } from 'chai'
-import { Liquid } from '../..'
+import { Liquid } from '../../../src/liquid'
 
 const liquid = new Liquid()
 

test/integration/util/error.ts

@@ -41,11 +41,6 @@ describe('error', function () {
       const err = await expect(engine.parseAndRender(html)).be.rejected
       expect(err.token.input).to.equal(html)
     })
-    it('should contain line number in err.token.line', async function () {
-      const err = await expect(engine.parseAndRender('1\n2\n{% . a %}\n4')).be.rejected
-      expect(err.name).to.equal('TokenizationError')
-      expect(err.token.line).to.equal(3)
-    })
     it('should contain stack in err.stack', async function () {
       const err = await expect(engine.parseAndRender('{% . a %}')).be.rejected
       expect(err.message).to.contain('illegal tag syntax')
@@ -58,11 +53,11 @@ describe('error', function () {
         expect(err.stack).to.not.contain('at Object.parse')
       })
     })
-    it('should throw error with line and pos if tag unmatched', async function () {
+    it('should throw error with [line, col] if tag unmatched', async function () {
       const err = await expect(engine.parseAndRender('1\n2\nfoo{% assign a = 4 }\n4')).be.rejected
+      console.log(err.stack)
       expect(err.name).to.equal('TokenizationError')
-      expect(err.token.line).to.equal(3)
+      expect(err.message).to.equal('tag "{% assign a =..." not closed, line:3, col:4')
-      expect(err.token.col).to.equal(4)
     })
   })
 
@@ -175,12 +170,6 @@ describe('error', function () {
       expect(err.stack).to.contain(message.join('\n'))
       expect(err.name).to.equal('RenderError')
     })
-    it('should contain line number in err.token.line', async function () {
-      const src = '1\n2\n{{1|throwingFilter}}\n4'
-      const err = await expect(engine.parseAndRender(src)).be.rejected
-      expect(err.token.line).to.equal(3)
-      expect(err.name).to.equal('RenderError')
-    })
     it('should contain stack in err.stack', async function () {
       const err = await expect(engine.parseAndRender('{%rejectingTag%}')).be.rejected
       expect(err.message).to.contain('intended render reject')
@@ -256,12 +245,6 @@ describe('error', function () {
       expect(err.stack).to.contain(message.join('\n'))
     })
 
-    it('should contain line number in err.token.line', async function () {
-      const html = '<html>\n<head>\n\n{% raw %}\n\n'
-      const err = await expect(engine.parseAndRender(html)).be.rejected
-      expect(err.token.line).to.equal(4)
-    })
-
     it('should contain stack in err.stack', async function () {
       const err = await expect(engine.parseAndRender('{% -a %}')).be.rejected
       expect(err.stack).to.contain('ParseError: tag "-a" not found')

test/unit/context/context.ts

@@ -24,123 +24,49 @@ describe('Context', function () {
     ctx = new Context(scope)
   })
 
-  describe('#propertyAccessSeq()', function () {
-    it('should handle dot syntax', async function () {
-      expect(ctx.parseProp('foo.bar'))
-        .to.deep.equal(['foo', 'bar'])
-    })
-    it('should handle [<String>] syntax', async function () {
-      expect(ctx.parseProp('foo["bar"]'))
-        .to.deep.equal(['foo', 'bar'])
-    })
-    it('should handle [<Identifier>] syntax', async function () {
-      expect(ctx.parseProp('foo[foo]'))
-        .to.deep.equal(['foo', 'zoo'])
-    })
-    it('should handle nested access 1', async function () {
-      expect(ctx.parseProp('foo[bar.zoo]'))
-        .to.deep.equal(['foo', 'coo'])
-    })
-    it('should handle nested access 2', async function () {
-      expect(ctx.parseProp('foo[bar["zoo"]]'))
-        .to.deep.equal(['foo', 'coo'])
-    })
-    it('should handle nested access 3', async function () {
-      expect(ctx.parseProp('bar["foo"].zoo'))
-        .to.deep.equal(['bar', 'foo', 'zoo'])
-    })
-    it('should handle nested access 4', async function () {
-      expect(ctx.parseProp('foo[0].bar'))
-        .to.deep.equal(['foo', '0', 'bar'])
-    })
-    it('should handle nested access 5', async function () {
-      expect(ctx.parseProp('foo[one].bar'))
-        .to.deep.equal(['foo', '1', 'bar'])
-    })
-    it('should handle nested access 6', async function () {
-      expect(ctx.parseProp('foo[two].bar'))
-        .to.deep.equal(['foo', 'undefined', 'bar'])
-    })
-  })
-
   describe('#get()', function () {
     it('should get direct property', async function () {
-      expect(ctx.get('foo')).equal('zoo')
+      expect(ctx.get(['foo'])).equal('zoo')
+    })
+    it('should read nested property', async function () {
+      expect(ctx.get(['obj', 'first'])).to.equal('f')
+      expect(ctx.get(['obj', 'last'])).to.equal('l')
+      expect(ctx.get(['obj', 'size'])).to.equal(undefined)
     })
-
     it('undefined property should yield undefined', async function () {
-      expect(ctx.get('notdefined')).to.equal(undefined)
+      expect(ctx.get(['notdefined'])).to.equal(undefined)
-      expect(ctx.get(false as any)).to.equal(undefined)
+      expect(ctx.get([false as any])).to.equal(undefined)
-    })
-
-    it('should throw for invalid path', async function () {
-      expect(() => ctx.get('')).to.throw('invalid path:""')
-    })
-
-    it('should throw when [] unbalanced', async function () {
-      expect(() => ctx.get('foo[bar')).to.throw(/unbalanced \[\]/)
-    })
-
-    it('should throw when "" unbalanced', async function () {
-      expect(() => ctx.get('foo["bar]')).to.throw(/unbalanced "/)
-    })
-
-    it("should throw when '' unbalanced", async function () {
-      expect(() => ctx.get("foo['bar]")).to.throw(/unbalanced '/)
     })
     it('should respect to toLiquid', async function () {
       const scope = new Context({ foo: {
         toLiquid: () => ({ bar: 'BAR' }),
         bar: 'bar'
       } })
-      expect(scope.get('foo.bar')).to.equal('BAR')
+      expect(scope.get(['foo', 'bar'])).to.equal('BAR')
     })
-
-    it('should access child property via dot syntax', async function () {
-      expect(ctx.get('bar.zoo')).to.equal('coo')
-      expect(ctx.get('bar.arr')).to.deep.equal(['a', 'b'])
-    })
-
-    it('should access child property via [<String>] syntax', async function () {
-      expect(ctx.get('bar["zoo"]')).to.equal('coo')
-    })
-
-    it('should access child property via [<Number>] syntax', async function () {
-      expect(ctx.get('bar.arr[0]')).to.equal('a')
-    })
-
-    it('should access child property via [<Identifier>] syntax', async function () {
-      expect(ctx.get('bar[foo]')).to.equal('coo')
-    })
-
     it('should return undefined when not exist', async function () {
-      expect(ctx.get('foo.foo.foo')).to.be.undefined
+      expect(ctx.get(['foo', 'foo', 'foo'])).to.be.undefined
     })
     it('should return string length as size', async function () {
-      expect(ctx.get('foo.size')).to.equal(3)
+      expect(ctx.get(['foo', 'size'])).to.equal(3)
     })
     it('should return array length as size', async function () {
-      expect(ctx.get('bar.arr.size')).to.equal(2)
+      expect(ctx.get(['bar', 'arr', 'size'])).to.equal(2)
-    })
-    it('should return size property if exists', async function () {
-      expect(ctx.get('zoo.size')).to.equal(4)
-    })
-    it('should return undefined if do not have size and length', async function () {
-      expect(ctx.get('one.size')).to.equal(undefined)
     })
     it('should read .first of array', async function () {
-      expect(ctx.get('bar.arr.first')).to.equal('a')
+      expect(ctx.get(['bar', 'arr', 'first'])).to.equal('a')
-    })
-    it('should read .first of object', async function () {
-      expect(ctx.get('obj.first')).to.equal('f')
     })
     it('should read .last of array', async function () {
-      expect(ctx.get('bar.arr.last')).to.equal('b')
+      expect(ctx.get(['bar', 'arr', 'last'])).to.equal('b')
-    })
-    it('should read .last of object', async function () {
-      expect(ctx.get('obj.last')).to.equal('l')
     })
   })
+
+  describe('#getFromScope()', function () {
+    it('should support string', () => {
+      expect(ctx.getFromScope({ obj: { foo: 'FOO' } }, 'obj.foo')).to.equal('FOO')
+    })
+  })
+
   describe('strictVariables', async function () {
     let ctx: Context
     beforeEach(function () {
@@ -149,22 +75,22 @@ describe('Context', function () {
       } as any)
     })
     it('should throw when variable not defined', function () {
-      return expect(() => ctx.get('notdefined')).to.throw(/undefined variable: notdefined/)
+      return expect(() => ctx.get(['notdefined'])).to.throw(/undefined variable: notdefined/)
     })
     it('should throw when deep variable not exist', async function () {
       ctx.push({ foo: 'FOO' })
-      return expect(() => ctx.get('foo.bar.not.defined')).to.throw(/undefined variable: bar/)
+      return expect(() => ctx.get(['foo', 'bar', 'not', 'defined'])).to.throw(/undefined variable: bar/)
     })
     it('should throw when itself not defined', async function () {
       ctx.push({ foo: 'FOO' })
-      return expect(() => ctx.get('foo.BAR')).to.throw(/undefined variable: BAR/)
+      return expect(() => ctx.get(['foo', 'BAR'])).to.throw(/undefined variable: BAR/)
     })
     it('should find variable in parent scope', async function () {
       ctx.push({ 'foo': 'foo' })
       ctx.push({
         'bar': 'bar'
       })
-      expect(ctx.get('foo')).to.equal('foo')
+      expect(ctx.get(['foo'])).to.equal('foo')
     })
   })
 
@@ -180,14 +106,14 @@ describe('Context', function () {
       ctx.push({
         foo: 'foo'
       })
-      expect(ctx.get('foo')).to.equal('foo')
+      expect(ctx.get(['foo'])).to.equal('foo')
-      expect(ctx.get('bar')).to.equal('bar')
+      expect(ctx.get(['bar'])).to.equal('bar')
     })
     it('should hide deep properties by push', async function () {
       ctx.push({ bar: { bar: 'bar' } })
       ctx.push({ bar: { foo: 'foo' } })
-      expect(ctx.get('bar.foo')).to.equal('foo')
+      expect(ctx.get(['bar', 'foo'])).to.equal('foo')
-      expect(ctx.get('bar.bar')).to.equal(undefined)
+      expect(ctx.get(['bar', 'bar'])).to.equal(undefined)
     })
   })
   describe('.pop()', function () {
@@ -196,7 +122,7 @@ describe('Context', function () {
         foo: 'foo'
       })
       ctx.pop()
-      expect(ctx.get('foo')).to.equal('zoo')
+      expect(ctx.get(['foo'])).to.equal('zoo')
     })
   })
 })

test/unit/parser/match-operator.ts

@@ -0,0 +1,26 @@
+import { expect } from 'chai'
+import { matchOperator } from '../../../src/parser/match-operator'
+
+describe('parser/matchOperator()', function () {
+  it('should match contains', () => {
+    expect(matchOperator('contains', 0)).to.equal(8)
+  })
+  it('should match comparision', () => {
+    expect(matchOperator('>', 0)).to.equal(1)
+    expect(matchOperator('>=', 0)).to.equal(2)
+    expect(matchOperator('<', 0)).to.equal(1)
+    expect(matchOperator('<=', 0)).to.equal(2)
+  })
+  it('should match binary logic', () => {
+    expect(matchOperator('and', 0)).to.equal(3)
+    expect(matchOperator('or', 0)).to.equal(2)
+  })
+  it('should not match if word not terminate', () => {
+    expect(matchOperator('true1', 0)).to.equal(-1)
+    expect(matchOperator('containsa', 0)).to.equal(-1)
+  })
+  it('should match if word boundary found', () => {
+    expect(matchOperator('>=1', 0)).to.equal(2)
+    expect(matchOperator('contains b', 0)).to.equal(8)
+  })
+})

test/unit/parser/parse-string-literal.ts

@@ -1,30 +1,5 @@
 import { expect } from 'chai'
-import { parseLiteral, parseStringLiteral } from '../../../src/parser/literal'
+import { parseStringLiteral } from '../../../src/parser/parse-string-literal'
-import { NullDrop } from '../../../src/drop/null-drop'
-
-describe('parseLiteral()', function () {
-  it('should eval boolean literal', async function () {
-    expect(parseLiteral('true')).to.equal(true)
-    expect(parseLiteral('TrUE')).to.equal(undefined)
-    expect(parseLiteral('false')).to.equal(false)
-  })
-  it('should eval number literal', async function () {
-    expect(parseLiteral('2.3')).to.equal(2.3)
-    expect(parseLiteral('.32')).to.equal(0.32)
-    expect(parseLiteral('-23.')).to.equal(-23)
-    expect(parseLiteral('23')).to.equal(23)
-  })
-  it('should eval string literal', async function () {
-    expect(parseLiteral('"ab\'c"')).to.equal("ab'c")
-    expect(parseLiteral("'ab\"c'")).to.equal('ab"c')
-  })
-  it('should eval nil literal', async function () {
-    expect(parseLiteral('nil')).to.be.instanceOf(NullDrop)
-  })
-  it('should eval null literal', async function () {
-    expect(parseLiteral('null')).to.be.instanceOf(NullDrop)
-  })
-})
 
 describe('parseStringLiteral()', function () {
   it('should parse octal escape', () => {
@@ -36,7 +11,7 @@ describe('parseStringLiteral()', function () {
   it('should skip invalid octal escape', () => {
     expect(parseStringLiteral(String.raw`"\9"`)).to.equal('9')
   })
-  it('should parse \n, \t, \r', () => {
+  it('should parse \\n, \\t, \\r', () => {
     expect(parseStringLiteral(String.raw`"fo\no"`)).to.equal('fo\no')
     expect(parseStringLiteral(String.raw`'fo\to'`)).to.equal('fo\to')
     expect(parseStringLiteral(String.raw`'fo\ro'`)).to.equal('fo\ro')

test/unit/parser/tokenizer.ts

@@ -1,210 +1,431 @@
 import { expect } from 'chai'
+import { WordToken } from '../../../src/tokens/word-token'
+import { NumberToken } from '../../../src/tokens/number-token'
+import { PropertyAccessToken } from '../../../src/tokens/property-access-token'
+import { RangeToken } from '../../../src/tokens/range-token'
+import { OperatorToken } from '../../../src/tokens/operator-token'
 import { Tokenizer } from '../../../src/parser/tokenizer'
-import { TagToken } from '../../../src/parser/tag-token'
+import { TagToken } from '../../../src/tokens/tag-token'
-import { OutputToken } from '../../../src/parser/output-token'
+import { QuotedToken } from '../../../src/tokens/quoted-token'
-import { HTMLToken } from '../../../src/parser/html-token'
+import { OutputToken } from '../../../src/tokens/output-token'
+import { HTMLToken } from '../../../src/tokens/html-token'
 
 describe('Tokenize', function () {
   it('should read quoted', () => {
-    expect(new Tokenizer('"foo" ff').readQuoted().toString()).to.equal('"foo"')
+    expect(new Tokenizer('"foo" ff').readQuoted()!.getText()).to.equal('"foo"')
-    expect(new Tokenizer(' "foo"ff').readQuoted().toString()).to.equal('"foo"')
+    expect(new Tokenizer(' "foo"ff').readQuoted()!.getText()).to.equal('"foo"')
-  })
-  it('should read property access', () => {
-    expect(new Tokenizer('a[ b][ "c d" ]').readPropertyAccess().toString()).to.equal('a[ b][ "c d" ]')
-    expect(new Tokenizer('a.b[c[d.e]]').readPropertyAccess().toString()).to.equal('a.b[c[d.e]]')
   })
   it('should read value', () => {
-    expect(new Tokenizer('2.33.2').readValue().toString()).to.equal('2.33.2')
+    expect(new Tokenizer('a[ b][ "c d" ]').readValueOrThrow().getText()).to.equal('a[ b][ "c d" ]')
-    expect(new Tokenizer('"foo"a').readValue().toString()).to.equal('"foo"')
+    expect(new Tokenizer('a.b[c[d.e]]').readValueOrThrow().getText()).to.equal('a.b[c[d.e]]')
-    expect(new Tokenizer('a[b]["c d"]').readValue().toString()).to.equal('a[b]["c d"]')
+  })
+  it('should read number value', () => {
+    const token: NumberToken = new Tokenizer('2.33.2').readValueOrThrow() as any
+    expect(token).to.be.instanceOf(NumberToken)
+    expect(token.whole.getText()).to.equal('2')
+    expect(token.decimal!.getText()).to.equal('33')
+    expect(token.getText()).to.equal('2.33')
+  })
+  it('should read quoted value', () => {
+    const value = new Tokenizer('"foo"a').readValue()
+    expect(value).to.be.instanceOf(QuotedToken)
+    expect(value!.getText()).to.equal('"foo"')
+  })
+  it('should read property access value', () => {
+    expect(new Tokenizer('a[b]["c d"]').readValueOrThrow().getText()).to.equal('a[b]["c d"]')
   })
   it('should read hash', () => {
-    expect(new Tokenizer('foo: 3').readHash()).to.deep.equal(['foo', '3'])
+    const hash1 = new Tokenizer('foo: 3').readHash()
-    expect(new Tokenizer(', foo: a[ "bar"]').readHash()).to.deep.equal(['foo', 'a[ "bar"]'])
+    expect(hash1!.name.content).to.equal('foo')
+    expect(hash1!.value!.getText()).to.equal('3')
+
+    const hash2 = new Tokenizer(', foo: a[ "bar"]').readHash()
+    expect(hash2!.name.content).to.equal('foo')
+    expect(hash2!.value!.getText()).to.equal('a[ "bar"]')
   })
-  it('should read hashs', () => {
+  it('should read multiple hashs', () => {
-    expect(new Tokenizer(', limit: 3 reverse offset:off').readHashes())
+    const hashes = new Tokenizer(', limit: 3 reverse offset:off').readHashes()
-      .to.deep.equal([['limit', '3'], ['reverse', ''], ['offset', 'off']])
+    expect(hashes).to.have.lengthOf(3)
-    expect(new Tokenizer('cols: 2, rows: data["rows"]').readHashes())
+    const [limit, reverse, offset] = hashes
-      .to.deep.equal([['cols', '2'], ['rows', 'data["rows"]']])
+    expect(limit.name.content).to.equal('limit')
+    expect(limit.value!.getText()).to.equal('3')
+
+    expect(reverse.name.content).to.equal('reverse')
+    expect(reverse.value).to.be.undefined
+
+    expect(offset.name.content).to.equal('offset')
+    expect(offset.value!.getText()).to.equal('off')
+  })
+  it('should read hash value with property access', () => {
+    const hashes = new Tokenizer('cols: 2, rows: data["rows"]').readHashes()
+    expect(hashes).to.have.lengthOf(2)
+    const [cols, rols] = hashes
+
+    expect(cols.name.content).to.equal('cols')
+    expect(cols.value!.getText()).to.equal('2')
+
+    expect(rols.name.content).to.equal('rows')
+    expect(rols.value!.getText()).to.equal('data["rows"]')
   })
   it('should read HTML token', function () {
     const html = '<html><body><p>Lorem Ipsum</p></body></html>'
     const tokenizer = new Tokenizer(html)
-    const tokens = tokenizer.readTokens()
+    const tokens = tokenizer.readTopLevelTokens()
 
     expect(tokens.length).to.equal(1)
-    expect(tokens[0].content).to.equal(html)
     expect(tokens[0]).instanceOf(HTMLToken)
+    expect((tokens[0] as HTMLToken).getContent()).to.equal(html)
   })
   it('should read tag token', function () {
     const html = '<p>{% for p in a[1]%}</p>'
     const tokenizer = new Tokenizer(html)
-    const tokens = tokenizer.readTokens()
+    const tokens = tokenizer.readTopLevelTokens()
 
     expect(tokens.length).to.equal(3)
-    expect(tokens[1]).instanceOf(TagToken)
+    const tag = tokens[1] as TagToken
-    expect(tokens[1].content).to.equal('for p in a[1]')
+    expect(tag).instanceOf(TagToken)
+    expect(tag.name).to.equal('for')
+    expect(tag.args).to.equal('p in a[1]')
   })
-  it('should read value token', function () {
+  it('should read output token', function () {
     const html = '<p>{{foo | date: "%Y-%m-%d"}}</p>'
     const tokenizer = new Tokenizer(html)
-    const tokens = tokenizer.readTokens()
+    const tokens = tokenizer.readTopLevelTokens()
 
     expect(tokens.length).to.equal(3)
-    expect(tokens[1]).instanceOf(OutputToken)
+    const output = tokens[1] as OutputToken
-    expect(tokens[1].content).to.equal('foo | date: "%Y-%m-%d"')
+    expect(output).instanceOf(OutputToken)
+    expect(output.content).to.equal('foo | date: "%Y-%m-%d"')
   })
   it('should handle consecutive value and tags', function () {
     const html = '{{foo}}{{bar}}{%foo%}{%bar%}'
     const tokenizer = new Tokenizer(html)
-    const tokens = tokenizer.readTokens()
+    const tokens = tokenizer.readTopLevelTokens()
 
     expect(tokens.length).to.equal(4)
-    expect(tokens[0]).instanceOf(OutputToken)
+    const o1 = tokens[0] as OutputToken
-    expect(tokens[2]).instanceOf(TagToken)
+    const o2 = tokens[1] as OutputToken
+    const t1 = tokens[2] as TagToken
+    const t2 = tokens[3] as TagToken
+    expect(o1).instanceOf(OutputToken)
+    expect(o2).instanceOf(OutputToken)
+    expect(t1).instanceOf(TagToken)
+    expect(t2).instanceOf(TagToken)
 
-    expect(tokens[1].content).to.equal('bar')
+    expect(o1.content).to.equal('foo')
-    expect(tokens[2].content).to.equal('foo')
+    expect(o2.content).to.equal('bar')
+    expect(t1.name).to.equal('foo')
+    expect(t1.args).to.equal('')
+    expect(t2.name).to.equal('bar')
+    expect(t2.args).to.equal('')
   })
   it('should keep white spaces and newlines', function () {
     const html = '{%foo%}\n{%bar %}  \n {%alice%}'
     const tokenizer = new Tokenizer(html)
-    const tokens = tokenizer.readTokens()
+    const tokens = tokenizer.readTopLevelTokens()
     expect(tokens.length).to.equal(5)
     expect(tokens[1]).instanceOf(HTMLToken)
-    expect(tokens[1].raw).to.equal('\n')
+    expect(tokens[1].getText()).to.equal('\n')
     expect(tokens[3]).instanceOf(HTMLToken)
-    expect(tokens[3].raw).to.equal('  \n ')
+    expect(tokens[3].getText()).to.equal('  \n ')
   })
   it('should handle multiple lines tag', function () {
     const html = '{%foo\na:a\nb:1.23\n%}'
     const tokenizer = new Tokenizer(html)
-    const tokens = tokenizer.readTokens()
+    const tokens = tokenizer.readTopLevelTokens()
     expect(tokens.length).to.equal(1)
     expect(tokens[0]).instanceOf(TagToken)
     expect((tokens[0] as TagToken).args).to.equal('a:a\nb:1.23')
-    expect(tokens[0].raw).to.equal('{%foo\na:a\nb:1.23\n%}')
+    expect(tokens[0].getText()).to.equal('{%foo\na:a\nb:1.23\n%}')
   })
   it('should handle multiple lines value', function () {
     const html = '{{foo\n|date:\n"%Y-%m-%d"\n}}'
     const tokenizer = new Tokenizer(html)
-    const tokens = tokenizer.readTokens()
+    const tokens = tokenizer.readTopLevelTokens()
     expect(tokens.length).to.equal(1)
     expect(tokens[0]).instanceOf(OutputToken)
-    expect(tokens[0].raw).to.equal('{{foo\n|date:\n"%Y-%m-%d"\n}}')
+    expect(tokens[0].getText()).to.equal('{{foo\n|date:\n"%Y-%m-%d"\n}}')
   })
   it('should handle complex object property access', function () {
     const html = '{{ obj["my:property with anything"] }}'
     const tokenizer = new Tokenizer(html)
-    const tokens = tokenizer.readTokens()
+    const tokens = tokenizer.readTopLevelTokens()
     expect(tokens.length).to.equal(1)
-    expect(tokens[0]).instanceOf(OutputToken)
+    const output = tokens[0] as OutputToken
-    expect(tokens[0].content).to.equal('obj["my:property with anything"]')
+    expect(output).instanceOf(OutputToken)
+    expect(output.content).to.equal('obj["my:property with anything"]')
   })
   it('should throw if tag not closed', function () {
     const html = '{% assign foo = bar {{foo}}'
     const tokenizer = new Tokenizer(html)
-    expect(() => tokenizer.readTokens()).to.throw(/tag "{% assign foo..." not closed/)
+    expect(() => tokenizer.readTopLevelTokens()).to.throw(/tag "{% assign foo..." not closed/)
   })
   it('should throw if output not closed', function () {
     const tokenizer = new Tokenizer('{{name}')
-    expect(() => tokenizer.readTokens()).to.throw(/output "{{name}" not closed/)
+    expect(() => tokenizer.readTopLevelTokens()).to.throw(/output "{{name}" not closed/)
   })
-  it('should read a simple filter', function () {
+  describe('#readRange()', () => {
-    const tokenizer = new Tokenizer('| plus')
+    it('should read `(1..3)`', () => {
-    const token = tokenizer.readFilterToken()
+      const range = new Tokenizer('(1..3)').readRange()
-    expect(token).to.have.property('name', 'plus')
+      expect(range).to.be.instanceOf(RangeToken)
-    expect(token).to.have.property('args').to.deep.equal([])
+      expect(range!.getText()).to.deep.equal('(1..3)')
+      const { lhs, rhs } = range!
+      expect(lhs).to.be.instanceOf(NumberToken)
+      expect(lhs.getText()).to.equal('1')
+      expect(rhs).to.be.instanceOf(NumberToken)
+      expect(rhs.getText()).to.equal('3')
+    })
+    it('should throw for `(..3)`', () => {
+      expect(() => new Tokenizer('(..3)').readRange()).to.throw('unexpected token "..3)", value expected')
+    })
+    it('should read `(a.b..c["..d"])`', () => {
+      const range = new Tokenizer('(a.b..c["..d"])').readRange()
+      expect(range).to.be.instanceOf(RangeToken)
+      expect(range!.getText()).to.deep.equal('(a.b..c["..d"])')
+    })
   })
-  it('should read a filter with argument', function () {
+  describe('#readFilter()', () => {
-    const tokenizer = new Tokenizer(' | plus: 1')
+    it('should read a simple filter', function () {
-    const token = tokenizer.readFilterToken()
+      const tokenizer = new Tokenizer('| plus')
-    expect(token).to.have.property('name', 'plus')
+      const token = tokenizer.readFilter()
-    expect(token).to.have.property('args').to.deep.equal(['1'])
+      expect(token).to.have.property('name', 'plus')
-  })
+      expect(token).to.have.property('args').to.deep.equal([])
-  it('should read a filter with colon but no argument', function () {
+    })
-    const tokenizer = new Tokenizer('| plus:')
+    it('should read a filter with argument', function () {
-    const token = tokenizer.readFilterToken()
+      const tokenizer = new Tokenizer(' | plus: 1')
-    expect(token).to.have.property('name', 'plus')
+      const token = tokenizer.readFilter()
-    expect(token).to.have.property('args').to.deep.equal([])
+      expect(token).to.have.property('name', 'plus')
-  })
+      expect(token!.args).to.have.lengthOf(1)
-  it('should read a filter with k/v argument', function () {
-    const tokenizer = new Tokenizer(' | plus: a:1')
-    const token = tokenizer.readFilterToken()
-    expect(token).to.have.property('name', 'plus')
-    expect(token).to.have.property('args').to.deep.equal([['a', '1']])
-  })
-  it('should read a filter with "arr[0]" argument', function () {
-    const tokenizer = new Tokenizer('| plus: arr[0]')
-    const token = tokenizer.readFilterToken()
-    expect(token).to.have.property('name', 'plus')
-    expect(token).to.have.property('args').to.deep.equal(['arr[0]'])
-  })
-  it('should read a filter with obj.foo argument', function () {
-    const tokenizer = new Tokenizer('| plus: obj.foo')
-    const token = tokenizer.readFilterToken()
-    expect(token).to.have.property('name', 'plus')
-    expect(token).to.have.property('args').to.deep.equal(['obj.foo'])
-  })
-  it('should read a filter with obj["foo"] argument', function () {
-    const tokenizer = new Tokenizer('| plus: obj["good luck"]')
-    const token = tokenizer.readFilterToken()
-    expect(token).to.have.property('name', 'plus')
-    expect(token).to.have.property('args').to.deep.equal(['obj["good luck"]'])
-  })
-  it('should read simple filters', function () {
-    const tokenizer = new Tokenizer('| plus: 3 | capitalize')
-    const tokens = tokenizer.readFilterTokens()
 
-    expect(tokens).to.have.lengthOf(2)
+      const one: NumberToken = token!.args[0] as any
-    expect(tokens[0]).to.have.property('name', 'plus')
+      expect(one).to.be.instanceOf(NumberToken)
-    expect(tokens[0]).to.have.property('args').to.deep.equal(['3'])
+      expect(one.getText()).to.equal('1')
-    expect(tokens[1]).to.have.property('name', 'capitalize')
+    })
-    expect(tokens[1]).to.have.property('args').to.deep.equal([])
+    it('should read a filter with colon but no argument', function () {
-  })
+      const tokenizer = new Tokenizer('| plus:')
-  it('should read filters', function () {
+      const token = tokenizer.readFilter()
-    const tokenizer = new Tokenizer('| plus: 3 | capitalize | append: foo[a.b["c d"]]')
+      expect(token).to.have.property('name', 'plus')
-    const tokens = tokenizer.readFilterTokens()
+      expect(token).to.have.property('args').to.deep.equal([])
+    })
+    it('should read a filter with k/v argument', function () {
+      const tokenizer = new Tokenizer(' | plus: a:1')
+      const token = tokenizer.readFilter()
+      expect(token).to.have.property('name', 'plus')
+      expect(token!.args).to.have.lengthOf(1)
 
-    expect(tokens).to.have.lengthOf(3)
+      const [k, v]: [string, NumberToken] = token!.args[0] as any
-    expect(tokens[0]).to.have.property('name', 'plus')
+      expect(k).to.equal('a')
-    expect(tokens[0]).to.have.property('args').to.deep.equal(['3'])
+      expect(v).to.be.instanceOf(NumberToken)
-    expect(tokens[1]).to.have.property('name', 'capitalize')
+      expect(v.getText()).to.equal('1')
-    expect(tokens[1]).to.have.property('args').to.deep.equal([])
+    })
-    expect(tokens[2]).to.have.property('name', 'append')
+    it('should read a filter with "arr[0]" argument', function () {
-    expect(tokens[2]).to.have.property('args').to.deep.equal(['foo[a.b["c d"]]'])
+      const tokenizer = new Tokenizer('| plus: arr[0]')
+      const token = tokenizer.readFilter()
+      expect(token).to.have.property('name', 'plus')
+      expect(token!.args).to.have.lengthOf(1)
+
+      const pa: PropertyAccessToken = token!.args[0] as any
+      expect(token!.args[0]).to.be.instanceOf(PropertyAccessToken)
+      expect(pa.variable.content).to.equal('arr')
+      expect(pa.props).to.have.lengthOf(1)
+      expect(pa.props[0]).to.be.instanceOf(NumberToken)
+      expect(pa.props[0].getText()).to.equal('0')
+    })
+    it('should read a filter with obj.foo argument', function () {
+      const tokenizer = new Tokenizer('| plus: obj.foo')
+      const token = tokenizer.readFilter()
+      expect(token).to.have.property('name', 'plus')
+      expect(token!.args).to.have.lengthOf(1)
+
+      const pa: PropertyAccessToken = token!.args[0] as any
+      expect(token!.args[0]).to.be.instanceOf(PropertyAccessToken)
+      expect(pa.variable.content).to.equal('obj')
+      expect(pa.props).to.have.lengthOf(1)
+      expect(pa.props[0]).to.be.instanceOf(WordToken)
+      expect(pa.props[0].getText()).to.equal('foo')
+    })
+    it('should read a filter with obj["foo"] argument', function () {
+      const tokenizer = new Tokenizer('| plus: obj["good luck"]')
+      const token = tokenizer.readFilter()
+      expect(token).to.have.property('name', 'plus')
+      expect(token!.args).to.have.lengthOf(1)
+
+      const pa: PropertyAccessToken = token!.args[0] as any
+      expect(token!.args[0]).to.be.instanceOf(PropertyAccessToken)
+      expect(pa.getText()).to.equal('obj["good luck"]')
+      expect(pa.variable.content).to.equal('obj')
+      expect(pa.props[0].getText()).to.equal('"good luck"')
+    })
   })
-  it('should read expression `a==b`', () => {
+  describe('#readFilters()', () => {
-    const exp = new Tokenizer('a==b').readExpression()
+    it('should read simple filters', function () {
-    expect([...exp]).to.deep.equal(['a', '==', 'b'])
+      const tokenizer = new Tokenizer('| plus: 3 | capitalize')
+      const tokens = tokenizer.readFilters()
+
+      expect(tokens).to.have.lengthOf(2)
+      expect(tokens[0]).to.have.property('name', 'plus')
+      expect(tokens[0].args).to.have.lengthOf(1)
+      expect(tokens[0].args[0]).to.be.instanceOf(NumberToken)
+      expect((tokens[0].args[0] as any).getText()).to.equal('3')
+
+      expect(tokens[1]).to.have.property('name', 'capitalize')
+      expect(tokens[1].args).to.have.lengthOf(0)
+    })
+    it('should read filters', function () {
+      const tokenizer = new Tokenizer('| plus: a:3 | capitalize | append: foo[a.b["c d"]]')
+      const tokens = tokenizer.readFilters()
+
+      expect(tokens).to.have.lengthOf(3)
+      expect(tokens[0]).to.have.property('name', 'plus')
+      expect(tokens[0].args).to.have.lengthOf(1)
+      const [k, v]: [string, NumberToken] = tokens[0].args[0] as any
+      expect(k).to.equal('a')
+      expect(v).to.be.instanceOf(NumberToken)
+      expect(v.getText()).to.equal('3')
+
+      expect(tokens[1]).to.have.property('name', 'capitalize')
+      expect(tokens[1].args).to.have.lengthOf(0)
+
+      expect(tokens[2]).to.have.property('name', 'append')
+      expect(tokens[2].args).to.have.lengthOf(1)
+      expect(tokens[2].args[0]).to.be.instanceOf(PropertyAccessToken)
+      expect((tokens[2].args[0] as any).getText()).to.equal('foo[a.b["c d"]]')
+      expect((tokens[2].args[0] as any).props[0].getText()).to.equal('a.b["c d"]')
+    })
   })
-  it('should read expression `^`', () => {
+  describe('#readExpression()', () => {
-    const exp = new Tokenizer('^').readExpression()
+    it('should read expression `a `', () => {
-    expect([...exp]).to.deep.equal([])
+      const exp = [...new Tokenizer('a ').readExpression()]
-  })
+
-  it('should read expression `a == b`', () => {
+      expect(exp).to.have.lengthOf(1)
-    const exp = new Tokenizer('a == b').readExpression()
+      expect(exp[0]).to.be.instanceOf(PropertyAccessToken)
-    expect([...exp]).to.deep.equal(['a', '==', 'b'])
+      expect(exp[0].getText()).to.deep.equal('a')
-  })
+    })
-  it('should read expression `(1..3) contains 3`', () => {
+    it('should read expression `a[][b]`', () => {
-    const exp = new Tokenizer('(1..3) contains 3').readExpression()
+      const exp = [...new Tokenizer('a[][b]').readExpression()]
-    expect([...exp]).to.deep.equal(['(1..3)', 'contains', '3'])
+
-  })
+      expect(exp).to.have.lengthOf(1)
-  it('should read expression `a[b] = c`', () => {
+      const pa = exp[0] as PropertyAccessToken
-    const exp = new Tokenizer('a[b] = c').readExpression()
+      expect(pa).to.be.instanceOf(PropertyAccessToken)
-    expect([...exp]).to.deep.equal(['a[b]', '=', 'c'])
+      expect(pa.variable.content).to.deep.equal('a')
-  })
+      expect(pa.props).to.have.lengthOf(2)
-  it('should read expression `c[a["b"]] >= c`', () => {
+
-    const exp = new Tokenizer('c[a["b"]] >= c').readExpression()
+      const [p1, p2] = pa.props
-    expect([...exp]).to.deep.equal(['c[a["b"]]', '>=', 'c'])
+      expect(p1).to.be.instanceOf(WordToken)
-  })
+      expect(p1.getText()).to.equal('')
-  it('should read expression `"][" == var`', () => {
+      expect(p2).to.be.instanceOf(PropertyAccessToken)
-    const exp = new Tokenizer('"][" == var').readExpression()
+      expect(p2.getText()).to.equal('b')
-    expect([...exp]).to.deep.equal(['"]["', '==', 'var'])
+    })
-  })
+    it('should read expression `a.`', () => {
-  it('should read expression `"\\\'" == "\\""`', () => {
+      const exp = [...new Tokenizer('a.').readExpression()]
-    const exp = new Tokenizer('"\\\'" == "\\""').readExpression()
+
-    expect([...exp]).to.deep.equal(['"\\\'"', '==', '"\\""'])
+      expect(exp).to.have.lengthOf(1)
+      const pa = exp[0] as PropertyAccessToken
+      expect(pa).to.be.instanceOf(PropertyAccessToken)
+      expect(pa.variable.content).to.deep.equal('a')
+      expect(pa.props).to.have.lengthOf(0)
+    })
+    it('should read expression `a ==`', () => {
+      const exp = [...new Tokenizer('a ==').readExpression()]
+
+      expect(exp).to.have.lengthOf(1)
+      expect(exp[0]).to.be.instanceOf(PropertyAccessToken)
+      expect(exp[0].getText()).to.deep.equal('a')
+    })
+    it('should read expression `a==b`', () => {
+      const exp = new Tokenizer('a==b').readExpression()
+      const [a, equals, b] = exp
+
+      expect(a).to.be.instanceOf(PropertyAccessToken)
+      expect(a.getText()).to.deep.equal('a')
+
+      expect(equals).to.be.instanceOf(OperatorToken)
+      expect(equals.getText()).to.equal('==')
+
+      expect(b).to.be.instanceOf(PropertyAccessToken)
+      expect(b.getText()).to.deep.equal('b')
+    })
+    it('should read expression `^`', () => {
+      const exp = new Tokenizer('^').readExpression()
+      expect([...exp]).to.deep.equal([])
+    })
+    it('should read expression `a == b`', () => {
+      const exp = new Tokenizer('a == b').readExpression()
+      const [a, equals, b] = exp
+
+      expect(a).to.be.instanceOf(PropertyAccessToken)
+      expect(a.getText()).to.deep.equal('a')
+
+      expect(equals).to.be.instanceOf(OperatorToken)
+      expect(equals.getText()).to.equal('==')
+
+      expect(b).to.be.instanceOf(PropertyAccessToken)
+      expect(b.getText()).to.deep.equal('b')
+    })
+    it('should read expression `(1..3) contains 3`', () => {
+      const exp = new Tokenizer('(1..3) contains 3').readExpression()
+      const [range, contains, rhs] = exp
+
+      expect(range).to.be.instanceOf(RangeToken)
+      expect(range.getText()).to.deep.equal('(1..3)')
+
+      expect(contains).to.be.instanceOf(OperatorToken)
+      expect(contains.getText()).to.equal('contains')
+
+      expect(rhs).to.be.instanceOf(NumberToken)
+      expect(rhs.getText()).to.deep.equal('3')
+    })
+    it('should read expression `a[b] == c`', () => {
+      const exp = new Tokenizer('a[b] == c').readExpression()
+      const [lhs, contains, rhs] = exp
+
+      expect(lhs).to.be.instanceOf(PropertyAccessToken)
+      expect(lhs.getText()).to.deep.equal('a[b]')
+
+      expect(contains).to.be.instanceOf(OperatorToken)
+      expect(contains.getText()).to.equal('==')
+
+      expect(rhs).to.be.instanceOf(PropertyAccessToken)
+      expect(rhs.getText()).to.deep.equal('c')
+    })
+    it('should read expression `c[a["b"]] >= c`', () => {
+      const exp = new Tokenizer('c[a["b"]] >= c').readExpression()
+      const [lhs, op, rhs] = exp
+
+      expect(lhs).to.be.instanceOf(PropertyAccessToken)
+      expect(lhs.getText()).to.deep.equal('c[a["b"]]')
+
+      expect(op).to.be.instanceOf(OperatorToken)
+      expect(op.getText()).to.equal('>=')
+
+      expect(rhs).to.be.instanceOf(PropertyAccessToken)
+      expect(rhs.getText()).to.deep.equal('c')
+    })
+    it('should read expression `"][" == var`', () => {
+      const exp = new Tokenizer('"][" == var').readExpression()
+      const [lhs, equals, rhs] = exp
+
+      expect(lhs).to.be.instanceOf(QuotedToken)
+      expect(lhs.getText()).to.deep.equal('"]["')
+
+      expect(equals).to.be.instanceOf(OperatorToken)
+      expect(equals.getText()).to.equal('==')
+
+      expect(rhs).to.be.instanceOf(PropertyAccessToken)
+      expect(rhs.getText()).to.deep.equal('var')
+    })
+    it('should read expression `"\\\'" == "\\""`', () => {
+      const exp = new Tokenizer('"\\\'" == "\\""').readExpression()
+      const [lhs, equals, rhs] = exp
+
+      expect(lhs).to.be.instanceOf(QuotedToken)
+      expect(lhs.getText()).to.deep.equal('"\\\'"')
+
+      expect(equals).to.be.instanceOf(OperatorToken)
+      expect(equals.getText()).to.equal('==')
+
+      expect(rhs).to.be.instanceOf(QuotedToken)
+      expect(rhs.getText()).to.deep.equal('"\\""')
+    })
   })
 })

test/unit/render/expression.ts

@@ -4,59 +4,115 @@ import { Context } from '../../../src/context/context'
 import { toThenable } from '../../../src/util/async'
 
 describe('Expression', function () {
-  let ctx: Context
+  const ctx = new Context({})
-
-  beforeEach(function () {
-    ctx = new Context({
-      one: 1,
-      two: 2,
-      empty: '',
-      quote: '"',
-      space: ' ',
-      x: 'XXX',
-      y: undefined,
-      z: null,
-      obj: {
-        ']': 'right bracket'
-      }
-    })
-  })
 
   it('should throw when context not defined', done => {
-    toThenable(new Expression().value(undefined!)).catch(err => {
+    toThenable(new Expression('foo').value(undefined!))
-      expect(err.message).to.match(/context not defined/)
+      .then(() => done(new Error('should not resolved')))
-      done()
+      .catch(err => {
-      return 0 as any
+        expect(err.message).to.match(/context not defined/)
+        done()
+      })
+  })
+
+  describe('single value', function () {
+    it('should eval literal', async function () {
+      expect(await toThenable(new Expression('2.4').value(ctx))).to.equal(2.4)
+      expect(await toThenable(new Expression('"foo"').value(ctx))).to.equal('foo')
+      expect(await toThenable(new Expression('false').value(ctx))).to.equal(false)
+    })
+    it('should eval range expression', async function () {
+      const ctx = new Context({ two: 2 })
+      expect(await toThenable(new Expression('(2..4)').value(ctx))).to.deep.equal([2, 3, 4])
+      expect(await toThenable(new Expression('(two..4)').value(ctx))).to.deep.equal([2, 3, 4])
+    })
+    it('should eval literal', async function () {
+      expect(await toThenable(new Expression('2.4').value(ctx))).to.equal(2.4)
+      expect(await toThenable(new Expression('"foo"').value(ctx))).to.equal('foo')
+      expect(await toThenable(new Expression('false').value(ctx))).to.equal(false)
+    })
+
+    it('should eval property access', async function () {
+      const ctx = new Context({
+        foo: { bar: 'BAR' },
+        coo: 'bar',
+        doo: { foo: 'bar', bar: { foo: 'bar' } }
+      })
+      expect(await toThenable(new Expression('foo.bar').value(ctx))).to.equal('BAR')
+      expect(await toThenable(new Expression('foo["bar"]').value(ctx))).to.equal('BAR')
+      expect(await toThenable(new Expression('foo[coo]').value(ctx))).to.equal('BAR')
+      expect(await toThenable(new Expression('foo[doo.foo]').value(ctx))).to.equal('BAR')
+      expect(await toThenable(new Expression('foo[doo["foo"]]').value(ctx))).to.equal('BAR')
+      expect(await toThenable(new Expression('doo[coo].foo').value(ctx))).to.equal('bar')
     })
   })
 
-  it('should eval simple expression', async function () {
+  describe('simple expression', function () {
-    expect(await toThenable(new Expression('1==2').value(ctx))).to.equal(false)
+    it('should return false for "1==2"', async () => {
-    expect(await toThenable(new Expression('1<2').value(ctx))).to.equal(true)
+      expect(await toThenable(new Expression('1==2').value(ctx))).to.equal(false)
-    expect(await toThenable(new Expression('1 < 2').value(ctx))).to.equal(true)
+    })
-    expect(await toThenable(new Expression('1   <   2').value(ctx))).to.equal(true)
+    it('should return true for "1<2"', async () => {
-    expect(await toThenable(new Expression('2 <= 2').value(ctx))).to.equal(true)
+      expect(await toThenable(new Expression('1<2').value(ctx))).to.equal(true)
-    expect(await toThenable(new Expression('one <= two').value(ctx))).to.equal(true)
+    })
-    expect(await toThenable(new Expression('x contains "x"').value(ctx))).to.equal(false)
+    it('should return true for "1 < 2"', async () => {
-    expect(await toThenable(new Expression('x contains "X"').value(ctx))).to.equal(true)
+      expect(await toThenable(new Expression('1 < 2').value(ctx))).to.equal(true)
-    expect(await toThenable(new Expression('1 contains "x"').value(ctx))).to.equal(false)
+    })
-    expect(await toThenable(new Expression('y contains "x"').value(ctx))).to.equal(false)
+    it('should return true for "1   <   2"', async () => {
-    expect(await toThenable(new Expression('z contains "x"').value(ctx))).to.equal(false)
+      expect(await toThenable(new Expression('1   <   2').value(ctx))).to.equal(true)
-    expect(await toThenable(new Expression('(1..5) contains 3').value(ctx))).to.equal(true)
+    })
-    expect(await toThenable(new Expression('(1..5) contains 6').value(ctx))).to.equal(false)
+    it('should return true for "2 <= 2"', async () => {
-    expect(await toThenable(new Expression('"<=" == "<="').value(ctx))).to.equal(true)
+      expect(await toThenable(new Expression('2 <= 2').value(ctx))).to.equal(true)
+    })
+    it('should return true for "one <= two"', async () => {
+      const ctx = new Context({ one: 1, two: 2 })
+      expect(await toThenable(new Expression('one <= two').value(ctx))).to.equal(true)
+    })
+    it('should return false for "x contains "x""', async () => {
+      const ctx = new Context({ x: 'XXX' })
+      expect(await toThenable(new Expression('x contains "x"').value(ctx))).to.equal(false)
+    })
+    it('should return true for "x contains "X""', async () => {
+      const ctx = new Context({ x: 'XXX' })
+      expect(await toThenable(new Expression('x contains "X"').value(ctx))).to.equal(true)
+    })
+    it('should return false for "1 contains "x""', async () => {
+      const ctx = new Context({ x: 'XXX' })
+      expect(await toThenable(new Expression('1 contains "x"').value(ctx))).to.equal(false)
+    })
+    it('should return false for "y contains "x""', async () => {
+      const ctx = new Context({ x: 'XXX' })
+      expect(await toThenable(new Expression('y contains "x"').value(ctx))).to.equal(false)
+    })
+    it('should return false for "z contains "x""', async () => {
+      const ctx = new Context({ x: 'XXX' })
+      expect(await toThenable(new Expression('z contains "x"').value(ctx))).to.equal(false)
+    })
+    it('should return true for "(1..5) contains 3"', async () => {
+      const ctx = new Context({ x: 'XXX' })
+      expect(await toThenable(new Expression('(1..5) contains 3').value(ctx))).to.equal(true)
+    })
+    it('should return false for "(1..5) contains 6"', async () => {
+      const ctx = new Context({ x: 'XXX' })
+      expect(await toThenable(new Expression('(1..5) contains 6').value(ctx))).to.equal(false)
+    })
+    it('should return true for ""<=" == "<=""', async () => {
+      expect(await toThenable(new Expression('"<=" == "<="').value(ctx))).to.equal(true)
+    })
   })
 
   it('should allow space in quoted value', async function () {
+    const ctx = new Context({ space: ' ' })
     expect(await toThenable(new Expression('" " == space').value(ctx))).to.equal(true)
   })
 
   describe('escape', () => {
     it('should escape quote', async function () {
+      const ctx = new Context({ quote: '"' })
       expect(await toThenable(new Expression('"\\"" == quote').value(ctx))).to.equal(true)
     })
     it('should escape square bracket', async function () {
-      expect(await toThenable(new Expression('obj["]"] == "right bracket"').value(ctx))).to.equal(true)
+      const ctx = new Context({ obj: { ']': 'bracket' } })
+      expect(await toThenable(new Expression('obj["]"] == "bracket"').value(ctx))).to.equal(true)
     })
   })
 
@@ -71,6 +127,7 @@ describe('Expression', function () {
       expect(await toThenable(new Expression('1 < 2 or x contains "x"').value(ctx))).to.equal(true)
     })
     it('should support value and !=', async function () {
+      const ctx = new Context({ empty: '' })
       expect(await toThenable(new Expression('empty and empty != ""').value(ctx))).to.equal(false)
     })
     it('should recognize quoted value', async function () {
@@ -84,10 +141,9 @@ describe('Expression', function () {
       const ctx = new Context({ obj: { foo: true } })
       expect(await toThenable(new Expression('obj["foo"] and true').value(ctx))).to.equal(true)
     })
-  })
+    it('should allow nested property access', async function () {
-
+      const ctx = new Context({ obj: { foo: 'FOO' }, keys: { "what's this": 'foo' } })
-  it('should eval range expression', async function () {
+      expect(await toThenable(new Expression('obj[keys["what\'s this"]]').value(ctx))).to.equal('FOO')
-    expect(await toThenable(new Expression('(2..4)').value(ctx))).to.deep.equal([2, 3, 4])
+    })
-    expect(await toThenable(new Expression('(two..4)').value(ctx))).to.deep.equal([2, 3, 4])
   })
 })

test/unit/render/render.ts

@@ -1,6 +1,6 @@
 import { expect } from 'chai'
 import { Context } from '../../../src/context/context'
-import { Token } from '../../../src/parser/token'
+import { HTMLToken } from '../../../src/tokens/html-token'
 import { Render } from '../../../src/render/render'
 import { HTML } from '../../../src/template/html'
 import { toThenable } from '../../../src/util/async'
@@ -14,7 +14,7 @@ describe('render', function () {
   describe('.renderTemplates()', function () {
     it('should render html', async function () {
       const scope = new Context()
-      const token = { content: '<p>' } as Token
+      const token = { getContent: () => '<p>' } as HTMLToken
       const html = await toThenable(render.renderTemplates([new HTML(token)], scope))
       return expect(html).to.equal('<p>')
     })

test/unit/template/filter/filter.ts

@@ -3,6 +3,9 @@ import * as sinon from 'sinon'
 import * as sinonChai from 'sinon-chai'
 import { Context } from '../../../../src/context/context'
 import { toThenable } from '../../../../src/util/async'
+import { NumberToken } from '../../../../src/tokens/number-token'
+import { QuotedToken } from '../../../../src/tokens/quoted-token'
+import { WordToken } from '../../../../src/tokens/word-token'
 import { FilterMap } from '../../../../src/template/filter/filter-map'
 
 chai.use(sinonChai)
@@ -27,13 +30,15 @@ describe('filter', function () {
   it('should call filter impl with correct arguments', async function () {
     const spy = sinon.spy()
     filters.set('foo', spy)
-    await toThenable(filters.create('foo', ['33']).render('foo', ctx))
+    const thirty = new NumberToken(new WordToken('30', 0, 2), undefined)
-    expect(spy).to.have.been.calledWith('foo', 33)
+    await toThenable(filters.create('foo', [thirty]).render('foo', ctx))
+    expect(spy).to.have.been.calledWith('foo', 30)
   })
   it('should call filter impl with correct this arg', async function () {
     const spy = sinon.spy()
     filters.set('foo', spy)
-    await toThenable(filters.create('foo', ['33']).render('foo', ctx))
+    const thirty = new NumberToken(new WordToken('33', 0, 2), undefined)
+    await toThenable(filters.create('foo', [thirty]).render('foo', ctx))
     expect(spy).to.have.been.calledOn(sinon.match.has('context', ctx))
   })
   it('should render a simple filter', async function () {
@@ -43,12 +48,15 @@ describe('filter', function () {
 
   it('should render filters with argument', async function () {
     filters.set('add', (a, b) => a + b)
-    expect(await toThenable(filters.create('add', ['2']).render(3, ctx))).to.equal(5)
+    const two = new NumberToken(new WordToken('2', 0, 1), undefined)
+    expect(await toThenable(filters.create('add', [two]).render(3, ctx))).to.equal(5)
   })
 
   it('should render filters with multiple arguments', async function () {
     filters.set('add', (a, b, c) => a + b + c)
-    expect(await toThenable(filters.create('add', ['2', '"c"']).render(3, ctx))).to.equal('5c')
+    const two = new NumberToken(new WordToken('2', 0, 1), undefined)
+    const c = new QuotedToken('"c"', 0, 3)
+    expect(await toThenable(filters.create('add', [two, c]).render(3, ctx))).to.equal('5c')
   })
 
   it('should pass Objects/Drops as it is', async function () {
@@ -65,6 +73,7 @@ describe('filter', function () {
 
   it('should support key value pairs', async function () {
     filters.set('add', (a, b) => b[0] + ':' + (a + b[1]))
-    expect(await toThenable((filters.create('add', [['num', '2']]).render(3, ctx)))).to.equal('num:5')
+    const two = new NumberToken(new WordToken('2', 0, 1), undefined)
+    expect(await toThenable((filters.create('add', [['num', two]]).render(3, ctx)))).to.equal('num:5')
   })
 })

test/unit/template/hash.ts

@@ -28,7 +28,8 @@ describe('Hash', function () {
     expect(hash.num).to.equal(2.3)
   })
   it('should parse "num:bar.coo"', async function () {
-    const hash = await toThenable(new Hash('num:bar.coo').render(new Context({ bar: { coo: 3 } })))
+    const pending = new Hash('num:bar.coo').render(new Context({ bar: { coo: 3 } }))
+    const hash = await toThenable(pending)
     expect(hash.num).to.equal(3)
   })
   it('should parse "num1:2.3 reverse,num2:bar.coo\n num3: arr[0]"', async function () {

test/unit/template/output.ts

@@ -2,7 +2,7 @@ import * as chai from 'chai'
 import { toThenable } from '../../../src/util/async'
 import { Context } from '../../../src/context/context'
 import { Output } from '../../../src/template/output'
-import { OutputToken } from '../../../src/parser/output-token'
+import { OutputToken } from '../../../src/tokens/output-token'
 import { FilterMap } from '../../../src/template/filter/filter-map'
 
 const expect = chai.expect

test/unit/template/tag.ts

@@ -3,7 +3,7 @@ import { Tag } from '../../../src/template/tag/tag'
 import { Context } from '../../../src/context/context'
 import * as sinon from 'sinon'
 import * as sinonChai from 'sinon-chai'
-import { TagToken } from '../../../src/parser/tag-token'
+import { TagToken } from '../../../src/tokens/tag-token'
 import { toThenable } from '../../../src/util/async'
 
 chai.use(sinonChai)

test/unit/template/value.ts

@@ -1,4 +1,5 @@
 import * as chai from 'chai'
+import { QuotedToken } from '../../../src/tokens/quoted-token'
 import { toThenable } from '../../../src/util/async'
 import { FilterMap } from '../../../src/template/filter/filter-map'
 import * as sinonChai from 'sinon-chai'
@@ -15,71 +16,17 @@ describe('Value', function () {
     const filterMap = new FilterMap(false)
     it('should parse "foo', function () {
       const tpl = new Value('foo', filterMap)
-      expect(tpl.initial).to.equal('foo')
+      expect(tpl.initial!.getText()).to.equal('foo')
       expect(tpl.filters).to.deep.equal([])
     })
-
+    it('should parse filters in value content', function () {
-    it('should parse "foo | add"', function () {
-      const tpl = new Value('foo | add', filterMap)
-      expect(tpl.initial).to.equal('foo')
-      expect(tpl.filters.length).to.equal(1)
-      expect(tpl.filters[0].args).to.eql([])
-    })
-    it('should parse "foo,foo | add"', function () {
-      const tpl = new Value('foo,foo | add', filterMap)
-      expect(tpl.initial).to.equal('foo')
-      expect(tpl.filters.length).to.equal(1)
-      expect(tpl.filters[0].args).to.eql([])
-    })
-    it('should parse "foo | add: 3, false"', function () {
-      const tpl = new Value('foo | add: 3, "foo"', filterMap)
-      expect(tpl.initial).to.equal('foo')
-      expect(tpl.filters.length).to.equal(1)
-      expect(tpl.filters[0].args).to.eql(['3', '"foo"'])
-    })
-    it('should parse "foo | add: "foo" bar, 3"', function () {
-      const tpl = new Value('foo | add: "foo" bar, 3', filterMap)
-      expect(tpl.initial).to.equal('foo')
-      expect(tpl.filters.length).to.equal(1)
-      expect(tpl.filters[0].name).to.eql('add')
-      expect(tpl.filters[0].args).to.eql(['"foo"', '3'])
-    })
-    it('should parse "foo | add: "|", 3', function () {
-      const tpl = new Value('foo | add: "|", 3', filterMap)
-      expect(tpl.initial).to.equal('foo')
-      expect(tpl.filters.length).to.equal(1)
-      expect(tpl.filters[0].args).to.eql(['"|"', '3'])
-    })
-    it('should parse "foo | add: "|", 3', function () {
-      const tpl = new Value('foo | add: "|", 3', filterMap)
-      expect(tpl.initial).to.equal('foo')
-      expect(tpl.filters.length).to.equal(1)
-      expect(tpl.filters[0].args).to.eql(['"|"', '3'])
-    })
-    it('should support arguments as named key/values', function () {
-      const f = new Value('o | foo: key1: "literal1", key2: value2', filterMap)
-      expect(f.filters[0].name).to.equal('foo')
-      expect(f.filters[0].args).to.eql([['key1', '"literal1"'], ['key2', 'value2']])
-    })
-    it('should support arguments as named key/values with inline literals', function () {
-      const f = new Value('o | foo: "test0", key1: "literal1", key2: value2', filterMap)
-      expect(f.filters[0].name).to.equal('foo')
-      expect(f.filters[0].args).to.deep.equal(['"test0"', ['key1', '"literal1"'], ['key2', 'value2']])
-    })
-    it('should support arguments as named key/values with inline values', function () {
-      const f = new Value('o | foo: test0, key1: "literal1", key2: value2', filterMap)
-      expect(f.filters[0].name).to.equal('foo')
-      expect(f.filters[0].args).to.deep.equal(['test0', ['key1', '"literal1"'], ['key2', 'value2']])
-    })
-    it('should support argument values named same as keys', function () {
-      const f = new Value('o | foo: a: a', filterMap)
-      expect(f.filters[0].name).to.equal('foo')
-      expect(f.filters[0].args).to.deep.equal([['a', 'a']])
-    })
-    it('should support argument literals named same as keys', function () {
       const f = new Value('o | foo: a: "a"', filterMap)
       expect(f.filters[0].name).to.equal('foo')
-      expect(f.filters[0].args).to.deep.equal([['a', '"a"']])
+      expect(f.filters[0].args).to.have.lengthOf(1)
+      const [k, v] = f.filters[0].args[0] as any
+      expect(k).to.equal('a')
+      expect(v).to.be.instanceOf(QuotedToken)
+      expect((v as QuotedToken).getText()).to.equal('"a"')
     })
   })
 

test/unit/util/assert.ts

@@ -5,11 +5,11 @@ const expect = chai.expect
 
 describe('assert', function () {
   it('should not throw if predicate is truthy', function () {
-    const fn = () => assert('foo', 'bar')
+    const fn = () => assert('foo', () => 'bar')
     expect(fn).to.not.throw()
   })
   it('should not throw if predicate is truthy', function () {
-    const fn = () => assert('', 'bar')
+    const fn = () => assert('', () => 'bar')
     expect(fn).to.throw(/bar/)
   })
   it('should populate default message', function () {