feat: static variable analysis (#770)

* feat: static variable analysis

* Accept any iterable from `children`, `arguments`, etc.

* Test analysis of standard tags

* Use `TagToken.tokenizer` instead of creating a new one

* Test analysis of netsted tags

* Group variables by their root value

* Test analysis of nested globals and locals

* Analyze included and rendered templates WIP

* Use existing tokenizer when constructing `Hash`

* Improve test coverage

* Analyze variables from `layout` and `block` tags

* Test analysis of Jekyll style includes

* Handle variables that start with a nested variable

* Async analysis

* Test non-standard tag end to end

* Implement convenience analysis methods on the `Liquid` class

* More analysis convenience methods

* Accept string or template array

* Draft static analysis docs

* Deduplicate variables names

* Fix isolated scope global variable map

* Coerce variables to strings instead of extending String

* Private map instead of extending Map

* Fix e2e test

* Tentatively implement analysis of aliased variables

* Fix nested variable segments array

* Update docs sidebar

src/index.ts

@@ -6,9 +6,9 @@ export { Drop } from './drop'
 export { Emitter } from './emitters'
 export { defaultOperators, Operators, evalToken, evalQuotedToken, Expression, isFalsy, isTruthy } from './render'
 export { Context, Scope } from './context'
-export { Value, Hash, Template, FilterImplOptions, Tag, Filter, Output } from './template'
+export { Value, Hash, Template, FilterImplOptions, Tag, Filter, Output, Variable, VariableLocation, VariableSegments, Variables, StaticAnalysis, StaticAnalysisOptions, analyze, analyzeSync, Arguments, PartialScope } from './template'
 export { Token, TopLevelToken, TagToken, ValueToken } from './tokens'
-export { TokenKind, Tokenizer, ParseStream } from './parser'
+export { TokenKind, Tokenizer, ParseStream, Parser } from './parser'
 export { filters } from './filters'
 export * from './tags'
 export { defaultOptions, LiquidOptions } from './liquid-options'

src/liquid.ts

@@ -1,6 +1,6 @@
 import { Context } from './context'
-import { toPromise, toValueSync, isFunction, forOwn } from './util'
-import { TagClass, createTagClass, TagImplOptions, FilterImplOptions, Template, Value } from './template'
+import { toPromise, toValueSync, isFunction, forOwn, isString, strictUniq } from './util'
+import { TagClass, createTagClass, TagImplOptions, FilterImplOptions, Template, Value, StaticAnalysisOptions, StaticAnalysis, analyze, analyzeSync, SegmentArray } from './template'
 import { LookupType } from './fs/loader'
 import { Render } from './render'
 import { Parser } from './parser'
@@ -122,4 +122,92 @@ export class Liquid {
       self.renderFile(filePath, ctx).then(html => callback(null, html) as any, callback as any)
     }
   }
+
+  public async analyze (template: Template[], options: StaticAnalysisOptions = {}): Promise<StaticAnalysis> {
+    return analyze(template, options)
+  }
+
+  public analyzeSync (template: Template[], options: StaticAnalysisOptions = {}): StaticAnalysis {
+    return analyzeSync(template, options)
+  }
+
+  public async parseAndAnalyze (html: string, filename?: string, options: StaticAnalysisOptions = {}): Promise<StaticAnalysis> {
+    return analyze(this.parse(html, filename), options)
+  }
+
+  public parseAndAnalyzeSync (html: string, filename?: string, options: StaticAnalysisOptions = {}): StaticAnalysis {
+    return analyzeSync(this.parse(html, filename), options)
+  }
+
+  /** Return an array of all variables without their properties. */
+  public async variables (template: string | Template[], options: StaticAnalysisOptions = {}): Promise<string[]> {
+    const analysis = await analyze(isString(template) ? this.parse(template) : template, options)
+    return Object.keys(analysis.variables)
+  }
+
+  /** Return an array of all variables without their properties. */
+  public variablesSync (template: string | Template[], options: StaticAnalysisOptions = {}): string[] {
+    const analysis = analyzeSync(isString(template) ? this.parse(template) : template, options)
+    return Object.keys(analysis.variables)
+  }
+
+  /** Return an array of all variables including their properties/paths. */
+  public async fullVariables (template: string | Template[], options: StaticAnalysisOptions = {}): Promise<string[]> {
+    const analysis = await analyze(isString(template) ? this.parse(template) : template, options)
+    return Array.from(new Set(Object.values(analysis.variables).flatMap((a) => a.map((v) => String(v)))))
+  }
+
+  /** Return an array of all variables including their properties/paths. */
+  public fullVariablesSync (template: string | Template[], options: StaticAnalysisOptions = {}): string[] {
+    const analysis = analyzeSync(isString(template) ? this.parse(template) : template, options)
+    return Array.from(new Set(Object.values(analysis.variables).flatMap((a) => a.map((v) => String(v)))))
+  }
+
+  /** Return an array of all variables, each as an array of properties/segments. */
+  public async variableSegments (template: string | Template[], options: StaticAnalysisOptions = {}): Promise<Array<SegmentArray>> {
+    const analysis = await analyze(isString(template) ? this.parse(template) : template, options)
+    return Array.from(strictUniq(Object.values(analysis.variables).flatMap((a) => a.map((v) => v.toArray()))))
+  }
+
+  /** Return an array of all variables, each as an array of properties/segments. */
+  public variableSegmentsSync (template: string | Template[], options: StaticAnalysisOptions = {}): Array<SegmentArray> {
+    const analysis = analyzeSync(isString(template) ? this.parse(template) : template, options)
+    return Array.from(strictUniq(Object.values(analysis.variables).flatMap((a) => a.map((v) => v.toArray()))))
+  }
+
+  /** Return an array of all expected context variables without their properties. */
+  public async globalVariables (template: string | Template[], options: StaticAnalysisOptions = {}): Promise<string[]> {
+    const analysis = await analyze(isString(template) ? this.parse(template) : template, options)
+    return Object.keys(analysis.globals)
+  }
+
+  /** Return an array of all expected context variables without their properties. */
+  public globalVariablesSync (template: string | Template[], options: StaticAnalysisOptions = {}): string[] {
+    const analysis = analyzeSync(isString(template) ? this.parse(template) : template, options)
+    return Object.keys(analysis.globals)
+  }
+
+  /** Return an array of all expected context variables including their properties/paths. */
+  public async globalFullVariables (template: string | Template[], options: StaticAnalysisOptions = {}): Promise<string[]> {
+    const analysis = await analyze(isString(template) ? this.parse(template) : template, options)
+    return Array.from(new Set(Object.values(analysis.globals).flatMap((a) => a.map((v) => String(v)))))
+  }
+
+  /** Return an array of all expected context variables including their properties/paths. */
+  public globalFullVariablesSync (template: string | Template[], options: StaticAnalysisOptions = {}): string[] {
+    const analysis = analyzeSync(isString(template) ? this.parse(template) : template, options)
+    return Array.from(new Set(Object.values(analysis.globals).flatMap((a) => a.map((v) => String(v)))))
+  }
+
+  /** Return an array of all expected context variables, each as an array of properties/segments. */
+  public async globalVariableSegments (template: string | Template[], options: StaticAnalysisOptions = {}): Promise<Array<SegmentArray>> {
+    const analysis = await analyze(isString(template) ? this.parse(template) : template, options)
+    return Array.from(strictUniq(Object.values(analysis.globals).flatMap((a) => a.map((v) => v.toArray()))))
+  }
+
+  /** Return an array of all expected context variables, each as an array of properties/segments. */
+  public globalVariableSegmentsSync (template: string | Template[], options: StaticAnalysisOptions = {}): Array<SegmentArray> {
+    const analysis = analyzeSync(isString(template) ? this.parse(template) : template, options)
+    return Array.from(strictUniq(Object.values(analysis.globals).flatMap((a) => a.map((v) => v.toArray()))))
+  }
 }

src/render/expression.ts

@@ -5,7 +5,7 @@ import type { UnaryOperatorHandler } from '../render'
 import { Drop } from '../drop'
 
 export class Expression {
-  private postfix: Token[]
+  readonly postfix: Token[]
 
   public constructor (tokens: IterableIterator<Token>) {
     this.postfix = [...toPostfix(tokens)]

src/tags/assign.ts

@@ -1,11 +1,16 @@
 import { Value, Liquid, TopLevelToken, TagToken, Context, Tag } from '..'
+import { Arguments } from '../template'
+import { IdentifierToken } from '../tokens'
+
 export default class extends Tag {
   private key: string
   private value: Value
+  private identifier: IdentifierToken
 
   constructor (token: TagToken, remainTokens: TopLevelToken[], liquid: Liquid) {
     super(token, remainTokens, liquid)
-    this.key = this.tokenizer.readIdentifier().content
+    this.identifier = this.tokenizer.readIdentifier()
+    this.key = this.identifier.content
     this.tokenizer.assert(this.key, 'expected variable name')
 
     this.tokenizer.skipBlank()
@@ -17,4 +22,12 @@ export default class extends Tag {
   * render (ctx: Context): Generator<unknown, void, unknown> {
     ctx.bottom()[this.key] = yield this.value.value(ctx, this.liquid.options.lenientIf)
   }
+
+  public * arguments (): Arguments {
+    yield this.value
+  }
+
+  public * localScope (): Iterable<IdentifierToken> {
+    yield this.identifier
+  }
 }

src/tags/block.ts

@@ -42,4 +42,12 @@ export default class extends Tag {
       ? (superBlock: BlockDrop, emitter: Emitter) => renderChild(new BlockDrop(() => renderCurrent(superBlock, emitter)), emitter)
       : renderCurrent
   }
+
+  public * children (): Generator<unknown, Template[]> {
+    return this.templates
+  }
+
+  public blockScope (): Iterable<string> {
+    return ['block']
+  }
 }

src/tags/capture.ts

@@ -1,14 +1,16 @@
 import { Liquid, Tag, Template, Context, TagToken, TopLevelToken } from '..'
 import { Parser } from '../parser'
-import { evalQuotedToken } from '../render'
+import { IdentifierToken, QuotedToken } from '../tokens'
 import { isTagToken } from '../util'
 
 export default class extends Tag {
+  identifier: IdentifierToken | QuotedToken
   variable: string
   templates: Template[] = []
   constructor (tagToken: TagToken, remainTokens: TopLevelToken[], liquid: Liquid, parser: Parser) {
     super(tagToken, remainTokens, liquid)
-    this.variable = this.readVariableName()
+    this.identifier = this.readVariable()
+    this.variable = this.identifier.content
 
     while (remainTokens.length) {
       const token = remainTokens.shift()!
@@ -17,16 +19,26 @@ export default class extends Tag {
     }
     throw new Error(`tag ${tagToken.getText()} not closed`)
   }
+
+  private readVariable (): IdentifierToken | QuotedToken {
+    let ident: IdentifierToken | QuotedToken | undefined = this.tokenizer.readIdentifier()
+    if (ident.content) return ident
+    ident = this.tokenizer.readQuoted()
+    if (ident) return ident
+    throw this.tokenizer.error('invalid capture name')
+  }
+
   * render (ctx: Context): Generator<unknown, void, string> {
     const r = this.liquid.renderer
     const html = yield r.renderTemplates(this.templates, ctx)
     ctx.bottom()[this.variable] = html
   }
-  private readVariableName () {
-    const word = this.tokenizer.readIdentifier().content
-    if (word) return word
-    const quoted = this.tokenizer.readQuoted()
-    if (quoted) return evalQuotedToken(quoted)
-    throw this.tokenizer.error('invalid capture name')
+
+  public * children (): Generator<unknown, Template[]> {
+    return this.templates
+  }
+
+  public * localScope (): Iterable<string | IdentifierToken | QuotedToken> {
+    yield this.identifier
   }
 }

src/tags/case.ts

@@ -1,6 +1,7 @@
 import { ValueToken, Liquid, toValue, evalToken, Value, Emitter, TagToken, TopLevelToken, Context, Template, Tag, ParseStream } from '..'
 import { Parser } from '../parser'
 import { equals } from '../render'
+import { Arguments } from '../template'
 
 export default class extends Tag {
   value: Value
@@ -71,4 +72,17 @@ export default class extends Tag {
       yield r.renderTemplates(this.elseTemplates, ctx, emitter)
     }
   }
+
+  public * arguments (): Arguments {
+    yield this.value
+    yield * this.branches.flatMap(b => b.values)
+  }
+
+  public * children (): Generator<unknown, Template[]> {
+    const templates = this.branches.flatMap(b => b.templates)
+    if (this.elseTemplates) {
+      templates.push(...this.elseTemplates)
+    }
+    return templates
+  }
 }

src/tags/cycle.ts

@@ -1,4 +1,5 @@
 import { TopLevelToken, Liquid, ValueToken, evalToken, Emitter, TagToken, Context, Tag } from '..'
+import { Arguments } from '../template'
 
 export default class extends Tag {
   private candidates: ValueToken[] = []
@@ -38,4 +39,12 @@ export default class extends Tag {
     groups[fingerprint] = idx
     return yield evalToken(candidate, ctx)
   }
+
+  public * arguments (): Arguments {
+    yield * this.candidates
+
+    if (this.group) {
+      yield this.group
+    }
+  }
 }

src/tags/decrement.ts

@@ -1,11 +1,14 @@
 import { Tag, Liquid, TopLevelToken, Emitter, TagToken, Context } from '..'
+import { IdentifierToken } from '../tokens'
 import { isNumber, stringify } from '../util'
 
 export default class extends Tag {
+  private identifier: IdentifierToken
   private variable: string
   constructor (token: TagToken, remainTokens: TopLevelToken[], liquid: Liquid) {
     super(token, remainTokens, liquid)
-    this.variable = this.tokenizer.readIdentifier().content
+    this.identifier = this.tokenizer.readIdentifier()
+    this.variable = this.identifier.content
   }
   render (context: Context, emitter: Emitter) {
     const scope = context.environments
@@ -14,4 +17,8 @@ export default class extends Tag {
     }
     emitter.write(stringify(--scope[this.variable]))
   }
+
+  public * localScope (): Iterable<string | IdentifierToken> {
+    yield this.identifier
+  }
 }

src/tags/echo.ts

@@ -1,4 +1,5 @@
 import { Liquid, TopLevelToken, Emitter, Value, TagToken, Context, Tag } from '..'
+import { Arguments } from '../template'
 
 export default class extends Tag {
   private value?: Value
@@ -15,4 +16,10 @@ export default class extends Tag {
     const val = yield this.value.value(ctx, false)
     emitter.write(val)
   }
+
+  public * arguments (): Arguments {
+    if (this.value) {
+      yield this.value
+    }
+  }
 }

src/tags/for.ts

@@ -1,7 +1,8 @@
 import { Hash, ValueToken, Liquid, Tag, evalToken, Emitter, TagToken, TopLevelToken, Context, Template, ParseStream } from '..'
-import { assertEmpty, toEnumerable } from '../util'
+import { assertEmpty, isValueToken, toEnumerable } from '../util'
 import { ForloopDrop } from '../drop/forloop-drop'
 import { Parser } from '../parser'
+import { Arguments } from '../template'
 
 const MODIFIERS = ['offset', 'limit', 'reversed']
 
@@ -25,7 +26,7 @@ export default class extends Tag {
 
     this.variable = variable.content
     this.collection = collection
-    this.hash = new Hash(this.tokenizer.remaining(), liquid.options.keyValueSeparator)
+    this.hash = new Hash(this.tokenizer, liquid.options.keyValueSeparator)
     this.templates = []
     this.elseTemplates = []
 
@@ -75,6 +76,28 @@ export default class extends Tag {
     }
     ctx.pop()
   }
+
+  public * children (): Generator<unknown, Template[]> {
+    const templates = this.templates.slice()
+    if (this.elseTemplates) {
+      templates.push(...this.elseTemplates)
+    }
+    return templates
+  }
+
+  public * arguments (): Arguments {
+    yield this.collection
+
+    for (const v of Object.values(this.hash.hash)) {
+      if (isValueToken(v)) {
+        yield v
+      }
+    }
+  }
+
+  public blockScope (): Iterable<string> {
+    return [this.variable, 'forloop']
+  }
 }
 
 function reversed<T> (arr: Array<T>) {

src/tags/if.ts

@@ -1,5 +1,6 @@
 import { Liquid, Tag, Value, Emitter, isTruthy, TagToken, TopLevelToken, Context, Template } from '..'
 import { Parser } from '../parser'
+import { Arguments } from '../template'
 import { assert, assertEmpty } from '../util'
 
 export default class extends Tag {
@@ -11,13 +12,13 @@ export default class extends Tag {
     let p: Template[] = []
     parser.parseStream(remainTokens)
       .on('start', () => this.branches.push({
-        value: new Value(tagToken.args, this.liquid),
+        value: new Value(tagToken.tokenizer.readFilteredValue(), this.liquid),
         templates: (p = [])
       }))
       .on('tag:elsif', (token: TagToken) => {
         assert(!this.elseTemplates, 'unexpected elsif after else')
         this.branches.push({
-          value: new Value(token.args, this.liquid),
+          value: new Value(token.tokenizer.readFilteredValue(), this.liquid),
           templates: (p = [])
         })
       })
@@ -44,4 +45,16 @@ export default class extends Tag {
     }
     yield r.renderTemplates(this.elseTemplates || [], ctx, emitter)
   }
+
+  public * children (): Generator<unknown, Template[]> {
+    const templates = this.branches.flatMap(b => b.templates)
+    if (this.elseTemplates) {
+      templates.push(...this.elseTemplates)
+    }
+    return templates
+  }
+
+  public arguments (): Arguments {
+    return this.branches.map(b => b.value)
+  }
 }

src/tags/include.ts

@@ -1,6 +1,8 @@
 import { Template, ValueToken, TopLevelToken, Liquid, Tag, assert, evalToken, Hash, Emitter, TagToken, Context } from '..'
 import { BlockMode, Scope } from '../context'
 import { Parser } from '../parser'
+import { Argument, Arguments, PartialScope } from '../template'
+import { isString, isValueToken } from '../util'
 import { parseFilePath, renderFilePath } from './render'
 
 export default class extends Tag {
@@ -21,7 +23,7 @@ export default class extends Tag {
       } else tokenizer.p = begin
     } else tokenizer.p = begin
 
-    this.hash = new Hash(tokenizer.remaining(), liquid.options.jekyllInclude || liquid.options.keyValueSeparator)
+    this.hash = new Hash(tokenizer, liquid.options.jekyllInclude || liquid.options.keyValueSeparator)
   }
   * render (ctx: Context, emitter: Emitter): Generator<unknown, void, unknown> {
     const { liquid, hash, withVar } = this
@@ -40,4 +42,40 @@ export default class extends Tag {
     ctx.pop()
     ctx.restoreRegister(saved)
   }
+
+  public * children (partials: boolean, sync: boolean): Generator<unknown, Template[]> {
+    if (partials && isString(this['file'])) {
+      return (yield this.liquid._parsePartialFile(this['file'], sync, this['currentFile'])) as Template[]
+    }
+    return []
+  }
+
+  public partialScope (): PartialScope | undefined {
+    if (isString(this['file'])) {
+      let names: Array<string | [string, Argument]>
+
+      if (this.liquid.options.jekyllInclude) {
+        names = ['include']
+      } else {
+        names = Object.keys(this.hash.hash)
+        if (this.withVar) {
+          names.push([this['file'], this.withVar])
+        }
+      }
+
+      return { name: this['file'], isolated: false, scope: names }
+    }
+  }
+
+  public * arguments (): Arguments {
+    yield * Object.values(this.hash.hash).filter(isValueToken)
+
+    if (isValueToken(this['file'])) {
+      yield this['file']
+    }
+
+    if (isValueToken(this.withVar)) {
+      yield this.withVar
+    }
+  }
 }

src/tags/increment.ts

@@ -1,11 +1,14 @@
 import { isNumber, stringify } from '../util'
 import { Tag, Liquid, TopLevelToken, Emitter, TagToken, Context } from '..'
+import { IdentifierToken } from '../tokens'
 
 export default class extends Tag {
+  private identifier: IdentifierToken
   private variable: string
   constructor (token: TagToken, remainTokens: TopLevelToken[], liquid: Liquid) {
     super(token, remainTokens, liquid)
-    this.variable = this.tokenizer.readIdentifier().content
+    this.identifier = this.tokenizer.readIdentifier()
+    this.variable = this.identifier.content
   }
   render (context: Context, emitter: Emitter) {
     const scope = context.environments
@@ -16,4 +19,8 @@ export default class extends Tag {
     scope[this.variable]++
     emitter.write(stringify(val))
   }
+
+  public * localScope (): Iterable<string | IdentifierToken> {
+    yield this.identifier
+  }
 }

src/tags/layout.ts

@@ -3,6 +3,8 @@ import { BlockMode } from '../context'
 import { parseFilePath, renderFilePath, ParsedFileName } from './render'
 import { BlankDrop } from '../drop'
 import { Parser } from '../parser'
+import { Arguments, PartialScope } from '../template'
+import { isString, isValueToken } from '../util'
 
 export default class extends Tag {
   args: Hash
@@ -12,7 +14,7 @@ export default class extends Tag {
     super(token, remainTokens, liquid)
     this.file = parseFilePath(this.tokenizer, this.liquid, parser)
     this['currentFile'] = token.file
-    this.args = new Hash(this.tokenizer.remaining(), liquid.options.keyValueSeparator)
+    this.args = new Hash(this.tokenizer, liquid.options.keyValueSeparator)
     this.templates = parser.parseTokens(remainTokens)
   }
   * render (ctx: Context, emitter: Emitter): Generator<unknown, unknown, unknown> {
@@ -41,4 +43,32 @@ export default class extends Tag {
     yield renderer.renderTemplates(templates, ctx, emitter)
     ctx.pop()
   }
+
+  public * children (partials: boolean): Generator<unknown, Template[]> {
+    const templates = this.templates.slice()
+
+    if (partials && isString(this.file)) {
+      templates.push(...(yield this.liquid._parsePartialFile(this.file, true, this['currentFile'])) as Template[])
+    }
+
+    return templates
+  }
+
+  public * arguments (): Arguments {
+    for (const v of Object.values(this.args.hash)) {
+      if (isValueToken(v)) {
+        yield v
+      }
+    }
+
+    if (isValueToken(this.file)) {
+      yield this.file
+    }
+  }
+
+  public partialScope (): PartialScope | undefined {
+    if (isString(this.file)) {
+      return { name: this.file, isolated: false, scope: Object.keys(this.args.hash) }
+    }
+  }
 }

src/tags/liquid.ts

@@ -11,4 +11,8 @@ export default class extends Tag {
   * render (ctx: Context, emitter: Emitter): Generator<unknown, void, unknown> {
     yield this.liquid.renderer.renderTemplates(this.templates, ctx, emitter)
   }
+
+  public * children (): Generator<unknown, Template[]> {
+    return this.templates
+  }
 }

src/tags/render.ts

@@ -1,8 +1,9 @@
 import { __assign } from 'tslib'
 import { ForloopDrop } from '../drop'
-import { toEnumerable } from '../util'
+import { isString, isValueToken, toEnumerable } from '../util'
 import { TopLevelToken, assert, Liquid, Token, Template, evalQuotedToken, TypeGuards, Tokenizer, evalToken, Hash, Emitter, TagToken, Context, Tag } from '..'
 import { Parser } from '../parser'
+import { Argument, Arguments, PartialScope } from '../template'
 
 export type ParsedFileName = Template[] | Token | string | undefined
 
@@ -45,7 +46,7 @@ export default class extends Tag {
       tokenizer.p = begin
       break
     }
-    this.hash = new Hash(tokenizer.remaining(), liquid.options.keyValueSeparator)
+    this.hash = new Hash(tokenizer, liquid.options.keyValueSeparator)
   }
   * render (ctx: Context, emitter: Emitter): Generator<unknown, void, unknown> {
     const { liquid, hash } = this
@@ -75,6 +76,61 @@ export default class extends Tag {
       yield liquid.renderer.renderTemplates(templates, childCtx, emitter)
     }
   }
+
+  public * children (partials: boolean, sync: boolean): Generator<unknown, Template[]> {
+    if (partials && isString(this['file'])) {
+      return (yield this.liquid._parsePartialFile(this['file'], sync, this['currentFile'])) as Template[]
+    }
+    return []
+  }
+
+  public partialScope (): PartialScope | undefined {
+    if (isString(this['file'])) {
+      const names: Array<string | [string, Argument]> = Object.keys(this.hash.hash)
+
+      if (this['with']) {
+        const { value, alias } = this['with']
+        if (isString(alias)) {
+          names.push([alias, value])
+        } else if (isString(this.file)) {
+          names.push([this.file, value])
+        }
+      }
+
+      if (this['for']) {
+        const { value, alias } = this['for']
+        if (isString(alias)) {
+          names.push([alias, value])
+        } else if (isString(this.file)) {
+          names.push([this.file, value])
+        }
+      }
+
+      return { name: this['file'], isolated: true, scope: names }
+    }
+  }
+
+  public * arguments (): Arguments {
+    for (const v of Object.values(this.hash.hash)) {
+      if (isValueToken(v)) {
+        yield v
+      }
+    }
+
+    if (this['with']) {
+      const { value } = this['with']
+      if (isValueToken(value)) {
+        yield value
+      }
+    }
+
+    if (this['for']) {
+      const { value } = this['for']
+      if (isValueToken(value)) {
+        yield value
+      }
+    }
+  }
 }
 
 /**

src/tags/tablerow.ts

@@ -1,7 +1,8 @@
-import { toEnumerable } from '../util'
+import { isValueToken, toEnumerable } from '../util'
 import { ValueToken, Liquid, Tag, evalToken, Emitter, Hash, TagToken, TopLevelToken, Context, Template, ParseStream } from '..'
 import { TablerowloopDrop } from '../drop/tablerowloop-drop'
 import { Parser } from '../parser'
+import { Arguments } from '../template'
 
 export default class extends Tag {
   variable: string
@@ -21,7 +22,7 @@ export default class extends Tag {
 
     this.variable = variable.content
     this.collection = collectionToken
-    this.args = new Hash(this.tokenizer.remaining(), liquid.options.keyValueSeparator)
+    this.args = new Hash(this.tokenizer, liquid.options.keyValueSeparator)
     this.templates = []
 
     let p
@@ -63,4 +64,22 @@ export default class extends Tag {
     if (collection.length) emitter.write('</tr>')
     ctx.pop()
   }
+
+  public * children (): Generator<unknown, Template[]> {
+    return this.templates
+  }
+
+  public * arguments (): Arguments {
+    yield this.collection
+
+    for (const v of Object.values(this.args.hash)) {
+      if (isValueToken(v)) {
+        yield v
+      }
+    }
+  }
+
+  public blockScope (): string[] {
+    return [this.variable, 'tablerowloop']
+  }
 }

src/tags/unless.ts

@@ -1,5 +1,6 @@
 import { Liquid, Tag, Value, TopLevelToken, Template, Emitter, isTruthy, isFalsy, Context, TagToken } from '..'
 import { Parser } from '../parser'
+import { Arguments } from '../template'
 
 export default class extends Tag {
   branches: { value: Value, test: (val: any, ctx: Context) => boolean, templates: Template[] }[] = []
@@ -10,7 +11,7 @@ export default class extends Tag {
     let elseCount = 0
     parser.parseStream(remainTokens)
       .on('start', () => this.branches.push({
-        value: new Value(tagToken.args, this.liquid),
+        value: new Value(tagToken.tokenizer.readFilteredValue(), this.liquid),
         test: isFalsy,
         templates: (p = [])
       }))
@@ -20,7 +21,7 @@ export default class extends Tag {
           return
         }
         this.branches.push({
-          value: new Value(token.args, this.liquid),
+          value: new Value(token.tokenizer.readFilteredValue(), this.liquid),
           test: isTruthy,
           templates: (p = [])
         })
@@ -52,4 +53,16 @@ export default class extends Tag {
 
     yield r.renderTemplates(this.elseTemplates, ctx, emitter)
   }
+
+  public * children (): Generator<unknown, Template[]> {
+    const children = this.branches.flatMap(b => b.templates)
+    if (this.elseTemplates) {
+      children.push(...this.elseTemplates)
+    }
+    return children
+  }
+
+  public arguments (): Arguments {
+    return this.branches.map(b => b.value)
+  }
 }

src/template/analysis.spec.ts

@@ -0,0 +1,51 @@
+import { Variable, VariableMap } from './analysis'
+
+describe('Analysis variable', () => {
+  const mockLocation = { row: 1, col: 1, file: undefined }
+
+  it('should coerce to a string', () => {
+    const v = new Variable(['foo', 'bar'], mockLocation)
+    expect(String(v)).toBe('foo.bar')
+  })
+
+  it('should represent nested variables', () => {
+    const nested = new Variable(['bar', 1], mockLocation)
+    const v = new Variable(['foo', nested], mockLocation)
+    expect(`${v}`).toBe('foo[bar[1]]')
+  })
+
+  it('should represent bracketed segments', () => {
+    const v = new Variable(['foo', 'bar baz'], mockLocation)
+    expect(`${v}`).toBe("foo['bar baz']")
+  })
+
+  it('should represent bracketed root', () => {
+    const v = new Variable(['foo bar'], mockLocation)
+    expect(`${v}`).toBe("['foo bar']")
+  })
+
+  it('should have a segments property', () => {
+    const v = new Variable(['foo', 'bar'], mockLocation)
+    expect(v.segments).toStrictEqual(['foo', 'bar'])
+  })
+
+  it('should have a location property', () => {
+    const v = new Variable(['foo', 'bar'], mockLocation)
+    expect(v.location).toStrictEqual(mockLocation)
+  })
+})
+
+describe('Variable map', () => {
+  it('should coerce variables to their string representation', () => {
+    const v = new Variable(['foo', 'bar'], { row: 1, col: 1, file: undefined })
+    const mapping = new VariableMap()
+    mapping.push(v)
+    expect(mapping.has(v)).toBe(true)
+  })
+
+  it('should return an empty array if a variable is not in the map', () => {
+    const v = new Variable(['foo', 'bar'], { row: 1, col: 1, file: undefined })
+    const mapping = new VariableMap()
+    expect(mapping.get(v)).toStrictEqual([])
+  })
+})

src/template/analysis.ts

@@ -0,0 +1,448 @@
+import { Argument, Template, Value } from '.'
+import { isKeyValuePair } from '../parser/filter-arg'
+import { PropertyAccessToken, ValueToken } from '../tokens'
+import {
+  isNumberToken,
+  isPropertyAccessToken,
+  isQuotedToken,
+  isRangeToken,
+  isString,
+  isValueToken,
+  isWordToken,
+  toPromise,
+  toValueSync
+} from '../util'
+
+/**
+ * Row, column and file name where a variable was found.
+ */
+export interface VariableLocation {
+  row: number;
+  col: number;
+  file?: string;
+}
+
+/**
+ * A variable's segments as an array, possibly with nested arrays of segments.
+ */
+export type SegmentArray = Array<string | number | SegmentArray>
+
+/**
+ * A variable's segments and location, which can be coerced to a string.
+ */
+export class Variable {
+  constructor (
+    readonly segments: Array<string | number | Variable>,
+    readonly location: VariableLocation
+  ) {}
+
+  public toString (): string {
+    return segmentsString(this.segments, true)
+  }
+
+  /** Return this variable's segments as an array, possibly with nested arrays for nested paths. */
+  public toArray (): SegmentArray {
+    function * _visit (...segments: Array<string | number | Variable>): Generator<string | number | SegmentArray> {
+      for (const segment of segments) {
+        if (segment instanceof Variable) {
+          yield Array.from(_visit(...segment.segments))
+        } else {
+          yield segment
+        }
+      }
+    }
+    return Array.from(_visit(...this.segments))
+  }
+}
+
+/**
+ * Property names and array indexes that make up a path to a variable.
+ */
+export type VariableSegments = Array<string | number | Variable>;
+
+/**
+ * A mapping of variable names to an array of locations at which the variable was found.
+ */
+export type Variables = { [key: string]: Variable[] };
+
+/**
+ * Group variables by the string representation of their root segment.
+ */
+export class VariableMap {
+  private map: Map<string, Variable[]>
+
+  constructor () {
+    this.map = new Map()
+  }
+
+  public get (key: Variable): Variable[] {
+    const k = segmentsString([key.segments[0]])
+    if (!this.map.has(k)) {
+      this.map.set(k, [])
+    }
+    return this.map.get(k) as Variable[]
+  }
+
+  public has (key: Variable): boolean {
+    return this.map.has(segmentsString([key.segments[0]]))
+  }
+
+  public push (variable: Variable): void {
+    this.get(variable).push(variable)
+  }
+
+  public asObject (): Variables {
+    return Object.fromEntries(this.map)
+  }
+}
+
+/**
+ * The result of calling `analyze()` or `analyzeSync()`.
+ */
+export interface StaticAnalysis {
+  /**
+   * All variables, whether they are in scope or not. Including references to names
+   * such as `forloop` from the `for` tag.
+   */
+  variables: Variables;
+
+  /**
+   * Variables that are not in scope. These could be a "global" variables that are
+   * expected to be provided by the application developer, or possible mistakes
+   * from the template author.
+   *
+   * If a variable is referenced before and after assignment, you should expect
+   * that variable to be included in `globals`, `variables` and `locals`, each with
+   * a different location.
+   */
+  globals: Variables;
+
+  /**
+   * Template variables that are added to the template local scope using tags like
+   * `assign`, `capture` or `increment`.
+   */
+  locals: Variables;
+}
+
+export interface StaticAnalysisOptions {
+  /**
+   * When `true` (the default), try to load partial templates and analyze them too.
+   */
+  partials?: boolean;
+}
+
+export const defaultStaticAnalysisOptions: StaticAnalysisOptions = {
+  partials: true
+}
+
+function * _analyze (templates: Template[], partials: boolean, sync: boolean): Generator<unknown, StaticAnalysis> {
+  const variables = new VariableMap()
+  const globals = new VariableMap()
+  const locals = new VariableMap()
+
+  const rootScope = new DummyScope(new Set())
+
+  // Names of partial templates that we've already analyzed.
+  const seen: Set<string | undefined> = new Set()
+
+  function updateVariables (variable: Variable, scope: DummyScope) {
+    variables.push(variable)
+    const aliased = scope.alias(variable)
+
+    if (aliased !== undefined) {
+      const root = aliased.segments[0]
+      // TODO: What if a a template renders a rendered template? Do we need scope.parent?
+      if (isString(root) && !rootScope.has(root)) {
+        globals.push(aliased)
+      }
+    } else {
+      const root = variable.segments[0]
+      if (isString(root) && !scope.has(root)) {
+        globals.push(variable)
+      }
+    }
+
+    // Recurse for nested Variables
+    for (const segment of variable.segments) {
+      if (segment instanceof Variable) {
+        updateVariables(segment, scope)
+      }
+    }
+  }
+
+  function * visit (template: Template, scope: DummyScope): Generator<unknown, void> {
+    if (template.arguments) {
+      for (const arg of template.arguments()) {
+        for (const variable of extractVariables(arg)) {
+          updateVariables(variable, scope)
+        }
+      }
+    }
+
+    if (template.localScope) {
+      for (const ident of template.localScope()) {
+        scope.add(ident.content)
+        scope.deleteAlias(ident.content)
+        const [row, col] = ident.getPosition()
+        locals.push(new Variable([ident.content], { row, col, file: ident.file }))
+      }
+    }
+
+    if (template.children) {
+      if (template.partialScope) {
+        const partial = template.partialScope()
+
+        if (partial === undefined) {
+          // Layouts, for example, can have children that are not partials.
+          for (const child of (yield template.children(partials, sync)) as Template[]) {
+            yield visit(child, scope)
+          }
+          return
+        }
+
+        if (seen.has(partial.name)) return
+
+        const partialScopeNames: Set<string> = new Set()
+        const partialScope = partial.isolated
+          ? new DummyScope(partialScopeNames)
+          : scope.push(partialScopeNames)
+
+        for (const name of partial.scope) {
+          if (isString(name)) {
+            partialScopeNames.add(name)
+          } else {
+            const [alias, argument] = name
+            partialScopeNames.add(alias)
+            const variables = Array.from(extractVariables(argument))
+            if (variables.length) {
+              partialScope.setAlias(alias, variables[0].segments)
+            }
+          }
+        }
+
+        for (const child of (yield template.children(partials, sync)) as Template[]) {
+          yield visit(child, partialScope)
+          seen.add(partial.name)
+        }
+
+        partialScope.pop()
+      } else {
+        if (template.blockScope) {
+          scope.push(new Set(template.blockScope()))
+        }
+
+        for (const child of (yield template.children(partials, sync)) as Template[]) {
+          yield visit(child, scope)
+        }
+
+        if (template.blockScope) {
+          scope.pop()
+        }
+      }
+    }
+  }
+
+  for (const template of templates) {
+    yield visit(template, rootScope)
+  }
+
+  return {
+    variables: variables.asObject(),
+    globals: globals.asObject(),
+    locals: locals.asObject()
+  }
+}
+
+/**
+ * Statically analyze a template and report variable usage.
+ */
+export function analyze (template: Template[], options: StaticAnalysisOptions = {}): Promise<StaticAnalysis> {
+  const opts = { ...defaultStaticAnalysisOptions, ...options } as Required<StaticAnalysisOptions>
+  return toPromise(_analyze(template, opts.partials, false))
+}
+
+/**
+ * Statically analyze a template and report variable usage.
+ */
+export function analyzeSync (template: Template[], options: StaticAnalysisOptions = {}): StaticAnalysis {
+  const opts = { ...defaultStaticAnalysisOptions, ...options } as Required<StaticAnalysisOptions>
+  return toValueSync(_analyze(template, opts.partials, true))
+}
+
+interface ScopeStackItem {
+  names: Set<string>;
+  aliases: Map<string, VariableSegments>;
+}
+
+/**
+ * A stack to manage scopes while traversing templates during static analysis.
+ */
+class DummyScope {
+  private stack: Array<ScopeStackItem>
+
+  constructor (globals: Set<string>) {
+    this.stack = [{ names: globals, aliases: new Map() }]
+  }
+
+  /** Return true if `name` is in scope.  */
+  public has (name: string): boolean {
+    for (const scope of this.stack) {
+      if (scope.names.has(name)) {
+        return true
+      }
+    }
+    return false
+  }
+
+  public push (scope: Set<string>): DummyScope {
+    this.stack.push({ names: scope, aliases: new Map() })
+    return this
+  }
+
+  public pop (): Set<string> | undefined {
+    return this.stack.pop()?.names
+  }
+
+  // Add a name to the template scope.
+  public add (name: string): void {
+    this.stack[0].names.add(name)
+  }
+
+  /** Return the variable that `variable` aliases, or `variable` if it doesn't alias anything. */
+  public alias (variable: Variable): Variable | undefined {
+    const root = variable.segments[0]
+    if (!isString(root)) return undefined
+    const alias = this.getAlias(root)
+    if (alias === undefined) return undefined
+    return new Variable([...alias, ...variable.segments.slice(1)], variable.location)
+  }
+
+  // TODO: `from` could be a path with multiple segments, like `include.x`.
+  public setAlias (from: string, to: VariableSegments): void {
+    this.stack[this.stack.length - 1].aliases.set(from, to)
+  }
+
+  public deleteAlias (name: string): void {
+    this.stack[this.stack.length - 1].aliases.delete(name)
+  }
+
+  private getAlias (name: string): VariableSegments | undefined {
+    for (const scope of this.stack) {
+      if (scope.aliases.has(name)) {
+        return scope.aliases.get(name)
+      }
+
+      // If a scope has defined `name`, then it masks aliases in parent scopes.
+      if (scope.names.has(name)) {
+        return undefined
+      }
+    }
+    return undefined
+  }
+}
+
+function * extractVariables (value: Argument): Generator<Variable> {
+  if (isValueToken(value)) {
+    yield * extractValueTokenVariables(value)
+  } else if (value instanceof Value) {
+    yield * extractFilteredValueVariables(value)
+  }
+}
+
+function * extractFilteredValueVariables (value: Value): Generator<Variable> {
+  for (const token of value.initial.postfix) {
+    if (isValueToken(token)) {
+      yield * extractValueTokenVariables(token)
+    }
+  }
+
+  for (const filter of value.filters) {
+    for (const arg of filter.args) {
+      if (isKeyValuePair(arg) && arg[1]) {
+        yield * extractValueTokenVariables(arg[1])
+      } else if (isValueToken(arg)) {
+        yield * extractValueTokenVariables(arg)
+      }
+    }
+  }
+}
+
+function * extractValueTokenVariables (token: ValueToken): Generator<Variable> {
+  if (isRangeToken(token)) {
+    yield * extractValueTokenVariables(token.lhs)
+    yield * extractValueTokenVariables(token.rhs)
+  } else if (isPropertyAccessToken(token)) {
+    yield extractPropertyAccessVariable(token)
+  }
+}
+
+function extractPropertyAccessVariable (token: PropertyAccessToken): Variable {
+  const segments: VariableSegments = []
+
+  // token is not guaranteed to have `file` set. We'll try to get it from a prop if not.
+  let file: string | undefined = token.file
+
+  // Here we're flattening the first segment of a path if it is a nested path.
+  const root = token.props[0]
+  file = file || root.file
+  if (isQuotedToken(root) || isNumberToken(root) || isWordToken(root)) {
+    segments.push(root.content)
+  } else if (isPropertyAccessToken(root)) {
+    // Flatten paths that start with a nested path.
+    segments.push(...extractPropertyAccessVariable(root).segments)
+  }
+
+  for (const prop of token.props.slice(1)) {
+    file = file || prop.file
+    if (isQuotedToken(prop) || isNumberToken(prop) || isWordToken(prop)) {
+      segments.push(prop.content)
+    } else if (isPropertyAccessToken(prop)) {
+      segments.push(extractPropertyAccessVariable(prop))
+    }
+  }
+
+  const [row, col] = token.getPosition()
+  return new Variable(segments, {
+    row,
+    col,
+    file
+  })
+}
+
+// This is used to detect segments that can be represented with dot notation
+// when creating a string representation of VariableSegments.
+const RE_PROPERTY = /^[\u0080-\uFFFFa-zA-Z_][\u0080-\uFFFFa-zA-Z0-9_-]*$/
+
+/**
+ * Return a string representation of segments using dot notation where possible.
+ * @param segments - The property names and array indices that make up a path to a variable.
+ * @param bracketedRoot - If false (the default), don't surround the root segment with square brackets.
+ */
+function segmentsString (segments: VariableSegments, bracketedRoot = false): string {
+  const buf: string[] = []
+
+  const root = segments[0]
+  if (isString(root)) {
+    if (!bracketedRoot || root.match(RE_PROPERTY)) {
+      buf.push(`${root}`)
+    } else {
+      buf.push(`['${root}']`)
+    }
+  }
+
+  for (const segment of segments.slice(1)) {
+    if (segment instanceof Variable) {
+      buf.push(`[${segmentsString(segment.segments)}]`)
+    } else if (isString(segment)) {
+      if (segment.match(RE_PROPERTY)) {
+        buf.push(`.${segment}`)
+      } else {
+        buf.push(`['${segment}']`)
+      }
+    } else {
+      buf.push(`[${segment}]`)
+    }
+  }
+
+  return buf.join('')
+}

src/template/hash.spec.ts

@@ -1,6 +1,7 @@
 import { toPromise } from '../util'
 import { Hash } from './hash'
 import { Context } from '../context'
+import { Tokenizer } from '../parser'
 
 describe('Hash', function () {
   it('should parse "reverse"', async function () {
@@ -43,4 +44,10 @@ describe('Hash', function () {
     const hash = await toPromise(new Hash('num=2.3', '=').render(new Context()))
     expect(hash.num).toBe(2.3)
   })
+  it('should accept an existing tokenizer', async function () {
+    const tokenizer = new Tokenizer('a:1, b:2')
+    const hash = await toPromise(new Hash(tokenizer).render(new Context()))
+    expect(hash.a).toBe(1)
+    expect(hash.b).toBe(2)
+  })
 })

src/template/hash.ts

@@ -15,12 +15,14 @@ type HashValueTokens = Record<string, Token | undefined>
  */
 export class Hash {
   hash: HashValueTokens = {}
-  constructor (markup: string, jekyllStyle?: boolean | string) {
-    const tokenizer = new Tokenizer(markup, {})
+
+  constructor (input: string | Tokenizer, jekyllStyle?: boolean | string) {
+    const tokenizer = input instanceof Tokenizer ? input : new Tokenizer(input, {})
     for (const hash of tokenizer.readHashes(jekyllStyle)) {
       this.hash[hash.name.content] = hash.value
     }
   }
+
   * render (ctx: Context): Generator<unknown, Record<string, any>, unknown> {
     const hash = {}
     for (const key of Object.keys(this.hash)) {

src/template/index.ts

@@ -8,3 +8,4 @@ export * from './hash'
 export * from './value'
 export * from './output'
 export * from './html'
+export * from './analysis'

src/template/output.ts

@@ -1,5 +1,5 @@
 import { Value } from './value'
-import { Template, TemplateImpl } from '../template'
+import { Arguments, Template, TemplateImpl } from '../template'
 import { Context } from '../context/context'
 import { Emitter } from '../emitters/emitter'
 import { OutputToken } from '../tokens/output-token'
@@ -25,4 +25,8 @@ export class Output extends TemplateImpl<OutputToken> implements Template {
     const val = yield this.value.value(ctx, false)
     emitter.write(val)
   }
+
+  public * arguments (): Arguments {
+    yield this.value
+  }
 }

src/template/template.ts

@@ -1,8 +1,41 @@
 import { Context } from '../context/context'
 import { Token } from '../tokens/token'
 import { Emitter } from '../emitters/emitter'
+import { IdentifierToken, QuotedToken, ValueToken } from '../tokens'
+import { Value } from './value'
+
+export type Argument = Value | ValueToken
+export type Arguments = Iterable<Argument>
+
+/** Scope information used when analyzing partial templates. */
+export interface PartialScope {
+  /**
+   * The name of the partial template. We need this to make sure we only analyze
+   * each template once.
+   * */
+  name: string;
+
+  /**
+   * If `true`, names in `scope` will be added to a new, isolated scope before
+   * analyzing any child templates, without access to the parent template's scope.
+   */
+  isolated: boolean;
+
+  /**
+   * A list of names that will be in scope for the child template.
+   *
+   * If an item is a [string, Argument] tuple, the string is considered an alias
+   * for the argument.
+   */
+  scope: Iterable<string | [string, Argument]>;
+}
 
 export interface Template {
   token: Token;
   render(ctx: Context, emitter: Emitter): any;
+  children?(partials: boolean, sync: boolean): Generator<unknown, Template[]> ;
+  arguments?(): Arguments;
+  blockScope?(): Iterable<string>;
+  localScope?(): Iterable<IdentifierToken | QuotedToken>;
+  partialScope?(): PartialScope | undefined;
 }

src/template/value.ts

@@ -20,6 +20,7 @@ export class Value {
     this.initial = token.initial
     this.filters = token.filters.map(token => new Filter(token, this.getFilter(liquid, token.name), liquid))
   }
+
   public * value (ctx: Context, lenient?: boolean): Generator<unknown, unknown, unknown> {
     lenient = lenient || (ctx.opts.lenientIf && this.filters.length > 0 && this.filters[0].name === 'default')
     let val = yield this.initial.evaluate(ctx, lenient)
@@ -29,6 +30,7 @@ export class Value {
     }
     return val
   }
+
   private getFilter (liquid: Liquid, name: string) {
     const impl = liquid.filters[name]
     assert(impl || !liquid.options.strictFilters, () => `undefined filter: ${name}`)

src/tokens/liquid-tag-token.ts

@@ -7,7 +7,6 @@ import { Tokenizer, TokenKind } from '../parser'
  */
 export class LiquidTagToken extends DelimitedToken {
   public name: string
-  public args: string
   public tokenizer: Tokenizer
   public constructor (
     input: string,
@@ -17,12 +16,13 @@ export class LiquidTagToken extends DelimitedToken {
     file?: string
   ) {
     super(TokenKind.Tag, [begin, end], input, begin, end, false, false, file)
-
     this.tokenizer = new Tokenizer(input, options.operators, file, this.contentRange)
     this.name = this.tokenizer.readTagName()
     this.tokenizer.assert(this.name, 'illegal liquid tag syntax')
-
     this.tokenizer.skipBlank()
-    this.args = this.tokenizer.remaining()
+  }
+
+  get args (): string {
+    return this.tokenizer.input.slice(this.tokenizer.p, this.contentRange[1])
   }
 }

src/tokens/tag-token.ts

@@ -21,6 +21,7 @@ export class TagToken extends DelimitedToken {
     this.tokenizer.assert(this.name, `illegal tag syntax, tag name expected`)
     this.tokenizer.skipBlank()
   }
+
   get args (): string {
     return this.tokenizer.input.slice(this.tokenizer.p, this.contentRange[1])
   }

src/util/type-guards.ts

@@ -1,4 +1,4 @@
-import { RangeToken, NumberToken, QuotedToken, LiteralToken, PropertyAccessToken, OutputToken, HTMLToken, TagToken, IdentifierToken, DelimitedToken, OperatorToken } from '../tokens'
+import { RangeToken, NumberToken, QuotedToken, LiteralToken, PropertyAccessToken, OutputToken, HTMLToken, TagToken, IdentifierToken, DelimitedToken, OperatorToken, ValueToken } from '../tokens'
 import { TokenKind } from '../parser'
 
 export function isDelimitedToken (val: any): val is DelimitedToken {
@@ -45,6 +45,11 @@ export function isRangeToken (val: any): val is RangeToken {
   return getKind(val) === TokenKind.Range
 }
 
+export function isValueToken (val: any): val is ValueToken {
+  // valueTokenBitMask = TokenKind.Number | TokenKind.Literal | TokenKind.Quoted | TokenKind.PropertyAccess | TokenKind.Range
+  return (getKind(val) & 1667) > 0
+}
+
 function getKind (val: any) {
   return val ? val.kind : -1
 }

src/util/underscore.ts

@@ -194,3 +194,16 @@ export function argumentsToValue<F extends (...args: any) => any, T> (fn: F) {
 export function escapeRegExp (text: string) {
   return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&')
 }
+
+/** Return an array containing unique elements from _array_. Works with nested arrays and objects. */
+export function * strictUniq<T> (array: Array<T>): Generator<T> {
+  const seen = new Set()
+
+  for (const element of array) {
+    const key = JSON.stringify(element)
+    if (!seen.has(key)) {
+      seen.add(key)
+      yield element
+    }
+  }
+}