feat: static variable analysis (#770)

* feat: static variable analysis

* Accept any iterable from `children`, `arguments`, etc.

* Test analysis of standard tags

* Use `TagToken.tokenizer` instead of creating a new one

* Test analysis of netsted tags

* Group variables by their root value

* Test analysis of nested globals and locals

* Analyze included and rendered templates WIP

* Use existing tokenizer when constructing `Hash`

* Improve test coverage

* Analyze variables from `layout` and `block` tags

* Test analysis of Jekyll style includes

* Handle variables that start with a nested variable

* Async analysis

* Test non-standard tag end to end

* Implement convenience analysis methods on the `Liquid` class

* More analysis convenience methods

* Accept string or template array

* Draft static analysis docs

* Deduplicate variables names

* Fix isolated scope global variable map

* Coerce variables to strings instead of extending String

* Private map instead of extending Map

* Fix e2e test

* Tentatively implement analysis of aliased variables

* Fix nested variable segments array

* Update docs sidebar

test/e2e/parse-and-analyze.spec.ts

@@ -0,0 +1,70 @@
+import { Context, Emitter, Hash, Liquid, Scope, Tag, TagToken, Template, TopLevelToken, ParseStream, Parser, Arguments, analyzeSync, Variable, StaticAnalysisOptions, StaticAnalysis } from '../..'
+
+class MockTag extends Tag {
+  private args: Hash
+  private templates: Template[]
+
+  constructor (token: TagToken, remainTokens: TopLevelToken[], liquid: Liquid, parser: Parser) {
+    super(token, remainTokens, liquid)
+    this.args = new Hash(token.tokenizer)
+    this.templates = []
+
+    const stream: ParseStream = parser.parseStream(remainTokens)
+      .on<TagToken>('tag:endmock', () => { stream.stop() })
+      .on('template', (tpl: Template) => this.templates.push(tpl))
+      .on('end', () => { throw new Error(`tag ${token.getText()} not closed`) })
+
+    stream.start()
+  }
+
+  public * render (ctx: Context, emitter: Emitter): Generator<unknown, void, string> {
+    const scope = (yield this.args.render(ctx)) as unknown as Scope
+    ctx.push(scope)
+    yield this.liquid.renderer.renderTemplates(this.templates, ctx, emitter)
+    ctx.pop()
+  }
+
+  public * children (): Generator<unknown, Template[]> {
+    return this.templates
+  }
+
+  public * arguments (): Arguments {
+    // XXX: tokens and type guards are not exported
+    yield * Object.values(this.args.hash).filter((el) => el !== undefined) as Arguments
+  }
+
+  public blockScope (): Iterable<string> {
+    return Object.keys(this.args.hash)
+  }
+}
+
+describe('Static analysis', () => {
+  it('should report variables from non-standard tags', () => {
+    const engine = new Liquid()
+    engine.registerTag('mock', MockTag)
+
+    const template = engine.parse('{% mock a:b x:y %}{{ x }}{{ z }}{% endmock %}')
+    const analysis = analyzeSync(template)
+
+    const b = [new Variable(['b'], { row: 1, col: 11, file: undefined })]
+    const x = [new Variable(['x'], { row: 1, col: 22, file: undefined })]
+    const y = [new Variable(['y'], { row: 1, col: 15, file: undefined })]
+    const z = [new Variable(['z'], { row: 1, col: 29, file: undefined })]
+
+    expect(analysis).toStrictEqual({
+      variables: { b, x, y, z },
+      globals: { b, y, z },
+      locals: { }
+    })
+  })
+
+  it('should export analysis interfaces', () => {
+    const engine = new Liquid()
+    const template = engine.parse('{% include nothing %}')
+    const options: StaticAnalysisOptions = { partials: false }
+    const analysis: StaticAnalysis = analyzeSync(template, options)
+    const vars: Variable[] = analysis.variables['nothing'] || []
+    const v: Variable = vars[0]
+    expect(String(v)).toBe('nothing')
+  })
+})

test/integration/liquid/liquid.spec.ts

@@ -229,4 +229,113 @@ describe('Liquid', function () {
       expect(drainStream(stream)).rejects.toThrow(/intended render error/)
     })
   })
+  describe('#analyze', () => {
+    const engine = new Liquid()
+    it('should analyze templates asynchronously', () => {
+      const template = engine.parse('{{ a }}{{ b }}')
+      expect(engine.analyze(template).then((a) => Object.keys(a.variables))).resolves.toStrictEqual(['a', 'b'])
+    })
+  })
+  describe('#analyzeSync', () => {
+    const engine = new Liquid()
+    it('should analyze templates synchronously', () => {
+      const template = engine.parse('{{ a }}{{ b }}')
+      expect(Object.keys(engine.analyzeSync(template).variables)).toStrictEqual(['a', 'b'])
+    })
+  })
+  describe('#parseAndAnalyze', () => {
+    const engine = new Liquid()
+    it('should parse and analyze templates asynchronously', () => {
+      expect(engine.parseAndAnalyze('{{ a }}{{ b }}').then((a) => Object.keys(a.variables))).resolves.toStrictEqual(['a', 'b'])
+    })
+  })
+  describe('#parseAndAnalyzeSync', () => {
+    const engine = new Liquid()
+    it('should analyze templates synchronously', () => {
+      expect(Object.keys(engine.parseAndAnalyzeSync('{{ a }}{{ b }}').variables)).toStrictEqual(['a', 'b'])
+    })
+  })
+  describe('Convenience analysis', () => {
+    const engine = new Liquid()
+
+    it('should list all variables without their properties', () => {
+      expect(engine.variables('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}')).resolves.toStrictEqual(['a', 'c'])
+      expect(engine.variables(engine.parse('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}'))).resolves.toStrictEqual(['a', 'c'])
+    })
+
+    it('should list all variables without their properties synchronously', () => {
+      expect(engine.variablesSync('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}')).toStrictEqual(['a', 'c'])
+      expect(engine.variablesSync(engine.parse('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}'))).toStrictEqual(['a', 'c'])
+    })
+
+    it('should list global variables without their properties', () => {
+      expect(engine.globalVariables('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}')).resolves.toStrictEqual(['a'])
+      expect(engine.globalVariables(engine.parse('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}'))).resolves.toStrictEqual(['a'])
+    })
+
+    it('should list global variables without their properties synchronously', () => {
+      expect(engine.globalVariablesSync('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}')).toStrictEqual(['a'])
+      expect(engine.globalVariablesSync(engine.parse('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}'))).toStrictEqual(['a'])
+    })
+
+    it('should list all variables with their properties', () => {
+      expect(engine.fullVariables('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}')).resolves.toStrictEqual(['a.b', 'c'])
+      expect(engine.fullVariables(engine.parse('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}'))).resolves.toStrictEqual(['a.b', 'c'])
+    })
+
+    it('should list all variables with their properties synchronously', () => {
+      expect(engine.fullVariablesSync('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}')).toStrictEqual(['a.b', 'c'])
+      expect(engine.fullVariablesSync(engine.parse('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}'))).toStrictEqual(['a.b', 'c'])
+    })
+
+    it('should list global variables with their properties', () => {
+      expect(engine.globalFullVariables('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}')).resolves.toStrictEqual(['a.b'])
+      expect(engine.globalFullVariables(engine.parse('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}'))).resolves.toStrictEqual(['a.b'])
+    })
+
+    it('should list global variables with their properties synchronously', () => {
+      expect(engine.globalFullVariablesSync('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}')).toStrictEqual(['a.b'])
+      expect(engine.globalFullVariablesSync(engine.parse('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}'))).toStrictEqual(['a.b'])
+    })
+
+    it('should list all variables as an array of segments', () => {
+      expect(engine.variableSegments('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}')).resolves.toStrictEqual([['a', 'b'], ['c']])
+      expect(engine.variableSegments(engine.parse('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}'))).resolves.toStrictEqual([['a', 'b'], ['c']])
+    })
+
+    it('should list all variables as an array of segments synchronously', () => {
+      expect(engine.variableSegmentsSync('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}')).toStrictEqual([['a', 'b'], ['c']])
+      expect(engine.variableSegmentsSync(engine.parse('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}'))).toStrictEqual([['a', 'b'], ['c']])
+    })
+
+    it('should list all variables as an array of segments with nested variables as arrays', () => {
+      expect(engine.variableSegments('{{ a[b.c].d }}')).resolves.toStrictEqual([['a', ['b', 'c'], 'd'], ['b', 'c']])
+      expect(engine.variableSegments(engine.parse('{{ a[b.c].d }}'))).resolves.toStrictEqual([['a', ['b', 'c'], 'd'], ['b', 'c']])
+    })
+
+    it('should list all variables synchronously as an array of segments with nested variables as arrays', () => {
+      expect(engine.variableSegmentsSync('{{ a[b.c].d }}')).toStrictEqual([['a', ['b', 'c'], 'd'], ['b', 'c']])
+      expect(engine.variableSegmentsSync(engine.parse('{{ a[b.c].d }}'))).toStrictEqual([['a', ['b', 'c'], 'd'], ['b', 'c']])
+    })
+
+    it('should list global variables as an array of segments', () => {
+      expect(engine.globalVariableSegments('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}')).resolves.toStrictEqual([['a', 'b']])
+      expect(engine.globalVariableSegments(engine.parse('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}'))).resolves.toStrictEqual([['a', 'b']])
+    })
+
+    it('should list global variables as an array of segments synchronously', () => {
+      expect(engine.globalVariableSegmentsSync('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}')).toStrictEqual([['a', 'b']])
+      expect(engine.globalVariableSegmentsSync(engine.parse('{% assign c = 1 %}{{ a.b }}{{ c }}{{ c }}'))).toStrictEqual([['a', 'b']])
+    })
+
+    it('should list global variables as an array of segments with nested variables as arrays', () => {
+      expect(engine.globalVariableSegments('{{ a[b.c].d }}')).resolves.toStrictEqual([['a', ['b', 'c'], 'd'], ['b', 'c']])
+      expect(engine.globalVariableSegments(engine.parse('{{ a[b.c].d }}'))).resolves.toStrictEqual([['a', ['b', 'c'], 'd'], ['b', 'c']])
+    })
+
+    it('should list global variables synchronously as an array of segments with nested variables as arrays', () => {
+      expect(engine.globalVariableSegmentsSync('{{ a[b.c].d }}')).toStrictEqual([['a', ['b', 'c'], 'd'], ['b', 'c']])
+      expect(engine.globalVariableSegmentsSync(engine.parse('{{ a[b.c].d }}'))).toStrictEqual([['a', ['b', 'c'], 'd'], ['b', 'c']])
+    })
+  })
 })

test/integration/util/error.spec.ts

@@ -264,7 +264,7 @@ describe('error', function () {
     it('should throw ParseError when tag value not specified', async function () {
       await expect(engine.parseAndRender('{% if %}{% endif %}')).rejects.toMatchObject({
         name: 'TokenizationError',
-        message: 'invalid value expression: "", line:1, col:1'
+        message: 'invalid value expression: "", line:1, col:6'
       })
     })
     it('should throw ParseError when tag parse throws', async function () {