feat: precise line/col for tokenization Error, #613

src/template/output.spec.ts

@@ -7,31 +7,32 @@ import { defaultOptions } from '../liquid-options'
 describe('Output', function () {
   const emitter: any = { write: (html: string) => (emitter.html += html), html: '' }
   const liquid = { options: {} } as any
+  const token = { content: 'obj', input: 'obj' } as OutputToken
   beforeEach(() => { emitter.html = '' })
 
   it('should stringify objects', async function () {
     const scope = new Context({
-      foo: { obj: { arr: ['a', 2] } }
+      obj: { foo: { arr: ['a', 2] } }
     })
-    const output = new Output({ content: 'foo' } as OutputToken, liquid)
+    const output = new Output(token, liquid)
     await toPromise(output.render(scope, emitter))
     return expect(emitter.html).toBe('[object Object]')
   })
   it('should skip function property', async function () {
     const scope = new Context({ obj: { foo: 'foo', bar: (x: any) => x } })
-    const output = new Output({ content: 'obj' } as OutputToken, liquid)
+    const output = new Output(token, liquid)
     await toPromise(output.render(scope, emitter))
     return expect(emitter.html).toBe('[object Object]')
   })
   it('should respect to .toString()', async () => {
     const scope = new Context({ obj: { toString: () => 'FOO' } })
-    const output = new Output({ content: 'obj' } as OutputToken, liquid)
+    const output = new Output(token, liquid)
     await toPromise(output.render(scope, emitter))
     return expect(emitter.html).toBe('FOO')
   })
   it('should respect to .toString()', async () => {
     const scope = new Context({ obj: { toString: () => 'FOO' } })
-    const output = new Output({ content: 'obj' } as OutputToken, liquid)
+    const output = new Output(token, liquid)
     await toPromise(output.render(scope, emitter))
     return expect(emitter.html).toBe('FOO')
   })
@@ -47,6 +48,7 @@ describe('Output', function () {
       html: '',
       keepOutputType: true
     }
+    const token = { content: 'foo', input: 'foo' } as OutputToken
 
     beforeEach(() => { emitter.html = '' })
 
@@ -54,7 +56,7 @@ describe('Output', function () {
       const scope = new Context({
         foo: 42
       }, { ...defaultOptions, keepOutputType: true })
-      const output = new Output({ content: 'foo' } as OutputToken, liquid)
+      const output = new Output(token, liquid)
       await toPromise(output.render(scope, emitter))
       return expect(emitter.html).toBe(42)
     })
@@ -62,7 +64,7 @@ describe('Output', function () {
       const scope = new Context({
         foo: true
       }, { ...defaultOptions, keepOutputType: true })
-      const output = new Output({ content: 'foo' } as OutputToken, liquid)
+      const output = new Output(token, liquid)
       await toPromise(output.render(scope, emitter))
       return expect(emitter.html).toBe(true)
     })
@@ -70,7 +72,7 @@ describe('Output', function () {
       const scope = new Context({
         foo: 'test'
       }, { ...defaultOptions, keepOutputType: true })
-      const output = new Output({ content: 'foo' } as OutputToken, liquid)
+      const output = new Output(token, liquid)
       await toPromise(output.render(scope, emitter))
       return expect(emitter.html).toBe('test')
     })
@@ -78,7 +80,7 @@ describe('Output', function () {
       const scope = new Context({
         foo: { a: { b: 42 } }
       }, { ...defaultOptions, keepOutputType: true })
-      const output = new Output({ content: 'foo' } as OutputToken, liquid)
+      const output = new Output(token, liquid)
       await toPromise(output.render(scope, emitter))
       return expect(emitter.html).toEqual({ a: { b: 42 } })
     })

src/template/output.ts

@@ -3,6 +3,7 @@ import { Template, TemplateImpl } from '../template'
 import { Context } from '../context/context'
 import { Emitter } from '../emitters/emitter'
 import { OutputToken } from '../tokens/output-token'
+import { Tokenizer } from '../parser'
 import { Liquid } from '../liquid'
 import { Filter } from './filter'
 
@@ -10,7 +11,8 @@ export class Output extends TemplateImpl<OutputToken> implements Template {
   value: Value
   public constructor (token: OutputToken, liquid: Liquid) {
     super(token)
-    this.value = new Value(token.content, liquid)
+    const tokenizer = new Tokenizer(token.input, liquid.options.operators, token.file, token.contentRange)
+    this.value = new Value(tokenizer.readFilteredValue(), liquid)
     const filters = this.value.filters
     const outputEscape = liquid.options.outputEscape
     if (!filters[filters.length - 1]?.raw && outputEscape) {

src/template/tag.ts

@@ -1,5 +1,6 @@
 import { TemplateImpl } from './template-impl'
 import type { Emitter } from '../emitters/emitter'
+import type { Tokenizer } from '../parser'
 import type { Context } from '../context/context'
 import type { TopLevelToken, TagToken } from '../tokens'
 import type { Template } from './template'
@@ -10,11 +11,13 @@ export type TagRenderReturn = Generator<unknown, unknown, unknown> | Promise<unk
 export abstract class Tag extends TemplateImpl<TagToken> implements Template {
   public name: string
   public liquid: Liquid
+  protected tokenizer: Tokenizer
 
   public constructor (token: TagToken, remainTokens: TopLevelToken[], liquid: Liquid) {
     super(token)
     this.name = token.name
     this.liquid = liquid
+    this.tokenizer = token.tokenizer
   }
   public abstract render (ctx: Context, emitter: Emitter): TagRenderReturn;
 }

src/template/value.ts

@@ -2,6 +2,7 @@ import { Filter } from './filter'
 import { Expression } from '../render'
 import { Tokenizer } from '../parser'
 import { assert } from '../util'
+import type { FilteredValueToken } from '../tokens'
 import type { Liquid } from '../liquid'
 import type { Context } from '../context'
 
@@ -12,10 +13,12 @@ export class Value {
   /**
    * @param str the value to be valuated, eg.: "foobar" | truncate: 3
    */
-  public constructor (str: string, liquid: Liquid) {
-    const tokenizer = new Tokenizer(str, liquid.options.operators)
-    this.initial = tokenizer.readExpression()
-    this.filters = tokenizer.readFilters().map(({ name, args }) => new Filter(name, this.getFilter(liquid, name), args, liquid))
+  public constructor (input: string | FilteredValueToken, liquid: Liquid) {
+    const token: FilteredValueToken = typeof input === 'string'
+      ? new Tokenizer(input, liquid.options.operators).readFilteredValue()
+      : input
+    this.initial = token.initial
+    this.filters = token.filters.map(({ name, args }) => new Filter(name, this.getFilter(liquid, name), args, liquid))
   }
   public * value (ctx: Context, lenient?: boolean): Generator<unknown, unknown, unknown> {
     lenient = lenient || (ctx.opts.lenientIf && this.filters.length > 0 && this.filters[0].name === 'default')