feat: precise line/col for tokenization Error, #613

src/tokens/delimited-token.ts

@@ -1,14 +1,14 @@
 import { Token } from './token'
 import { TokenKind } from '../parser'
-import { last } from '../util'
+import { TYPES, BLANK } from '../util'
 
 export abstract class DelimitedToken extends Token {
   public trimLeft = false
   public trimRight = false
-  public content: string
+  public contentRange: [number, number]
   public constructor (
     kind: TokenKind,
-    content: string,
+    [contentBegin, contentEnd]: [number, number],
     input: string,
     begin: number,
     end: number,
@@ -17,16 +17,19 @@ export abstract class DelimitedToken extends Token {
     file?: string
   ) {
     super(kind, input, begin, end, file)
-    this.content = this.getText()
-    const tl = content[0] === '-'
-    const tr = last(content) === '-'
-    this.content = content
-      .slice(
-        tl ? 1 : 0,
-        tr ? -1 : content.length
-      )
-      .trim()
+    const tl = input[contentBegin] === '-'
+    const tr = input[contentEnd - 1] === '-'
+
+    let l = tl ? contentBegin + 1 : contentBegin
+    let r = tr ? contentEnd - 1 : contentEnd
+    while (l < r && (TYPES[input.charCodeAt(l)] & BLANK)) l++
+    while (r > l && (TYPES[input.charCodeAt(r - 1)] & BLANK)) r--
+
+    this.contentRange = [l, r]
     this.trimLeft = tl || trimLeft
     this.trimRight = tr || trimRight
   }
+  get content () {
+    return this.input.slice(this.contentRange[0], this.contentRange[1])
+  }
 }

src/tokens/filtered-value-token.ts

@@ -0,0 +1,22 @@
+import { Token } from './token'
+import { FilterToken } from './filter-token'
+import { TokenKind } from '../parser'
+import { Expression } from '../render'
+
+/**
+ * value expression with optional filters
+ * e.g.
+ * {% assign foo="bar" | append: "coo" %}
+ */
+export class FilteredValueToken extends Token {
+  constructor (
+    public initial: Expression,
+    public filters: FilterToken[],
+    public input: string,
+    public begin: number,
+    public end: number,
+    public file?: string
+  ) {
+    super(TokenKind.FilteredValue, input, begin, end, file)
+  }
+}

src/tokens/index.ts

@@ -15,3 +15,4 @@ export * from './range-token'
 export * from './value-token'
 export * from './liquid-tag-token'
 export * from './delimited-token'
+export * from './filtered-value-token'

src/tokens/liquid-tag-token.ts

@@ -1,11 +1,14 @@
 import { DelimitedToken } from './delimited-token'
-import { TokenizationError } from '../util'
 import { NormalizedFullOptions } from '../liquid-options'
 import { Tokenizer, TokenKind } from '../parser'
 
+/**
+ * LiquidTagToken is different from TagToken by not having delimiters `{%` or `%}`
+ */
 export class LiquidTagToken extends DelimitedToken {
   public name: string
   public args: string
+  public tokenizer: Tokenizer
   public constructor (
     input: string,
     begin: number,
@@ -13,20 +16,13 @@ export class LiquidTagToken extends DelimitedToken {
     options: NormalizedFullOptions,
     file?: string
   ) {
-    const value = input.slice(begin, end)
-    super(TokenKind.Tag, value, input, begin, end, false, false, file)
+    super(TokenKind.Tag, [begin, end], input, begin, end, false, false, file)
 
-    if (!/\S/.test(value)) {
-      // A line that contains only whitespace.
-      this.name = ''
-      this.args = ''
-    } else {
-      const tokenizer = new Tokenizer(this.content, options.operators)
-      this.name = tokenizer.readTagName()
-      if (!this.name) throw new TokenizationError(`illegal liquid tag syntax`, this)
+    this.tokenizer = new Tokenizer(input, options.operators, file, this.contentRange)
+    this.name = this.tokenizer.readTagName()
+    this.tokenizer.assert(this.name, 'illegal liquid tag syntax')
 
-      tokenizer.skipBlank()
-      this.args = tokenizer.remaining()
-    }
+    this.tokenizer.skipBlank()
+    this.args = this.tokenizer.remaining()
   }
 }

src/tokens/output-token.ts

@@ -11,7 +11,7 @@ export class OutputToken extends DelimitedToken {
     file?: string
   ) {
     const { trimOutputLeft, trimOutputRight, outputDelimiterLeft, outputDelimiterRight } = options
-    const value = input.slice(begin + outputDelimiterLeft.length, end - outputDelimiterRight.length)
-    super(TokenKind.Output, value, input, begin, end, trimOutputLeft, trimOutputRight, file)
+    const valueRange: [number, number] = [begin + outputDelimiterLeft.length, end - outputDelimiterRight.length]
+    super(TokenKind.Output, valueRange, input, begin, end, trimOutputLeft, trimOutputRight, file)
   }
 }

src/tokens/tag-token.ts

@@ -1,11 +1,10 @@
 import { DelimitedToken } from './delimited-token'
-import { TokenizationError } from '../util/error'
 import { Tokenizer, TokenKind } from '../parser'
 import type { NormalizedFullOptions } from '../liquid-options'
 
 export class TagToken extends DelimitedToken {
   public name: string
-  public args: string
+  public tokenizer: Tokenizer
   public constructor (
     input: string,
     begin: number,
@@ -14,14 +13,15 @@ export class TagToken extends DelimitedToken {
     file?: string
   ) {
     const { trimTagLeft, trimTagRight, tagDelimiterLeft, tagDelimiterRight } = options
-    const value = input.slice(begin + tagDelimiterLeft.length, end - tagDelimiterRight.length)
-    super(TokenKind.Tag, value, input, begin, end, trimTagLeft, trimTagRight, file)
+    const [valueBegin, valueEnd] = [begin + tagDelimiterLeft.length, end - tagDelimiterRight.length]
+    super(TokenKind.Tag, [valueBegin, valueEnd], input, begin, end, trimTagLeft, trimTagRight, file)
 
-    const tokenizer = new Tokenizer(this.content, options.operators)
-    this.name = tokenizer.readTagName()
-    if (!this.name) throw new TokenizationError(`illegal tag syntax`, this)
-
-    tokenizer.skipBlank()
-    this.args = tokenizer.remaining()
+    this.tokenizer = new Tokenizer(input, options.operators, file, this.contentRange)
+    this.name = this.tokenizer.readTagName()
+    this.tokenizer.assert(this.name, `illegal tag syntax, tag name expected`)
+    this.tokenizer.skipBlank()
+  }
+  get args (): string {
+    return this.tokenizer.input.slice(this.tokenizer.p, this.contentRange[1])
   }
 }