feat: throw an Error if delimiter not matched

src/parser/token.ts

@@ -1,11 +1,13 @@
 export default class Token {
   type: string
   line: number
+  col: number
   raw: string
   input: string
   file: string
   value: string
-  constructor (raw, pos, input, file, line) {
+  constructor (raw, col, input, file, line) {
+    this.col = col
     this.line = line
     this.raw = raw
     this.input = input

src/parser/tokenizer.ts

@@ -1,7 +1,9 @@
 import whiteSpaceCtrl from './whitespace-ctrl'
 import HTMLToken from './html-token'
 import TagToken from './tag-token'
+import Token from './token'
 import OutputToken from './output-token'
+import { TokenizationError } from 'src/util/error'
 import { LiquidOptions, defaultOptions } from 'src/liquid-options'
 
 enum ParseState { HTML, OUTPUT, TAG }
@@ -14,43 +16,59 @@ export default class Tokenizer {
   tokenize (input: string, file?: string) {
     const tokens = []
     let p = 0
-    let line = 1
+    let curLine = 1
     let state = ParseState.HTML
     let buffer = ''
-    let bufferBegin = 0
+    let lineBegin = 0
+    let line = 1
+    let col = 1
 
     while (p < input.length) {
-      if (input[p] === '\n') line++
+      if (input[p] === '\n') {
+        curLine++
+        lineBegin = p + 1
+      }
       const bin = input.substr(p, 2)
       if (state === ParseState.HTML) {
         if (bin === '{{' || bin === '{%') {
-          if (buffer) tokens.push(new HTMLToken(buffer, bufferBegin, input, file, line))
+          if (buffer) tokens.push(new HTMLToken(buffer, col, input, file, line))
           buffer = bin
-          bufferBegin = p
+          line = curLine
+          col = p - lineBegin + 1
           p += 2
           state = bin === '{{' ? ParseState.OUTPUT : ParseState.TAG
           continue
         }
       } else if (state === ParseState.OUTPUT && bin === '}}') {
         buffer += '}}'
-        tokens.push(new OutputToken(buffer, bufferBegin, input, file, line))
+        tokens.push(new OutputToken(buffer, col, input, file, line))
         p += 2
         buffer = ''
-        bufferBegin = p
+        line = curLine
+        col = p - lineBegin + 1
         state = ParseState.HTML
         continue
       } else if (bin === '%}') {
         buffer += '%}'
-        tokens.push(new TagToken(buffer, bufferBegin, input, file, line))
+        tokens.push(new TagToken(buffer, col, input, file, line))
         p += 2
         buffer = ''
-        bufferBegin = p
+        line = curLine
+        col = p - lineBegin + 1
         state = ParseState.HTML
         continue
       }
       buffer += input[p++]
     }
-    if (buffer) tokens.push(new HTMLToken(buffer, bufferBegin, input, file, line))
+    if (state !== ParseState.HTML) {
+      const t = state === ParseState.OUTPUT ? 'output' : 'tag'
+      const str = buffer.length > 16 ? buffer.slice(0, 13) + '...' : buffer
+      throw new TokenizationError(
+        new Error(`${t} "${str}" not closed`),
+        new Token(buffer, col, input, file, line)
+      )
+    }
+    if (buffer) tokens.push(new HTMLToken(buffer, col, input, file, line))
 
     whiteSpaceCtrl(tokens, this.options)
     return tokens