feat: throw an Error if delimiter not matched

src/parser/token.ts

@@ -1,11 +1,13 @@
 export default class Token {
   type: string
   line: number
+  col: number
   raw: string
   input: string
   file: string
   value: string
-  constructor (raw, pos, input, file, line) {
+  constructor (raw, col, input, file, line) {
+    this.col = col
     this.line = line
     this.raw = raw
     this.input = input

src/parser/tokenizer.ts

@@ -1,7 +1,9 @@
 import whiteSpaceCtrl from './whitespace-ctrl'
 import HTMLToken from './html-token'
 import TagToken from './tag-token'
+import Token from './token'
 import OutputToken from './output-token'
+import { TokenizationError } from 'src/util/error'
 import { LiquidOptions, defaultOptions } from 'src/liquid-options'
 
 enum ParseState { HTML, OUTPUT, TAG }
@@ -14,43 +16,59 @@ export default class Tokenizer {
   tokenize (input: string, file?: string) {
     const tokens = []
     let p = 0
-    let line = 1
+    let curLine = 1
     let state = ParseState.HTML
     let buffer = ''
-    let bufferBegin = 0
+    let lineBegin = 0
+    let line = 1
+    let col = 1
 
     while (p < input.length) {
-      if (input[p] === '\n') line++
+      if (input[p] === '\n') {
+        curLine++
+        lineBegin = p + 1
+      }
       const bin = input.substr(p, 2)
       if (state === ParseState.HTML) {
         if (bin === '{{' || bin === '{%') {
-          if (buffer) tokens.push(new HTMLToken(buffer, bufferBegin, input, file, line))
+          if (buffer) tokens.push(new HTMLToken(buffer, col, input, file, line))
           buffer = bin
-          bufferBegin = p
+          line = curLine
+          col = p - lineBegin + 1
           p += 2
           state = bin === '{{' ? ParseState.OUTPUT : ParseState.TAG
           continue
         }
       } else if (state === ParseState.OUTPUT && bin === '}}') {
         buffer += '}}'
-        tokens.push(new OutputToken(buffer, bufferBegin, input, file, line))
+        tokens.push(new OutputToken(buffer, col, input, file, line))
         p += 2
         buffer = ''
-        bufferBegin = p
+        line = curLine
+        col = p - lineBegin + 1
         state = ParseState.HTML
         continue
       } else if (bin === '%}') {
         buffer += '%}'
-        tokens.push(new TagToken(buffer, bufferBegin, input, file, line))
+        tokens.push(new TagToken(buffer, col, input, file, line))
         p += 2
         buffer = ''
-        bufferBegin = p
+        line = curLine
+        col = p - lineBegin + 1
         state = ParseState.HTML
         continue
       }
       buffer += input[p++]
     }
-    if (buffer) tokens.push(new HTMLToken(buffer, bufferBegin, input, file, line))
+    if (state !== ParseState.HTML) {
+      const t = state === ParseState.OUTPUT ? 'output' : 'tag'
+      const str = buffer.length > 16 ? buffer.slice(0, 13) + '...' : buffer
+      throw new TokenizationError(
+        new Error(`${t} "${str}" not closed`),
+        new Token(buffer, col, input, file, line)
+      )
+    }
+    if (buffer) tokens.push(new HTMLToken(buffer, col, input, file, line))
 
     whiteSpaceCtrl(tokens, this.options)
     return tokens

src/util/error.ts

@@ -11,14 +11,12 @@ abstract class LiquidError {
   name: string
   message: string
   stack: string
-  private line: string
   private file: string
   private input: string
   private token: Token
   private originalError: Error
   constructor (err, token) {
     this.input = token.input
-    this.line = token.line
     this.file = token.file
     this.originalError = err
     this.token = token
@@ -28,7 +26,7 @@ abstract class LiquidError {
 
     captureStack.call(obj)
     const err = this.originalError
-    const context = mkContext(this.input, this.line)
+    const context = mkContext(this.input, this.token.line)
     this.message = mkMessage(err.message, this.token)
     this.stack = this.message + '\n' + context +
       '\n' + (this.stack || this.message) +
@@ -110,7 +108,7 @@ function mkMessage (msg, token) {
     msg += ', file:' + token.file
   }
   if (token.line) {
-    msg += ', line:' + token.line
+    msg += `, line:${token.line}, col:${token.col}`
   }
   return msg
 }

test/unit/parser/tokenizer.ts

@@ -6,7 +6,7 @@ import HTMLToken from 'src/parser/html-token'
 
 describe('tokenizer', function () {
   const tokenizer = new Tokenizer()
-  describe('parse', function () {
+  describe('#tokenize()', function () {
     it('should handle plain HTML', function () {
       const html = '<html><body><p>Lorem Ipsum</p></body></html>'
       const tokens = tokenizer.tokenize(html)
@@ -66,5 +66,15 @@ describe('tokenizer', function () {
       expect(tokens[0]).instanceOf(OutputToken)
       expect(tokens[0].raw).to.equal('{{foo\n|date:\n"%Y-%m-%d"\n}}')
     })
+    it('should throw if tag not closed', function () {
+      expect(() => {
+        tokenizer.tokenize('{% assign foo = bar {{foo}}')
+      }).to.throw(/tag "{% assign foo..." not closed/)
+    })
+    it('should throw if output not closed', function () {
+      expect(() => {
+        tokenizer.tokenize('{{name}')
+      }).to.throw(/output "{{name}" not closed/)
+    })
   })
 })

test/unit/util/error.ts

@@ -28,7 +28,7 @@ describe('error', function () {
         'TokenizationError'
       ]
       const err = await expect(engine.parseAndRender(html.join('\n'))).be.rejected
-      expect(err.message).to.equal('illegal tag syntax, line:3')
+      expect(err.message).to.equal('illegal tag syntax, line:3, col:2')
       expect(err.stack).to.contain(message.join('\n'))
       expect(err.name).to.equal('TokenizationError')
     })
@@ -37,10 +37,10 @@ describe('error', function () {
       const err = await expect(engine.parseAndRender(html)).be.rejected
       expect(err.input).to.equal(html)
     })
-    it('should contain line number in err.line', async function () {
+    it('should contain line number in err.token.line', async function () {
       const err = await expect(engine.parseAndRender('1\n2\n{% . a %}\n4')).be.rejected
       expect(err.name).to.equal('TokenizationError')
-      expect(err.line).to.equal(3)
+      expect(err.token.line).to.equal(3)
     })
     it('should contain stack in err.stack', async function () {
       const err = await expect(engine.parseAndRender('{% . a %}')).be.rejected
@@ -68,6 +68,12 @@ describe('error', function () {
       expect(err.name).to.equal('TokenizationError')
       expect(err.file).to.equal(path.resolve('/foo.html'))
     })
+    it('should throw error with line and pos if tag unmatched', async function () {
+      const err = await expect(engine.parseAndRender('1\n2\nfoo{% assign a = 4 }\n4')).be.rejected
+      expect(err.name).to.equal('TokenizationError')
+      expect(err.token.line).to.equal(3)
+      expect(err.token.col).to.equal(4)
+    })
   })
 
   describe('RenderError', function () {
@@ -128,7 +134,7 @@ describe('error', function () {
         'RenderError'
       ]
       const err = await expect(engine.parseAndRender(html.join('\n'))).be.rejected
-      expect(err.message).to.equal('intended render error, line:4')
+      expect(err.message).to.equal('intended render error, line:4, col:2')
       expect(err.stack).to.contain(message.join('\n'))
       expect(err.name).to.equal('RenderError')
     })
@@ -157,7 +163,7 @@ describe('error', function () {
       const err = await expect(engine.parseAndRender(html)).be.rejected
       console.log(err.message)
       console.log(err.stack)
-      expect(err.message).to.equal(`intended render error, file:${path.resolve('/throwing-tag.html')}, line:4`)
+      expect(err.message).to.equal(`intended render error, file:${path.resolve('/throwing-tag.html')}, line:4, col:2`)
       expect(err.stack).to.contain(message.join('\n'))
       expect(err.name).to.equal('RenderError')
     })
@@ -177,7 +183,7 @@ describe('error', function () {
         'RenderError'
       ]
       const err = await expect(engine.parseAndRender(html)).be.rejected
-      expect(err.message).to.equal(`intended render error, file:${path.resolve('/throwing-tag.html')}, line:4`)
+      expect(err.message).to.equal(`intended render error, file:${path.resolve('/throwing-tag.html')}, line:4, col:2`)
       expect(err.stack).to.contain(message.join('\n'))
       expect(err.name).to.equal('RenderError')
     })
@@ -187,10 +193,10 @@ describe('error', function () {
       expect(err.input).to.equal(html)
       expect(err.name).to.equal('RenderError')
     })
-    it('should contain line number in err.line', async function () {
+    it('should contain line number in err.token.line', async function () {
       const src = '1\n2\n{{1|throwingFilter}}\n4'
       const err = await expect(engine.parseAndRender(src)).be.rejected
-      expect(err.line).to.equal(3)
+      expect(err.token.line).to.equal(3)
       expect(err.name).to.equal('RenderError')
     })
     it('should contain stack in err.stack', async function () {
@@ -260,7 +266,7 @@ describe('error', function () {
         'ParseError: tag a not found'
       ]
       const err = await expect(engine.parseAndRender(html.join('\n'))).be.rejected
-      expect(err.message).to.equal('tag a not found, line:4')
+      expect(err.message).to.equal('tag a not found, line:4, col:2')
       expect(err.stack).to.contain(message.join('\n'))
       expect(err.name).to.equal('ParseError')
     })
@@ -275,14 +281,14 @@ describe('error', function () {
         'ParseError: tag a not found'
       ]
       const err = await expect(engine.parseAndRender(html.join('\n'))).be.rejected
-      expect(err.message).to.equal('tag a not found, line:2')
+      expect(err.message).to.equal('tag a not found, line:2, col:2')
       expect(err.stack).to.contain(message.join('\n'))
     })
 
-    it('should contain line number in err.line', async function () {
+    it('should contain line number in err.token.line', async function () {
       const html = '<html>\n<head>\n\n{% raw %}\n\n'
       const err = await expect(engine.parseAndRender(html)).be.rejected
-      expect(err.line).to.equal(4)
+      expect(err.token.line).to.equal(4)
     })
 
     it('should contain stack in err.stack', async function () {