fix: expression and string literal parser, #186

test/unit/parser/expression-tokenizer.ts

@@ -0,0 +1,43 @@
+import { tokenize } from '../../../src/parser/expression-tokenizer'
+import { expect } from 'chai'
+
+describe('expression tokenizer', () => {
+  describe('spaces', () => {
+    it('should tokenize a + b', () => {
+      expect([...tokenize('a + b')]).to.deep.equal(['a', '+', 'b'])
+    })
+    it('should tokenize a==1', () => {
+      expect([...tokenize('a==1')]).to.deep.equal(['a', '==', '1'])
+    })
+  })
+
+  describe('range', () => {
+    it('should tokenize (1..3) contains 3', () => {
+      expect([...tokenize('(1..3)')]).to.deep.equal(['(1..3)'])
+    })
+  })
+
+  describe('bracket', () => {
+    it('should tokenize a[b] = c', () => {
+      expect([...tokenize('a[b] = c')]).to.deep.equal(['a[b]', '=', 'c'])
+    })
+    it('should tokenize c[a["b"]] < c', () => {
+      expect([...tokenize('c[a["b"]] < c')]).to.deep.equal(['c[a["b"]]', '<', 'c'])
+    })
+    it('should tokenize "][" == var', () => {
+      expect([...tokenize('"][" == var')]).to.deep.equal(['"]["', '==', 'var'])
+    })
+  })
+
+  describe('quotes', () => {
+    it('should tokenize " " == var', () => {
+      expect([...tokenize('" " == var')]).to.deep.equal(['" "', '==', 'var'])
+    })
+    it('should tokenize "\\\'" == var', () => {
+      expect([...tokenize('"\\\'" == var')]).to.deep.equal(['"\\\'"', '==', 'var'])
+    })
+    it('should tokenize "\\"" == var', () => {
+      expect([...tokenize('"\\"" == var')]).to.deep.equal(['"\\""', '==', 'var'])
+    })
+  })
+})

test/unit/parser/literal.ts

@@ -1,8 +1,8 @@
 import { expect } from 'chai'
-import { parseLiteral } from '../../../src/parser/literal'
+import { parseLiteral, parseStringLiteral } from '../../../src/parser/literal'
 import { NullDrop } from '../../../src/drop/null-drop'
 
-describe('parseLiteral', function () {
+describe('parseLiteral()', function () {
   it('should eval boolean literal', async function () {
     expect(parseLiteral('true')).to.equal(true)
     expect(parseLiteral('TrUE')).to.equal(undefined)
@@ -25,3 +25,36 @@ describe('parseLiteral', function () {
     expect(parseLiteral('null')).to.be.instanceOf(NullDrop)
   })
 })
+
+describe('parseStringLiteral()', function () {
+  it('should parse octal escape', () => {
+    expect(parseStringLiteral(String.raw`"\1010"`)).to.equal('A0')
+    expect(parseStringLiteral(String.raw`"\12"`)).to.equal('\n')
+    expect(parseStringLiteral(String.raw`"\01"`)).to.equal('\u0001')
+    expect(parseStringLiteral(String.raw`"\0"`)).to.equal('\0')
+  })
+  it('should skip invalid octal escape', () => {
+    expect(parseStringLiteral(String.raw`"\9"`)).to.equal('9')
+  })
+  it('should parse \n, \t, \r', () => {
+    expect(parseStringLiteral(String.raw`"fo\no"`)).to.equal('fo\no')
+    expect(parseStringLiteral(String.raw`'fo\to'`)).to.equal('fo\to')
+    expect(parseStringLiteral(String.raw`'fo\ro'`)).to.equal('fo\ro')
+  })
+  it('should parse unicode(hex) escape', () => {
+    expect(parseStringLiteral('"\\u003C"')).to.equal('<')
+    expect(parseStringLiteral('"\\u003cZ"')).to.equal('<Z')
+    expect(parseStringLiteral('"\\u41"')).to.equal('A')
+  })
+  it('should skip invalid unicode(hex) escape', () => {
+    expect(parseStringLiteral('"\\u41Z"')).to.equal('AZ')
+    expect(parseStringLiteral('"\\uZ"')).to.equal('\0Z')
+  })
+  it('should parse quote escape', () => {
+    expect(parseStringLiteral(String.raw`"fo\'o"`)).to.equal("fo'o")
+    expect(parseStringLiteral(String.raw`'fo\"o'`)).to.equal('fo"o')
+  })
+  it('should parse slash escape', () => {
+    expect(parseStringLiteral(String.raw`'fo\\o'`)).to.equal('fo\\o')
+  })
+})