fix: expression and string literal parser, #186

src/parser/expression-tokenizer.ts

@@ -0,0 +1,50 @@
+const rBlank = /\s/
+const rPunctuation = /[<>=!]/
+
+enum ParseState {
+  INIT = 1,
+  SINGLE_QUOTE = 2,
+  DOUBLE_QUOTE = 4,
+  QUOTE = 6,
+  BRACKET = 8
+}
+
+export function * tokenize (expr: string): IterableIterator<string> {
+  const N = expr.length
+  const stack = [ParseState.INIT]
+  let str = ''
+  let lastIsPunc = false
+
+  for (let i = 0; i < N; i++) {
+    const c = expr[i]
+    const top = stack[stack.length - 1]
+    const isPunc = rPunctuation.test(c)
+    if (c === '\\') {
+      str += expr.substr(i++, 2)
+    } else if (top === ParseState.SINGLE_QUOTE && c === "'") {
+      str += c
+      stack.pop()
+    } else if (top === ParseState.DOUBLE_QUOTE && c === '"') {
+      str += c
+      stack.pop()
+    } else if (ParseState.QUOTE & top) {
+      str += c
+    } else if (top === ParseState.BRACKET && c === ']') {
+      str += c
+      stack.pop()
+    } else if (top === ParseState.INIT && rBlank.exec(c)) {
+      if (str) yield str
+      str = ''
+    } else if (top === ParseState.INIT && isPunc !== lastIsPunc) {
+      if (str) yield str
+      str = c
+    } else {
+      if (c === '"') stack.push(ParseState.DOUBLE_QUOTE)
+      else if (c === "'") stack.push(ParseState.SINGLE_QUOTE)
+      else if (c === '[') stack.push(ParseState.BRACKET)
+      str += c
+    }
+    lastIsPunc = isPunc
+  }
+  if (str) yield str
+}

src/parser/literal.ts

@@ -5,6 +5,24 @@ import { BlankDrop } from '../drop/blank-drop'
 
 type literal = true | false | NullDrop | EmptyDrop | BlankDrop | number | string
 
+const rHex = /[\da-fA-F]/
+const rOct = /[0-7]/
+const escapeChar = {
+  b: '\b',
+  f: '\f',
+  n: '\n',
+  r: '\r',
+  t: '\t',
+  v: '\x0B'
+}
+
+function hexVal (c: string) {
+  const code = c.charCodeAt(0)
+  if (code >= 97) return code - 87
+  if (code >= 65) return code - 55
+  return code - 48
+}
+
 export function parseLiteral (str: string): literal | undefined {
   str = str.trim()
 
@@ -14,5 +32,37 @@ export function parseLiteral (str: string): literal | undefined {
   if (str === 'empty') return new EmptyDrop()
   if (str === 'blank') return new BlankDrop()
   if (!isNaN(Number(str))) return Number(str)
-  if ((str[0] === '"' || str[0] === "'") && str[0] === last(str)) return str.slice(1, -1)
+  if ((str[0] === '"' || str[0] === "'") && str[0] === last(str)) return parseStringLiteral(str)
+}
+
+export function parseStringLiteral (str: string): string {
+  let ret = ''
+  for (let i = 1; i < str.length - 1; i++) {
+    if (str[i] !== '\\') {
+      ret += str[i]
+      continue
+    }
+    if (escapeChar[str[i + 1]] !== undefined) {
+      ret += escapeChar[str[++i]]
+    } else if (str[i + 1] === 'u') {
+      let val = 0
+      let j = i + 2
+      while (j <= i + 5 && rHex.test(str[j])) {
+        val = val * 16 + hexVal(str[j++])
+      }
+      i = j - 1
+      ret += String.fromCharCode(val)
+    } else if (!rOct.test(str[i + 1])) {
+      ret += str[++i]
+    } else {
+      let j = i + 1
+      let val = 0
+      while (j <= i + 3 && rOct.test(str[j])) {
+        val = val * 8 + hexVal(str[j++])
+      }
+      i = j - 1
+      ret += String.fromCharCode(val)
+    }
+  }
+  return ret
 }