perf: introduce AST to avoid reparse

src/parser/delimited-token.ts

@@ -1,27 +0,0 @@
-import { Token } from './token'
-import { last } from '../util/underscore'
-
-export class DelimitedToken extends Token {
-  public constructor (
-    raw: string,
-    content: string,
-    input: string,
-    line: number,
-    pos: number,
-    trimLeft: boolean,
-    trimRight: boolean,
-    file?: string
-  ) {
-    super(raw, input, line, pos, file)
-    const tl = content[0] === '-'
-    const tr = last(content) === '-'
-    this.content = content
-      .slice(
-        tl ? 1 : 0,
-        tr ? -1 : content.length
-      )
-      .trim()
-    this.trimLeft = tl || trimLeft
-    this.trimRight = tr || trimRight
-  }
-}

src/parser/filter-arg.ts

@@ -1,9 +1,10 @@
 import { isArray } from '../util/underscore'
+import { ValueToken } from '../tokens/value-token'
 
-type KeyValuePair = [string?, string?]
+type KeyValuePair = [string?, ValueToken?]
 
-export type FilterArg = string|KeyValuePair
+export type FilterArg = ValueToken | KeyValuePair
 
-export function isKeyValuePair (arr: FilterArg): arr is KeyValuePair {
+export function isKeyValuePair (arr: FilterArg): arr is KeyValuePair { // TODO check
   return isArray(arr)
 }

src/parser/filter-token.ts

@@ -1,16 +0,0 @@
-import { Token } from './token'
-import { FilterArg } from './filter-arg'
-
-export class FilterToken extends Token {
-  public constructor (
-    public name: string,
-    public args: FilterArg[],
-    raw: string,
-    input: string,
-    line: number,
-    col: number,
-    file?: string
-  ) {
-    super(raw, input, line, col, file)
-  }
-}

src/parser/html-token.ts

@@ -1,11 +0,0 @@
-import { Token } from './token'
-
-export class HTMLToken extends Token {
-  public constructor (str: string, input: string, line: number, col: number, file?: string) {
-    super(str, input, line, col, file)
-    this.content = str
-  }
-  public static is (token: Token): token is HTMLToken {
-    return token instanceof HTMLToken
-  }
-}

src/parser/lexical.ts

@@ -1,29 +0,0 @@
-// quote related
-const singleQuoted = /'[^']*'/
-const doubleQuoted = /"[^"]*"/
-export const quoted = new RegExp(`${singleQuoted.source}|${doubleQuoted.source}`)
-export const quoteBalanced = new RegExp(`(?:${quoted.source}|[^'"])*`)
-
-// basic types
-export const number = /[+-]?(?:\d+\.?\d*|\.?\d+)/
-export const bool = /true|false/
-
-// property access
-export const identifier = /[\w-]+[?]?/
-export const subscript = new RegExp(`\\[(?:${quoted.source}|[\\w-\\.]+)\\]`)
-export const literal = new RegExp(`(?:${quoted.source}|${bool.source}|${number.source})`)
-export const variable = new RegExp(`${identifier.source}(?:\\.${identifier.source}|${subscript.source})*`)
-
-// range related
-export const rangeLimit = new RegExp(`(?:${variable.source}|${number.source})`)
-export const range = new RegExp(`\\(${rangeLimit.source}\\.\\.${rangeLimit.source}\\)`)
-export const rangeCapture = new RegExp(`\\((${rangeLimit.source})\\.\\.(${rangeLimit.source})\\)`)
-
-export const value = new RegExp(`(?:${variable.source}|${literal.source}|${range.source})`)
-
-// full match
-export const tagLine = new RegExp(`^\\s*(${identifier.source})\\s*([\\s\\S]*?)\\s*$`)
-export const numberLine = new RegExp(`^${number.source}$`)
-export const boolLine = new RegExp(`^${bool.source}$`, 'i')
-export const quotedLine = new RegExp(`^${quoted.source}$`)
-export const rangeLine = new RegExp(`^${rangeCapture.source}$`)

src/parser/match-operator.ts

@@ -0,0 +1,24 @@
+import { VARIABLE } from '../util/character'
+
+const trie = {
+  a: { n: { d: { end: true, needBoundary: true } } },
+  o: { r: { end: true, needBoundary: true } },
+  c: { o: { n: { t: { a: { i: { n: { s: { end: true, needBoundary: true } } } } } } } },
+  '=': { '=': { end: true } },
+  '!': { '=': { end: true } },
+  '>': { end: true, '=': { end: true } },
+  '<': { end: true, '=': { end: true } }
+}
+
+export function matchOperator (str: string, begin: number, end = str.length) {
+  let node = trie
+  let i = begin
+  let info
+  while (node[str[i]] && i < end) {
+    node = node[str[i++]]
+    if (node['end']) info = node
+  }
+  if (!info) return -1
+  if (info['needBoundary'] && str.charCodeAt(i) & VARIABLE) return -1
+  return i
+}

src/parser/output-token.ts

@@ -1,20 +0,0 @@
-import { DelimitedToken } from './delimited-token'
-import { Token } from './token'
-import { NormalizedFullOptions } from '../liquid-options'
-
-export class OutputToken extends DelimitedToken {
-  public constructor (
-    raw: string,
-    value: string,
-    input: string,
-    line: number,
-    pos: number,
-    options: NormalizedFullOptions,
-    file?: string
-  ) {
-    super(raw, value, input, line, pos, options.trimOutputLeft, options.trimOutputRight, file)
-  }
-  public static is (token: Token): token is OutputToken {
-    return token instanceof OutputToken
-  }
-}

src/parser/parse-stream.ts

@@ -1,20 +1,21 @@
-import { Token } from '../parser/token'
+import { Token } from '../tokens/token'
 import { Template } from '../template/template'
-import { TagToken } from './tag-token'
+import { isTagToken } from '../util/type-guards'
+import { TopLevelToken } from '../tokens/toplevel-token'
 
-type ParseToken = ((token: Token, remainTokens: Token[]) => Template)
+type ParseToken<T extends Token> = ((token: T, remainTokens: T[]) => Template)
 
-export class ParseStream {
-  private tokens: Token[]
+export class ParseStream<T extends Token = TopLevelToken> {
+  private tokens: T[]
   private handlers: {[key: string]: (arg: any) => void} = {}
   private stopRequested = false
-  private parseToken: ParseToken
+  private parseToken: ParseToken<T>
 
-  public constructor (tokens: Token[], parseToken: ParseToken) {
+  public constructor (tokens: T[], parseToken: ParseToken<T>) {
     this.tokens = tokens
     this.parseToken = parseToken
   }
-  public on<T extends Template | Token | undefined> (name: string, cb: (arg: T) => void): ParseStream {
+  public on<T2 extends Template | T | undefined> (name: string, cb: (arg: T2) => void): ParseStream<T> {
     this.handlers[name] = cb
     return this
   }
@@ -24,10 +25,10 @@ export class ParseStream {
   }
   public start () {
     this.trigger('start')
-    let token: Token | undefined
+    let token: T | undefined
     while (!this.stopRequested && (token = this.tokens.shift())) {
       if (this.trigger('token', token)) continue
-      if (TagToken.is(token) && this.trigger(`tag:${token.name}`, token)) {
+      if (isTagToken(token) && this.trigger(`tag:${token.name}`, token)) {
         continue
       }
       const template = this.parseToken(token, this.tokens)

src/parser/parse-string-literal.ts

@@ -1,10 +1,3 @@
-import { last } from '../util/underscore'
-import { NullDrop } from '../drop/null-drop'
-import { EmptyDrop } from '../drop/empty-drop'
-import { BlankDrop } from '../drop/blank-drop'
-
-type literal = true | false | NullDrop | EmptyDrop | BlankDrop | number | string
-
 const rHex = /[\da-fA-F]/
 const rOct = /[0-7]/
 const escapeChar = {
@@ -23,18 +16,6 @@ function hexVal (c: string) {
   return code - 48
 }
 
-export function parseLiteral (str: string): literal | undefined {
-  str = str.trim()
-
-  if (str === 'true') return true
-  if (str === 'false') return false
-  if (str === 'nil' || str === 'null') return new NullDrop()
-  if (str === 'empty') return new EmptyDrop()
-  if (str === 'blank') return new BlankDrop()
-  if (!isNaN(Number(str))) return Number(str)
-  if ((str[0] === '"' || str[0] === "'") && str[0] === last(str)) return parseStringLiteral(str)
-}
-
 export function parseStringLiteral (str: string): string {
   let ret = ''
   for (let i = 1; i < str.length - 1; i++) {
@@ -66,3 +47,4 @@ export function parseStringLiteral (str: string): string {
   }
   return ret
 }
+

src/parser/parser.ts

@@ -1,13 +1,13 @@
 import { ParseError } from '../util/error'
 import { Liquid } from '../liquid'
 import { ParseStream } from './parse-stream'
-import { Token } from './token'
-import { TagToken } from './tag-token'
-import { OutputToken } from './output-token'
+import { isTagToken, isOutputToken } from '../util/type-guards'
+import { OutputToken } from '../tokens/output-token'
 import { Tag } from '../template/tag/tag'
 import { Output } from '../template/output'
 import { HTML } from '../template/html'
 import { Template } from '../template/template'
+import { TopLevelToken } from '../tokens/toplevel-token'
 
 export default class Parser {
   private liquid: Liquid
@@ -15,7 +15,7 @@ export default class Parser {
   public constructor (liquid: Liquid) {
     this.liquid = liquid
   }
-  public parse (tokens: Token[]) {
+  public parse (tokens: TopLevelToken[]) {
     let token
     const templates: Template[] = []
     while ((token = tokens.shift())) {
@@ -23,12 +23,12 @@ export default class Parser {
     }
     return templates
   }
-  public parseToken (token: Token, remainTokens: Token[]) {
+  public parseToken (token: TopLevelToken, remainTokens: TopLevelToken[]) {
     try {
-      if (TagToken.is(token)) {
+      if (isTagToken(token)) {
         return new Tag(token, remainTokens, this.liquid)
       }
-      if (OutputToken.is(token)) {
+      if (isOutputToken(token)) {
         return new Output(token as OutputToken, this.liquid.filters)
       }
       return new HTML(token)
@@ -36,7 +36,7 @@ export default class Parser {
       throw new ParseError(e, token)
     }
   }
-  public parseStream (tokens: Token[]) {
+  public parseStream (tokens: TopLevelToken[]) {
     return new ParseStream(tokens, (token, tokens) => this.parseToken(token, tokens))
   }
 }

src/parser/substr.ts

@@ -1,19 +0,0 @@
-export class Substr {
-  constructor (
-    public str: string,
-    public begin: number,
-    public end: number = begin
-  ) {}
-  size () {
-    return this.end - this.begin
-  }
-  toString () {
-    return this.str.slice(this.begin, this.end)
-  }
-  first () {
-    return this.str[this.begin]
-  }
-  last () {
-    return this.str[this.end - 1]
-  }
-}

src/parser/tag-token.ts

@@ -1,30 +0,0 @@
-import { DelimitedToken } from './delimited-token'
-import { Token } from './token'
-import { TokenizationError } from '../util/error'
-import * as lexical from './lexical'
-import { NormalizedFullOptions } from '../liquid-options'
-
-export class TagToken extends DelimitedToken {
-  public name: string
-  public args: string
-  public constructor (
-    raw: string,
-    value: string,
-    input: string,
-    line: number,
-    pos: number,
-    options: NormalizedFullOptions,
-    file?: string
-  ) {
-    super(raw, value, input, line, pos, options.trimTagLeft, options.trimTagRight, file)
-    const match = this.content.match(lexical.tagLine)
-    if (!match) {
-      throw new TokenizationError(`illegal tag syntax`, this)
-    }
-    this.name = match[1]
-    this.args = match[2]
-  }
-  public static is (token: Token): token is TagToken {
-    return token instanceof TagToken
-  }
-}

src/parser/token-kind.ts

@@ -0,0 +1,14 @@
+export enum TokenKind {
+  Number,
+  Literal,
+  Tag,
+  Output,
+  HTML,
+  Filter,
+  Hash,
+  PropertyAccess,
+  Word,
+  Range,
+  Quoted,
+  Operator
+}

src/parser/token.ts

@@ -1,14 +0,0 @@
-export class Token {
-  public trimLeft = false
-  public trimRight = false
-  public content: string
-  public constructor (
-    public raw: string,
-    public input: string,
-    public line: number,
-    public col: number,
-    public file?: string
-  ) {
-    this.content = raw
-  }
-}

src/parser/tokenizer.ts

@@ -1,158 +1,158 @@
 import { whiteSpaceCtrl } from './whitespace-ctrl'
-import { Substr } from './substr'
+import { NumberToken } from '../tokens/number-token'
+import { WordToken } from '../tokens/word-token'
+import { literalValues } from '../util/literal'
+import { LiteralToken } from '../tokens/literal-token'
+import { OperatorToken } from '../tokens/operator-token'
+import { PropertyAccessToken } from '../tokens/property-access-token'
+import { assert } from '../util/assert'
+import { TopLevelToken } from '../tokens/toplevel-token'
 import { FilterArg } from './filter-arg'
-import { FilterToken } from './filter-token'
+import { FilterToken } from '../tokens/filter-token'
+import { HashToken } from '../tokens/hash-token'
+import { QuotedToken } from '../tokens/quoted-token'
 import { ellipsis } from '../util/underscore'
-import { HTMLToken } from './html-token'
-import { TagToken } from './tag-token'
-import { Token } from './token'
-import { OutputToken } from './output-token'
+import { HTMLToken } from '../tokens/html-token'
+import { TagToken } from '../tokens/tag-token'
+import { Token } from '../tokens/token'
+import { RangeToken } from '../tokens/range-token'
+import { ValueToken } from '../tokens/value-token'
+import { OutputToken } from '../tokens/output-token'
 import { TokenizationError } from '../util/error'
 import { NormalizedFullOptions, defaultOptions } from '../liquid-options'
-
-// bitmask character types to boost performance
-// generated by bin/char-types.js
-const TYPES = '00000000044004000000000000000000428000080000010011111111110022210111111111111111111111111110000101111111111111111111111111100000'
-const VARIABLE = 1
-const OPERATOR = 2
-const BLANK = 4
-const QUOTE = 8
+import { TYPES, QUOTE, BLANK, VARIABLE } from '../util/character'
+import { matchOperator } from './match-operator'
 
 export class Tokenizer {
-  private p = 0
-  private N: number
-  private line = 1
-  private col = 1
+  p = 0
+  N: number
   constructor (
     private input: string,
-    private file: string = '',
-    private options: NormalizedFullOptions = defaultOptions
+    private file: string = ''
   ) {
     this.N = input.length
   }
 
-  * readExpression (): IterableIterator<string> {
+  * readExpression (): IterableIterator<Token> {
+    const operand = this.readValue()
+    if (!operand) return
+
+    yield operand
+
     while (this.p < this.N) {
+      const operator = this.readOperator()
+      if (!operator) return
+
       const operand = this.readValue()
-      if (operand.size()) {
-        yield operand.toString()
-        continue
-      }
-      this.readBlank()
-      const operator = new Substr(this.input, this.p)
-      while (OPERATOR & this.peekType()) operator.end = this.read()
-      if (operator.size()) {
-        yield operator.toString()
-        continue
-      }
-      this.read()
+      if (!operand) return
+
+      yield operator
+      yield operand
     }
   }
-
-  readFilterTokens (): FilterToken[] {
+  readOperator (): OperatorToken | undefined {
+    this.skipBlank()
+    const end = matchOperator(this.input, this.p, this.p + 8)
+    if (end === -1) return
+    return new OperatorToken(this.input, this.p, (this.p = end), this.file)
+  }
+  readFilters (): FilterToken[] {
     const filters = []
     while (true) {
-      const filter = this.readFilterToken()
+      const filter = this.readFilter()
       if (!filter) return filters
       filters.push(filter)
     }
   }
-
-  // | foo
-  // | foo: a
-  // | foo: a, b
-  // | foo: a, b: 1
-  readFilterToken (): FilterToken | null {
+  readFilter (): FilterToken | null {
     this.readTo('|')
     const begin = this.p
-    const name = this.readVariable().toString()
-    if (!name) return null
+    const name = this.readWord()
+    if (!name.size()) return null
     const args = []
-    this.readBlank()
+    this.skipBlank()
     if (this.peek() === ':') {
       do {
-        this.read()
+        ++this.p
         const arg = this.readFilterArg()
         arg && args.push(arg)
-        while (this.p < this.N && this.peek() !== ',' && this.peek() !== '|') this.read()
+        while (this.p < this.N && this.peek() !== ',' && this.peek() !== '|') ++this.p
       } while (this.peek() === ',')
     }
-    const raw = this.input.slice(begin, this.p)
-    return new FilterToken(name, args, raw, this.input, this.line, this.col, this.file)
+    return new FilterToken(name.getText(), args, this.input, begin, this.p, this.file)
   }
 
-  readFilterArg (): FilterArg | null {
+  readFilterArg (): FilterArg | undefined {
     const key = this.readValue()
-    if (!key.size()) return null
-    this.readBlank()
-    if (this.peek() === ':') {
-      this.read()
-      return [key.toString(), this.readValue().toString()]
-    }
-    return key.toString()
+    if (!key) return
+    this.skipBlank()
+    if (this.peek() !== ':') return key
+    ++this.p
+    const value = this.readValue()
+    return [key.getText(), value]
   }
 
-  readTokens (): Token[] {
-    const tokens: Token[] = []
+  readTopLevelTokens (options: NormalizedFullOptions = defaultOptions): TopLevelToken[] {
+    const tokens: TopLevelToken[] = []
     while (this.p < this.N) {
-      const token = this.readToken()
+      const token = this.readTopLevelToken(options)
       tokens.push(token)
     }
-    whiteSpaceCtrl(tokens, this.options)
+    whiteSpaceCtrl(tokens, options)
     return tokens
   }
 
-  readToken (): Token {
-    const { tagDelimiterLeft, outputDelimiterLeft } = this.options
-    if (this.matchWord(tagDelimiterLeft)) return this.readTagToken()
-    if (this.matchWord(outputDelimiterLeft)) return this.readOutputToken()
-    return this.readHTMLToken()
+  readTopLevelToken (options: NormalizedFullOptions): TopLevelToken {
+    const { tagDelimiterLeft, outputDelimiterLeft } = options
+    if (this.matchWord(tagDelimiterLeft)) return this.readTagToken(options)
+    if (this.matchWord(outputDelimiterLeft)) return this.readOutputToken(options)
+    return this.readHTMLToken(options)
   }
 
-  readHTMLToken (): HTMLToken {
-    const html = new Substr(this.input, this.p)
+  readHTMLToken (options: NormalizedFullOptions): HTMLToken {
+    const begin = this.p
     while (this.p < this.N) {
-      const { tagDelimiterLeft, outputDelimiterLeft } = this.options
+      const { tagDelimiterLeft, outputDelimiterLeft } = options
       if (this.matchWord(tagDelimiterLeft)) break
       if (this.matchWord(outputDelimiterLeft)) break
-      html.end = this.read()
+      ++this.p
     }
-    return new HTMLToken(html.toString(), this.input, this.line, this.col, this.file)
+    return new HTMLToken(this.input, begin, this.p, this.file)
   }
 
-  readTagToken (): TagToken {
-    const { line, col, file, input, options } = this
-    const { tagDelimiterLeft, tagDelimiterRight } = options
-    const buffer = this.readTo(tagDelimiterRight).toString()
-    if (!this.reverseMatchWord(tagDelimiterRight, buffer)) {
-      throw new TokenizationError(
-        `tag "${ellipsis(buffer, 16)}" not closed`,
-        new Token(buffer, input, line, col, file)
-      )
+  readTagToken (options: NormalizedFullOptions): TagToken {
+    const { file, input } = this
+    const { tagDelimiterRight } = options
+    const begin = this.p
+    if (this.readTo(tagDelimiterRight) === -1) {
+      this.mkError(`tag "${this.ellipsis(begin)}" not closed`, begin)
     }
-    const value = buffer.slice(tagDelimiterLeft.length, -tagDelimiterRight.length)
-    return new TagToken(buffer, value, input, line, col, options, file)
+    return new TagToken(input, begin, this.p, options, file)
   }
 
-  readOutputToken (): OutputToken {
-    const { line, col, file, input, options } = this
-    const { outputDelimiterLeft, outputDelimiterRight } = options
-    const buffer = this.readTo(outputDelimiterRight).toString()
-    if (!this.reverseMatchWord(outputDelimiterRight, buffer)) {
-      throw new TokenizationError(
-        `output "${ellipsis(buffer, 16)}" not closed`,
-        new Token(buffer, input, line, col, file)
-      )
+  readOutputToken (options: NormalizedFullOptions): OutputToken {
+    const { file, input } = this
+    const { outputDelimiterRight } = options
+    const begin = this.p
+    if (this.readTo(outputDelimiterRight) === -1) {
+      this.mkError(`output "${this.ellipsis(begin)}" not closed`, begin)
     }
-    const value = buffer.slice(outputDelimiterLeft.length, -outputDelimiterRight.length)
-    return new OutputToken(buffer, value, input, line, col, options, file)
+    return new OutputToken(input, begin, this.p, options, file)
   }
 
-  readVariable (): Substr {
-    this.readBlank()
-    const ans = new Substr(this.input, this.p)
-    while (this.peekType() & VARIABLE) ans.end = this.read()
-    return ans
+  mkError (msg: string, begin: number) {
+    throw new TokenizationError(msg, new WordToken(this.input, begin, this.N, this.file))
+  }
+
+  ellipsis (begin: number = this.p) {
+    return ellipsis(this.input.slice(begin), 16)
+  }
+
+  readWord (): WordToken { // rename to identifier
+    this.skipBlank()
+    const begin = this.p
+    while (this.peekType() & VARIABLE) ++this.p
+    return new WordToken(this.input, begin, this.p, this.file)
   }
 
   readHashes () {
@@ -164,133 +164,135 @@ export class Tokenizer {
     }
   }
 
-  readHash () {
-    this.readBlank()
-    if (this.peek() === ',') this.read()
-    const name = this.readVariable().toString()
-    if (!name) return null
+  readHash (): HashToken | undefined {
+    this.skipBlank()
+    if (this.peek() === ',') ++this.p
+    const begin = this.p
+    const name = this.readWord()
+    if (!name.size()) return
+    let value
 
-    this.readBlank()
-    let value = ''
+    this.skipBlank()
     if (this.peek() === ':') {
-      this.read()
-      value = this.readValue().toString()
+      ++this.p
+      value = this.readValue()
     }
-    return [name, value]
+    return new HashToken(this.input, begin, this.p, name, value, this.file)
   }
 
-  readPropertyAccess (): Substr {
-    this.readBlank()
-    const ans = new Substr(this.input, this.p)
-    let nested = 0
+  remaining () {
+    return this.input.slice(this.p)
+  }
+
+  advance (i = 1) {
+    this.p += i
+  }
+
+  end () {
+    return this.p >= this.N
+  }
+
+  readTo (end: string): number {
     while (this.p < this.N) {
-      const c = this.peek()
-      const code = this.peekType()
-      if (c === '[') {
-        this.read()
-        ans.end = this.readValue().end
-        nested++
-      } else if (c === ']') {
-        if (!nested) break
-        ans.end = this.read()
-        nested--
-      } else if (c === '.') {
-        if (this.peekType(1) & VARIABLE) {
-          this.read()
-          ans.end = this.readVariable().end
-        } else break
-      } else if (code & VARIABLE) {
-        ans.end = this.read()
-      } else {
-        if (nested) this.read()
-        else break
-      }
+      ++this.p
+      if (this.reverseMatchWord(end)) return this.p
     }
-    return ans
+    return -1
   }
-  readTo (end: string): Substr {
-    const ans = new Substr(this.input, this.p)
-    while (this.p < this.N) {
-      ans.end = this.read()
-      if (this.reverseMatchWord(end)) break
+
+  readValue (): ValueToken | undefined {
+    const value = this.readQuoted() || this.readRange()
+    if (value) return value
+
+    const variable = this.readWord()
+    if (!variable.size()) return
+
+    let isNumber = variable.isNumber(true)
+    const props: (QuotedToken | WordToken)[] = []
+    while (true) {
+      if (this.peek() === '[') {
+        isNumber = false
+        this.p++
+        const prop = this.readValue() || new WordToken(this.input, this.p, this.p, this.file)
+        this.readTo(']')
+        props.push(prop)
+      } else if (this.peek() === '.' && this.peek(1) !== '.') { // skip range syntax
+        this.p++
+        const prop = this.readWord()
+        if (!prop.size()) break
+        if (!prop.isNumber()) isNumber = false
+        props.push(prop)
+      } else break
     }
-    return ans
+    if (!props.length && literalValues.hasOwnProperty(variable.content)) {
+      return new LiteralToken(this.input, variable.begin, variable.end, this.file)
+    }
+    if (isNumber) return new NumberToken(variable, props[0] as WordToken)
+    return new PropertyAccessToken(variable, props, this.p)
   }
-  readValue (): Substr {
-    let val = this.readQuoted()
-    if (val.size()) return val
-    val = this.readBoolean()
-    if (val.size()) return val
-    val = this.readPropertyAccess()
-    if (val.size()) return val
-    return this.readRange()
+
+  readRange (): RangeToken | undefined {
+    this.skipBlank()
+    const begin = this.p
+    if (this.peek() !== '(') return
+    ++this.p
+    const lhs = this.readValueOrThrow()
+    this.p += 2
+    const rhs = this.readValueOrThrow()
+    ++this.p
+    return new RangeToken(this.input, begin, this.p, lhs, rhs, this.file)
   }
-  readRange (): Substr {
-    this.readBlank()
-    const ans = new Substr(this.input, this.p)
-    if (this.peek() !== '(') return ans
-    this.read()
-    this.readValue()
-    this.read(2)
-    this.readValue()
-    ans.end = this.read()
-    return ans
+
+  readValueOrThrow (): ValueToken {
+    const value = this.readValue()
+    assert(value, () => `unexpected token "${this.ellipsis()}", value expected`)
+    return value!
   }
-  readBoolean (): Substr {
-    this.readBlank()
-    const ans = new Substr(this.input, this.p)
-    if (this.matchWord('true') && !(this.peekType(4) & VARIABLE)) ans.end = this.read(4)
-    else if (this.matchWord('false') && !(this.peekType(5) & VARIABLE)) ans.end = this.read(5)
-    return ans
-  }
-  readQuoted (): Substr {
-    this.readBlank()
-    const ans = new Substr(this.input, this.p)
-    if (!(this.peekType() & QUOTE)) return ans
-    ans.end = this.read()
+
+  readQuoted (): QuotedToken | undefined {
+    this.skipBlank()
+    const begin = this.p
+    if (!(this.peekType() & QUOTE)) return
+    ++this.p
     let escaped = false
     while (this.p < this.N) {
-      ans.end = this.read()
-      if (ans.last() === ans.first() && !escaped) break
+      ++this.p
+      if (this.input[this.p - 1] === this.input[begin] && !escaped) break
       if (escaped) escaped = false
-      else if (ans.last() === '\\') escaped = true
+      else if (this.input[this.p - 1] === '\\') escaped = true
     }
-    return ans
+    return new QuotedToken(this.input, begin, this.p, this.file)
   }
-  read (n = 1): number {
-    if (n > 1) this.read(n - 1)
-    const c = this.input[this.p++]
-    if (c === '\n') {
-      this.line++
-      this.col = 1
-    } else {
-      this.col++
-    }
-    return this.p
+
+  readFileName (): WordToken {
+    const begin = this.p
+    while (!(this.peekType() & BLANK) && this.peek() !== ',' && this.p < this.N) this.p++
+    return new WordToken(this.input, begin, this.p, this.file)
   }
+
   matchWord (word: string) {
     for (let i = 0; i < word.length; i++) {
       if (word[i] !== this.input[this.p + i]) return false
     }
     return true
   }
-  reverseMatchWord (word: string, buffer?: string) {
-    const str = buffer || this.input
-    const end = buffer === undefined ? this.p : buffer.length
+
+  reverseMatchWord (word: string) {
     for (let i = 0; i < word.length; i++) {
-      if (word[word.length - 1 - i] !== str[end - 1 - i]) return false
+      if (word[word.length - 1 - i] !== this.input[this.p - 1 - i]) return false
     }
     return true
   }
+
   peekType (n = 0) {
-    return +TYPES[this.input.charCodeAt(this.p + n)]
+    return TYPES[this.input.charCodeAt(this.p + n)]
   }
+
   peek (n = 0) {
     return this.input[this.p + n]
   }
-  readBlank () {
-    let ans = ''
-    while (this.peekType() & BLANK) ans += this.read()
-    return ans
+
+  skipBlank () {
+    while (this.peekType() & BLANK) ++this.p
   }
 }

src/parser/whitespace-ctrl.ts

@@ -1,7 +1,8 @@
-import { Token } from '../parser/token'
-import { TagToken } from '../parser/tag-token'
-import { HTMLToken } from '../parser/html-token'
+import { Token } from '../tokens/token'
+import { DelimitedToken } from '../tokens/delimited-token'
+import { isTagToken, isHTMLToken } from '../util/type-guards'
 import { NormalizedFullOptions } from '../liquid-options'
+import { TYPES, INLINE_BLANK, BLANK } from '../util/character'
 
 export function whiteSpaceCtrl (tokens: Token[], options: NormalizedFullOptions) {
   options = { greedy: true, ...options }
@@ -9,11 +10,12 @@ export function whiteSpaceCtrl (tokens: Token[], options: NormalizedFullOptions)
 
   for (let i = 0; i < tokens.length; i++) {
     const token = tokens[i]
+    if (!(token instanceof DelimitedToken)) continue
     if (!inRaw && token.trimLeft) {
       trimLeft(tokens[i - 1], options.greedy)
     }
 
-    if (TagToken.is(token)) {
+    if (isTagToken(token)) {
       if (token.name === 'raw') inRaw = true
       else if (token.name === 'endraw') inRaw = false
     }
@@ -25,15 +27,16 @@ export function whiteSpaceCtrl (tokens: Token[], options: NormalizedFullOptions)
 }
 
 function trimLeft (token: Token, greedy: boolean) {
-  if (!token || !HTMLToken.is(token)) return
+  if (!token || !isHTMLToken(token)) return
 
-  const rLeft = greedy ? /\s+$/g : /[\t\r ]*$/g
-  token.content = token.content.replace(rLeft, '')
+  const mask = greedy ? BLANK : INLINE_BLANK
+  while (TYPES[token.input.charCodeAt(token.end - 1 - token.trimRight)] & mask) token.trimRight++
 }
 
 function trimRight (token: Token, greedy: boolean) {
-  if (!token || !HTMLToken.is(token)) return
+  if (!token || !isHTMLToken(token)) return
 
-  const rRight = greedy ? /^\s+/g : /^[\t\r ]*\n?/g
-  token.content = token.content.replace(rRight, '')
+  const mask = greedy ? BLANK : INLINE_BLANK
+  while (TYPES[token.input.charCodeAt(token.begin + token.trimLeft)] & mask) token.trimLeft++
+  if (token.input.charAt(token.begin + token.trimLeft) === '\n') token.trimLeft++
 }