mirror of
https://github.com/harttle/liquidjs.git
synced 2026-09-16 12:50:38 -07:00
feat: with & for in render tag, closes #195
This commit is contained in:
harttle
+263
-77
@@ -1,93 +1,279 @@
|
||||
import { whiteSpaceCtrl } from './whitespace-ctrl'
|
||||
import { FilterArg } from './filter-arg'
|
||||
import { FilterToken } from './filter-token'
|
||||
import { ellipsis } from '../util/underscore'
|
||||
import { HTMLToken } from './html-token'
|
||||
import { TagToken } from './tag-token'
|
||||
import { Token } from './token'
|
||||
import { OutputToken } from './output-token'
|
||||
import { TokenizationError } from '../util/error'
|
||||
import { NormalizedFullOptions, applyDefault } from '../liquid-options'
|
||||
import { flatten } from './flatten/node'
|
||||
import { NormalizedFullOptions, defaultOptions } from '../liquid-options'
|
||||
|
||||
enum ParseState { HTML, OUTPUT, TAG }
|
||||
// bitmask character types to boost performance
|
||||
// generated by bin/char-types.js
|
||||
const TYPES = '00000000044004000000000000000000428000080000010011111111110022210111111111111111111111111110000101111111111111111111111111100000'
|
||||
const VARIABLE = 1
|
||||
const OPERATOR = 2
|
||||
const BLANK = 4
|
||||
const QUOTE = 8
|
||||
|
||||
export class Tokenizer {
|
||||
private options: NormalizedFullOptions
|
||||
public constructor (options?: NormalizedFullOptions) {
|
||||
this.options = applyDefault(options)
|
||||
private p = 0
|
||||
private N: number
|
||||
private line = 1
|
||||
private col = 1
|
||||
constructor (
|
||||
private input: string,
|
||||
private file: string = '',
|
||||
private options: NormalizedFullOptions = defaultOptions
|
||||
) {
|
||||
this.N = input.length
|
||||
}
|
||||
public tokenize (input: string, file?: string) {
|
||||
|
||||
* readExpression (): IterableIterator<string> {
|
||||
while (this.p < this.N) {
|
||||
let val = this.readValue()
|
||||
if (val) {
|
||||
yield val
|
||||
continue
|
||||
}
|
||||
this.readBlank()
|
||||
while (OPERATOR & this.peekType()) val += this.read()
|
||||
if (val) {
|
||||
yield val
|
||||
continue
|
||||
}
|
||||
this.read()
|
||||
}
|
||||
}
|
||||
|
||||
readFilterTokens (): FilterToken[] {
|
||||
const filters = []
|
||||
while (true) {
|
||||
const filter = this.readFilterToken()
|
||||
if (!filter) return filters
|
||||
filters.push(filter)
|
||||
}
|
||||
}
|
||||
|
||||
// | foo
|
||||
// | foo: a
|
||||
// | foo: a, b
|
||||
// | foo: a, b: 1
|
||||
readFilterToken (): FilterToken | null {
|
||||
this.readTo('|')
|
||||
const begin = this.p
|
||||
const name = this.readVariable()
|
||||
if (!name) return null
|
||||
const args = []
|
||||
this.readBlank()
|
||||
if (this.peek() === ':') {
|
||||
do {
|
||||
this.read()
|
||||
const arg = this.readFilterArg()
|
||||
arg && args.push(arg)
|
||||
while (this.p < this.N && this.peek() !== ',' && this.peek() !== '|') this.read()
|
||||
} while (this.peek() === ',')
|
||||
}
|
||||
const raw = this.input.slice(begin, this.p)
|
||||
return new FilterToken(name, args, raw, this.input, this.line, this.col, this.file)
|
||||
}
|
||||
|
||||
readFilterArg (): FilterArg | null {
|
||||
const key = this.readValue()
|
||||
if (!key) return null
|
||||
this.readBlank()
|
||||
if (this.peek() === ':') {
|
||||
this.read()
|
||||
return [key, this.readValue()]
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
readTokens (): Token[] {
|
||||
const tokens: Token[] = []
|
||||
const {
|
||||
tagDelimiterLeft,
|
||||
tagDelimiterRight,
|
||||
outputDelimiterLeft,
|
||||
outputDelimiterRight
|
||||
} = this.options
|
||||
let p = 0
|
||||
let curLine = 1
|
||||
let state = ParseState.HTML
|
||||
let buffer = ''
|
||||
let lineBegin = 0
|
||||
let line = 1
|
||||
let col = 1
|
||||
|
||||
while (p < input.length) {
|
||||
if (input[p] === '\n') {
|
||||
curLine++
|
||||
lineBegin = p + 1
|
||||
}
|
||||
if (state === ParseState.HTML) {
|
||||
if (input.substr(p, outputDelimiterLeft.length) === outputDelimiterLeft) {
|
||||
if (buffer) tokens.push(new HTMLToken(flatten(buffer), input, line, col, file))
|
||||
buffer = outputDelimiterLeft
|
||||
line = curLine
|
||||
col = p - lineBegin + 1
|
||||
p += outputDelimiterLeft.length
|
||||
state = ParseState.OUTPUT
|
||||
continue
|
||||
} else if (input.substr(p, tagDelimiterLeft.length) === tagDelimiterLeft) {
|
||||
if (buffer) tokens.push(new HTMLToken(flatten(buffer), input, line, col, file))
|
||||
buffer = tagDelimiterLeft
|
||||
line = curLine
|
||||
col = p - lineBegin + 1
|
||||
p += tagDelimiterLeft.length
|
||||
state = ParseState.TAG
|
||||
continue
|
||||
}
|
||||
} else if (
|
||||
state === ParseState.OUTPUT &&
|
||||
input.substr(p, outputDelimiterRight.length) === outputDelimiterRight
|
||||
) {
|
||||
buffer += outputDelimiterRight
|
||||
tokens.push(new OutputToken(flatten(buffer), buffer.slice(outputDelimiterLeft.length, -outputDelimiterRight.length), input, line, col, this.options, file))
|
||||
p += outputDelimiterRight.length
|
||||
buffer = ''
|
||||
line = curLine
|
||||
col = p - lineBegin + 1
|
||||
state = ParseState.HTML
|
||||
continue
|
||||
} else if (input.substr(p, tagDelimiterRight.length) === tagDelimiterRight) {
|
||||
buffer += tagDelimiterRight
|
||||
tokens.push(new TagToken(flatten(buffer), buffer.slice(tagDelimiterLeft.length, -tagDelimiterRight.length), input, line, col, this.options, file))
|
||||
p += tagDelimiterRight.length
|
||||
buffer = ''
|
||||
line = curLine
|
||||
col = p - lineBegin + 1
|
||||
state = ParseState.HTML
|
||||
continue
|
||||
}
|
||||
buffer += input[p++]
|
||||
while (this.p < this.N) {
|
||||
const token = this.readToken()
|
||||
tokens.push(token)
|
||||
}
|
||||
if (state !== ParseState.HTML) {
|
||||
const t = state === ParseState.OUTPUT ? 'output' : 'tag'
|
||||
const str = buffer.length > 16 ? buffer.slice(0, 13) + '...' : buffer
|
||||
throw new TokenizationError(
|
||||
`${t} "${str}" not closed`,
|
||||
new Token(flatten(buffer), input, line, col, file)
|
||||
)
|
||||
}
|
||||
if (buffer) tokens.push(new HTMLToken(flatten(buffer), input, line, col, file))
|
||||
|
||||
whiteSpaceCtrl(tokens, this.options)
|
||||
return tokens
|
||||
}
|
||||
|
||||
readToken (): Token {
|
||||
const { tagDelimiterLeft, outputDelimiterLeft } = this.options
|
||||
if (this.peekWord(tagDelimiterLeft.length) === tagDelimiterLeft) return this.readTagToken()
|
||||
if (this.peekWord(outputDelimiterLeft.length) === outputDelimiterLeft) return this.readOutputToken()
|
||||
return this.readHTMLToken()
|
||||
}
|
||||
|
||||
readHTMLToken (): HTMLToken {
|
||||
let html = ''
|
||||
while (this.p < this.N) {
|
||||
const { tagDelimiterLeft, outputDelimiterLeft } = this.options
|
||||
if (this.peekWord(tagDelimiterLeft.length) === tagDelimiterLeft) break
|
||||
if (this.peekWord(outputDelimiterLeft.length) === outputDelimiterLeft) break
|
||||
html += this.read()
|
||||
}
|
||||
return new HTMLToken(html, this.input, this.line, this.col, this.file)
|
||||
}
|
||||
|
||||
readTagToken (): TagToken {
|
||||
const { line, col, file, input, options } = this
|
||||
const { tagDelimiterLeft, tagDelimiterRight } = options
|
||||
const buffer = this.readTo(tagDelimiterRight)
|
||||
if (buffer.slice(-tagDelimiterRight.length) !== tagDelimiterRight) {
|
||||
throw new TokenizationError(
|
||||
`tag "${ellipsis(buffer, 16)}" not closed`,
|
||||
new Token(buffer, input, line, col, file)
|
||||
)
|
||||
}
|
||||
const value = buffer.slice(tagDelimiterLeft.length, -tagDelimiterRight.length)
|
||||
return new TagToken(buffer, value, input, line, col, options, file)
|
||||
}
|
||||
|
||||
readOutputToken (): OutputToken {
|
||||
const { line, col, file, input, options } = this
|
||||
const { outputDelimiterLeft, outputDelimiterRight } = options
|
||||
const buffer = this.readTo(outputDelimiterRight)
|
||||
if (buffer.slice(-outputDelimiterRight.length) !== outputDelimiterRight) {
|
||||
throw new TokenizationError(
|
||||
`output "${ellipsis(buffer, 16)}" not closed`,
|
||||
new Token(buffer, input, line, col, file)
|
||||
)
|
||||
}
|
||||
const value = buffer.slice(outputDelimiterLeft.length, -outputDelimiterRight.length)
|
||||
return new OutputToken(buffer, value, input, line, col, options, file)
|
||||
}
|
||||
|
||||
readVariable () {
|
||||
this.readBlank()
|
||||
let ans = ''
|
||||
while (this.peekType() & VARIABLE) ans += this.read()
|
||||
return ans
|
||||
}
|
||||
|
||||
readHashes () {
|
||||
const hashes = []
|
||||
while (true) {
|
||||
const hash = this.readHash()
|
||||
if (!hash) return hashes
|
||||
hashes.push(hash)
|
||||
}
|
||||
}
|
||||
|
||||
readHash () {
|
||||
this.readBlank()
|
||||
if (this.peek() === ',') this.read()
|
||||
const name = this.readVariable()
|
||||
if (!name) return null
|
||||
|
||||
this.readBlank()
|
||||
let value = ''
|
||||
if (this.peek() === ':') {
|
||||
this.read()
|
||||
value = this.readValue()
|
||||
}
|
||||
return [name, value]
|
||||
}
|
||||
|
||||
readPropertyAccess () {
|
||||
this.readBlank()
|
||||
let ans = ''
|
||||
let nested = 0
|
||||
while (this.p < this.N) {
|
||||
const c = this.peek()
|
||||
const code = this.peekType()
|
||||
if (c === '[') {
|
||||
ans += this.read() + this.readValue()
|
||||
nested++
|
||||
} else if (c === ']') {
|
||||
if (!nested) break
|
||||
ans += this.read()
|
||||
nested--
|
||||
} else if (c === '.') {
|
||||
if (this.peekType(1) & VARIABLE) {
|
||||
ans += this.read()
|
||||
ans += this.readVariable()
|
||||
} else break
|
||||
} else if (code & VARIABLE) {
|
||||
ans += this.read()
|
||||
} else {
|
||||
if (nested) this.read()
|
||||
else break
|
||||
}
|
||||
}
|
||||
return ans
|
||||
}
|
||||
readTo (end: string) {
|
||||
let ans = ''
|
||||
while (this.p < this.N) {
|
||||
ans += this.read()
|
||||
if (ans.slice(-end.length) === end) break
|
||||
}
|
||||
return ans
|
||||
}
|
||||
readValue () {
|
||||
let val = this.readQuoted()
|
||||
if (val) return val
|
||||
val = this.readBoolean()
|
||||
if (val) return val
|
||||
val = this.readPropertyAccess()
|
||||
if (val) return val
|
||||
return this.readRange()
|
||||
}
|
||||
readRange () {
|
||||
this.readBlank()
|
||||
if (this.peek() !== '(') return ''
|
||||
let ans = this.read()
|
||||
ans += this.readValue()
|
||||
ans += this.read(2)
|
||||
ans += this.readValue()
|
||||
ans += this.read()
|
||||
return ans
|
||||
}
|
||||
readBoolean () {
|
||||
this.readBlank()
|
||||
if (this.peekWord(4) === 'true' && !(this.peekType(4) & VARIABLE)) return this.read(4)
|
||||
if (this.peekWord(5) === 'false' && !(this.peekType(5) & VARIABLE)) return this.read(5)
|
||||
return ''
|
||||
}
|
||||
readQuoted () {
|
||||
this.readBlank()
|
||||
if (!(this.peekType() & QUOTE)) return ''
|
||||
let ans = this.read()
|
||||
let escaped = false
|
||||
while (this.p < this.N) {
|
||||
const c = this.read()
|
||||
ans += c
|
||||
if (c === ans[0] && !escaped) break
|
||||
if (escaped) escaped = false
|
||||
else if (c === '\\') escaped = true
|
||||
}
|
||||
return ans
|
||||
}
|
||||
read (n = 1): string {
|
||||
const c = this.input[this.p++]
|
||||
if (c === '\n') {
|
||||
this.line++
|
||||
this.col = 1
|
||||
} else {
|
||||
this.col++
|
||||
}
|
||||
return n === 1 ? c : c + this.read(n - 1)
|
||||
}
|
||||
peekWord (n: number) {
|
||||
return this.input.substr(this.p, n)
|
||||
}
|
||||
peekType (n = 0) {
|
||||
return +TYPES[this.input.charCodeAt(this.p + n)]
|
||||
}
|
||||
peek (n = 0) {
|
||||
return this.input[this.p + n]
|
||||
}
|
||||
readBlank () {
|
||||
let ans = ''
|
||||
while (this.peekType() & BLANK) ans += this.read()
|
||||
return ans
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user