Class: Tokenizer

Hierarchy

• Tokenizer

Constructors

constructor

+ new Tokenizer(input: string, trie: Trie, file: string): Tokenizer

Defined in src/parser/tokenizer.ts:31

Parameters:

Name	Type	Default
input	string	-
trie	Trie	-
file	string	“”

Returns: Tokenizer

Properties

N

• N: number

Defined in src/parser/tokenizer.ts:30

---

file

• file: string

Defined in src/parser/tokenizer.ts:36

---

input

• input: string

Defined in src/parser/tokenizer.ts:34

---

p

• p: number = 0

Defined in src/parser/tokenizer.ts:29

---

Private rawBeginAt

• rawBeginAt: number = -1

Defined in src/parser/tokenizer.ts:31

---

Private trie

• trie: Trie

Defined in src/parser/tokenizer.ts:35

Methods

advance

▸ advance(i: number): void

Defined in src/parser/tokenizer.ts:246

Parameters:

Name	Type	Default
i	number	1

Returns: void

---

end

▸ end(): boolean

Defined in src/parser/tokenizer.ts:250

Returns: boolean

---

match

▸ match(word: string): boolean

Defined in src/parser/tokenizer.ts:347

Parameters:

Name	Type
word	string

Returns: boolean

---

mkError

▸ mkError(msg: string, begin: number): TokenizationError‹›

Defined in src/parser/tokenizer.ts:194

Parameters:

Name	Type
msg	string
begin	number

Returns: TokenizationError‹›

---

peek

▸ peek(n: number): string

Defined in src/parser/tokenizer.ts:365

Parameters:

Name	Type	Default
n	number	0

Returns: string

---

peekType

▸ peekType(n: number): number

Defined in src/parser/tokenizer.ts:361

Parameters:

Name	Type	Default
n	number	0

Returns: number

---

readEndrawOrRawContent

▸ readEndrawOrRawContent(options: NormalizedFullOptions): HTMLToken | TagToken

Defined in src/parser/tokenizer.ts:167

Parameters:

Name	Type
options	NormalizedFullOptions

Returns: HTMLToken | TagToken

---

readExpression

▸ readExpression(): Expression‹›

Defined in src/parser/tokenizer.ts:41

Returns: Expression‹›

---

readExpressionTokens

▸ readExpressionTokens(): IterableIterator‹Token›

Defined in src/parser/tokenizer.ts:45

Returns: IterableIterator‹Token›

---

readFileNameTemplate

▸ readFileNameTemplate(options: NormalizedFullOptions): IterableIterator‹TopLevelToken›

Defined in src/parser/tokenizer.ts:335

Parameters:

Name	Type
options	NormalizedFullOptions

Returns: IterableIterator‹TopLevelToken›

---

readFilter

▸ readFilter(): FilterToken | null

Defined in src/parser/tokenizer.ts:76

Returns: FilterToken | null

---

readFilterArg

▸ readFilterArg(): FilterArg | undefined

Defined in src/parser/tokenizer.ts:97

Returns: FilterArg | undefined

---

readFilters

▸ readFilters(): FilterToken[]

Defined in src/parser/tokenizer.ts:68

Returns: FilterToken[]

---

readHTMLToken

▸ readHTMLToken(stopStrings: string[]): HTMLToken

Defined in src/parser/tokenizer.ts:125

Parameters:

Name	Type
stopStrings	string[]

Returns: HTMLToken

---

readHash

▸ readHash(): HashToken | undefined

Defined in src/parser/tokenizer.ts:226

Returns: HashToken | undefined

---

readHashes

▸ readHashes(): HashToken‹›[]

Defined in src/parser/tokenizer.ts:217

Returns: HashToken‹›[]

---

readIdentifier

▸ readIdentifier(): IdentifierToken

Defined in src/parser/tokenizer.ts:210

Returns: IdentifierToken

---

readOperator

▸ readOperator(): OperatorToken | undefined

Defined in src/parser/tokenizer.ts:62

Returns: OperatorToken | undefined

---

readOutputToken

▸ readOutputToken(options: NormalizedFullOptions): OutputToken

Defined in src/parser/tokenizer.ts:157

Parameters:

Name	Type	Default
options	NormalizedFullOptions	defaultOptions

Returns: OutputToken

---

readQuoted

▸ readQuoted(): QuotedToken | undefined

Defined in src/parser/tokenizer.ts:320

Returns: QuotedToken | undefined

---

readRange

▸ readRange(): RangeToken | undefined

Defined in src/parser/tokenizer.ts:302

Returns: RangeToken | undefined

---

readTagToken

▸ readTagToken(options: NormalizedFullOptions): TagToken

Defined in src/parser/tokenizer.ts:134

Parameters:

Name	Type	Default
options	NormalizedFullOptions	defaultOptions

Returns: TagToken

---

readTo

▸ readTo(end: string): number

Defined in src/parser/tokenizer.ts:254

Parameters:

Name	Type
end	string

Returns: number

---

readToDelimiter

▸ readToDelimiter(delimiter: string): number

Defined in src/parser/tokenizer.ts:145

Parameters:

Name	Type
delimiter	string

Returns: number

---

readTopLevelToken

▸ readTopLevelToken(options: NormalizedFullOptions): TopLevelToken

Defined in src/parser/tokenizer.ts:117

Parameters:

Name	Type
options	NormalizedFullOptions

Returns: TopLevelToken

---

readTopLevelTokens

▸ readTopLevelTokens(options: NormalizedFullOptions): TopLevelToken[]

Defined in src/parser/tokenizer.ts:107

Parameters:

Name	Type	Default
options	NormalizedFullOptions	defaultOptions

Returns: TopLevelToken[]

---

readValue

▸ readValue(): ValueToken | undefined

Defined in src/parser/tokenizer.ts:262

Returns: ValueToken | undefined

---

readValueOrThrow

▸ readValueOrThrow(): ValueToken

Defined in src/parser/tokenizer.ts:314

Returns: ValueToken

---

readWord

▸ readWord(): IdentifierToken‹›

Defined in src/parser/tokenizer.ts:205

deprecated

Returns: IdentifierToken‹›

---

remaining

▸ remaining(): string

Defined in src/parser/tokenizer.ts:242

Returns: string

---

rmatch

▸ rmatch(pattern: string): boolean

Defined in src/parser/tokenizer.ts:354

Parameters:

Name	Type
pattern	string

Returns: boolean

---

skipBlank

▸ skipBlank(): void

Defined in src/parser/tokenizer.ts:369

Returns: void

---

snapshot

▸ snapshot(begin: number): string

Defined in src/parser/tokenizer.ts:198

Parameters:

Name	Type	Default
begin	number	this.p

Returns: string