Hierarchy
- Tokenizer
Constructors
constructor
+ new Tokenizer(input: string, operators: Operators, file?: undefined | string): Tokenizer
Defined in src/parser/tokenizer.ts:13
Parameters:
| Name | Type | Default |
|---|---|---|
input |
string | - |
operators |
Operators | defaultOptions.operators |
file? |
undefined | string | - |
Returns: Tokenizer
Properties
N
• N: number
Defined in src/parser/tokenizer.ts:11
Optional file
• file? : undefined | string
Defined in src/parser/tokenizer.ts:18
input
• input: string
Defined in src/parser/tokenizer.ts:16
Private opTrie
• opTrie: Trie
Defined in src/parser/tokenizer.ts:13
p
• p: number = 0
Defined in src/parser/tokenizer.ts:10
Private rawBeginAt
• rawBeginAt: number = -1
Defined in src/parser/tokenizer.ts:12
Methods
advance
▸ advance(i: number): void
Defined in src/parser/tokenizer.ts:253
Parameters:
| Name | Type | Default |
|---|---|---|
i |
number | 1 |
Returns: void
end
▸ end(): boolean
Defined in src/parser/tokenizer.ts:257
Returns: boolean
match
▸ match(word: string): boolean
Defined in src/parser/tokenizer.ts:354
Parameters:
| Name | Type |
|---|---|
word |
string |
Returns: boolean
mkError
▸ mkError(msg: string, begin: number): TokenizationError‹›
Defined in src/parser/tokenizer.ts:193
Parameters:
| Name | Type |
|---|---|
msg |
string |
begin |
number |
Returns: TokenizationError‹›
peek
▸ peek(n: number): string
Defined in src/parser/tokenizer.ts:372
Parameters:
| Name | Type | Default |
|---|---|---|
n |
number | 0 |
Returns: string
peekType
▸ peekType(n: number): number
Defined in src/parser/tokenizer.ts:368
Parameters:
| Name | Type | Default |
|---|---|---|
n |
number | 0 |
Returns: number
readEndrawOrRawContent
▸ readEndrawOrRawContent(options: NormalizedFullOptions): HTMLToken | TagToken
Defined in src/parser/tokenizer.ts:149
Parameters:
| Name | Type |
|---|---|
options |
NormalizedFullOptions |
readExpression
▸ readExpression(): Expression‹›
Defined in src/parser/tokenizer.ts:24
Returns: Expression‹›
readExpressionTokens
▸ readExpressionTokens(): IterableIterator‹Token›
Defined in src/parser/tokenizer.ts:28
Returns: IterableIterator‹Token›
readFileNameTemplate
▸ readFileNameTemplate(options: NormalizedFullOptions): IterableIterator‹TopLevelToken›
Defined in src/parser/tokenizer.ts:342
Parameters:
| Name | Type |
|---|---|
options |
NormalizedFullOptions |
Returns: IterableIterator‹TopLevelToken›
readFilter
▸ readFilter(): FilterToken | null
Defined in src/parser/tokenizer.ts:57
Returns: FilterToken | null
readFilterArg
▸ readFilterArg(): FilterArg | undefined
Defined in src/parser/tokenizer.ts:79
Returns: FilterArg | undefined
readFilters
▸ readFilters(): FilterToken[]
Defined in src/parser/tokenizer.ts:49
Returns: FilterToken[]
readHTMLToken
▸ readHTMLToken(stopStrings: string[]): HTMLToken
Defined in src/parser/tokenizer.ts:107
Parameters:
| Name | Type |
|---|---|
stopStrings |
string[] |
Returns: HTMLToken
readHash
▸ readHash(jekyllStyle?: undefined | false | true): HashToken | undefined
Defined in src/parser/tokenizer.ts:232
Parameters:
| Name | Type |
|---|---|
jekyllStyle? |
undefined | false | true |
Returns: HashToken | undefined
readHashes
▸ readHashes(jekyllStyle?: undefined | false | true): HashToken‹›[]
Defined in src/parser/tokenizer.ts:223
Parameters:
| Name | Type |
|---|---|
jekyllStyle? |
undefined | false | true |
Returns: HashToken‹›[]
readIdentifier
▸ readIdentifier(): IdentifierToken
Defined in src/parser/tokenizer.ts:209
Returns: IdentifierToken
readLiquidTagToken
▸ readLiquidTagToken(options: NormalizedFullOptions): LiquidTagToken
Defined in src/parser/tokenizer.ts:185
Parameters:
| Name | Type |
|---|---|
options |
NormalizedFullOptions |
Returns: LiquidTagToken
readLiquidTagTokens
▸ readLiquidTagTokens(options: NormalizedFullOptions): LiquidTagToken[]
Defined in src/parser/tokenizer.ts:176
Parameters:
| Name | Type | Default |
|---|---|---|
options |
NormalizedFullOptions | defaultOptions |
Returns: LiquidTagToken[]
readOperator
▸ readOperator(): OperatorToken | undefined
Defined in src/parser/tokenizer.ts:43
Returns: OperatorToken | undefined
readOutputToken
▸ readOutputToken(options: NormalizedFullOptions): OutputToken
Defined in src/parser/tokenizer.ts:139
Parameters:
| Name | Type | Default |
|---|---|---|
options |
NormalizedFullOptions | defaultOptions |
Returns: OutputToken
readQuoted
▸ readQuoted(): QuotedToken | undefined
Defined in src/parser/tokenizer.ts:327
Returns: QuotedToken | undefined
readRange
▸ readRange(): RangeToken | undefined
Defined in src/parser/tokenizer.ts:309
Returns: RangeToken | undefined
readTagName
▸ readTagName(): string
Defined in src/parser/tokenizer.ts:216
Returns: string
readTagToken
▸ readTagToken(options: NormalizedFullOptions): TagToken
Defined in src/parser/tokenizer.ts:116
Parameters:
| Name | Type | Default |
|---|---|---|
options |
NormalizedFullOptions | defaultOptions |
Returns: TagToken
readTo
▸ readTo(end: string): number
Defined in src/parser/tokenizer.ts:261
Parameters:
| Name | Type |
|---|---|
end |
string |
Returns: number
readToDelimiter
▸ readToDelimiter(delimiter: string): number
Defined in src/parser/tokenizer.ts:127
Parameters:
| Name | Type |
|---|---|
delimiter |
string |
Returns: number
readTopLevelToken
▸ readTopLevelToken(options: NormalizedFullOptions): TopLevelToken
Defined in src/parser/tokenizer.ts:99
Parameters:
| Name | Type |
|---|---|
options |
NormalizedFullOptions |
Returns: TopLevelToken
readTopLevelTokens
▸ readTopLevelTokens(options: NormalizedFullOptions): TopLevelToken[]
Defined in src/parser/tokenizer.ts:89
Parameters:
| Name | Type | Default |
|---|---|---|
options |
NormalizedFullOptions | defaultOptions |
Returns: TopLevelToken[]
readValue
▸ readValue(): ValueToken | undefined
Defined in src/parser/tokenizer.ts:269
Returns: ValueToken | undefined
readValueOrThrow
▸ readValueOrThrow(): ValueToken
Defined in src/parser/tokenizer.ts:321
Returns: ValueToken
readWord
▸ readWord(): IdentifierToken‹›
Defined in src/parser/tokenizer.ts:204
deprecated
Returns: IdentifierToken‹›
remaining
▸ remaining(): string
Defined in src/parser/tokenizer.ts:249
Returns: string
rmatch
▸ rmatch(pattern: string): boolean
Defined in src/parser/tokenizer.ts:361
Parameters:
| Name | Type |
|---|---|
pattern |
string |
Returns: boolean
skipBlank
▸ skipBlank(): void
Defined in src/parser/tokenizer.ts:376
Returns: void
snapshot
▸ snapshot(begin: number): string
Defined in src/parser/tokenizer.ts:197
Parameters:
| Name | Type | Default |
|---|---|---|
begin |
number | this.p |
Returns: string