Hierarchy
- Tokenizer
Constructors
constructor
+ new Tokenizer(input: string, operators: Operators, file?: undefined | string): Tokenizer
Defined in src/parser/tokenizer.ts:13
Parameters:
| Name | Type | Default |
|---|---|---|
input |
string | - |
operators |
Operators | defaultOptions.operators |
file? |
undefined | string | - |
Returns: Tokenizer
Properties
N
• N: number
Defined in src/parser/tokenizer.ts:11
Optional file
• file? : undefined | string
Defined in src/parser/tokenizer.ts:18
input
• input: string
Defined in src/parser/tokenizer.ts:16
Private opTrie
• opTrie: Trie
Defined in src/parser/tokenizer.ts:13
p
• p: number = 0
Defined in src/parser/tokenizer.ts:10
Private rawBeginAt
• rawBeginAt: number = -1
Defined in src/parser/tokenizer.ts:12
Methods
advance
▸ advance(i: number): void
Defined in src/parser/tokenizer.ts:255
Parameters:
| Name | Type | Default |
|---|---|---|
i |
number | 1 |
Returns: void
end
▸ end(): boolean
Defined in src/parser/tokenizer.ts:259
Returns: boolean
match
▸ match(word: string): boolean
Defined in src/parser/tokenizer.ts:356
Parameters:
| Name | Type |
|---|---|
word |
string |
Returns: boolean
mkError
▸ mkError(msg: string, begin: number): TokenizationError‹›
Defined in src/parser/tokenizer.ts:195
Parameters:
| Name | Type |
|---|---|
msg |
string |
begin |
number |
Returns: TokenizationError‹›
peek
▸ peek(n: number): string
Defined in src/parser/tokenizer.ts:374
Parameters:
| Name | Type | Default |
|---|---|---|
n |
number | 0 |
Returns: string
peekType
▸ peekType(n: number): number
Defined in src/parser/tokenizer.ts:370
Parameters:
| Name | Type | Default |
|---|---|---|
n |
number | 0 |
Returns: number
readEndrawOrRawContent
▸ readEndrawOrRawContent(options: NormalizedFullOptions): HTMLToken | TagToken
Defined in src/parser/tokenizer.ts:151
Parameters:
| Name | Type |
|---|---|
options |
NormalizedFullOptions |
readExpression
▸ readExpression(): Expression‹›
Defined in src/parser/tokenizer.ts:24
Returns: Expression‹›
readExpressionTokens
▸ readExpressionTokens(): IterableIterator‹Token›
Defined in src/parser/tokenizer.ts:28
Returns: IterableIterator‹Token›
readFileNameTemplate
▸ readFileNameTemplate(options: NormalizedFullOptions): IterableIterator‹TopLevelToken›
Defined in src/parser/tokenizer.ts:344
Parameters:
| Name | Type |
|---|---|
options |
NormalizedFullOptions |
Returns: IterableIterator‹TopLevelToken›
readFilter
▸ readFilter(): FilterToken | null
Defined in src/parser/tokenizer.ts:59
Returns: FilterToken | null
readFilterArg
▸ readFilterArg(): FilterArg | undefined
Defined in src/parser/tokenizer.ts:81
Returns: FilterArg | undefined
readFilters
▸ readFilters(): FilterToken[]
Defined in src/parser/tokenizer.ts:51
Returns: FilterToken[]
readHTMLToken
▸ readHTMLToken(stopStrings: string[]): HTMLToken
Defined in src/parser/tokenizer.ts:109
Parameters:
| Name | Type |
|---|---|
stopStrings |
string[] |
Returns: HTMLToken
readHash
▸ readHash(jekyllStyle?: undefined | false | true): HashToken | undefined
Defined in src/parser/tokenizer.ts:234
Parameters:
| Name | Type |
|---|---|
jekyllStyle? |
undefined | false | true |
Returns: HashToken | undefined
readHashes
▸ readHashes(jekyllStyle?: undefined | false | true): HashToken‹›[]
Defined in src/parser/tokenizer.ts:225
Parameters:
| Name | Type |
|---|---|
jekyllStyle? |
undefined | false | true |
Returns: HashToken‹›[]
readIdentifier
▸ readIdentifier(): IdentifierToken
Defined in src/parser/tokenizer.ts:211
Returns: IdentifierToken
readLiquidTagToken
▸ readLiquidTagToken(options: NormalizedFullOptions): LiquidTagToken
Defined in src/parser/tokenizer.ts:187
Parameters:
| Name | Type |
|---|---|
options |
NormalizedFullOptions |
Returns: LiquidTagToken
readLiquidTagTokens
▸ readLiquidTagTokens(options: NormalizedFullOptions): LiquidTagToken[]
Defined in src/parser/tokenizer.ts:178
Parameters:
| Name | Type | Default |
|---|---|---|
options |
NormalizedFullOptions | defaultOptions |
Returns: LiquidTagToken[]
readOperator
▸ readOperator(): OperatorToken | undefined
Defined in src/parser/tokenizer.ts:45
Returns: OperatorToken | undefined
readOutputToken
▸ readOutputToken(options: NormalizedFullOptions): OutputToken
Defined in src/parser/tokenizer.ts:141
Parameters:
| Name | Type | Default |
|---|---|---|
options |
NormalizedFullOptions | defaultOptions |
Returns: OutputToken
readQuoted
▸ readQuoted(): QuotedToken | undefined
Defined in src/parser/tokenizer.ts:329
Returns: QuotedToken | undefined
readRange
▸ readRange(): RangeToken | undefined
Defined in src/parser/tokenizer.ts:311
Returns: RangeToken | undefined
readTagName
▸ readTagName(): string
Defined in src/parser/tokenizer.ts:218
Returns: string
readTagToken
▸ readTagToken(options: NormalizedFullOptions): TagToken
Defined in src/parser/tokenizer.ts:118
Parameters:
| Name | Type | Default |
|---|---|---|
options |
NormalizedFullOptions | defaultOptions |
Returns: TagToken
readTo
▸ readTo(end: string): number
Defined in src/parser/tokenizer.ts:263
Parameters:
| Name | Type |
|---|---|
end |
string |
Returns: number
readToDelimiter
▸ readToDelimiter(delimiter: string): number
Defined in src/parser/tokenizer.ts:129
Parameters:
| Name | Type |
|---|---|
delimiter |
string |
Returns: number
readTopLevelToken
▸ readTopLevelToken(options: NormalizedFullOptions): TopLevelToken
Defined in src/parser/tokenizer.ts:101
Parameters:
| Name | Type |
|---|---|
options |
NormalizedFullOptions |
Returns: TopLevelToken
readTopLevelTokens
▸ readTopLevelTokens(options: NormalizedFullOptions): TopLevelToken[]
Defined in src/parser/tokenizer.ts:91
Parameters:
| Name | Type | Default |
|---|---|---|
options |
NormalizedFullOptions | defaultOptions |
Returns: TopLevelToken[]
readValue
▸ readValue(): ValueToken | undefined
Defined in src/parser/tokenizer.ts:271
Returns: ValueToken | undefined
readValueOrThrow
▸ readValueOrThrow(): ValueToken
Defined in src/parser/tokenizer.ts:323
Returns: ValueToken
readWord
▸ readWord(): IdentifierToken‹›
Defined in src/parser/tokenizer.ts:206
deprecated
Returns: IdentifierToken‹›
remaining
▸ remaining(): string
Defined in src/parser/tokenizer.ts:251
Returns: string
rmatch
▸ rmatch(pattern: string): boolean
Defined in src/parser/tokenizer.ts:363
Parameters:
| Name | Type |
|---|---|
pattern |
string |
Returns: boolean
skipBlank
▸ skipBlank(): void
Defined in src/parser/tokenizer.ts:378
Returns: void
snapshot
▸ snapshot(begin: number): string
Defined in src/parser/tokenizer.ts:199
Parameters:
| Name | Type | Default |
|---|---|---|
begin |
number | this.p |
Returns: string