Class: Tokenizer

Hierarchy

  • Tokenizer

Constructors

constructor

+ new Tokenizer(input: string, operators: Operators, file?: undefined | string): Tokenizer

Defined in src/parser/tokenizer.ts:13

Parameters:

Name Type Default
input string -
operators Operators defaultOptions.operators
file? undefined | string -

Returns: Tokenizer

Properties

N

• N: number

Defined in src/parser/tokenizer.ts:11

Optional file

• file? : undefined | string

Defined in src/parser/tokenizer.ts:18

input

• input: string

Defined in src/parser/tokenizer.ts:16

Private opTrie

• opTrie: Trie

Defined in src/parser/tokenizer.ts:13

p

• p: number = 0

Defined in src/parser/tokenizer.ts:10

Private rawBeginAt

• rawBeginAt: number = -1

Defined in src/parser/tokenizer.ts:12

Methods

advance

▸ advance(i: number): void

Defined in src/parser/tokenizer.ts:255

Parameters:

Name Type Default
i number 1

Returns: void

end

▸ end(): boolean

Defined in src/parser/tokenizer.ts:259

Returns: boolean

match

▸ match(word: string): boolean

Defined in src/parser/tokenizer.ts:356

Parameters:

Name Type
word string

Returns: boolean

mkError

▸ mkError(msg: string, begin: number): TokenizationError‹›

Defined in src/parser/tokenizer.ts:195

Parameters:

Name Type
msg string
begin number

Returns: TokenizationError‹›

peek

▸ peek(n: number): string

Defined in src/parser/tokenizer.ts:374

Parameters:

Name Type Default
n number 0

Returns: string

peekType

▸ peekType(n: number): number

Defined in src/parser/tokenizer.ts:370

Parameters:

Name Type Default
n number 0

Returns: number

readEndrawOrRawContent

▸ readEndrawOrRawContent(options: NormalizedFullOptions): HTMLToken | TagToken

Defined in src/parser/tokenizer.ts:151

Parameters:

Name Type
options NormalizedFullOptions

Returns: HTMLToken | TagToken

readExpression

▸ readExpression(): Expression‹›

Defined in src/parser/tokenizer.ts:24

Returns: Expression‹›

readExpressionTokens

▸ readExpressionTokens(): IterableIterator‹Token›

Defined in src/parser/tokenizer.ts:28

Returns: IterableIterator‹Token›

readFileNameTemplate

▸ readFileNameTemplate(options: NormalizedFullOptions): IterableIterator‹TopLevelToken›

Defined in src/parser/tokenizer.ts:344

Parameters:

Name Type
options NormalizedFullOptions

Returns: IterableIterator‹TopLevelToken›

readFilter

▸ readFilter(): FilterToken | null

Defined in src/parser/tokenizer.ts:59

Returns: FilterToken | null

readFilterArg

▸ readFilterArg(): FilterArg | undefined

Defined in src/parser/tokenizer.ts:81

Returns: FilterArg | undefined

readFilters

▸ readFilters(): FilterToken[]

Defined in src/parser/tokenizer.ts:51

Returns: FilterToken[]

readHTMLToken

▸ readHTMLToken(stopStrings: string[]): HTMLToken

Defined in src/parser/tokenizer.ts:109

Parameters:

Name Type
stopStrings string[]

Returns: HTMLToken

readHash

▸ readHash(jekyllStyle?: undefined | false | true): HashToken | undefined

Defined in src/parser/tokenizer.ts:234

Parameters:

Name Type
jekyllStyle? undefined | false | true

Returns: HashToken | undefined

readHashes

▸ readHashes(jekyllStyle?: undefined | false | true): HashToken‹›[]

Defined in src/parser/tokenizer.ts:225

Parameters:

Name Type
jekyllStyle? undefined | false | true

Returns: HashToken‹›[]

readIdentifier

▸ readIdentifier(): IdentifierToken

Defined in src/parser/tokenizer.ts:211

Returns: IdentifierToken

readLiquidTagToken

▸ readLiquidTagToken(options: NormalizedFullOptions): LiquidTagToken

Defined in src/parser/tokenizer.ts:187

Parameters:

Name Type
options NormalizedFullOptions

Returns: LiquidTagToken

readLiquidTagTokens

▸ readLiquidTagTokens(options: NormalizedFullOptions): LiquidTagToken[]

Defined in src/parser/tokenizer.ts:178

Parameters:

Name Type Default
options NormalizedFullOptions defaultOptions

Returns: LiquidTagToken[]

readOperator

▸ readOperator(): OperatorToken | undefined

Defined in src/parser/tokenizer.ts:45

Returns: OperatorToken | undefined

readOutputToken

▸ readOutputToken(options: NormalizedFullOptions): OutputToken

Defined in src/parser/tokenizer.ts:141

Parameters:

Name Type Default
options NormalizedFullOptions defaultOptions

Returns: OutputToken

readQuoted

▸ readQuoted(): QuotedToken | undefined

Defined in src/parser/tokenizer.ts:329

Returns: QuotedToken | undefined

readRange

▸ readRange(): RangeToken | undefined

Defined in src/parser/tokenizer.ts:311

Returns: RangeToken | undefined

readTagName

▸ readTagName(): string

Defined in src/parser/tokenizer.ts:218

Returns: string

readTagToken

▸ readTagToken(options: NormalizedFullOptions): TagToken

Defined in src/parser/tokenizer.ts:118

Parameters:

Name Type Default
options NormalizedFullOptions defaultOptions

Returns: TagToken

readTo

▸ readTo(end: string): number

Defined in src/parser/tokenizer.ts:263

Parameters:

Name Type
end string

Returns: number

readToDelimiter

▸ readToDelimiter(delimiter: string): number

Defined in src/parser/tokenizer.ts:129

Parameters:

Name Type
delimiter string

Returns: number

readTopLevelToken

▸ readTopLevelToken(options: NormalizedFullOptions): TopLevelToken

Defined in src/parser/tokenizer.ts:101

Parameters:

Name Type
options NormalizedFullOptions

Returns: TopLevelToken

readTopLevelTokens

▸ readTopLevelTokens(options: NormalizedFullOptions): TopLevelToken[]

Defined in src/parser/tokenizer.ts:91

Parameters:

Name Type Default
options NormalizedFullOptions defaultOptions

Returns: TopLevelToken[]

readValue

▸ readValue(): ValueToken | undefined

Defined in src/parser/tokenizer.ts:271

Returns: ValueToken | undefined

readValueOrThrow

▸ readValueOrThrow(): ValueToken

Defined in src/parser/tokenizer.ts:323

Returns: ValueToken

readWord

▸ readWord(): IdentifierToken‹›

Defined in src/parser/tokenizer.ts:206

deprecated

Returns: IdentifierToken‹›

remaining

▸ remaining(): string

Defined in src/parser/tokenizer.ts:251

Returns: string

rmatch

▸ rmatch(pattern: string): boolean

Defined in src/parser/tokenizer.ts:363

Parameters:

Name Type
pattern string

Returns: boolean

skipBlank

▸ skipBlank(): void

Defined in src/parser/tokenizer.ts:378

Returns: void

snapshot

▸ snapshot(begin: number): string

Defined in src/parser/tokenizer.ts:199

Parameters:

Name Type Default
begin number this.p

Returns: string

Next