mirror of
https://github.com/Shopify/liquid.git
synced 2026-09-16 01:10:41 -07:00
Dylan Thacker-Smith
e20df18eca
It transforms liquid code to equivalently parsed code, that doesn't depend on lax parsing quirks. Note that this transformation can hide the intent of the original code, so it is best to review the changes, since removed syntax often would indicate a bug in the original liquid code.
50 lines
1006 B
Ruby
50 lines
1006 B
Ruby
# frozen_string_literal: true
|
|
|
|
module Liquid
|
|
class Tokenizer
|
|
attr_reader :line_number, :for_liquid_tag
|
|
|
|
def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
|
|
@source = source.to_s.to_str
|
|
@line_number = line_number || (line_numbers ? 1 : nil)
|
|
@for_liquid_tag = for_liquid_tag
|
|
@offset = 0
|
|
@tokens = tokenize
|
|
end
|
|
|
|
def shift
|
|
token = @tokens[@offset]
|
|
return nil unless token
|
|
|
|
@offset += 1
|
|
|
|
if @line_number
|
|
@line_number += @for_liquid_tag ? 1 : token.count("\n")
|
|
end
|
|
|
|
token
|
|
end
|
|
|
|
def more?
|
|
@offset < @tokens.length
|
|
end
|
|
|
|
private
|
|
|
|
def tokenize
|
|
return [] if @source.empty?
|
|
|
|
return @source.split("\n") if @for_liquid_tag
|
|
|
|
tokens = @source.split(TemplateParser)
|
|
|
|
# removes the rogue empty element at the beginning of the array
|
|
if tokens[0]&.empty?
|
|
@offset += 1
|
|
end
|
|
|
|
tokens
|
|
end
|
|
end
|
|
end
|