use String#split for liquid tag tokenization

This commit is contained in:
Michael Go
2024-11-01 17:11:10 -03:00
parent f49ceed826
commit 7ac53a3639
+1 -23
View File
@@ -84,9 +84,7 @@ module Liquid
def tokenize
if @for_liquid_tag
until @ss.eos?
@tokens << shift_liquid_tag
end
@tokens = @source.split("\n")
else
until @ss.eos?
@tokens << shift_normal
@@ -104,26 +102,6 @@ module Liquid
token
end
def shift_liquid_tag
token = next_liquid_token
return unless token
token
end
def next_liquid_token
# read until we find a \n
start = @ss.pos
if @ss.skip_until(NEWLINE).nil?
token = @ss.rest
@ss.terminate
return token
end
@source.byteslice(start, @ss.pos - start - 1)
end
def next_token
# possible states: :text, :tag, :variable
byte_a = @ss.peek_byte