Labyricorn/liquid
1
0
Fork 0
mirror of https://github.com/Shopify/liquid.git synced 2026-10-03 09:05:13 -07:00
Code Issues Packages Projects Releases Wiki Activity

passing tokenizer unit test

Browse Source
This commit is contained in:
Michael Go
2024-10-24 19:57:02 -03:00
parent b9ac58e31c
commit 9323a5f4ef
1 changed files with 24 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/liquid/tokenizer.rb
+24 -36
View File
@@ -8,24 +8,15 @@ module Liquid
@source = source @source = source
@line_number = line_number || (line_numbers ? 1 : nil) @line_number = line_number || (line_numbers ? 1 : nil)
@for_liquid_tag = for_liquid_tag @for_liquid_tag = for_liquid_tag
@offset = 0 @offset = 0
@state = :outside @state = :outside
@ss = StringScanner.new(source) @ss = StringScanner.new(source)
@n = 0
end end
def shift def shift
@n += 1
puts @ss.string if @n > 10000
puts @ss.pos if @n > 10000
return nil if @ss.eos? return nil if @ss.eos?
token = +"" token = take_new_next
if (t = take_new_next)
token << t
end
until @state == :outside || @ss.eos? until @state == :outside || @ss.eos?
token << if @state == :inside_variable || @state == :inside_tag token << if @state == :inside_variable || @state == :inside_tag
@@ -41,41 +32,37 @@ module Liquid
private private
def take_rest def take_rest
puts "take_rest: #{@state}"
xtake_rest
end
def take_new_next
puts "take_new_next: #{@state}"
xtake_new_next
end
def take_inside
puts "take_inside: #{@state}"
xtake_inside
end
def xtake_rest
token = @ss.rest token = @ss.rest
@ss.terminate @ss.terminate
@state = :outside @state = :outside
token token
end end
"{% foo %}" def take_new_next
def xtake_new_next
start_pos = @ss.pos start_pos = @ss.pos
if (n = @ss.skip_until(VariableStart)) if (n = @ss.skip_until(VariableStart))
@state = :inside_variable unless n == 2 if n == 2
end_pos = @ss.pos @state = :inside_variable
end_pos = @ss.pos
token = @source.byteslice(start_pos, end_pos - start_pos) token = @source.byteslice(start_pos, end_pos - start_pos)
else
@ss.pos -= 2
token = @source.byteslice(start_pos, n - 2)
end
@line_number += @for_liquid_tag ? 1 : token.count("\n") if @line_number @line_number += @for_liquid_tag ? 1 : token.count("\n") if @line_number
token token
elsif (n = @ss.skip_until(TagStart)) elsif (n = @ss.skip_until(TagStart))
@state = :inside_tag unless n == 2 if n == 2
end_pos = @ss.pos @state = :inside_tag
token = @source.byteslice(start_pos, end_pos - start_pos) end_pos = @ss.pos
token = @source.byteslice(start_pos, end_pos - start_pos)
else
@ss.pos -= 2
token = @source.byteslice(start_pos, n - 2)
end
@line_number += @for_liquid_tag ? 1 : token.count("\n") if @line_number @line_number += @for_liquid_tag ? 1 : token.count("\n") if @line_number
token token
else else
@@ -83,19 +70,20 @@ module Liquid
end end
end end
def xtake_inside def take_inside
start_pos = @ss.pos start_pos = @ss.pos
if @state == :inside_variable if @state == :inside_variable
if (n = @ss.skip_until(VariableEnd)) if (n = @ss.skip_until(VariableEnd))
@state = :outside @state = :outside
token = @source.byteslice(start_pos, start_pos + n) token = @source.byteslice(start_pos, n)
@line_number += @for_liquid_tag ? 1 : token.count("\n") if @line_number @line_number += @for_liquid_tag ? 1 : token.count("\n") if @line_number
token token
end end
elsif @state == :inside_tag elsif @state == :inside_tag
if (n = @ss.skip_until(TagEnd)) if (n = @ss.skip_until(TagEnd))
@state = :outside @state = :outside
token = @source.byteslice(start_pos, start_pos + n) token = @source.byteslice(start_pos, n)
@line_number += @for_liquid_tag ? 1 : token.count("\n") if @line_number @line_number += @for_liquid_tag ? 1 : token.count("\n") if @line_number
token token
end end