Labyricorn/liquid
1
0
Fork 0
mirror of https://github.com/Shopify/liquid.git synced 2026-09-17 18:00:41 -07:00
Code Issues Packages Projects Releases Wiki Activity

Initial concept

Browse Source
This commit is contained in:
Mike Angell
2019-09-20 15:35:12 +10:00
parent f4d134cd5c
commit 3ffae9a6e4
7 changed files with 190 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/liquid/tokenizer.rb
+76 -1
View File
@@ -28,12 +28,87 @@ module Liquid
return @source.split("\n") if @for_liquid_tag
tokens = @source.split(TemplateParser)
tokens = tokenize_new(@source)
# tokens = @source.split(TemplateParser)
# removes the rogue empty element at the beginning of the array
tokens.shift if tokens[0]&.empty?
tokens
end
T_TAG_OPEN = "{%"
T_VAR_OPEN = "{{"
T_SIN_QUOT = "'"
T_DOU_QUOT = '"'
T_TAG_CLOS = "%}"
T_VAR_CLOS = "}}"
T_VAR_CLO2 = "}"
S_NIL = 0
S_TAG = 1
S_VAR = 2
S_TAG_SIN = 3
S_TAG_DOU = 4
S_VAR_SIN = 5
S_VAR_DOU = 6
def tokenize_new(source)
output = []
s = S_NIL
current = +""
source.split(/({%|{{|"|'|}}|%}|})/om).each do |t|
if t == T_TAG_OPEN && s <= S_VAR
s = S_TAG
output << current
current = t
elsif t == T_VAR_OPEN && s <= S_VAR
s = S_VAR
output << current
current = t
elsif t == T_VAR_OPEN && s == S_TAG
s = S_VAR
current += t
elsif t == T_SIN_QUOT && s == S_TAG
s = S_TAG_SIN
current += t
elsif t == T_SIN_QUOT && s == S_TAG_SIN
s = S_TAG
current += t
elsif t == T_DOU_QUOT && s == S_TAG
s = S_TAG_DOU
current += t
elsif t == T_DOU_QUOT && s == S_TAG_DOU
s = S_TAG
current += t
elsif t == T_SIN_QUOT && s == S_VAR
s = S_VAR_SIN
current += t
elsif t == T_SIN_QUOT && s == S_VAR_SIN
s = S_VAR
current += t
elsif t == T_DOU_QUOT && s == S_VAR
s = S_VAR_DOU
current += t
elsif t == T_DOU_QUOT && s == S_VAR_DOU
s = S_VAR
current += t
elsif t == T_TAG_CLOS && s == S_TAG
s = S_NIL
current += t
output << current
current = +""
elsif (t == T_VAR_CLOS || t == T_VAR_CLO2) && s == S_VAR
s = S_NIL
current += t
output << current
current = +""
else
current += t
end
end
output << current unless current == ""
output
end
end
end