diff --git a/lib/liquid/tokenizer.rb b/lib/liquid/tokenizer.rb index a6f378a4..dcd001f8 100644 --- a/lib/liquid/tokenizer.rb +++ b/lib/liquid/tokenizer.rb @@ -84,9 +84,7 @@ module Liquid def tokenize if @for_liquid_tag - until @ss.eos? - @tokens << shift_liquid_tag - end + @tokens = @source.split("\n") else until @ss.eos? @tokens << shift_normal @@ -104,26 +102,6 @@ module Liquid token end - def shift_liquid_tag - token = next_liquid_token - - return unless token - - token - end - - def next_liquid_token - # read until we find a \n - start = @ss.pos - if @ss.skip_until(NEWLINE).nil? - token = @ss.rest - @ss.terminate - return token - end - - @source.byteslice(start, @ss.pos - start - 1) - end - def next_token # possible states: :text, :tag, :variable byte_a = @ss.peek_byte