diff --git a/lib/liquid/lexer.rb b/lib/liquid/lexer.rb index 228b998e..6f1fabe3 100644 --- a/lib/liquid/lexer.rb +++ b/lib/liquid/lexer.rb @@ -25,7 +25,7 @@ module Liquid } def initialize(input) - @input = input.chars.to_a + @input = input end def tokenize @@ -33,18 +33,26 @@ module Liquid @output = [] loop do - consume_whitespace - c = @input[@p] + tok = next_token + return @output unless tok + @output << tok + end + end - # are we out of input? - return @output unless c + def next_token + consume_whitespace + c = @input[@p] + return nil unless c - if identifier?(c) - @output << consume_identifier - elsif s = SPECIALS[c] - @output << Token[s] - @p += 1 - end + if identifier?(c) + identifier + elsif c == '"' || c == '\'' + string_literal + elsif s = SPECIALS[c] + @p += 1 + Token[s] + else + raise SyntaxError, "Unexpected character #{c}." end end @@ -56,19 +64,39 @@ module Liquid c =~ /^\s$/ end + def consume + c = @input[@p] + @p += 1 + c + end + def consume_whitespace while whitespace?(@input[@p]) @p += 1 end end - def consume_identifier + def identifier str = "" while identifier?(@input[@p]) str << @input[@p] @p += 1 end - Token[:identifier, str] + Token[:id, str] end + + def string_literal + quote = consume() + + start = @p + while @input[@p] != quote + @p += 1 + end + @p += 1 # closing quote + + Token[:string, @input[start..(@p-2)]] + end + + def number_literal end end