Inline Parser#next_token to avoid method dispatch

lib/liquid/lexer.rb

@@ -23,20 +23,10 @@ module Liquid
       @output = []
 
       loop do
-        tok = next_token
-        unless tok
-          @output << [:end_of_string]
-          return @output
-        end
-        @output << tok
-      end
-    end
-
-    def next_token
         @ss.skip(/\s*/)
-      return if @ss.eos?
 
-      case
+        tok = case
+        when @ss.eos? then nil
         when t = @ss.scan(COMPARISON_OPERATOR) then [:comparison, t]
         when t = @ss.scan(SINGLE_STRING_LITERAL) then [:string, t]
         when t = @ss.scan(DOUBLE_STRING_LITERAL) then [:string, t]
@@ -44,18 +34,20 @@ module Liquid
         when t = @ss.scan(INTEGER_LITERAL) then [:integer, t]
         when t = @ss.scan(IDENTIFIER) then [:id, t]
         else
-        lex_specials
-      end
-    end
-
-    protected
-    def lex_specials
           c = @ss.getch
           if s = SPECIALS[c]
-        return [s,c]
+            [s,c]
-      end
+          else
-
             raise SyntaxError, "Unexpected character #{c}."
           end
         end
+
+        unless tok
+          @output << [:end_of_string]
+          return @output
+        end
+        @output << tok
+      end
+    end
+  end
 end

test/liquid/lexer_test.rb

@@ -45,11 +45,4 @@ class LexerTest < Test::Unit::TestCase
       Lexer.new("%").tokenize
     end
   end
-
-  def test_next_token
-    l = Lexer.new('hi 5.0')
-    assert_equal [:id, 'hi'], l.next_token
-    assert_equal [:float, '5.0'], l.next_token
-    assert_nil l.next_token
-  end
 end