refactor Lexer to be static class function

lib/liquid/lexer.rb

@@ -25,38 +25,37 @@ module Liquid
     COMPARISON_OPERATOR   = /==|!=|<>|<=?|>=?|contains(?=\s)/
     WHITESPACE_OR_NOTHING = /\s*/
 
-    def initialize(input)
+    class << self
-      @ss = StringScanner.new(input)
+      def tokenize(input)
-    end
+        ss = StringScanner.new(input)
+        output = []
 
-    def tokenize
+        until ss.eos?
-      @output = []
+          ss.skip(WHITESPACE_OR_NOTHING)
-
+          break if ss.eos?
-      until @ss.eos?
+          tok      = if (t = ss.scan(COMPARISON_OPERATOR))
-        @ss.skip(WHITESPACE_OR_NOTHING)
+            [:comparison, t]
-        break if @ss.eos?
+          elsif (t = ss.scan(STRING_LITERAL))
-        tok      = if (t = @ss.scan(COMPARISON_OPERATOR))
+            [:string, t]
-          [:comparison, t]
+          elsif (t = ss.scan(NUMBER_LITERAL))
-        elsif (t = @ss.scan(STRING_LITERAL))
+            [:number, t]
-          [:string, t]
+          elsif (t = ss.scan(IDENTIFIER))
-        elsif (t = @ss.scan(NUMBER_LITERAL))
+            [:id, t]
-          [:number, t]
+          elsif (t = ss.scan(DOTDOT))
-        elsif (t = @ss.scan(IDENTIFIER))
+            [:dotdot, t]
-          [:id, t]
-        elsif (t = @ss.scan(DOTDOT))
-          [:dotdot, t]
-        else
-          c     = @ss.getch
-          if (s = SPECIALS[c])
-            [s, c]
           else
-            raise SyntaxError, "Unexpected character #{c}"
+            c     = ss.getch
+            if (s = SPECIALS[c])
+              [s, c]
+            else
+              raise SyntaxError, "Unexpected character #{c}"
+            end
           end
+          output << tok
         end
-        @output << tok
-      end
 
-      @output << [:end_of_string]
+        output << [:end_of_string]
+      end
     end
   end
 
@@ -157,82 +156,79 @@ module Liquid
       table.freeze
     end
 
-    def initialize(input)
-      @input = input
-    end
-
     # rubocop:disable Metrics/BlockNesting
-    def tokenize
+    class << self
-      ss = StringScannerPool.pop(@input)
+      def tokenize(input)
-      @output = []
+        ss = StringScannerPool.pop(input)
+        output = []
 
-      until ss.eos?
+        until ss.eos?
-        ss.skip(WHITESPACE_OR_NOTHING)
+          ss.skip(WHITESPACE_OR_NOTHING)
 
-        break if ss.eos?
+          break if ss.eos?
 
-        start_pos = ss.pos
+          start_pos = ss.pos
-        peeked = ss.peek_byte
+          peeked = ss.peek_byte
 
-        if (special = SPECIAL_TABLE[peeked])
+          if (special = SPECIAL_TABLE[peeked])
-          ss.scan_byte
-          # Special case for ".."
-          if special == DOT && ss.peek_byte == DOT_ORD
             ss.scan_byte
-            @output << DOTDOT
+            # Special case for ".."
-          elsif special == DASH
+            if special == DOT && ss.peek_byte == DOT_ORD
-            # Special case for negative numbers
+              ss.scan_byte
-            if (peeked_byte = ss.peek_byte) && NUMBER_TABLE[peeked_byte]
+              output << DOTDOT
-              ss.pos -= 1
+            elsif special == DASH
-              @output << [:number, ss.scan(NUMBER_LITERAL)]
+              # Special case for negative numbers
+              if (peeked_byte = ss.peek_byte) && NUMBER_TABLE[peeked_byte]
+                ss.pos -= 1
+                output << [:number, ss.scan(NUMBER_LITERAL)]
+              else
+                output << special
+              end
             else
-              @output << special
+              output << special
+            end
+          elsif (sub_table = TWO_CHARS_COMPARISON_JUMP_TABLE[peeked])
+            ss.scan_byte
+            if (peeked_byte = ss.peek_byte) && (found = sub_table[peeked_byte])
+              output << found
+              ss.scan_byte
+            else
+              raise_syntax_error(start_pos, ss)
+            end
+          elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
+            ss.scan_byte
+            if (peeked_byte = ss.peek_byte) && (found = sub_table[peeked_byte])
+              output << found
+              ss.scan_byte
+            else
+              output << SINGLE_COMPARISON_TOKENS[peeked]
             end
           else
-            @output << special
+            type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked]
-          end
-        elsif (sub_table = TWO_CHARS_COMPARISON_JUMP_TABLE[peeked])
-          ss.scan_byte
-          if (peeked_byte = ss.peek_byte) && (found = sub_table[peeked_byte])
-            @output << found
-            ss.scan_byte
-          else
-            raise_syntax_error(start_pos, ss)
-          end
-        elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
-          ss.scan_byte
-          if (peeked_byte = ss.peek_byte) && (found = sub_table[peeked_byte])
-            @output << found
-            ss.scan_byte
-          else
-            @output << SINGLE_COMPARISON_TOKENS[peeked]
-          end
-        else
-          type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked]
 
-          if type && (t = ss.scan(pattern))
+            if type && (t = ss.scan(pattern))
-            # Special case for "contains"
+              # Special case for "contains"
-            @output << if type == :id && t == "contains" && @output.last&.first != :dot
+              output << if type == :id && t == "contains" && output.last&.first != :dot
-              COMPARISON_CONTAINS
+                COMPARISON_CONTAINS
+              else
+                [type, t]
+              end
             else
-              [type, t]
+              raise_syntax_error(start_pos, ss)
             end
-          else
-            raise_syntax_error(start_pos, ss)
           end
         end
+        # rubocop:enable Metrics/BlockNesting
+        output << EOS
+      ensure
+        StringScannerPool.release(ss)
       end
-      # rubocop:enable Metrics/BlockNesting
 
-      @output << EOS
+      def raise_syntax_error(start_pos, ss)
-    ensure
+        ss.pos = start_pos
-      StringScannerPool.release(ss)
+        # the character could be a UTF-8 character, use getch to get all the bytes
-    end
+        raise SyntaxError, "Unexpected character #{ss.getch}"
-
+      end
-    def raise_syntax_error(start_pos, ss)
-      ss.pos = start_pos
-      # the character could be a UTF-8 character, use getch to get all the bytes
-      raise SyntaxError, "Unexpected character #{ss.getch}"
     end
   end
 

lib/liquid/parser.rb

@@ -3,8 +3,7 @@
 module Liquid
   class Parser
     def initialize(input)
-      l       = Lexer.new(input)
+      @tokens = Lexer.tokenize(input)
-      @tokens = l.tokenize
       @p      = 0 # pointer to current location
     end
 

lib/liquid/string_scanner_pool.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 module Liquid
   class StringScannerPool
     class << self
       def pop(input)
-        @ss_pool ||= [StringScanner.new("")] * 5
+        @ss_pool ||= 5.times.each_with_object([]) { |_i, arr| arr << StringScanner.new("") }
 
         if @ss_pool.empty?
           StringScanner.new(input)
@@ -14,7 +16,6 @@ module Liquid
       end
 
       def release(ss)
-        binding.irb if ss.nil?
         @ss_pool ||= []
         @ss_pool << ss
       end

test/unit/lexer_unit_test.rb

@@ -134,6 +134,6 @@ class LexerUnitTest < Minitest::Test
   private
 
   def tokenize(input)
-    Lexer.new(input).tokenize
+    Lexer.tokenize(input)
   end
 end