remove StringScanner version check in Lexer and Tokenizer

lib/liquid/lexer.rb

@@ -1,62 +1,7 @@
 # frozen_string_literal: true
 
 module Liquid
-  class Lexer1
+  class Lexer
-    SPECIALS = {
-      '|' => :pipe,
-      '.' => :dot,
-      ':' => :colon,
-      ',' => :comma,
-      '[' => :open_square,
-      ']' => :close_square,
-      '(' => :open_round,
-      ')' => :close_round,
-      '?' => :question,
-      '-' => :dash,
-    }.freeze
-    IDENTIFIER            = /[a-zA-Z_][\w-]*\??/
-    SINGLE_STRING_LITERAL = /'[^\']*'/
-    DOUBLE_STRING_LITERAL = /"[^\"]*"/
-    STRING_LITERAL        = Regexp.union(SINGLE_STRING_LITERAL, DOUBLE_STRING_LITERAL)
-    NUMBER_LITERAL        = /-?\d+(\.\d+)?/
-    DOTDOT                = /\.\./
-    COMPARISON_OPERATOR   = /==|!=|<>|<=?|>=?|contains(?=\s)/
-    WHITESPACE_OR_NOTHING = /\s*/
-
-    class << self
-      def tokenize(ss)
-        output = []
-
-        until ss.eos?
-          ss.skip(WHITESPACE_OR_NOTHING)
-          break if ss.eos?
-          tok      = if (t = ss.scan(COMPARISON_OPERATOR))
-            [:comparison, t]
-          elsif (t = ss.scan(STRING_LITERAL))
-            [:string, t]
-          elsif (t = ss.scan(NUMBER_LITERAL))
-            [:number, t]
-          elsif (t = ss.scan(IDENTIFIER))
-            [:id, t]
-          elsif (t = ss.scan(DOTDOT))
-            [:dotdot, t]
-          else
-            c     = ss.getch
-            if (s = SPECIALS[c])
-              [s, c]
-            else
-              raise SyntaxError, "Unexpected character #{c}"
-            end
-          end
-          output << tok
-        end
-
-        output << [:end_of_string]
-      end
-    end
-  end
-
-  class Lexer2
     CLOSE_ROUND = [:close_round, ")"].freeze
     CLOSE_SQUARE = [:close_square, "]"].freeze
     COLON = [:colon, ":"].freeze
@@ -225,7 +170,4 @@ module Liquid
       end
     end
   end
-
-  # Remove this once we can depend on strscan >= 3.1.1
-  Lexer = HAS_STRING_SCANNER_SCAN_BYTE ? Lexer2 : Lexer1
 end

lib/liquid/tokenizer.rb

@@ -3,55 +3,7 @@
 require "strscan"
 
 module Liquid
-  class Tokenizer1
+  class Tokenizer
-    attr_reader :line_number, :for_liquid_tag
-
-    def initialize(
-      source:,
-      string_scanner:, # this is not used
-      line_numbers: false,
-      line_number: nil,
-      for_liquid_tag: false
-    )
-      @source         = source.to_s.to_str
-      @line_number    = line_number || (line_numbers ? 1 : nil)
-      @for_liquid_tag = for_liquid_tag
-      @offset         = 0
-      @tokens         = tokenize
-    end
-
-    def shift
-      token = @tokens[@offset]
-      return nil unless token
-
-      @offset += 1
-
-      if @line_number
-        @line_number += @for_liquid_tag ? 1 : token.count("\n")
-      end
-
-      token
-    end
-
-    private
-
-    def tokenize
-      return [] if @source.empty?
-
-      return @source.split("\n") if @for_liquid_tag
-
-      tokens = @source.split(TemplateParser)
-
-      # removes the rogue empty element at the beginning of the array
-      if tokens[0]&.empty?
-        @offset += 1
-      end
-
-      tokens
-    end
-  end
-
-  class Tokenizer2
     attr_reader :line_number, :for_liquid_tag
 
     TAG_END = /%\}/
@@ -71,14 +23,15 @@ module Liquid
     )
       @line_number = line_number || (line_numbers ? 1 : nil)
       @for_liquid_tag = for_liquid_tag
-      @source = source
+      @source = source.to_s.to_str
       @offset = 0
       @tokens = []
 
-      @ss = string_scanner
+      if @source
-      @ss.string = @source
+        @ss = string_scanner
-
+        @ss.string = @source
-      tokenize
+        tokenize
+      end
     end
 
     def shift
@@ -199,7 +152,4 @@ module Liquid
       @source.byteslice(start, @ss.pos - start)
     end
   end
-
-  # Remove this once we can depend on strscan >= 3.1.1
-  Tokenizer = StringScanner.instance_methods.include?(:scan_byte) ? Tokenizer2 : Tokenizer1
 end

liquid.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |s|
 
   s.require_path = "lib"
 
-  s.add_dependency("strscan")
+  s.add_dependency("strscan", ">= 3.1.1")
   s.add_dependency("bigdecimal")
   s.add_dependency("lru_redux")