refactor tokenizer to simulate original regex properly

lib/liquid/tokenizer.rb

@@ -14,9 +14,6 @@ module Liquid
     CLOSE_CURLEY = "}".ord
     PERCENTAGE = "%".ord
 
-    CLOSE_CURLEY_FOLLOWED_BY_CLOSE_CURLEY = (CLOSE_CURLEY << 8) | CLOSE_CURLEY
-    OPEN_CURLEY_FOLLOWED_BY_PERCENTAGE = (OPEN_CURLEY << 8) | PERCENTAGE
-
     def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
       @line_number    = line_number || (line_numbers ? 1 : nil)
       @for_liquid_tag = for_liquid_tag
@@ -107,7 +104,6 @@ module Liquid
     def next_variable_token
       start = @ss.pos - 2
 
-      # it is possible to see a {% before a }} so we need to check for that
       byte_a = @ss.scan_byte
       byte_b = byte_a
 
@@ -116,19 +112,28 @@ module Liquid
 
         break unless byte_a
 
+        if @ss.eos?
+          if byte_a == CLOSE_CURLEY
+            return @source.byteslice(start, @ss.pos - start)
+          else
+            break
+          end
+        end
+
         byte_b = @ss.scan_byte
 
-        if byte_b > CLOSE_CURLEY || (byte_b != CLOSE_CURLEY && byte_b != PERCENTAGE)
+        if byte_a == CLOSE_CURLEY
-          byte_a = byte_b
+          if byte_b == CLOSE_CURLEY
-          next
+            return @source.byteslice(start, @ss.pos - start)
-        end
+          elsif byte_b != CLOSE_CURLEY
-
+            @ss.pos -= 1
-        val = (byte_a << 8) | byte_b
+            return @source.byteslice(start, @ss.pos - start)
-        if val == CLOSE_CURLEY_FOLLOWED_BY_CLOSE_CURLEY
+          end
-          return @source.byteslice(start, @ss.pos - start)
+        elsif byte_a == OPEN_CURLEY && byte_b == PERCENTAGE
-        elsif val == OPEN_CURLEY_FOLLOWED_BY_PERCENTAGE
           return next_tag_token_with_start(start)
         end
+
+        byte_a = byte_b
       end
 
       "{{"

test/integration/tags/raw_tag_test.rb

@@ -16,6 +16,7 @@ class RawTagTest < Minitest::Test
     assert_template_result('>{{ test }}<', '> {%- raw -%}{{ test }}{%- endraw -%} <')
     assert_template_result("> inner  <", "> {%- raw -%} inner {%- endraw %} <")
     assert_template_result("> inner <", "> {%- raw -%} inner {%- endraw -%} <")
+    assert_template_result("{Hello}", "{% raw %}{{% endraw %}Hello{% raw %}}{% endraw %}")
   end
 
   def test_open_tag_in_raw

test/unit/tokenizer_unit_test.rb

@@ -31,6 +31,19 @@ class TokenizerTest < Minitest::Test
     assert_equal([1, 1, 3], tokenize_line_numbers(" {{\n funk \n}} "))
   end
 
+  def test_incomplete_curly_braces
+    assert_equal(["{{.}", " "], tokenize('{{.} '))
+    assert_equal(["{{}", "%}"], tokenize('{{}%}'))
+    assert_equal(["{{}}", "}"], tokenize('{{}}}'))
+  end
+
+  def test_unmatching_start_and_end
+    assert_equal(["{{%}"], tokenize('{{%}'))
+    assert_equal(["{{%%%}}"], tokenize('{{%%%}}'))
+    assert_equal(["{%", "}}"], tokenize('{%}}'))
+    assert_equal(["{%%}", "}"], tokenize('{%%}}'))
+  end
+
   private
 
   def new_tokenizer(source, parse_context: Liquid::ParseContext.new, start_line_number: nil)