convert template source to string to ensure encoding validity

check template UTF8 validity before parsing

README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://api.travis-ci.org/Shopify/liquid.svg?branch=master)](http://travis-ci.org/Shopify/liquid)
+[![Build status](https://github.com/Shopify/liquid/actions/workflows/liquid.yml/badge.svg)](https://github.com/Shopify/liquid/actions/workflows/liquid.yml)
 [![Inline docs](http://inch-ci.org/github/Shopify/liquid.svg?branch=master)](http://inch-ci.org/github/Shopify/liquid)
 
 # Liquid template engine

lib/liquid/block_body.rb

@@ -6,6 +6,7 @@ module Liquid
   class BlockBody
     LiquidTagToken      = /\A\s*(#{TagName})\s*(.*?)\z/o
     FullToken           = /\A#{TagStart}#{WhitespaceControl}?(\s*)(#{TagName})(\s*)(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
+    FullTokenPossiblyInvalid = /\A(.*)#{TagStart}#{WhitespaceControl}?\s*(\w+)\s*(.*)?#{WhitespaceControl}?#{TagEnd}\z/om
     ContentOfVariable   = /\A#{VariableStart}#{WhitespaceControl}?(.*?)#{WhitespaceControl}?#{VariableEnd}\z/om
     WhitespaceOrNothing = /\A\s*\z/
     TAGSTART            = "{%"

lib/liquid/condition.rb

@@ -112,8 +112,6 @@ module Liquid
       if left.is_a?(MethodLiteral)
         if right.respond_to?(left.method_name)
           return right.send(left.method_name)
-        elsif (res = fallback_simulation_of_active_support(left.method_name, right))
-          return res
         else
           return nil
         end
@@ -122,8 +120,6 @@ module Liquid
       if right.is_a?(MethodLiteral)
         if left.respond_to?(right.method_name)
           return left.send(right.method_name)
-        elsif (res = fallback_simulation_of_active_support(right.method_name, left))
-          return res
         else
           return nil
         end
@@ -132,17 +128,6 @@ module Liquid
       left == right
     end
 
-    # ActiveSupport creates #blank? as an alias for #empty? on Hash and Array.
-    # Without this simulation, [] == blank behaves differently when AS is loaded vs. not.
-    def fallback_simulation_of_active_support(method_name, obj)
-      return nil unless method_name == :blank?
-
-      case obj
-      when Array, Hash
-        obj.empty?
-      end # else nil
-    end
-
     def interpret_condition(left, right, op, context)
       # If the operator is empty this means that the decision statement is just
       # a single variable. We can just poll this variable from the context and

lib/liquid/locales/en.yml

@@ -15,6 +15,7 @@
       include: "Error in tag 'include' - Valid syntax: include '[template]' (with|for) [object|collection]"
       inline_comment_invalid: "Syntax error in tag '#' - Each line of comments must be prefixed by the '#' character"
       invalid_delimiter: "'%{tag}' is not a valid delimiter for %{block_name} tags. use %{block_delimiter}"
+      invalid_template_encoding: "Invalid template encoding"
       render: "Syntax error in tag 'render' - Template name must be a quoted string"
       table_row: "Syntax Error in 'table_row loop' - Valid syntax: table_row [item] in [collection] cols=3"
       tag_never_closed: "'%{block_name}' tag was never closed"

lib/liquid/standardfilters.rb

@@ -29,6 +29,19 @@ module Liquid
     )
     STRIP_HTML_TAGS = /<.*?>/m
 
+    class << self
+      def try_coerce_encoding(input, encoding:)
+        original_encoding = input.encoding
+        if input.encoding != encoding
+          input.force_encoding(encoding)
+          unless input.valid_encoding?
+            input.force_encoding(original_encoding)
+          end
+        end
+        input
+      end
+    end
+
     # @liquid_public_docs
     # @liquid_type filter
     # @liquid_category array
@@ -150,7 +163,8 @@ module Liquid
     # @liquid_syntax string | base64_decode
     # @liquid_return [string]
     def base64_decode(input)
-      Base64.strict_decode64(input.to_s)
+      input = input.to_s
+      StandardFilters.try_coerce_encoding(Base64.strict_decode64(input), encoding: input.encoding)
     rescue ::ArgumentError
       raise Liquid::ArgumentError, "invalid base64 provided to base64_decode"
     end
@@ -174,7 +188,8 @@ module Liquid
     # @liquid_syntax string | base64_url_safe_decode
     # @liquid_return [string]
     def base64_url_safe_decode(input)
-      Base64.urlsafe_decode64(input.to_s)
+      input = input.to_s
+      StandardFilters.try_coerce_encoding(Base64.urlsafe_decode64(input), encoding: input.encoding)
     rescue ::ArgumentError
       raise Liquid::ArgumentError, "invalid base64 provided to base64_url_safe_decode"
     end
@@ -892,9 +907,11 @@ module Liquid
         raise_property_error(property)
       end
 
-      InputIterator.new(values_for_sum, context).sum do |item|
+      result = InputIterator.new(values_for_sum, context).sum do |item|
         Utils.to_number(item)
       end
+
+      result.is_a?(BigDecimal) ? result.to_f : result
     end
 
     private

lib/liquid/tag.rb

@@ -54,7 +54,8 @@ module Liquid
     # of the `render_to_output_buffer` method will become the default and the `render`
     # method will be removed.
     def render_to_output_buffer(context, output)
-      output << render(context)
+      render_result = render(context)
+      output << render_result if render_result
       output
     end
 

lib/liquid/tags/comment.rb

@@ -25,6 +25,65 @@ module Liquid
     def blank?
       true
     end
+
+    private
+
+    def parse_body(body, tokenizer)
+      if parse_context.depth >= MAX_DEPTH
+        raise StackLevelError, "Nesting too deep"
+      end
+
+      parse_context.depth += 1
+      comment_tag_depth = 1
+
+      begin
+        # Consume tokens without creating child nodes.
+        # The children tag doesn't require to be a valid Liquid except the comment and raw tag.
+        # The child comment and raw tag must be closed.
+        while (token = tokenizer.send(:shift))
+          tag_name = if tokenizer.for_liquid_tag
+            next if token.empty? || token.match?(BlockBody::WhitespaceOrNothing)
+
+            tag_name_match = BlockBody::LiquidTagToken.match(token)
+
+            next if tag_name_match.nil?
+
+            tag_name_match[1]
+          else
+            token =~ BlockBody::FullToken
+            Regexp.last_match(2)
+          end
+
+          case tag_name
+          when "raw"
+            parse_raw_tag_body(tokenizer)
+          when "comment"
+            comment_tag_depth += 1
+          when "endcomment"
+            comment_tag_depth -= 1
+          end
+
+          if comment_tag_depth.zero?
+            parse_context.trim_whitespace = (token[-3] == WhitespaceControl) unless tokenizer.for_liquid_tag
+            return false
+          end
+        end
+
+        raise_tag_never_closed(block_name)
+      ensure
+        parse_context.depth -= 1
+      end
+
+      false
+    end
+
+    def parse_raw_tag_body(tokenizer)
+      while (token = tokenizer.send(:shift))
+        return if token =~ BlockBody::FullTokenPossiblyInvalid && "endraw" == Regexp.last_match(2)
+      end
+
+      raise_tag_never_closed("raw")
+    end
   end
 
   Template.register_tag('comment', Comment)

lib/liquid/tags/raw.rb

@@ -14,7 +14,6 @@ module Liquid
   # @liquid_syntax_keyword expression The expression to be output without being rendered.
   class Raw < Block
     Syntax = /\A\s*\z/
-    FullTokenPossiblyInvalid = /\A(.*)#{TagStart}#{WhitespaceControl}?\s*(\w+)\s*(.*)?#{WhitespaceControl}?#{TagEnd}\z/om
 
     def initialize(tag_name, markup, parse_context)
       super
@@ -25,7 +24,7 @@ module Liquid
     def parse(tokens)
       @body = +''
       while (token = tokens.shift)
-        if token =~ FullTokenPossiblyInvalid && block_delimiter == Regexp.last_match(2)
+        if token =~ BlockBody::FullTokenPossiblyInvalid && block_delimiter == Regexp.last_match(2)
           parse_context.trim_whitespace = (token[-3] == WhitespaceControl)
           @body << Regexp.last_match(1) if Regexp.last_match(1) != ""
           return

lib/liquid/template.rb

@@ -107,6 +107,12 @@ module Liquid
     # Returns self for easy chaining
     def parse(source, options = {})
       parse_context = configure_options(options)
+      source = source.to_s.to_str
+
+      unless source.valid_encoding?
+        raise SyntaxError, parse_context.locale.t("errors.syntax.invalid_template_encoding")
+      end
+
       tokenizer     = parse_context.new_tokenizer(source, start_line_number: @line_numbers && 1)
       @root         = Document.parse(tokenizer, parse_context)
       self

lib/liquid/tokenizer.rb

@@ -5,7 +5,7 @@ module Liquid
     attr_reader :line_number, :for_liquid_tag
 
     def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
-      @source         = source.to_s.to_str
+      @source         = source
       @line_number    = line_number || (line_numbers ? 1 : nil)
       @for_liquid_tag = for_liquid_tag
       @offset         = 0

test/integration/array_test.rb

@@ -1,103 +0,0 @@
-# frozen_string_literal: true
-
-require 'test_helper'
-
-class ArrayTest < Minitest::Test
-  include Liquid
-
-  def test_array_size
-    assert_template_result("0", "{{a.size}}", { 'a' => [] })
-    assert_template_result("2", "{{a.size}}", { 'a' => [1, 2] })
-  end
-
-  def test_array_first
-    assert_template_result("", "{{a.first}}", { 'a' => [] })
-    assert_template_result("1", "{{a.first}}", { 'a' => [1, 2] })
-  end
-
-  def test_array_last
-    assert_template_result("", "{{a.last}}", { 'a' => [] })
-    assert_template_result("2", "{{a.last}}", { 'a' => [1, 2] })
-  end
-
-  def test_array_index
-    assert_template_result("", "{{a[1]}}", { 'a' => [] })
-    assert_template_result("2", "{{a[1]}}", { 'a' => [1, 2] })
-  end
-
-  def test_negative_array_index
-    assert_template_result("3", "{{a[-1]}}", { 'a' => [1, 2, 3] })
-    assert_template_result("1", "{{a[-3]}}", { 'a' => [1, 2, 3] })
-    assert_template_result("", "{{a[-4]}}", { 'a' => [1, 2, 3] })
-  end
-
-  def test_array_to_s
-    arr = ["a", ["b", 1], "c"]
-    assert_template_result("ab1c", "{{a}}", { 'a' => arr })
-  end
-
-  def test_auto_methods
-    a1 = []
-    a2 = [1, 2, 3, "abc"]
-    assert_template_result("0,nonblank", "{{a.size}},{%if a['size'] == blank%}blank{%else%}nonblank{%endif%}", { 'a' => a1 })
-    assert_template_result("4,nonblank", "{{a.size}},{%if a['size'] == blank%}blank{%else%}nonblank{%endif%}", { 'a' => a2 })
-    assert_template_result(",", "{{a.first}},{{a['first']}}", { 'a' => a1 })
-    assert_template_result("1,", "{{a.first}},{{a['first']}}", { 'a' => a2 })
-    assert_template_result(",", "{{a.last}},{{a['last']}}", { 'a' => a1 })
-    assert_template_result("abc,", "{{a.last}},{{a['last']}}", { 'a' => a2 })
-  end
-
-  def test_array_equality
-    a0 = []
-    a1 = [1]
-    a2 = [1, 2]
-    a3 = [1]
-
-    arrays = [a0, a1, a2, a3]
-    arrays.each_with_index do |a, i|
-      arrays.each_with_index do |b, j|
-        prefix = "(#{i},#{j})"
-        assert_template_result(
-          prefix + (a == b ? "y" : "n"),
-          "#{prefix}{% if a == b %}y{%else%}n{%endif%}",
-          { "a" => a, "b" => b },
-        )
-      end
-    end
-  end
-
-  def test_contains
-    a0 = [1, "a", ["b", "c"], true]
-    a1 = ["b", "c"]
-    a2 = ["B", "C"]
-
-    tpl = "{%if a contains b%}y{%else%}n{%endif%}"
-
-    assert_template_result("y", tpl, { "a" => a0, "b" => 1 })
-    assert_template_result("n", tpl, { "a" => a0, "b" => 2 })
-    assert_template_result("y", tpl, { "a" => a0, "b" => a1 })
-    assert_template_result("n", tpl, { "a" => a0, "b" => a2 })
-  end
-
-  def test_empty
-    assert_template_result("y", "{%if a == empty%}y{%else%}n{%endif%}", { 'a' => [] })
-    assert_template_result("n", "{%if a == empty%}y{%else%}n{%endif%}", { 'a' => [1, 2] })
-  end
-
-  def test_blank
-    assert_template_result("y", "{%if a == blank%}y{%else%}n{%endif%}", { 'a' => [] })
-    assert_template_result("n", "{%if a == blank%}y{%else%}n{%endif%}", { 'a' => [1, 2] })
-  end
-
-  def test_iter
-    a0 = [1, "a"]
-    assert_template_result("1a", "{%for i in a%}{{i}}{%endfor%}", { 'a' => a0 })
-  end
-
-  def test_output
-    a0 = []
-    a1 = [1, "a", ["b", "c"], true]
-    assert_template_result("", "{{a}}", { 'a' => a0 })
-    assert_template_result("1abctrue", "{{a}}", { 'a' => a1 })
-  end
-end

test/integration/hash_test.rb

@@ -1,223 +0,0 @@
-# frozen_string_literal: true
-
-require "test_helper"
-require "json"
-
-class HashTest < Minitest::Test
-  include Liquid
-
-  class HashSubclass < Hash
-    def []=(key, value)
-      super(fmt_key(key), value)
-    end
-
-    def [](key)
-      super(fmt_key(key))
-    end
-
-    def key?(key)
-      super(fmt_key(key))
-    end
-
-    def include?(key)
-      super(fmt_key(key))
-    end
-
-    def to_s
-      super.upcase
-    end
-
-    private
-
-    def fmt_key(key)
-      return key unless key.is_a?(String)
-
-      key.upcase
-    end
-  end
-
-  def test_size
-    assert_template_result("0", "{{h.size}}", { "h" => {} })
-    assert_template_result("1", "{{h.size}}", { "h" => { "a" => 1 } })
-  end
-
-  def test_first
-    assert_template_result("", "{{h.first}}", { "h" => {} })
-    assert_template_result("a1", "{{h.first}}", { "h" => { "a" => 1 } })
-  end
-
-  def test_last
-    assert_template_result("", "{{h.last}}", { "h" => { "a" => 1 } })
-  end
-
-  def test_index
-    assert_template_result("", "{{h['a']}}", { "h" => {} })
-    assert_template_result("1", "{{h['a']}}", { "h" => { "a" => 1 } })
-  end
-
-  def test_to_s
-    h = { "a" => 1, "b" => [1, { "c" => 2 }] }
-    assert_template_result("{\"a\"=>1, \"b\"=>[1, {\"c\"=>2}]}", "{{h}}", { "h" => h })
-  end
-
-  def test_auto_methods
-    h1 = { "a" => 1 }
-    h2 = { "last" => 10, "first" => 11, "size" => 12 }
-    assert_template_result("1,nonblank", "{{h.size}},{%if h['size'] == blank%}blank{%else%}nonblank{%endif%}", { "h" => h1 })
-    assert_template_result("12,nonblank", "{{h.size}},{%if h['size'] == blank%}blank{%else%}nonblank{%endif%}", { "h" => h2 })
-    assert_template_result("a1,", "{{h.first}},{{h['first']}}", { "h" => h1 })
-    assert_template_result("11,11", "{{h.first}},{{h['first']}}", { "h" => h2 })
-    assert_template_result(",", "{{h.last}},{{h['last']}}", { "h" => h1 })
-    assert_template_result("10,10", "{{h.last}},{{h['last']}}", { "h" => h2 })
-  end
-
-  def test_equality
-    h0 = {}
-    h1 = { "a" => 1 }
-    h2 = { "a" => 1, "b" => 2 }
-    h3 = { "a" => 1 }
-
-    hashes = [h0, h1, h2, h3]
-    hashes.each_with_index do |a, i|
-      hashes.each_with_index do |b, j|
-        prefix = "(#{i},#{j})"
-        assert_template_result(
-          prefix + (a == b ? "y" : "n"),
-          "#{prefix}{% if a == b %}y{%else%}n{%endif%}",
-          { "a" => a, "b" => b },
-        )
-      end
-    end
-  end
-
-  def test_contains
-    h = { "a" => 1, "b" => { "c" => "d" }, "3" => ["f", 2] }
-
-    tpl = "{%if h contains b%}y{%else%}n{%endif%}"
-
-    assert_template_result("y", tpl, { "h" => h, "b" => "a" })
-    # assert_template_result("y", tpl, h, { "h" => { "b" => "a".html_safe } })
-    assert_template_result("y", tpl, { "h" => h, "b" => "a".b })
-    assert_template_result("n", tpl, { "h" => h, "b" => "A" })
-    # assert_template_result("n", tpl, h, { "h" => { "b" => "A".html_safe } })
-    assert_template_result("n", tpl, { "h" => h, "b" => "A".b })
-
-    assert_template_result("n", tpl, { "h" => h, "b" => 1 })
-    assert_template_result("n", tpl, { "h" => h, "b" => "1" })
-
-    assert_template_result("n", tpl, { "h" => h, "b" => 3 })
-    assert_template_result("y", tpl, { "h" => h, "b" => "3" })
-  end
-
-  def test_empty
-    assert_template_result("y", "{%if h == empty%}y{%else%}n{%endif%}", { "h" => {} })
-    assert_template_result("n", "{%if h == empty%}y{%else%}n{%endif%}", { "h" => { "a" => 1 } })
-  end
-
-  def test_blank
-    assert_template_result("y", "{%if h == blank%}y{%else%}n{%endif%}", { "h" => {} })
-    assert_template_result("n", "{%if h == blank%}y{%else%}n{%endif%}", { "h" => { "a" => 1 } })
-  end
-
-  def test_iter
-    h0 = { "a" => 1, "b" => { "c" => "d" }, "3" => ["f", 2] }
-    assert_template_result("a1b{\"c\"=>\"d\"}3f2", "{%for i in h%}{{i}}{%endfor%}", { "h" => h0 })
-  end
-
-  def test_output
-    h0 = {}
-    h1 = { "a" => 1, "b" => { "c" => "d" }, "3" => ["f", 2] }
-    assert_template_result("{}", "{{h}}", { "h" => h0 })
-    assert_template_result("{\"a\"=>1, \"b\"=>{\"c\"=>\"d\"}, \"3\"=>[\"f\", 2]}", "{{h}}", { "h" => h1 })
-  end
-
-  def test_integer_key
-    h = { 2 => "huh" }
-    assert_template_result("huh", "{{h[2]}}", { "h" => h })
-    assert_template_result("huh", "{{h[b]}}", { "h" => h, "b" => 2 })
-    assert_template_result("", "{{h[b]}}", { "h" => h, "b" => "2" })
-  end
-
-  def test_nil_key
-    assert_hash_roundtrip(nil, nil)
-    refute_hash_roundtrip(nil, "nil")
-    refute_hash_roundtrip(nil, "null")
-  end
-
-  def test_bool_key
-    assert_hash_roundtrip(true, true)
-    refute_hash_roundtrip(true, false)
-    refute_hash_roundtrip(true, "true")
-    refute_hash_roundtrip(true, nil)
-  end
-
-  def test_int_key
-    assert_hash_roundtrip(1, 1)
-    refute_hash_roundtrip(1, 2)
-    refute_hash_roundtrip(1, "1")
-    refute_hash_roundtrip(1, 1.0000000000001)
-  end
-
-  def test_string_key
-    assert_hash_roundtrip("word", "word")
-    # assert_hash_roundtrip("word", "word".html_safe)
-    assert_hash_roundtrip("word", "word".b)
-    refute_hash_roundtrip("word", "word ")
-    refute_hash_roundtrip("word", nil)
-    refute_hash_roundtrip("word", true)
-  end
-
-  def test_weird_keys
-    assert_hash_roundtrip([1, 2], [1, 2])
-    refute_hash_roundtrip([1, 2], [1, 3])
-    assert_hash_roundtrip({ "a" => 1 }, { "a" => 1 })
-    refute_hash_roundtrip({ "a" => 1 }, { "a" => 2 })
-    o = Struct.new(:to_liquid).new(Object.new)
-    # failed_hash_roundtrip(o, o)
-    refute_hash_roundtrip(o, 1)
-    refute_hash_roundtrip(o, nil)
-    assert_hash_roundtrip("\xff".b, "\xff".b)
-    assert_hash_roundtrip(1..2, 1..2)
-    refute_hash_roundtrip(1..2, 1...2)
-    assert_hash_roundtrip(
-      111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111,
-      111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111,
-    )
-    # f64 hashing isn't perfect but we can make some basic assertions.
-    assert_hash_roundtrip(1.1234, 1.1234)
-    refute_hash_roundtrip(1.1234, 1.1)
-  end
-
-  def test_hash_html_key
-    # I mean this is probably not ideal but this is the way liquid works.
-    assert_template_result("{\"a<script>b\"=>\"y\"}", "{{h}}", { "h" => { "a<script>b" => "y" } })
-  end
-
-  def test_hash_subclass
-    hs = HashSubclass.new
-    hs["A"] = "b"
-    assert_template_result("b", "{{h['a']}}", { "h" => hs })
-    assert_template_result("b", "{{h['A']}}", { "h" => hs })
-    assert_template_result("", "{{h['b']}}", { "h" => hs })
-    assert_template_result("yes", "{% if h contains 'a' %}yes{%else%}no{%endif%}", { "h" => hs })
-    assert_template_result('{"A"=>"B"}', "{{h}}", { "h" => hs })
-  end
-
-  private
-
-  def failed_hash_roundtrip(key, test)
-    assert_template_result(
-      "Liquid error (templates/index line 1): internal",
-      "{{h[b]}}",
-      { "h" => { key => "y" }, "b" => test },
-    )
-  end
-
-  def assert_hash_roundtrip(key, test)
-    assert_template_result("y", "{{h[b]}}", { "h" => { key => "y" }, "b" => test })
-  end
-
-  def refute_hash_roundtrip(key, test)
-    assert_template_result("", "{{h[b]}}", { "h" => { key => "y" }, "b" => test })
-  end
-end

test/integration/standard_filter_test.rb

@@ -176,7 +176,17 @@ class StandardFiltersTest < Minitest::Test
   end
 
   def test_base64_decode
-    assert_equal('one two three', @filters.base64_decode('b25lIHR3byB0aHJlZQ=='))
+    decoded = @filters.base64_decode('b25lIHR3byB0aHJlZQ==')
+    assert_equal('one two three', decoded)
+    assert_equal(Encoding::UTF_8, decoded.encoding)
+
+    decoded = @filters.base64_decode('4pyF')
+    assert_equal('✅', decoded)
+    assert_equal(Encoding::UTF_8, decoded.encoding)
+
+    decoded = @filters.base64_decode("/w==")
+    assert_equal(Encoding::ASCII_8BIT, decoded.encoding)
+    assert_equal((+"\xFF").force_encoding(Encoding::ASCII_8BIT), decoded)
 
     exception = assert_raises(Liquid::ArgumentError) do
       @filters.base64_decode("invalidbase64")
@@ -194,10 +204,21 @@ class StandardFiltersTest < Minitest::Test
   end
 
   def test_base64_url_safe_decode
+    decoded = @filters.base64_url_safe_decode('YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXogQUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVogMTIzNDU2Nzg5MCAhQCMkJV4mKigpLT1fKy8_Ljo7W117fVx8')
     assert_equal(
       'abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ 1234567890 !@#$%^&*()-=_+/?.:;[]{}\|',
-      @filters.base64_url_safe_decode('YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXogQUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVogMTIzNDU2Nzg5MCAhQCMkJV4mKigpLT1fKy8_Ljo7W117fVx8'),
+      decoded,
     )
+    assert_equal(Encoding::UTF_8, decoded.encoding)
+
+    decoded = @filters.base64_url_safe_decode('4pyF')
+    assert_equal('✅', decoded)
+    assert_equal(Encoding::UTF_8, decoded.encoding)
+
+    decoded = @filters.base64_url_safe_decode("_w==")
+    assert_equal(Encoding::ASCII_8BIT, decoded.encoding)
+    assert_equal((+"\xFF").force_encoding(Encoding::ASCII_8BIT), decoded)
+
     exception = assert_raises(Liquid::ArgumentError) do
       @filters.base64_url_safe_decode("invalidbase64")
     end
@@ -994,6 +1015,42 @@ class StandardFiltersTest < Minitest::Test
     assert(t.foo > 0)
   end
 
+  def test_sum_of_floats
+    input = [0.1, 0.2, 0.3]
+    assert_equal(0.6, @filters.sum(input))
+    assert_template_result("0.6", "{{ input | sum }}", { "input" => input })
+  end
+
+  def test_sum_of_negative_floats
+    input = [0.1, 0.2, -0.3]
+    assert_equal(0.0, @filters.sum(input))
+    assert_template_result("0.0", "{{ input | sum }}", { "input" => input })
+  end
+
+  def test_sum_with_float_strings
+    input = [0.1, "0.2", "0.3"]
+    assert_equal(0.6, @filters.sum(input))
+    assert_template_result("0.6", "{{ input | sum }}", { "input" => input })
+  end
+
+  def test_sum_resulting_in_negative_float
+    input = [0.1, -0.2, -0.3]
+    assert_equal(-0.4, @filters.sum(input))
+    assert_template_result("-0.4", "{{ input | sum }}", { "input" => input })
+  end
+
+  def test_sum_with_floats_and_indexable_map_values
+    input = [{ "quantity" => 1 }, { "quantity" => 0.2, "weight" => -0.3 }, { "weight" => 0.4 }]
+    assert_equal(0.0, @filters.sum(input))
+    assert_equal(1.2, @filters.sum(input, "quantity"))
+    assert_equal(0.1, @filters.sum(input, "weight"))
+    assert_equal(0.0, @filters.sum(input, "subtotal"))
+    assert_template_result("0", "{{ input | sum }}", { "input" => input })
+    assert_template_result("1.2", "{{ input | sum: 'quantity' }}", { "input" => input })
+    assert_template_result("0.1", "{{ input | sum: 'weight' }}", { "input" => input })
+    assert_template_result("0", "{{ input | sum: 'subtotal' }}", { "input" => input })
+  end
+
   private
 
   def with_timezone(tz)

test/integration/template_test.rb

@@ -337,4 +337,22 @@ class TemplateTest < Minitest::Test
     assert_equal("x=2", output)
     assert_instance_of(String, output)
   end
+
+  def test_raises_error_with_invalid_utf8
+    e = assert_raises(SyntaxError) do
+      Template.parse(<<~LIQUID)
+        {% comment %}
+          \xC0
+        {% endcomment %}
+      LIQUID
+    end
+
+    assert_equal('Liquid syntax error: Invalid template encoding', e.message)
+  end
+
+  def test_allows_non_string_values_as_source
+    assert_equal('', Template.parse(nil).render)
+    assert_equal('1', Template.parse(1).render)
+    assert_equal('true', Template.parse(true).render)
+  end
 end

test/unit/tag_unit_test.rb

@@ -20,4 +20,13 @@ class TagUnitTest < Minitest::Test
     tag = Tag.parse("some_tag", "", Tokenizer.new(""), ParseContext.new)
     assert_equal('some_tag', tag.tag_name)
   end
+
+  class CustomTag < Liquid::Tag
+    def render(_context); end
+  end
+
+  def test_tag_render_to_output_buffer_nil_value
+    custom_tag = CustomTag.parse("some_tag", "", Tokenizer.new(""), ParseContext.new)
+    assert_equal('some string', custom_tag.render_to_output_buffer(Context.new, "some string"))
+  end
 end

test/unit/tags/comment_tag_unit_test.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class CommentTagUnitTest < Minitest::Test
+  def test_comment_inside_liquid_tag
+    assert_template_result("", <<~LIQUID.chomp)
+      {% liquid
+        if 1 != 1
+        comment
+        else
+          echo 123
+        endcomment
+        endif
+      %}
+    LIQUID
+  end
+
+  def test_does_not_parse_nodes_inside_a_comment
+    assert_template_result("", <<~LIQUID.chomp)
+      {% comment %}
+        {% if true %}
+        {% if ... %}
+        {%- for ? -%}
+        {% while true %}
+        {%
+          unless if
+        %}
+        {% endcase %}
+      {% endcomment %}
+    LIQUID
+  end
+
+  def test_allows_unclosed_tags
+    assert_template_result('', <<~LIQUID.chomp)
+      {% comment %}
+        {% if true %}
+      {% endcomment %}
+    LIQUID
+  end
+
+  def test_open_tags_in_comment
+    assert_template_result('', <<~LIQUID.chomp)
+      {% comment %}
+        {% assign a = 123 {% comment %}
+      {% endcomment %}
+    LIQUID
+
+    assert_raises(Liquid::SyntaxError) do
+      assert_template_result("", <<~LIQUID.chomp)
+        {% comment %}
+          {% assign foo = "1"
+        {% endcomment %}
+      LIQUID
+    end
+
+    assert_raises(Liquid::SyntaxError) do
+      assert_template_result("", <<~LIQUID.chomp)
+        {% comment %}
+          {% comment %}
+            {% invalid
+          {% endcomment %}
+        {% endcomment %}
+      LIQUID
+    end
+
+    assert_raises(Liquid::SyntaxError) do
+      assert_template_result("", <<~LIQUID.chomp)
+        {% comment %}
+        {% {{ {%- endcomment %}
+      LIQUID
+    end
+  end
+
+  def test_child_comment_tags_need_to_be_closed
+    assert_template_result("", <<~LIQUID.chomp)
+      {% comment %}
+        {% comment %}
+          {% comment %}{%    endcomment     %}
+        {% endcomment %}
+      {% endcomment %}
+    LIQUID
+
+    assert_raises(Liquid::SyntaxError) do
+      assert_template_result("", <<~LIQUID.chomp)
+        {% comment %}
+          {% comment %}
+            {% comment %}
+          {% endcomment %}
+        {% endcomment %}
+      LIQUID
+    end
+  end
+
+  def test_child_raw_tags_need_to_be_closed
+    assert_template_result("", <<~LIQUID.chomp)
+      {% comment %}
+        {% raw %}
+          {% endcomment %}
+        {% endraw %}
+      {% endcomment %}
+    LIQUID
+
+    assert_raises(Liquid::SyntaxError) do
+      Liquid::Template.parse(<<~LIQUID.chomp)
+        {% comment %}
+          {% raw %}
+          {% endcomment %}
+        {% endcomment %}
+      LIQUID
+    end
+  end
+
+  def test_error_line_number_is_correct
+    template = Liquid::Template.parse(<<~LIQUID.chomp, line_numbers: true)
+      {% comment %}
+        {% if true %}
+      {% endcomment %}
+      {{ errors.standard_error }}
+    LIQUID
+
+    output = template.render('errors' => ErrorDrop.new)
+    expected = <<~TEXT.chomp
+
+      Liquid error (line 4): standard error
+    TEXT
+
+    assert_equal(expected, output)
+  end
+
+  def test_comment_tag_delimiter_with_extra_strings
+    assert_template_result(
+      '',
+      <<~LIQUID.chomp,
+        {% comment %}
+          {% comment %}
+          {% endcomment
+          {% if true %}
+          {% endif %}
+        {% endcomment %}
+      LIQUID
+    )
+  end
+
+  def test_nested_comment_tag_with_extra_strings
+    assert_template_result(
+      '',
+      <<~LIQUID.chomp,
+        {% comment %}
+          {% comment
+            {% assign foo = 1 %}
+          {% endcomment
+          {% assign foo = 1 %}
+        {% endcomment %}
+      LIQUID
+    )
+  end
+
+  def test_ignores_delimiter_with_extra_strings
+    assert_template_result(
+      '',
+      <<~LIQUID.chomp,
+        {% if true %}
+          {% comment %}
+            {% commentXXXXX %}wut{% endcommentXXXXX %}
+          {% endcomment %}
+        {% endif %}
+      LIQUID
+    )
+  end
+
+  def test_delimiter_can_have_extra_strings
+    assert_template_result('', "{% comment %}123{% endcomment xyz %}")
+    assert_template_result('', "{% comment %}123{% endcomment\txyz %}")
+    assert_template_result('', "{% comment %}123{% endcomment\nxyz %}")
+    assert_template_result('', "{% comment %}123{% endcomment\n   xyz  endcomment %}")
+    assert_template_result('', "{%comment}{% assign a = 1 %}{%endcomment}{% endif %}")
+  end
+
+  def test_with_whitespace_control
+    assert_template_result("Hello!", "      {%- comment -%}123{%- endcomment -%}Hello!")
+    assert_template_result("Hello!", "{%- comment -%}123{%- endcomment -%}     Hello!")
+    assert_template_result("Hello!", "      {%- comment -%}123{%- endcomment -%}     Hello!")
+
+    assert_template_result("Hello!", <<~LIQUID.chomp)
+      {%- comment %}Whitespace control!{% endcomment -%}
+      Hello!
+    LIQUID
+  end
+
+  def test_dont_override_liquid_tag_whitespace_control
+    assert_template_result("Hello!World!", <<~LIQUID.chomp)
+      Hello!
+      {%- liquid
+        comment
+         this is inside a liquid tag
+        endcomment
+      -%}
+      World!
+    LIQUID
+  end
+end