Add benchmark viewer

Gemfile

@@ -11,6 +11,7 @@ group :benchmark, :test do
   gem 'benchmark-ips'
   gem 'memory_profiler'
   gem 'terminal-table'
+  gem 'unicode_plot'
 
   install_if -> { RUBY_PLATFORM !~ /mingw|mswin|java/ && RUBY_ENGINE != 'truffleruby' } do
     gem 'stackprof'

Rakefile

@@ -92,6 +92,22 @@ namespace :benchmark do
     ruby "./performance/benchmark.rb lax"
   end
 
+  desc "Compare the render performance of compiled Liquid to standard Liquid and Liquid-C"
+  task :compare_render do
+    ENV.delete("LIQUID_C")
+    ENV.delete("RENDER_ONLY")
+
+    ENV["LIQUID_COMPILE"] = "1"
+    ruby "./performance/benchmark.rb strict"
+
+    ENV.delete("LIQUID_COMPILE")
+    ruby "./performance/benchmark.rb strict"
+
+    ENV["LIQUID_C"] = "1"
+    ruby "./performance/benchmark.rb strict"
+
+  end
+
   desc "Run the liquid benchmark with strict parsing"
   task :strict do
     ruby "./performance/benchmark.rb strict"

lib/liquid/compile.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
-require 'pry'
+require 'pry-byebug'
 
 module Liquid
-  class BlockBody
-    def render_to_output_buffer(context, output)
+  module CompileBlockBody
+    def parse(*)
+      super
+
       ruby = +"->(__context, __output, __nodes) {\n"
       compile(ruby)
       ruby << "\n}"
@@ -13,11 +15,13 @@ module Liquid
         hash[node.object_id] = node
       end
 
-      RubyVM::InstructionSequence
+      @instructions = RubyVM::InstructionSequence
           .compile(ruby)
           .eval
-          .call(context, output, nodes)
+    end
 
+    def render_to_output_buffer(context, output)
+      @instructions.call(context, output, nodes)
       output
     end
 
@@ -64,4 +68,8 @@ module Liquid
       ruby << "end\n"
     end
   end
+
+  class BlockBody
+    include CompileBlockBody
+  end
 end

performance/benchmark

@@ -0,0 +1,159 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "pry"
+require "unicode_plot"
+require "optparse"
+require "open3"
+require "csv"
+require "io/console"
+
+TERM_ROWS, TERM_COLS = IO.console.winsize
+
+def record
+  if ARGV.count != 2
+    STDERR.puts "Usage: benchmark.rb record [output_path]"
+    exit(1)
+  end
+  output_path = ARGV[1]
+  out, status = Open3.capture2("ruby", "#{__dir__}/benchmark_child.rb")
+  File.write(output_path, out)
+end
+
+def calc_stats(nums)
+  mean = nums.reduce(:+) / nums.length
+  variance = nums.map { |n| (n - mean).pow(2) }.reduce(:+) / nums.length
+  stddev = Math.sqrt(variance)
+  {
+    mean: mean,
+    variance: variance,
+    stddev: stddev,
+    normalized: normalize_outliers(mean, stddev, nums),
+    raw: nums
+  }
+end
+
+def normalize_outliers(mean, stddev, nums)
+  cutoff = stddev * 3
+  nums.map do |n|
+    if (n - mean).abs < cutoff
+      n
+    else
+      mean
+    end
+  end
+end
+
+
+def show
+  if ARGV.count < 2
+    STDERR.puts "Usage: benchmark.rb show [path1] [path2]? ..."
+    exit(1)
+  end
+
+  recordings = ARGV.drop(1).to_h do |path|
+    [File.basename(path), CSV.parse(
+      File.open(path),
+      col_sep: "\t",
+      headers: true,
+      converters: :integer
+    )]
+  end
+
+  runs = (1..1000).to_a
+  recordings.values.first.headers.each do |benchmark|
+    colors = [:green, :blue, :red]
+
+    10.times { puts }
+    title = "Benchmark: #{benchmark} (times in µs)"
+    print " " * (TERM_COLS / 2 - title.length)
+    puts title
+    puts
+
+    all_stats = recordings.transform_values do |csv|
+      stats = calc_stats(csv.map { |row| row[benchmark] })
+      stats[:color] = colors.shift
+      stats
+    end
+
+    line_plots = []
+    distributions = []
+    shared_line_plot = nil
+
+    max_mean_stats = all_stats.values.max_by { |stats| stats[:mean] }
+    max_y = (max_mean_stats[:mean] + max_mean_stats[:stddev] * 3).to_i
+
+    all_stats.each do |name, stats|
+      if shared_line_plot == nil
+        shared_line_plot = UnicodePlot.lineplot(
+          runs,
+          stats[:normalized],
+          name: name,
+          width: TERM_COLS - 25,
+          ylim: [0, max_y],
+          color: stats[:color]
+        )
+      else
+        UnicodePlot.lineplot!(shared_line_plot, stats[:normalized], name: name, color: stats[:color])
+      end
+
+      line_plots << render_to_s(UnicodePlot.lineplot(
+        runs,
+        stats[:normalized],
+        name: name,
+        color: stats[:color],
+        width: 40
+      ))
+
+      distributions << render_to_s(UnicodePlot.histogram(
+        stats[:normalized],
+        title: name,
+        color: stats[:color]
+      ))
+    end
+
+    shared_line_plot.render
+    print_columns(line_plots)
+    print_columns(distributions)
+
+    all_times = all_stats.transform_values { |stats| stats[:normalized] }
+    UnicodePlot.boxplot(data: all_times, title: "Comparison", width: TERM_COLS - 25).render
+  end
+end
+
+def render_to_s(plot)
+  io = StringIO.new
+  plot.render(io, color: true)
+  io.string
+end
+
+def visual_length(line)
+  line.gsub(/\e\[(\d+)m/, '').length
+end
+
+def print_columns(cols)
+  col_lines = cols.map { |col| col.split("\n") }
+  col_width = col_lines.map do |lines|
+    lines.map { |line| visual_length(line) }.max
+  end.max
+  col_height = col_lines.map { |lines| lines.length }.max
+
+  (0...col_height).each do |i|
+    col_lines.each do |lines|
+      line = lines[i] || ""
+      vis_length = visual_length(line)
+      print line
+      print(" " * (col_width - vis_length))
+    end
+    puts
+  end
+end
+
+case ARGV.first
+when "record", "r"
+  record
+when "show", "s"
+  show
+else
+  puts "Unknown option: #{ARGV.first}"
+end

performance/benchmark.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-require 'benchmark/ips'
-require_relative 'theme_runner'
-
-Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
-profiler = ThemeRunner.new
-
-Benchmark.ips do |x|
-  x.time   = 10
-  x.warmup = 5
-
-  puts
-  puts "Running benchmark for #{x.time} seconds (with #{x.warmup} seconds warmup)."
-  puts
-
-  x.report("parse:") { profiler.compile }
-  x.report("render:") { profiler.render }
-  x.report("parse & render:") { profiler.run }
-end

performance/benchmark_child.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'liquid'
+
+Liquid::Template.error_mode = :strict
+
+case ENV['ENGINE']
+when 'LIQUID_COMPILE'
+  require_relative '../lib/liquid/compile'
+when 'LIQUID_C'
+  require 'liquid/c'
+when 'LIQUID_RUBY'
+else
+  raise "Invalid engine: #{ENV['ENGINE'].inspect}"
+end
+
+OPTIONS = {
+  render_iters: 1000
+}
+
+def get_time_us
+  Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond)
+end
+
+def render_row(charts)
+  charts.map do |chart|
+    io = StringIO.new
+    chart.render(io)
+    puts io
+  end
+end
+
+Benchmarks = Class.new do
+  def initialize
+    @by_name = {}
+  end
+
+  def define(name, benchmark)
+    @by_name[name] = benchmark
+  end
+
+  def run
+    times = {}
+    @by_name.each do |name, benchmark|
+      benchmark.compile
+
+      times[name] = OPTIONS[:render_iters].times.map do
+        before = get_time_us
+        benchmark.render
+        get_time_us - before
+      end
+    end
+
+    puts times.keys.join("\t")
+    cols = times.values
+    OPTIONS[:render_iters].times do |i|
+      cols.each do |values|
+        print values[i]
+        print "\t" unless values == cols.last
+      end
+      puts
+    end
+  end
+end.new
+
+Dir[__dir__ + "/benchmarks/*.rb", base: __dir__].each do |path|
+  require_relative path
+end
+
+Benchmarks.run

performance/benchmarks/fizzbuzz_10000.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+Benchmarks.define('fizzbuzz_10000', Class.new do
+  TEMPLATE = <<~LIQUID
+  {% for i in (1..10000) %}{{ i }}
+  {% endfor %}
+  LIQUID
+
+  def compile
+    @parsed = Liquid::Template.parse(TEMPLATE)
+  end
+
+  def render
+    @parsed.render
+  end
+end.new)

performance/benchmarks/theme_runner.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative '../theme_runner'
+
+Benchmarks.define('theme_runner', ThemeRunner.new) 

performance/shopify/liquid.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 $LOAD_PATH.unshift(__dir__ + '/../../lib')
-require_relative '../../lib/liquid'
-
 require_relative 'comment_form'
 require_relative 'paginate'
 require_relative 'json_filter'

test/test_helper.rb

@@ -15,14 +15,12 @@ if (env_mode = ENV['LIQUID_PARSER_MODE'])
 end
 Liquid::Template.error_mode = mode
 
-if ENV['LIQUID_C'] == '1'
-  puts "-- LIQUID C"
-  require 'liquid/c'
-end
-
 if ENV['LIQUID_COMPILE'] == '1'
   puts "-- COMPILED"
   require 'liquid/compile'
+elsif ENV['LIQUID_C'] == '1'
+  puts "-- LIQUID C"
+  require 'liquid/c'
 end
 
 if Minitest.const_defined?('Test')