diff --git a/Gemfile b/Gemfile index daff580c..d9a2a564 100644 --- a/Gemfile +++ b/Gemfile @@ -11,6 +11,7 @@ group :benchmark, :test do gem 'benchmark-ips' gem 'memory_profiler' gem 'terminal-table' + gem 'unicode_plot' install_if -> { RUBY_PLATFORM !~ /mingw|mswin|java/ && RUBY_ENGINE != 'truffleruby' } do gem 'stackprof' diff --git a/Rakefile b/Rakefile index 99a06aa2..d9c30800 100755 --- a/Rakefile +++ b/Rakefile @@ -92,6 +92,22 @@ namespace :benchmark do ruby "./performance/benchmark.rb lax" end + desc "Compare the render performance of compiled Liquid to standard Liquid and Liquid-C" + task :compare_render do + ENV.delete("LIQUID_C") + ENV.delete("RENDER_ONLY") + + ENV["LIQUID_COMPILE"] = "1" + ruby "./performance/benchmark.rb strict" + + ENV.delete("LIQUID_COMPILE") + ruby "./performance/benchmark.rb strict" + + ENV["LIQUID_C"] = "1" + ruby "./performance/benchmark.rb strict" + + end + desc "Run the liquid benchmark with strict parsing" task :strict do ruby "./performance/benchmark.rb strict" diff --git a/lib/.DS_Store b/lib/.DS_Store new file mode 100644 index 00000000..6bcd8608 Binary files /dev/null and b/lib/.DS_Store differ diff --git a/lib/liquid/compile.rb b/lib/liquid/compile.rb index b32f8d52..8cbe6bbe 100644 --- a/lib/liquid/compile.rb +++ b/lib/liquid/compile.rb @@ -1,10 +1,12 @@ # frozen_string_literal: true -require 'pry' +require 'pry-byebug' module Liquid - class BlockBody - def render_to_output_buffer(context, output) + module CompileBlockBody + def parse(*) + super + ruby = +"->(__context, __output, __nodes) {\n" compile(ruby) ruby << "\n}" @@ -13,11 +15,13 @@ module Liquid hash[node.object_id] = node end - RubyVM::InstructionSequence + @instructions = RubyVM::InstructionSequence .compile(ruby) .eval - .call(context, output, nodes) + end + def render_to_output_buffer(context, output) + @instructions.call(context, output, nodes) output end @@ -64,4 +68,8 @@ module Liquid ruby << "end\n" end end + + class BlockBody + include CompileBlockBody + end end diff --git a/performance/benchmark b/performance/benchmark new file mode 100755 index 00000000..f087c65d --- /dev/null +++ b/performance/benchmark @@ -0,0 +1,159 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "pry" +require "unicode_plot" +require "optparse" +require "open3" +require "csv" +require "io/console" + +TERM_ROWS, TERM_COLS = IO.console.winsize + +def record + if ARGV.count != 2 + STDERR.puts "Usage: benchmark.rb record [output_path]" + exit(1) + end + output_path = ARGV[1] + out, status = Open3.capture2("ruby", "#{__dir__}/benchmark_child.rb") + File.write(output_path, out) +end + +def calc_stats(nums) + mean = nums.reduce(:+) / nums.length + variance = nums.map { |n| (n - mean).pow(2) }.reduce(:+) / nums.length + stddev = Math.sqrt(variance) + { + mean: mean, + variance: variance, + stddev: stddev, + normalized: normalize_outliers(mean, stddev, nums), + raw: nums + } +end + +def normalize_outliers(mean, stddev, nums) + cutoff = stddev * 3 + nums.map do |n| + if (n - mean).abs < cutoff + n + else + mean + end + end +end + + +def show + if ARGV.count < 2 + STDERR.puts "Usage: benchmark.rb show [path1] [path2]? ..." + exit(1) + end + + recordings = ARGV.drop(1).to_h do |path| + [File.basename(path), CSV.parse( + File.open(path), + col_sep: "\t", + headers: true, + converters: :integer + )] + end + + runs = (1..1000).to_a + recordings.values.first.headers.each do |benchmark| + colors = [:green, :blue, :red] + + 10.times { puts } + title = "Benchmark: #{benchmark} (times in µs)" + print " " * (TERM_COLS / 2 - title.length) + puts title + puts + + all_stats = recordings.transform_values do |csv| + stats = calc_stats(csv.map { |row| row[benchmark] }) + stats[:color] = colors.shift + stats + end + + line_plots = [] + distributions = [] + shared_line_plot = nil + + max_mean_stats = all_stats.values.max_by { |stats| stats[:mean] } + max_y = (max_mean_stats[:mean] + max_mean_stats[:stddev] * 3).to_i + + all_stats.each do |name, stats| + if shared_line_plot == nil + shared_line_plot = UnicodePlot.lineplot( + runs, + stats[:normalized], + name: name, + width: TERM_COLS - 25, + ylim: [0, max_y], + color: stats[:color] + ) + else + UnicodePlot.lineplot!(shared_line_plot, stats[:normalized], name: name, color: stats[:color]) + end + + line_plots << render_to_s(UnicodePlot.lineplot( + runs, + stats[:normalized], + name: name, + color: stats[:color], + width: 40 + )) + + distributions << render_to_s(UnicodePlot.histogram( + stats[:normalized], + title: name, + color: stats[:color] + )) + end + + shared_line_plot.render + print_columns(line_plots) + print_columns(distributions) + + all_times = all_stats.transform_values { |stats| stats[:normalized] } + UnicodePlot.boxplot(data: all_times, title: "Comparison", width: TERM_COLS - 25).render + end +end + +def render_to_s(plot) + io = StringIO.new + plot.render(io, color: true) + io.string +end + +def visual_length(line) + line.gsub(/\e\[(\d+)m/, '').length +end + +def print_columns(cols) + col_lines = cols.map { |col| col.split("\n") } + col_width = col_lines.map do |lines| + lines.map { |line| visual_length(line) }.max + end.max + col_height = col_lines.map { |lines| lines.length }.max + + (0...col_height).each do |i| + col_lines.each do |lines| + line = lines[i] || "" + vis_length = visual_length(line) + print line + print(" " * (col_width - vis_length)) + end + puts + end +end + +case ARGV.first +when "record", "r" + record +when "show", "s" + show +else + puts "Unknown option: #{ARGV.first}" +end \ No newline at end of file diff --git a/performance/benchmark.rb b/performance/benchmark.rb deleted file mode 100644 index 8bf23134..00000000 --- a/performance/benchmark.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -require 'benchmark/ips' -require_relative 'theme_runner' - -Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first -profiler = ThemeRunner.new - -Benchmark.ips do |x| - x.time = 10 - x.warmup = 5 - - puts - puts "Running benchmark for #{x.time} seconds (with #{x.warmup} seconds warmup)." - puts - - x.report("parse:") { profiler.compile } - x.report("render:") { profiler.render } - x.report("parse & render:") { profiler.run } -end diff --git a/performance/benchmark_child.rb b/performance/benchmark_child.rb new file mode 100644 index 00000000..22a7b430 --- /dev/null +++ b/performance/benchmark_child.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'liquid' + +Liquid::Template.error_mode = :strict + +case ENV['ENGINE'] +when 'LIQUID_COMPILE' + require_relative '../lib/liquid/compile' +when 'LIQUID_C' + require 'liquid/c' +when 'LIQUID_RUBY' +else + raise "Invalid engine: #{ENV['ENGINE'].inspect}" +end + +OPTIONS = { + render_iters: 1000 +} + +def get_time_us + Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond) +end + +def render_row(charts) + charts.map do |chart| + io = StringIO.new + chart.render(io) + puts io + end +end + +Benchmarks = Class.new do + def initialize + @by_name = {} + end + + def define(name, benchmark) + @by_name[name] = benchmark + end + + def run + times = {} + @by_name.each do |name, benchmark| + benchmark.compile + + times[name] = OPTIONS[:render_iters].times.map do + before = get_time_us + benchmark.render + get_time_us - before + end + end + + puts times.keys.join("\t") + cols = times.values + OPTIONS[:render_iters].times do |i| + cols.each do |values| + print values[i] + print "\t" unless values == cols.last + end + puts + end + end +end.new + +Dir[__dir__ + "/benchmarks/*.rb", base: __dir__].each do |path| + require_relative path +end + +Benchmarks.run \ No newline at end of file diff --git a/performance/benchmarks/fizzbuzz_10000.rb b/performance/benchmarks/fizzbuzz_10000.rb new file mode 100644 index 00000000..6e8ca7bd --- /dev/null +++ b/performance/benchmarks/fizzbuzz_10000.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +Benchmarks.define('fizzbuzz_10000', Class.new do + TEMPLATE = <<~LIQUID + {% for i in (1..10000) %}{{ i }} + {% endfor %} + LIQUID + + def compile + @parsed = Liquid::Template.parse(TEMPLATE) + end + + def render + @parsed.render + end +end.new) \ No newline at end of file diff --git a/performance/benchmarks/theme_runner.rb b/performance/benchmarks/theme_runner.rb new file mode 100644 index 00000000..997b9fad --- /dev/null +++ b/performance/benchmarks/theme_runner.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative '../theme_runner' + +Benchmarks.define('theme_runner', ThemeRunner.new) diff --git a/performance/shopify/liquid.rb b/performance/shopify/liquid.rb index 40444c3c..dc420832 100644 --- a/performance/shopify/liquid.rb +++ b/performance/shopify/liquid.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true $LOAD_PATH.unshift(__dir__ + '/../../lib') -require_relative '../../lib/liquid' - require_relative 'comment_form' require_relative 'paginate' require_relative 'json_filter' diff --git a/test/test_helper.rb b/test/test_helper.rb index 50a9e769..dc7c1c4f 100755 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -15,14 +15,12 @@ if (env_mode = ENV['LIQUID_PARSER_MODE']) end Liquid::Template.error_mode = mode -if ENV['LIQUID_C'] == '1' - puts "-- LIQUID C" - require 'liquid/c' -end - if ENV['LIQUID_COMPILE'] == '1' puts "-- COMPILED" require 'liquid/compile' +elsif ENV['LIQUID_C'] == '1' + puts "-- LIQUID C" + require 'liquid/c' end if Minitest.const_defined?('Test')