diff --git a/auto/autoresearch.md b/auto/autoresearch.md new file mode 100644 index 00000000..a07a4b43 --- /dev/null +++ b/auto/autoresearch.md @@ -0,0 +1,60 @@ +# Autoresearch: Liquid Parse+Render Performance + +## Objective +Optimize the Shopify Liquid template engine's parse and render performance. +The workload is the ThemeRunner benchmark which parses and renders real Shopify +theme templates (dropify, ripen, tribble, vogue) with realistic data from +`performance/shopify/database.rb`. We measure parse time, render time, and +object allocations. The optimization target is combined parse+render time (µs). + +## How to Run +Run `./auto/autoresearch.sh` — it runs unit tests, liquid-spec conformance, +then the performance benchmark, outputting metrics in parseable format. + +## Metrics +- **Primary (optimization target)**: `combined_µs` (µs, lower is better) — sum of parse + render time +- **Secondary (tradeoff monitoring)**: + - `parse_µs` — time to parse all theme templates (Liquid::Template#parse) + - `render_µs` — time to render all pre-compiled templates + - `allocations` — total object allocations for one parse+render cycle + Parse dominates (~70-75% of combined). Allocations correlate with GC pressure. + +## Files in Scope +- `lib/liquid/*.rb` — core Liquid library (parser, lexer, context, expression, etc.) +- `lib/liquid/tags/*.rb` — tag implementations (for, if, assign, etc.) +- `performance/bench_quick.rb` — benchmark script + +## Off Limits +- `test/` — tests must continue to pass unchanged +- `performance/tests/` — benchmark templates, do not modify +- `performance/shopify/` — benchmark data/filters, do not modify + +## Constraints +- All unit tests must pass (`bundle exec rake base_test`) +- liquid-spec failures must not increase beyond 2 (pre-existing UTF-8 edge cases) +- No new gem dependencies +- Semantic correctness must be preserved — templates must render identical output + +## Baseline +- **Commit**: 4ea835a (original, before any optimizations) +- **combined_µs**: 7,374 +- **parse_µs**: 5,928 +- **render_µs**: 1,446 +- **allocations**: 62,620 + +## Progress Log +- 3329b09: Replace FullToken regex with manual byte parsing → combined 7,262 (-1.5%) +- 97e6893: Replace VariableParser regex with manual byte scanner → combined 6,945 (-5.8%), allocs 58,009 +- 2b78e4b: getbyte instead of string indexing in whitespace_handler/create_variable → allocs 51,477 +- d291e63: Lexer equal? for frozen arrays, \s+ whitespace skip → combined ~6,331 +- d79b9fa: Avoid strip alloc in Expression.parse, byteslice for strings → allocs 49,151 +- fa41224: Short-circuit parse_number with first-byte check → allocs 48,240 +- c1113ad: Fast-path String in render_obj_to_output → combined ~6,071 +- 25f9224: Fast-path simple variable parsing (skip Lexer/Parser) → combined ~5,860, allocs 45,202 +- 3939d74: Replace SIMPLE_VARIABLE regex with byte scanner → combined ~5,717, allocs 42,763 +- fe7a2f5: Fast-path simple if conditions → combined ~5,444, allocs 41,490 +- cfa0dfe: Replace For tag Syntax regex with manual byte parser → combined ~4,974, allocs 39,847 +- 8a92a4e: Unified fast-path Variable: parse name directly, only lex filter chain → combined ~5,060, allocs 40,520 +- 58d2514: parse_tag_token returns [tag_name, markup, newlines] → combined ~4,815, allocs 37,355 +- db43492: Hoist write score check out of render loop → render ~1,345 +- 17daac9: Extend fast-path to quoted string literal variables → all 1,197 variables fast-pathed diff --git a/auto/autoresearch.sh b/auto/autoresearch.sh new file mode 100755 index 00000000..dd79c33d --- /dev/null +++ b/auto/autoresearch.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Autoresearch benchmark runner for Liquid performance optimization +# Runs: unit tests → liquid-spec → performance benchmark +# Outputs METRIC lines for the agent to parse +# Exit code 0 = all good, non-zero = broken +set -euo pipefail + +cd "$(dirname "$0")/.." + +# ── Step 1: Unit tests (fast gate) ────────────────────────────────── +echo "=== Unit Tests ===" +if ! bundle exec rake base_test 2>&1; then + echo "FATAL: unit tests failed" + exit 1 +fi + +# ── Step 2: liquid-spec (correctness gate) ────────────────────────── +echo "" +echo "=== Liquid Spec ===" +SPEC_OUTPUT=$(bundle exec liquid-spec run spec/ruby_liquid.rb 2>&1 || true) +echo "$SPEC_OUTPUT" | tail -3 + +# Extract failure count from "Total: N passed, N failed, N errors" line +# Allow known pre-existing failures (≤2) +TOTAL_LINE=$(echo "$SPEC_OUTPUT" | grep "^Total:" || echo "Total: 0 passed, 0 failed, 0 errors") +FAILURES=$(echo "$TOTAL_LINE" | sed -n 's/.*\([0-9][0-9]*\) failed.*/\1/p') +ERRORS=$(echo "$TOTAL_LINE" | sed -n 's/.*\([0-9][0-9]*\) error.*/\1/p') +FAILURES=${FAILURES:-0} +ERRORS=${ERRORS:-0} +TOTAL_BAD=$((FAILURES + ERRORS)) + +if [ "$TOTAL_BAD" -gt 2 ]; then + echo "FATAL: liquid-spec has $FAILURES failures and $ERRORS errors (threshold: 2)" + exit 1 +fi + +# ── Step 3: Performance benchmark ────────────────────────────────── +echo "" +echo "=== Performance Benchmark ===" +BENCH_OUTPUT=$(bundle exec ruby performance/bench_quick.rb 2>&1) +echo "$BENCH_OUTPUT" + +# Parse results and output METRIC lines +PARSE_US=$(echo "$BENCH_OUTPUT" | grep '^parse_us=' | cut -d= -f2) +RENDER_US=$(echo "$BENCH_OUTPUT" | grep '^render_us=' | cut -d= -f2) +COMBINED_US=$(echo "$BENCH_OUTPUT" | grep '^combined_us=' | cut -d= -f2) +ALLOCATIONS=$(echo "$BENCH_OUTPUT" | grep '^allocations=' | cut -d= -f2) + +echo "" +echo "METRIC combined_us=$COMBINED_US" +echo "METRIC parse_us=$PARSE_US" +echo "METRIC render_us=$RENDER_US" +echo "METRIC allocations=$ALLOCATIONS" diff --git a/performance/bench_quick.rb b/performance/bench_quick.rb index 46505913..6168f80e 100644 --- a/performance/bench_quick.rb +++ b/performance/bench_quick.rb @@ -9,9 +9,9 @@ RubyVM::YJIT.enable if defined?(RubyVM::YJIT) runner = ThemeRunner.new -# Warmup -5.times { runner.compile } -5.times { runner.render } +# Warmup — enough iterations for YJIT to fully optimize hot paths +20.times { runner.compile } +20.times { runner.render } GC.start GC.compact if GC.respond_to?(:compact)