from __future__ import annotations import base64 import html import io from pathlib import Path import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt import matplotlib.ticker as ticker from .models import BenchmarkResult from .report import ( LIKERT_QUESTIONS, _get_likert_scores, _parse_likert, load_latest_results, ) BASH_COLOR = "#4E79A7" LUSH_COLOR = "#E15759" NEUTRAL_COLOR = "#999999" def _fig_to_base64(fig: plt.Figure) -> str: buf = io.BytesIO() fig.savefig(buf, format="png", dpi=150, bbox_inches="tight", facecolor="white") plt.close(fig) buf.seek(0) return base64.b64encode(buf.read()).decode() def _aggregate_likert(results: list[BenchmarkResult]) -> dict[str, dict[str, float]]: """Return {question_key: {bash: avg, lush: avg}}.""" agg: dict[str, dict[str, list[float]]] = {} for key, _, _ in LIKERT_QUESTIONS: agg[key] = {"bash": [], "lush": []} for r in results: scores = _get_likert_scores(r) for key in scores: for lang in ("bash", "lush"): val = scores[key][lang] if val is not None: agg[key][lang].append(val) return { key: { lang: (sum(vals) / len(vals)) if vals else 0.0 for lang, vals in agg[key].items() } for key in agg } def chart_questionnaire_comparison(results: list[BenchmarkResult]) -> str: """Grouped horizontal bar chart comparing bash vs lush on each Likert metric.""" avgs = _aggregate_likert(results) labels = [label for _, label, _ in LIKERT_QUESTIONS] bash_vals = [avgs[key]["bash"] for key, _, _ in LIKERT_QUESTIONS] lush_vals = [avgs[key]["lush"] for key, _, _ in LIKERT_QUESTIONS] fig, ax = plt.subplots(figsize=(8, 7)) y = range(len(labels)) bar_h = 0.35 bars_bash = ax.barh([i + bar_h / 2 for i in y], bash_vals, bar_h, label="bash", color=BASH_COLOR) bars_lush = ax.barh([i - bar_h / 2 for i in y], lush_vals, bar_h, label="lush", color=LUSH_COLOR) ax.set_yticks(list(y)) ax.set_yticklabels(labels) ax.set_xlim(0, 5.5) ax.xaxis.set_major_locator(ticker.MultipleLocator(1)) ax.set_xlabel("Score (1-5)") ax.set_title("Questionnaire Scores: Bash vs Lush") ax.legend(loc="lower right") ax.invert_yaxis() for bar in bars_bash: w = bar.get_width() ax.text(w + 0.08, bar.get_y() + bar.get_height() / 2, f"{w:.1f}", va="center", fontsize=8) for bar in bars_lush: w = bar.get_width() ax.text(w + 0.08, bar.get_y() + bar.get_height() / 2, f"{w:.1f}", va="center", fontsize=8) ax.grid(axis="x", alpha=0.3) return _fig_to_base64(fig) def chart_turns_comparison(results: list[BenchmarkResult]) -> str: """Bar chart of agent turns per task for bash vs lush.""" # Only include tasks where the agent actually solved (solve mode) solve = [r for r in results if r.mode == "solve"] names = [r.task_name for r in solve] bash_turns = [r.bash_result.agent_turns if r.bash_result else 0 for r in solve] lush_turns = [r.lush_result.agent_turns if r.lush_result else 0 for r in solve] fig, ax = plt.subplots(figsize=(8, 4)) x = range(len(names)) bar_w = 0.35 ax.bar([i - bar_w / 2 for i in x], bash_turns, bar_w, label="bash", color=BASH_COLOR) ax.bar([i + bar_w / 2 for i in x], lush_turns, bar_w, label="lush", color=LUSH_COLOR) ax.set_xticks(list(x)) ax.set_xticklabels(names, rotation=35, ha="right", fontsize=8) ax.set_ylabel("Agent Turns") ax.set_title("Agent Turns to Solve (Solve Mode)") ax.yaxis.set_major_locator(ticker.MaxNLocator(integer=True)) ax.legend() ax.grid(axis="y", alpha=0.3) return _fig_to_base64(fig) def chart_per_task_heatmap(results: list[BenchmarkResult]) -> str: """Heatmap showing lush-minus-bash score diff per task and metric.""" labels = [label for _, label, _ in LIKERT_QUESTIONS] tasks = [r.task_name for r in results] data: list[list[float]] = [] for r in results: scores = _get_likert_scores(r) row = [] for key, _, _ in LIKERT_QUESTIONS: b = scores[key]["bash"] l = scores[key]["lush"] if b is not None and l is not None: row.append(l - b) else: row.append(0.0) data.append(row) fig, ax = plt.subplots(figsize=(10, max(4, len(tasks) * 0.45 + 1))) im = ax.imshow(data, cmap="RdYlGn", aspect="auto", vmin=-3, vmax=3) ax.set_xticks(range(len(labels))) ax.set_xticklabels(labels, rotation=45, ha="right", fontsize=7) ax.set_yticks(range(len(tasks))) ax.set_yticklabels(tasks, fontsize=8) for i in range(len(tasks)): for j in range(len(labels)): val = data[i][j] text = f"+{val:.0f}" if val > 0 else f"{val:.0f}" if val < 0 else "0" ax.text(j, i, text, ha="center", va="center", fontsize=7, color="white" if abs(val) >= 2 else "black") ax.set_title("Score Difference (Lush - Bash)") fig.colorbar(im, ax=ax, shrink=0.8, label="Lush advantage") return _fig_to_base64(fig) def chart_per_category_questionnaire(results: list[BenchmarkResult]) -> str: """Grouped bar chart: one cluster per category, bars for bash/lush avg scores.""" from collections import defaultdict by_cat: dict[str, list[BenchmarkResult]] = defaultdict(list) for r in results: by_cat[r.category].append(r) categories = sorted(by_cat) bash_avgs = [] lush_avgs = [] for cat in categories: b_scores: list[float] = [] l_scores: list[float] = [] for r in by_cat[cat]: scores = _get_likert_scores(r) for key in scores: if scores[key]["bash"] is not None: b_scores.append(scores[key]["bash"]) if scores[key]["lush"] is not None: l_scores.append(scores[key]["lush"]) bash_avgs.append(sum(b_scores) / len(b_scores) if b_scores else 0.0) lush_avgs.append(sum(l_scores) / len(l_scores) if l_scores else 0.0) fig, ax = plt.subplots(figsize=(8, 4)) x = range(len(categories)) bar_w = 0.35 bars_b = ax.bar([i - bar_w / 2 for i in x], bash_avgs, bar_w, label="bash", color=BASH_COLOR) bars_l = ax.bar([i + bar_w / 2 for i in x], lush_avgs, bar_w, label="lush", color=LUSH_COLOR) ax.set_xticks(list(x)) ax.set_xticklabels(categories, fontsize=9) ax.set_ylim(0, 5.5) ax.set_ylabel("Avg Score (1-5)") ax.set_title("Questionnaire Scores by Category") ax.legend() ax.grid(axis="y", alpha=0.3) for bar in bars_b: ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 0.08, f"{bar.get_height():.1f}", ha="center", va="bottom", fontsize=8) for bar in bars_l: ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 0.08, f"{bar.get_height():.1f}", ha="center", va="bottom", fontsize=8) return _fig_to_base64(fig) def chart_per_category_radar(results: list[BenchmarkResult]) -> list[tuple[str, str]]: """Small-multiples bar charts: one per category showing 12 Likert dimensions for bash vs lush.""" import numpy as np from collections import defaultdict by_cat: dict[str, list[BenchmarkResult]] = defaultdict(list) for r in results: by_cat[r.category].append(r) charts: list[tuple[str, str]] = [] labels = [label for _, label, _ in LIKERT_QUESTIONS] for cat in sorted(by_cat): cat_results = by_cat[cat] agg: dict[str, dict[str, list[float]]] = {} for key, _, _ in LIKERT_QUESTIONS: agg[key] = {"bash": [], "lush": []} for r in cat_results: scores = _get_likert_scores(r) for key in scores: for lang in ("bash", "lush"): val = scores[key][lang] if val is not None: agg[key][lang].append(val) bash_vals = [sum(agg[k]["bash"]) / len(agg[k]["bash"]) if agg[k]["bash"] else 0.0 for k, _, _ in LIKERT_QUESTIONS] lush_vals = [sum(agg[k]["lush"]) / len(agg[k]["lush"]) if agg[k]["lush"] else 0.0 for k, _, _ in LIKERT_QUESTIONS] fig, ax = plt.subplots(figsize=(7, 5)) y = range(len(labels)) bar_h = 0.35 ax.barh([i + bar_h / 2 for i in y], bash_vals, bar_h, label="bash", color=BASH_COLOR) ax.barh([i - bar_h / 2 for i in y], lush_vals, bar_h, label="lush", color=LUSH_COLOR) ax.set_yticks(list(y)) ax.set_yticklabels(labels, fontsize=8) ax.set_xlim(0, 5.5) ax.set_title(f"{cat}", fontsize=10) ax.legend(fontsize=8, loc="lower right") ax.invert_yaxis() ax.grid(axis="x", alpha=0.3) charts.append((cat, _fig_to_base64(fig))) return charts def _build_per_category_summary_html(results: list[BenchmarkResult]) -> str: """HTML table: rows=categories, columns=bash/lush pass rate, turns, scores.""" from collections import defaultdict by_cat: dict[str, list[BenchmarkResult]] = defaultdict(list) for r in results: by_cat[r.category].append(r) rows = [] for cat in sorted(by_cat): cat_results = by_cat[cat] b_passed = sum(1 for r in cat_results if r.bash_result and r.bash_result.all_passed) l_passed = sum(1 for r in cat_results if r.lush_result and r.lush_result.all_passed) b_total = sum(1 for r in cat_results if r.bash_result) l_total = sum(1 for r in cat_results if r.lush_result) b_turn_vals = [r.bash_result.agent_turns for r in cat_results if r.bash_result and r.bash_result.agent_turns > 0] l_turn_vals = [r.lush_result.agent_turns for r in cat_results if r.lush_result and r.lush_result.agent_turns > 0] b_turns_avg = sum(b_turn_vals) / len(b_turn_vals) if b_turn_vals else 0.0 l_turns_avg = sum(l_turn_vals) / len(l_turn_vals) if l_turn_vals else 0.0 b_scores: list[float] = [] l_scores: list[float] = [] for r in cat_results: scores = _get_likert_scores(r) for key in scores: if scores[key]["bash"] is not None: b_scores.append(scores[key]["bash"]) if scores[key]["lush"] is not None: l_scores.append(scores[key]["lush"]) b_avg = sum(b_scores) / len(b_scores) if b_scores else 0.0 l_avg = sum(l_scores) / len(l_scores) if l_scores else 0.0 rows.append(f""" {html.escape(cat)} {b_passed}/{b_total}{l_passed}/{l_total} {b_turns_avg:.1f}{l_turns_avg:.1f} {b_avg:.1f}{l_avg:.1f} """) return f"""{"".join(rows)}
Category Bash PassLush Pass Bash Avg TurnsLush Avg Turns Bash Avg ScoreLush Avg Score
""" def _build_summary_html(results: list[BenchmarkResult]) -> str: rows = [] for r in results: b = r.bash_result l = r.lush_result b_cls = "pass" if b and b.all_passed else "fail" l_cls = "pass" if l and l.all_passed else "fail" b_pass = "PASS" if b and b.all_passed else "FAIL" l_pass = "PASS" if l and l.all_passed else "FAIL" b_turns = str(b.agent_turns) if b else "-" l_turns = str(l.agent_turns) if l else "-" rows.append(f""" {html.escape(r.task_name)}{html.escape(r.category)} {b_pass}{b_turns} {l_pass}{l_turns} """) b_passed = sum(1 for r in results if r.bash_result and r.bash_result.all_passed) l_passed = sum(1 for r in results if r.lush_result and r.lush_result.all_passed) total = len(results) return f"""{"".join(rows)}
TaskCat BashTurns LushTurns
Total {b_passed}/{total} {l_passed}/{total}
""" def _build_detail_html(results: list[BenchmarkResult]) -> str: sections = [] for r in results: b_status = "PASS" if r.bash_result and r.bash_result.all_passed else "FAIL" l_status = "PASS" if r.lush_result and r.lush_result.all_passed else "FAIL" scores = _get_likert_scores(r) score_rows = [] for key, label, _ in LIKERT_QUESTIONS: b_val = scores[key]["bash"] l_val = scores[key]["lush"] b_str = f"{b_val:.0f}" if b_val is not None else "-" l_str = f"{l_val:.0f}" if l_val is not None else "-" if b_val is not None and l_val is not None: d = l_val - b_val d_str = f"+{d:.0f}" if d > 0 else f"{d:.0f}" d_cls = "pos" if d > 0 else "neg" if d < 0 else "" else: d_str = "-" d_cls = "" score_rows.append(f'{html.escape(label)}' f'{b_str}{l_str}' f'{d_str}') sections.append(f"""

{html.escape(r.task_name)} [{r.category}/{r.mode}] bash={b_status} lush={l_status}

{"".join(score_rows)}
MetricBashLushDiff
""") return "\n".join(sections) def export_html(results_dir: Path, output_path: Path) -> None: results = load_latest_results(results_dir) if not results: output_path.write_text("

No results found.

") return chart_questionnaire = chart_questionnaire_comparison(results) chart_turns = chart_turns_comparison(results) chart_heatmap = chart_per_task_heatmap(results) chart_cat_quest = chart_per_category_questionnaire(results) cat_radar_charts = chart_per_category_radar(results) summary_table = _build_summary_html(results) cat_summary_table = _build_per_category_summary_html(results) detail_html = _build_detail_html(results) model = results[0].model if results else "unknown" timestamp = max(r.timestamp for r in results) page = f""" Lush vs Bash Benchmark Report

Lush vs Bash Benchmark Report

Model: {html.escape(model)} · Latest run: {html.escape(timestamp)} · Tasks: {len(results)}

Summary

{summary_table}

Per-Category Summary

{cat_summary_table}

Questionnaire Scores

Questionnaire comparison

Questionnaire Scores by Category

Per-category questionnaire

Agent Turns (Solve Mode)

Turns comparison

Score Difference Heatmap (Lush - Bash)

Score heatmap

Per-Category Breakdown

{"".join(f'

{cat}

{cat} breakdown
' for cat, img in cat_radar_charts)}

Per-Task Detail

{detail_html} """ output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(page)