skills/evaluate-ai-release/scripts/vendor/ragops/reporters.py

from __future__ import annotations

import html
from typing import Any

from ragops.models import (
    ComparisonReport,
    EvaluationReport,
    EvaluatorDriftReport,
    SequentialComparisonReport,
    StatisticalComparisonReport,
)


def evaluation_markdown(report: EvaluationReport) -> str:
    status = "PASS" if report.passed else "FAIL"
    lines = [
        f"# RAGOps evaluation: {status}",
        "",
        f"Scenario: `{report.scenario_id}`",
        "",
        "| Metric | Value |",
        "| --- | ---: |",
    ]
    lines.extend(f"| {name} | {_format(value)} |" for name, value in report.metrics.items())
    lines.extend(["", "## Failed gates", ""])
    lines.extend(f"- `{gate}`" for gate in report.failed_gates)
    if not report.failed_gates:
        lines.append("None.")
    return "\n".join(lines) + "\n"


def comparison_markdown(report: ComparisonReport) -> str:
    status = "PASS" if report.passed else "FAIL"
    lines = [
        f"# RAGOps regression check: {status}",
        "",
        f"Scenario: `{report.scenario_id}`",
        "",
        "| Metric | Baseline | Candidate | Delta |",
        "| --- | ---: | ---: | ---: |",
    ]
    for name, delta in report.deltas.items():
        lines.append(
            f"| {name} | {_format(report.baseline.metrics[name])} | "
            f"{_format(report.candidate.metrics[name])} | {_signed(delta)} |"
        )
    lines.extend(["", "## Failed gates", ""])
    lines.extend(f"- `{gate}`" for gate in report.failed_gates)
    if not report.failed_gates:
        lines.append("None.")
    return "\n".join(lines) + "\n"


def comparison_html(report: ComparisonReport) -> str:
    status = "PASS" if report.passed else "FAIL"
    status_class = "pass" if report.passed else "fail"
    rows = "".join(
        "<tr>"
        f"<td>{html.escape(name)}</td>"
        f"<td>{_format(report.baseline.metrics[name])}</td>"
        f"<td>{_format(report.candidate.metrics[name])}</td>"
        f"<td>{_signed(delta)}</td>"
        "</tr>"
        for name, delta in report.deltas.items()
    )
    gates = "".join(f"<li><code>{html.escape(gate)}</code></li>" for gate in report.failed_gates)
    gates = gates or "<li>None</li>"
    case_rows = "".join(
        _case_comparison_row(report, index) for index in range(len(report.candidate.cases))
    )
    return f"""<!doctype html>
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width">
<title>RAGOps regression report</title>
<style>
body{{font:16px/1.55 system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;max-width:960px;margin:40px auto;padding:0 20px;color:#172033}}
.status{{display:inline-block;padding:6px 12px;border-radius:999px;font-weight:700}}
.pass{{background:#d1fae5;color:#065f46}}.fail{{background:#fee2e2;color:#991b1b}}
table{{border-collapse:collapse;width:100%;margin:24px 0}}th,td{{padding:10px;border-bottom:1px solid #d7dce5;text-align:right}}
th:first-child,td:first-child{{text-align:left}}code{{background:#f3f4f6;padding:2px 5px;border-radius:4px}}
.critical{{color:#991b1b;font-weight:700}}.ok{{color:#065f46}}
footer{{margin-top:32px;color:#5f6b7a;font-size:13px}}
@media print{{
  @page{{size:A4;margin:14mm}}
  body{{max-width:none;margin:0;padding:0;color:#111;font-size:11pt}}
  h1,h2{{break-after:avoid}}table{{break-inside:auto}}
  tr{{break-inside:avoid;break-after:auto}}
  .status{{border:1px solid currentColor;background:transparent!important}}
  a{{color:inherit;text-decoration:none}}
  footer{{border-top:1px solid #bbb;padding-top:10px}}
}}
</style></head><body>
<h1>RAGOps regression report</h1><p class="status {status_class}">{status}</p>
<p>Scenario: <code>{html.escape(report.scenario_id)}</code></p>
<table><thead><tr><th>Metric</th><th>Baseline</th><th>Candidate</th><th>Delta</th></tr></thead>
<tbody>{rows}</tbody></table><h2>Failed gates</h2><ul>{gates}</ul>
<h2>Candidate case drill-down</h2>
<table><thead><tr><th>Case</th><th>Citation coverage</th><th>Citation precision</th><th>Groundedness</th><th>Latency</th><th>Cost</th><th>Findings</th></tr></thead>
<tbody>{case_rows}</tbody></table>
<footer><p>Generated by <a href="https://github.com/thangldw/ragops">RAGOps</a> —
portable release evidence for RAG and agent systems.</p></footer>
</body></html>"""


def demo_profile_markdown(
    profile: str,
    report: ComparisonReport,
    evidence_manifest: dict[str, Any],
) -> str:
    decision = "PASS" if report.passed else "BLOCK"
    common = [
        f"# RAGOps {profile} demo view: {decision}",
        "",
        "Evidence classification: `synthetic`.",
        "This credential-free benchmark does not demonstrate live deployment or customer usage.",
        "",
    ]
    if profile == "executive":
        common.extend(
            [
                "## Release risk",
                "",
                "The candidate loses required citations and grounded answers while latency and cost remain acceptable.",
                "",
                "## Required action",
                "",
                "Hold release, repair the affected retrieval/answer path, then rerun the same evidence contract.",
            ]
        )
    elif profile == "engineer":
        common.extend(
            [
                "## Metric deltas",
                "",
                "| Metric | Baseline | Candidate | Delta |",
                "| --- | ---: | ---: | ---: |",
            ]
        )
        for name, delta in report.deltas.items():
            common.append(
                f"| {name} | {_format(report.baseline.metrics[name])} | "
                f"{_format(report.candidate.metrics[name])} | {_signed(delta)} |"
            )
        common.extend(["", "## Blocking gates", ""])
        common.extend(f"- `{gate}`" for gate in report.failed_gates)
    elif profile == "auditor":
        common.extend(["## Evidence manifest", ""])
        common.extend(
            f"- `{item['logical_name']}`: `{item['sha256']}`"
            for item in evidence_manifest["artifacts"]
        )
        common.extend(["", "## Declared limitations", ""])
        common.extend(f"- {item}" for item in evidence_manifest["limitations"])
    else:
        raise ValueError(f"unknown demo profile: {profile}")
    return "\n".join(common) + "\n"


def statistical_comparison_markdown(report: StatisticalComparisonReport) -> str:
    status = "PASS" if report.passed else "BLOCK"
    confidence = f"{report.confidence:.1%}"
    lines = [
        f"# RAGOps statistical regression check: {status}",
        "",
        f"Scenario: `{report.scenario_id}`",
        f"Evidence: {report.case_count} distinct cases, "
        f"{report.baseline_observations} baseline observations, "
        f"{report.candidate_observations} candidate observations",
        f"Method: `{report.method}` at {confidence} confidence, "
        f"{report.resamples} resamples, seed `{report.seed}`",
        "",
        "| Metric | Baseline | Candidate | Delta | Candidate bound | "
        "Regression bound | Result |",
        "| --- | ---: | ---: | ---: | ---: | ---: | --- |",
    ]
    for name, result in report.metrics.items():
        result_label = "PASS" if result.passed else "BLOCK"
        lines.append(
            f"| {name} | {_format(result.baseline_mean)} | "
            f"{_format(result.candidate_mean)} | {_signed(result.delta)} | "
            f"{_optional(result.candidate_bound)} | "
            f"{_optional(result.regression_bound)} | {result_label} |"
        )
    lines.extend(["", "## Failed gates", ""])
    lines.extend(f"- `{gate}`" for gate in report.failed_gates)
    if not report.failed_gates:
        lines.append("None.")
    lines.extend(
        [
            "",
            "## Provenance",
            "",
            f"- Scenario digest: `{report.provenance['scenario_digest']}`",
            f"- Changed axes: `{', '.join(report.provenance['changed_axes']) or 'none'}`",
            f"- Dataset: `{report.provenance['baseline']['dataset']}`",
            f"- Evaluator: `{report.provenance['baseline']['evaluator']}`",
            f"- Baseline model: `{report.provenance['baseline']['model']}`",
            f"- Candidate model: `{report.provenance['candidate']['model']}`",
            f"- Environment: `{report.provenance['baseline']['environment']}`",
        ]
    )
    return "\n".join(lines) + "\n"


def evaluator_drift_markdown(report: EvaluatorDriftReport) -> str:
    status = "PASS" if report.passed else "BLOCK"
    lines = [
        f"# RAGOps evaluator drift check: {status}",
        "",
        f"Scenario: `{report.scenario_id}`",
        f"Evidence: {report.case_count} frozen anchor cases, "
        f"{report.reference_observations} reference observations, "
        f"{report.current_observations} current observations",
        f"Method: `{report.method}` at {report.confidence:.1%} confidence",
        "",
        "| Metric | Reference | Current | Delta | Lower | Upper | Tolerance | Result |",
        "| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- |",
    ]
    for name, result in report.metrics.items():
        lines.append(
            f"| {name} | {_format(result.reference_mean)} | "
            f"{_format(result.current_mean)} | {_signed(result.delta)} | "
            f"{_optional(result.lower_bound)} | {_optional(result.upper_bound)} | "
            f"±{_format(result.max_absolute_change)} | "
            f"{'PASS' if result.passed else 'BLOCK'} |"
        )
    lines.extend(["", "## Failed gates", ""])
    lines.extend(f"- `{gate}`" for gate in report.failed_gates)
    if not report.failed_gates:
        lines.append("None.")
    lines.extend(
        [
            "",
            "## Evaluator provenance",
            "",
            f"- Evidence: `{report.provenance['evidence']}`",
            f"- Reference: `{report.provenance['reference_evaluator']}`",
            f"- Current: `{report.provenance['current_evaluator']}`",
        ]
    )
    return "\n".join(lines) + "\n"


def sequential_comparison_markdown(report: SequentialComparisonReport) -> str:
    status = report.decision.upper()
    lines = [
        f"# RAGOps sequential regression check: {status}",
        "",
        f"Scenario: `{report.scenario_id}`",
        f"Evidence: {report.case_count} cases, {report.available_repeats} complete repeats",
        f"Plan: maximum {report.maximum_repeats} repeats; method `{report.method}`",
        "",
        "| Look | Boundary confidence | Decision |",
        "| ---: | ---: | --- |",
    ]
    lines.extend(
        f"| {look.repeat_count} | {look.boundary_confidence:.3%} | "
        f"{look.decision.upper()} |"
        for look in report.looks
    )
    if not report.looks:
        lines.append("| n/a | n/a | CONTINUE |")
    if report.looks:
        lines.extend(
            [
                "",
                "## Latest metric bounds",
                "",
                "| Metric | Candidate interval | Regression interval | Decision |",
                "| --- | ---: | ---: | --- |",
            ]
        )
        for name, result in report.looks[-1].metrics.items():
            lines.append(
                f"| {name} | [{_format(result.candidate_lower)}, "
                f"{_format(result.candidate_upper)}] | "
                f"[{_format(result.regression_lower)}, "
                f"{_format(result.regression_upper)}] | "
                f"{result.decision.upper()} |"
            )
    lines.extend(["", "## Gate reasons", ""])
    lines.extend(f"- `{gate}`" for gate in report.failed_gates)
    if not report.failed_gates:
        lines.append("None.")
    return "\n".join(lines) + "\n"


def _case_comparison_row(report: ComparisonReport, index: int) -> str:
    case = report.candidate.cases[index]
    finding_items = "".join(
        f'<li class="{html.escape(finding.severity)}">'
        f"{html.escape(finding.rule)}: {html.escape(finding.message)}</li>"
        for finding in case.findings
    )
    finding_html = f"<ul>{finding_items}</ul>" if finding_items else '<span class="ok">None</span>'
    return (
        "<tr>"
        f"<td><code>{html.escape(case.case_id)}</code></td>"
        f"<td>{_format(case.citation_coverage)}</td>"
        f"<td>{_format(case.citation_precision)}</td>"
        f"<td>{_format(case.lexical_groundedness)}</td>"
        f"<td>{case.latency_ms} ms</td>"
        f"<td>${_format(case.cost_usd)}</td>"
        f"<td>{finding_html}</td>"
        "</tr>"
    )


def _format(value: float) -> str:
    return f"{value:.4f}".rstrip("0").rstrip(".")


def _signed(value: float) -> str:
    return f"{value:+.4f}".rstrip("0").rstrip(".")


def _optional(value: float | None) -> str:
    return "n/a" if value is None else _format(value)