← Files Web App QA & Release CopilotARCHIVED FILE

scripts/check_release_evidence.py

3.46 KB · Nov 20, 2026 · 06:40 UTC

↓ Download file

See the change to this file →

#!/usr/bin/env python3
"""Deterministic completeness checks for a release evidence JSON report.
This does not execute browsers or validate the truthfulness of supplied evidence.
"""
import argparse
import json
from pathlib import Path

STATUSES = {"pass", "fail", "unknown", "na"}
BLOCKER_SEVERITIES = {"P0", "P1"}


def evaluate(report):
    issues = []
    gates = report.get("gates", [])
    if not isinstance(gates, list):
        return {"verdict": "Not ready", "issues": ["gates must be a list"]}
    required = [g for g in gates if isinstance(g, dict) and g.get("required") is True]
    if not required:
        issues.append("at least one explicit mandatory gate is required")
    ids = [g.get("id") for g in gates if isinstance(g, dict)]
    if len(ids) != len(gates) or any(not isinstance(i, str) or not i.strip() for i in ids):
        issues.append("each gate must have a nonempty string id")
    if len(ids) != len(set(ids)):
        issues.append("duplicate gate id")
    for g in gates:
        if not isinstance(g, dict):
            issues.append("invalid gate object")
            continue
        gid = g.get("id", "unknown")
        status = g.get("status", "unknown")
        if status not in STATUSES:
            issues.append(f"{gid}: invalid status")
        if g.get("required") is True and status != "pass":
            issues.append(f"{gid}: mandatory gate is not verified passed")
        if status == "pass":
            ev = g.get("evidence")
            if not isinstance(ev, str) or not ev.strip():
                issues.append(f"{gid}: pass missing evidence reference")
        if status == "na" and not str(g.get("reason", "")).strip():
            issues.append(f"{gid}: not-applicable requires reason")
    blockers = report.get("blockers", [])
    if not isinstance(blockers, list):
        issues.append("blockers must be a list")
        blockers = []
    for b in blockers:
        if not isinstance(b, dict):
            issues.append("invalid blocker object")
        elif b.get("severity") in BLOCKER_SEVERITIES and b.get("resolved") is not True:
            issues.append(f"unresolved {b['severity']} blocker: {b.get('id','unknown')}")
    limitations = report.get("limitations", [])
    if not isinstance(limitations, list):
        issues.append("limitations must be a list")
        limitations = []
    for l in limitations:
        if not isinstance(l, dict) or l.get("accepted") is not True or not str(l.get("owner", "")).strip() or not str(l.get("follow_up", "")).strip():
            issues.append("limitation lacks acceptance, owner, or follow-up")
    if report.get("requires_deployment_approval") is True and report.get("deployment_approval") != "granted":
        issues.append("required deployment approval missing")
    if not isinstance(report.get("release_id"), str) or not report.get("release_id", "").strip():
        issues.append("release_id is required")
    verdict = "Not ready" if issues else ("Ready with known limitations" if limitations else "Ready")
    return {"verdict": verdict, "issues": issues, "mandatory_gate_count": len(required)}


def main():
    parser = argparse.ArgumentParser(description='Validate release-gate evidence completeness (not test execution).')
    parser.add_argument('report', type=Path)
    args = parser.parse_args()
    out = evaluate(json.loads(args.report.read_text(encoding='utf8')))
    print(json.dumps(out, indent=2))
    return 0 if out['verdict'] != 'Not ready' else 2


if __name__ == '__main__':
    raise SystemExit(main())

SHA-256: c2f14d031208f5de4b8cb1631125b04eb7f982cc497616d43c6f5585112d66e5