← Files Web App QA & Release CopilotARCHIVED FILE
scripts/check_release_evidence.py
3.46 KB · Nov 20, 2026 · 06:40 UTC
#!/usr/bin/env python3
"""Deterministic completeness checks for a release evidence JSON report.
This does not execute browsers or validate the truthfulness of supplied evidence.
"""
import argparse
import json
from pathlib import Path
STATUSES = {"pass", "fail", "unknown", "na"}
BLOCKER_SEVERITIES = {"P0", "P1"}
def evaluate(report):
issues = []
gates = report.get("gates", [])
if not isinstance(gates, list):
return {"verdict": "Not ready", "issues": ["gates must be a list"]}
required = [g for g in gates if isinstance(g, dict) and g.get("required") is True]
if not required:
issues.append("at least one explicit mandatory gate is required")
ids = [g.get("id") for g in gates if isinstance(g, dict)]
if len(ids) != len(gates) or any(not isinstance(i, str) or not i.strip() for i in ids):
issues.append("each gate must have a nonempty string id")
if len(ids) != len(set(ids)):
issues.append("duplicate gate id")
for g in gates:
if not isinstance(g, dict):
issues.append("invalid gate object")
continue
gid = g.get("id", "unknown")
status = g.get("status", "unknown")
if status not in STATUSES:
issues.append(f"{gid}: invalid status")
if g.get("required") is True and status != "pass":
issues.append(f"{gid}: mandatory gate is not verified passed")
if status == "pass":
ev = g.get("evidence")
if not isinstance(ev, str) or not ev.strip():
issues.append(f"{gid}: pass missing evidence reference")
if status == "na" and not str(g.get("reason", "")).strip():
issues.append(f"{gid}: not-applicable requires reason")
blockers = report.get("blockers", [])
if not isinstance(blockers, list):
issues.append("blockers must be a list")
blockers = []
for b in blockers:
if not isinstance(b, dict):
issues.append("invalid blocker object")
elif b.get("severity") in BLOCKER_SEVERITIES and b.get("resolved") is not True:
issues.append(f"unresolved {b['severity']} blocker: {b.get('id','unknown')}")
limitations = report.get("limitations", [])
if not isinstance(limitations, list):
issues.append("limitations must be a list")
limitations = []
for l in limitations:
if not isinstance(l, dict) or l.get("accepted") is not True or not str(l.get("owner", "")).strip() or not str(l.get("follow_up", "")).strip():
issues.append("limitation lacks acceptance, owner, or follow-up")
if report.get("requires_deployment_approval") is True and report.get("deployment_approval") != "granted":
issues.append("required deployment approval missing")
if not isinstance(report.get("release_id"), str) or not report.get("release_id", "").strip():
issues.append("release_id is required")
verdict = "Not ready" if issues else ("Ready with known limitations" if limitations else "Ready")
return {"verdict": verdict, "issues": issues, "mandatory_gate_count": len(required)}
def main():
parser = argparse.ArgumentParser(description='Validate release-gate evidence completeness (not test execution).')
parser.add_argument('report', type=Path)
args = parser.parse_args()
out = evaluate(json.loads(args.report.read_text(encoding='utf8')))
print(json.dumps(out, indent=2))
return 0 if out['verdict'] != 'Not ready' else 2
if __name__ == '__main__':
raise SystemExit(main())
SHA-256: c2f14d031208f5de4b8cb1631125b04eb7f982cc497616d43c6f5585112d66e5