← Files Codex SecurityARCHIVED FILE

scripts/workbench/retained_records.py

24.3 KB · Nov 18, 2026 · 06:11 UTC

↓ Download file

See the change to this file →

"""Identify and recover records retained from saved scan evidence."""

from __future__ import annotations

import copy
import re
import unicodedata
from collections import Counter
from collections.abc import Iterator
from pathlib import Path
from typing import Any

from finalize_scan_contract import (
    ContractError,
    _has_identity_text,
    _read_json,
    _recover_unsealed_coverage,
    _recover_unsealed_findings,
    _require_safe_json_value,
    _validate_schema_node,
    finding_candidate_id,
)
from finalize_scan_contract import _schema_values_equal as same_raw_value
from workbench.saved_coverage import _coverage_rows, _saved_coverage_id
from workbench.saved_sources import _digest, _encoded
from workbench_coverage_receipts import resolve_coverage_receipts

_FINDING_IDENTITY_SCHEMA = _read_json(
    Path(__file__).resolve().parents[2] / "schemas" / "findings.schema.json"
)["properties"]["findings"]["items"]["properties"]["identity"]


def _finding_identity(finding: dict[str, Any]) -> Any:
    provenance = finding.get("provenance")
    if isinstance(provenance, dict) and "preservedIdentity" in provenance:
        try:
            _validate_schema_node(
                provenance["preservedIdentity"], _FINDING_IDENTITY_SCHEMA, "finding.identity"
            )
        except ContractError:
            pass
        else:
            return provenance["preservedIdentity"]
    return finding.get("identity")


def _finding_key(finding: dict[str, Any], owner: str | None = None) -> str:
    # Wording and evidence may improve between checkpoints; distinct source locations
    # must not collide merely because two workers chose the same semantic identity.
    provenance = finding.get("provenance")
    identity = _finding_identity(finding)
    if not isinstance(identity, dict):
        normalized = dict(finding)
        normalized.pop("identity", None)
        _ensure_finding_identity(normalized)
        identity = normalized["identity"]
    owner = provenance.get("workerId", owner) if isinstance(provenance, dict) else owner
    parts = [finding.get("ruleId"), identity, _finding_locations(finding)]
    if owner is not None and finding_candidate_id(finding):
        parts.append(owner)
    return _digest(parts)


def _finding_locations(finding: dict[str, Any]) -> list[tuple[Any, Any, Any]]:
    locations = finding.get("locations", [])
    return sorted(
        (
            (
                location.get("path"),
                location.get("startLine"),
                location.get("endLine", location.get("startLine")),
            )
            for location in (locations if isinstance(locations, list) else [])
            if isinstance(location, dict)
        ),
        key=_encoded,
    )


def _worker_candidate_key(
    worker_id: str, candidate_id: str, finding: dict[str, Any]
) -> tuple[str, str, Any, Any, Any]:
    """Identify one worker-local candidate without merging unrelated locations."""
    identity = _finding_identity(finding)
    if not isinstance(identity, dict):
        normalized = dict(finding)
        _ensure_finding_identity(normalized)
        identity = normalized.get("identity")
    anchor = identity.get("anchor") if isinstance(identity, dict) else None
    instance = identity.get("instance") if isinstance(identity, dict) else None
    return worker_id, candidate_id, finding.get("ruleId"), anchor, instance


def _finding_content(finding: dict[str, Any]) -> dict[str, Any]:
    """Return substantive finding content without generated identity or provenance."""
    return {
        key: value
        for key, value in finding.items()
        if key not in {"findingId", "occurrenceId", "fingerprints", "identity", "provenance"}
    }


def _identity_candidate(finding: dict[str, Any]) -> str | None:
    return next(
        (
            value
            for section in (finding.get("extensions"), finding.get("provenance"))
            if isinstance(section, dict) and _has_identity_text(value := section.get("candidateId"))
        ),
        None,
    )


def _ensure_finding_identity(finding: Any) -> None:
    _ensure_finding_identities([finding])


def _semantic_identifier(value: str, fallback: str) -> str:
    normalized = re.sub(r"[\u0300-\u036f]", "", unicodedata.normalize("NFKD", value))
    identifier = re.sub(r"[^a-z0-9._/-]+", "-", normalized.lower())
    return re.sub(r"^[^a-z0-9]+|-+$", "", identifier) or fallback


def _semantic_identity_key(finding: dict[str, Any], identity: Any = None) -> bytes:
    identity = finding.get("identity") if identity is None else identity
    if isinstance(identity, dict):
        identity = {key: identity.get(key) for key in ("anchor", "instance")}
    return _encoded([finding.get("ruleId"), identity])


def _ensure_finding_identities(findings: list[Any], *, deep: bool = False) -> None:
    """Use the same candidate, anchor and sibling rules as draft publication."""
    anchors: dict[int, Any] = {}
    for index, finding in enumerate(findings):
        if not isinstance(finding, dict):
            continue
        identity = finding.get("identity")
        if "identity" in finding:
            anchors[index] = identity.get("anchor") if isinstance(identity, dict) else None
            continue
        candidate = _identity_candidate(finding)
        anchors[index] = _semantic_identifier(
            candidate or str(finding.get("title") or "finding"), f"finding-{index + 1}"
        )
    counts = Counter(
        _digest([findings[index].get("ruleId"), anchor]) for index, anchor in anchors.items()
    )

    authored = {
        _semantic_identity_key(finding)
        for finding in findings
        if isinstance(finding, dict) and "identity" in finding
    }
    generated = []
    for index, anchor in anchors.items():
        finding = findings[index]
        if "identity" in finding:
            continue
        identity = {"anchor": anchor}
        extensions = finding.get("extensions")
        sibling = next(
            (
                value
                for field in ("reportId", "ledgerRowId")
                if isinstance(extensions, dict)
                and _has_identity_text(value := extensions.get(field))
            ),
            None,
        )
        if sibling or counts[_digest([finding.get("ruleId"), anchor])] > 1:
            identity["instance"] = _semantic_identifier(
                sibling or str(finding.get("title") or "finding"), f"finding-{index + 1}"
            )
        finding["identity"] = identity
        generated.append(finding)

    reserved = {
        _semantic_identity_key(finding) for finding in findings if isinstance(finding, dict)
    }
    used = set() if deep else authored.copy()
    generated_ids = {id(finding) for finding in generated}
    for finding in findings if deep else generated:
        key = _semantic_identity_key(finding)
        if key in used or (id(finding) in generated_ids and key in authored):
            identity = finding["identity"]
            if id(finding) not in generated_ids:
                finding.setdefault("provenance", {}).setdefault(
                    "preservedIdentity", copy.deepcopy(identity)
                )
            base_instance = identity.get("instance", "saved")
            suffix = 2
            while True:
                identity["instance"] = f"{base_instance}-{suffix}"
                key = _semantic_identity_key(finding)
                if key not in reserved:
                    reserved.add(key)
                    break
                suffix += 1
        used.add(key)


def _retained_findings(finding: dict[str, Any]) -> Iterator[dict[str, Any]]:
    """Yield canonical and historical findings without trusting candidate IDs."""
    pending = [finding]
    seen: set[int] = set()
    while pending:
        current = pending.pop()
        marker = id(current)
        if marker in seen:
            continue
        seen.add(marker)
        yield current
        provenance = current.get("provenance")
        if not isinstance(provenance, dict):
            continue
        previous = provenance.get("previousFindings")
        if isinstance(previous, list):
            pending.extend(item for item in reversed(previous) if isinstance(item, dict))
        sources = provenance.get("sourceFindings")
        if isinstance(sources, list):
            pending.extend(
                source["finding"]
                for source in reversed(sources)
                if isinstance(source, dict) and isinstance(source.get("finding"), dict)
            )


def _append_previous_finding(
    item: dict[str, Any], finding: dict[str, Any], *, clone: bool = False
) -> None:
    if not isinstance(item.get("previousFindings"), list):
        item["previousFindings"] = []
    history = item["previousFindings"]
    if not any(
        isinstance(previous, dict)
        and _finding_key(previous) == _finding_key(finding)
        and _finding_content(previous) == _finding_content(finding)
        for previous in history
    ):
        history.append(copy.deepcopy(finding) if clone else finding)


def _recover_preserved_worker_source(
    scan_dir, relative, scan_id, binding, worker_id, worker, draft, warnings
):
    """Normalize findings and receipt references in the draft and return diagnostics for discarded findings."""
    discarded_findings = []
    # Recover imported records without relaxing canonical parent validation.
    if "complete" in draft and not isinstance(draft["complete"], bool):
        draft["complete"] = False
        warnings.append(f"{relative}: Invalid complete marker; retained as a progress checkpoint.")
    schema_dir = Path(__file__).resolve().parents[2] / "schemas"
    if "threatModel" in draft:
        try:
            model = draft["threatModel"]
            if isinstance(model, dict):
                model = {**model, "origin": "recovered"}
            schema = _read_json(schema_dir / "scan-manifest.schema.json")
            _validate_schema_node(
                model,
                schema["properties"]["scan"]["properties"]["threatModel"],
                "threatModel",
                schema,
            )
            _require_safe_json_value(model, "threatModel")
        except ContractError as exc:
            draft.pop("threatModel")
            warnings.append(f"{relative}: Skipped malformed threat model: {exc}.")
    directory = Path(relative).parent
    if directory.name == "checkpoints":
        directory = directory.parent
    recovered_findings = []
    for index, finding in enumerate(draft["findings"]):
        needs_identity = isinstance(finding, dict) and "identity" not in finding
        _ensure_finding_identity(finding)
        recovered = {"scanId": scan_id, "findings": [finding]}
        source_warnings: list[str] = []
        discarded = _recover_unsealed_findings(
            {"scan": {"id": scan_id, "target": binding["target"]}},
            recovered,
            schema_dir,
            scan_dir / directory,
            source_warnings,
            current_contract=True,
        )
        if discarded:
            discarded_findings.extend(discarded)
        for recovered_finding in recovered["findings"]:
            if needs_identity:
                # Reconcile anonymous revisions before assigning sibling identities.
                recovered_finding.pop("identity", None)
            writeup = recovered_finding.pop("writeup", None)
            if writeup is not None:
                source_warnings.append(
                    "Retained worker writeup at "
                    f"{directory.as_posix()}/{writeup['reportPath']}; "
                    "the parent report renders this finding inline."
                )
        recovered_findings.extend(recovered["findings"])
        warnings.extend(f"{relative} finding {index + 1}: {warning}" for warning in source_warnings)
    draft["findings"] = recovered_findings
    for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
        rows = draft["reviewState"].get(field)
        if not isinstance(rows, list):
            continue
        for index, row in enumerate(rows):
            if field == "openQuestions" and isinstance(row, str):
                row = rows[index] = {"question": row}
            if not isinstance(row, dict):
                continue
            row["workerId"] = worker_id
            if field == "surfaces":
                row["receiptRefs"] = resolve_coverage_receipts(
                    scan_dir,
                    row,
                    worker,
                    relative,
                )
    return discarded_findings


def _recover_preserved_worker_coverage(
    scan_dir, relative, worker_id, draft, discarded_findings, warnings
):
    """Repair coverage in the draft, append source warnings, and return whether coverage was discarded."""
    discarded_coverage = False
    for field in ("surfaces", "explicitExclusions", "deferred"):
        for row in _coverage_rows(draft["reviewState"], field):
            if isinstance(row, dict):
                row.setdefault("id", _saved_coverage_id(row))
                if field == "surfaces":
                    row.setdefault("receiptRefs", [])
    source_warnings = []
    discarded_fields = _recover_unsealed_coverage(
        draft["reviewState"],
        Path(__file__).resolve().parents[2] / "schemas",
        scan_dir,
        source_warnings,
        discarded_findings,
        # Reconcile local IDs and valid outcomes against worker history below.
        worker_draft=True,
    )
    if discarded_fields & {
        "surfaces",
        "explicitExclusions",
        "deferred",
        "resolvedDeferred",
    }:
        discarded_coverage = True
    for row in draft["reviewState"]["deferred"]:
        row["workerId"] = worker_id
    warnings.extend(f"{relative}: {warning}" for warning in source_warnings)
    return discarded_coverage


def identifiers(value: dict[str, Any]) -> tuple[Any, Any]:
    """Return the report ID and ledger row ID from extensions, using None for missing or blank IDs."""
    extensions = value.get("extensions")
    return tuple(
        identifier if _has_identity_text(identifier) else None
        for identifier in (
            extensions.get(field) if isinstance(extensions, dict) else None
            for field in ("reportId", "ledgerRowId")
        )
    )


def distinct_reports(left: dict[str, Any] | None, right: dict[str, Any] | None) -> bool:
    """Check the latest observations for conflicting report IDs or ledger row IDs."""
    return (
        left is not None
        and right is not None
        and any(
            a is not None and b is not None and a != b
            for a, b in zip(identifiers(left["latest"]), identifiers(right["latest"]), strict=True)
        )
    )


def raw_owner(value: dict[str, Any], source_owner: str | None) -> str | None:
    """Return the source owner, falling back to the worker ID recorded in finding provenance."""
    if source_owner is not None:
        return source_owner
    provenance = value.get("provenance", {})
    owner = provenance.get("workerId")
    return owner if isinstance(owner, str) else source_owner


def raw_scope(value: dict[str, Any]) -> tuple[Any, ...]:
    """Return the rule, path, and line range of the first location used to match anonymous reports."""
    # Match scanFindingIdentity's primary location for anonymous reports.
    location = value["locations"][0]
    return (
        value.get("ruleId"),
        location["path"],
        location["startLine"],
        location.get("endLine"),
    )


def same_raw_finding(left: dict[str, Any], right: dict[str, Any]) -> bool:
    """Check whether semantic identities match or, in their absence, whether candidate and report IDs are compatible."""
    if isinstance(left.get("identity"), dict) and isinstance(right.get("identity"), dict):
        return _semantic_identity_key(left, _finding_identity(left)) == _semantic_identity_key(
            right, _finding_identity(right)
        )
    candidates = (_identity_candidate(left), _identity_candidate(right))
    if all(candidates) and candidates[0] != candidates[1]:
        return False
    pairs = list(zip(identifiers(left), identifiers(right), strict=True))
    if any(a is not None for a, _ in pairs) and any(b is not None for _, b in pairs):
        if not any(a is not None and b is not None for a, b in pairs) or any(
            a is not None and b is not None and a != b for a, b in pairs
        ):
            return False
    return True


def same_raw_content(left: dict[str, Any], right: dict[str, Any]) -> bool:
    """Compare substantive finding fields while ignoring identity, provenance, and report-tracking IDs."""

    def content(value: dict[str, Any]) -> dict[str, Any]:
        result = _finding_content(value)
        if isinstance(result.get("extensions"), dict):
            result["extensions"] = {
                key: value
                for key, value in result["extensions"].items()
                if key not in {"candidateId", "reportId", "ledgerRowId"}
            }
            if not result["extensions"]:
                result.pop("extensions")
        return result

    return same_raw_value(content(left), content(right))


def contains_raw_value(current: Any, previous: Any) -> bool:
    """Check whether the current JSON value retains every part of the previous value.

    Match dictionary values by key and list elements by content, regardless of order.
    """
    if isinstance(previous, dict):
        return isinstance(current, dict) and all(
            key in current and contains_raw_value(current[key], value)
            for key, value in previous.items()
        )
    if isinstance(previous, list):
        return isinstance(current, list) and all(
            any(contains_raw_value(item, value) for item in current) for value in previous
        )
    return isinstance(current, bool) == isinstance(previous, bool) and current == previous


def _saved_outcome_receipts(scan_dir, relative, worker, field, item, coverage_schema):
    normalized = dict(item)
    if field == "surfaces":
        normalized.setdefault("id", _saved_coverage_id(item))
        normalized.setdefault("receiptRefs", [])
        if worker is not None:
            normalized["receiptRefs"] = resolve_coverage_receipts(
                scan_dir, normalized, worker, relative
            )
        verified = {
            "completeness": "partial",
            "surfaces": [normalized],
            "explicitExclusions": [],
            "deferred": [],
        }
        _recover_unsealed_coverage(
            verified, Path(__file__).resolve().parents[2] / "schemas", scan_dir, [], []
        )
        if (
            not verified["surfaces"]
            or verified["surfaces"][0]["disposition"] != item["disposition"]
        ):
            return None
        normalized = verified["surfaces"][0]
    try:
        _validate_schema_node(normalized, coverage_schema[field]["items"], f"coverage.{field}")
    except ContractError:
        return None
    return tuple(normalized.get("receiptRefs", [])) if field == "surfaces" else ()


def _conflicting_outcome_receipts(coverage, scan_dir, receipts):
    # Only a recovered surface can reserve the spelling that invalidates a second row.
    surfaces = _coverage_rows(coverage, "surfaces")
    projected = {
        "surfaces": [dict(row) if isinstance(row, dict) else row for row in surfaces],
        "deferred": [],
        "explicitExclusions": [],
    }
    _recover_unsealed_coverage(
        projected, Path(__file__).resolve().parents[2] / "schemas", scan_dir, [], []
    )
    kept = {ref.lower(): ref for row in projected["surfaces"] for ref in row["receiptRefs"]}
    for row in surfaces:
        if not (
            isinstance(row, dict)
            and isinstance(row.get("candidateId"), str)
            and row.get("disposition") in ("rejected", "not_applicable")
            and isinstance(row.get("receiptRefs"), list)
        ):
            continue
        normalized = {Path(ref).as_posix() for ref in row["receiptRefs"] if isinstance(ref, str)}
        if losing := {ref for ref in normalized & receipts if kept.get(ref.lower(), ref) != ref}:
            return losing
    return set()


def _saved_candidate_outcomes(
    scan_dir,
    current_drafts,
    valid_finding,
    coverage_candidate,
    candidate_order,
    workers_by_id,
    coverage_schema,
):
    """Return saved candidate outcomes with their evidence and the ordering of pending candidates by owner."""
    pending_candidate_orders: dict[tuple[str | None, str], tuple[int, int]] = {}
    outcomes = []
    for relative, draft, owner in current_drafts:
        reported_candidates = {
            finding_candidate_id(finding)
            for finding in draft["findings"]
            if isinstance(finding, dict) and valid_finding(finding)
        }
        pending_candidates = set()
        for field in ("deferred", "surfaces"):
            for item in _coverage_rows(draft["reviewState"], field):
                if isinstance(item, dict) and (
                    field == "deferred" or item.get("disposition") == "needs_follow_up"
                ):
                    if (pending := coverage_candidate(owner, item)) is not None:
                        if (
                            owner is not None
                            and pending[0] == owner
                            and pending[1] in reported_candidates
                        ):
                            continue
                        pending_candidates.add(pending)
                        order = candidate_order(relative, pending[0], item)
                        pending_candidate_orders[pending] = max(
                            pending_candidate_orders.get(pending, order), order
                        )
        for finding in draft["findings"]:
            if (
                isinstance(finding, dict)
                and valid_finding(finding)
                and (candidate_id := finding_candidate_id(finding))
            ):
                outcomes.append(
                    (
                        relative,
                        owner,
                        candidate_id,
                        "reported",
                        candidate_order(relative, owner),
                        (),
                    )
                )
        for field in ("surfaces", "explicitExclusions"):
            for item in _coverage_rows(draft["reviewState"], field):
                if (
                    isinstance(item, dict)
                    and isinstance(item.get("candidateId"), str)
                    and item.get("disposition") in ("reported", "rejected", "not_applicable")
                    and (candidate := coverage_candidate(owner, item)) is not None
                    and (item["disposition"] != "reported" or candidate[1] in reported_candidates)
                    and (candidate not in pending_candidates or item["disposition"] == "reported")
                ):
                    receipts = ()
                    if item["disposition"] != "reported":
                        receipts = _saved_outcome_receipts(
                            scan_dir,
                            relative,
                            workers_by_id.get(owner),
                            field,
                            item,
                            coverage_schema,
                        )
                        if receipts is None:
                            continue
                    outcomes.append(
                        (
                            relative,
                            candidate[0],
                            candidate[1],
                            item["disposition"],
                            candidate_order(relative, candidate[0], item),
                            receipts,
                        )
                    )
    return outcomes, pending_candidate_orders


def _competing_rejection_receipts(outcomes, all_sources, source_surface_receipts):
    receipt_spellings: dict[str, set[str]] = {}
    for _, _, _, _, _, receipts in outcomes:
        for ref in receipts:
            receipt_spellings.setdefault(ref.lower(), set()).add(ref)
    if receipt_spellings and any(
        isinstance(ref, str)
        and (spellings := receipt_spellings.get(Path(ref).as_posix().lower())) is not None
        and spellings != {Path(ref).as_posix()}
        for relative, draft, owner in all_sources
        for row in _coverage_rows(draft["reviewState"], "surfaces")
        if isinstance(row, dict)
        for refs in (source_surface_receipts(relative, row),)
        if isinstance(refs, list)
        for ref in refs
    ):
        return {ref for spellings in receipt_spellings.values() for ref in spellings}
    return set()

SHA-256: 98d1996f869bd0071510bc3c9dffadd7cf3e6d33c960e7c6f8ea090ba69b93ac