skills/proof-audit/evals/evals.json

{
  "skill_name": "proof-audit",
  "evals": [
    {
      "id": 1,
      "prompt": "Audit this claim in a commutative ring: if ab=ac then b=c. The proof cancels a from both sides without another hypothesis.",
      "expected_output": "Rejects the claim as written, identifies the missing cancellability/non-zero-divisor condition, and supplies a small counterexample.",
      "assertions": [
        "Normalizes the ring claim",
        "Identifies invalid cancellation",
        "Returns a refuted or restricted verdict",
        "Gives a valid small counterexample or exact missing hypothesis"
      ]
    },
    {
      "id": 2,
      "prompt": "Audit this proof. Its decisive leaf cites Theorem 4.3 for an equivariant integral statement, but the literature record has no exact theorem or version.",
      "expected_output": "Marks the external leaf conditional and routes the bounded theorem question through the available literature-check skill, including its authorized cache-first acquisition workflow, rather than relying on citation metadata or memory.",
      "assertions": [
        "Identifies the unverified external-source leaf",
        "Uses literature-check for the exact source question",
        "Checks the authorized local cache before fetching",
        "Does not pass the proof obligation from a snippet, citation, or memory"
      ]
    },
    {
      "id": 3,
      "prompt": "A ledger labels Claim A proved and independently audited, but the source proof file changed after the review. Audit A.",
      "expected_output": "Checks the actual current argument and treats earlier evidence/review as stale until revalidated.",
      "assertions": [
        "Does not certify mathematics from a ledger label",
        "Checks exact current hypotheses and proof bytes",
        "Separates evidence, freshness and review provenance"
      ]
    },
    {
      "id": 4,
      "prompt": "Audit a proof and numerical check about paths in the cone of positive-definite matrices. The supplied path crosses a singular matrix; the code silently replaces every small eigenvalue by a positive cutoff.",
      "expected_output": "Rejects both the path and the computation as evidence for the stated cone until a domain-valid path and a proved comparison with the regularized surrogate are supplied.",
      "assertions": [
        "Reconstructs the admissible domain from the definition",
        "Checks every intermediate point rather than only endpoints",
        "Does not treat the regularized surrogate as the claimed matrix cone",
        "States the strongest safe conclusion and the first repair obligation"
      ]
    },
    {
      "id": 5,
      "prompt": "Audit a report claiming an exhaustive identity check on 96 inputs. The driver iterates inputs[::5], reports that every assertion passed, and gives no reduction lemma.",
      "expected_output": "Finds that only a sample was checked, rejects the exhaustive label, and asks either for full enumeration or a proved symmetry reduction covering omitted representatives.",
      "assertions": [
        "Compares the claimed cardinality with the actual iterator",
        "Distinguishes passing sampled assertions from exhaustive coverage",
        "Does not infer omitted cases from unproved symmetry"
      ]
    },
    {
      "id": 6,
      "prompt": "Audit a graded chain-map argument whose regression matrices pass in degrees 1 through 4. The tests record only degree parity, while the proposed shift sends a degree-zero generator to absolute degree 2.",
      "expected_output": "Rejects the chain-map claim because the shift is not degree zero, checks the degree-zero boundary case, and explains why parity-only matrices could not detect the defect.",
      "assertions": [
        "Checks absolute degree and shift direction",
        "Includes the degree-zero case",
        "Does not let higher-arity parity tests certify a graded map"
      ]
    },
    {
      "id": 7,
      "prompt": "You are in a web host with read-only GitHub access and no script execution. The project's .mathbox ledger records evidence E000004 for claim C, pinning proofs/c.md, and its config opens reviews/ to deferred files. The user authorizes recording the audit. In its final step, proofs/c.md applies Lemma 2, which assumes injectivity, to a map not known to be injective.",
      "expected_output": "Audits the actual argument, isolates the misapplied Lemma 2 hypothesis as the failing leaf, and returns one deferred packet with a complete audit report under reviews/ and a separate failed review of E000004. States that the review has not been recorded.",
      "assertions": [
        "Finds the missing injectivity hypothesis from the proof text rather than ledger labels",
        "Writes the complete audit report as a new file under the configured reviews/ root",
        "Proposes a review of exactly E000004 with outcome fail and an honest independence declaration",
        "Invents no hashes or event IDs and states that the packet has not been applied"
      ]
    }
  ]
}