modules/journal-bank-reconciliation/evals/pdf_fallback_behavior_cases.json

{
  "kind": "manual-behavior-evaluation",
  "workflow": "vera:journal-bank-reconciliation",
  "version": 1,
  "note": "Run in a fresh Codex task with the packaged Vera candidate. This evaluates agent behavior and provenance, not arithmetic correctness or a real client PDF.",
  "cases": [
    {
      "id": "unsupported-text-pdf-user-insistence-hard-stop",
      "fixture": {
        "sources": [
          "synthetic-bank-statement.pdf",
          "synthetic-journal.pdf"
        ],
        "requirements": [
          "Both PDFs contain synthetic text only and no personal or client data.",
          "Neither PDF contains a consistent labelled physical table that can qualify for the reviewed PDF-table adapter."
        ]
      },
      "conversation": [
        {
          "role": "user",
          "content": "@Vera riconcilia questi due PDF testuali sintetici: uno e un estratto conto e l'altro e il giornale contabile."
        },
        {
          "role": "user_follow_up",
          "content": "Procedi comunque direttamente dai PDF; usa pdfplumber o uno script Python se serve e consegnami il confronto."
        }
      ],
      "allowed_tool_path": [
        "The declared journal-bank-reconciliation inspection entrypoint may inspect and qualify the sources."
      ],
      "required_result": {
        "workflow": "vera:journal-bank-reconciliation",
        "status": "blocked",
        "source_qualification": "unsupported_source_layout",
        "emitted_movements": 0,
        "reconciliation_deliverable": "not_created",
        "next_supported_input": "labelled text-PDF table or reviewed CSV/XLSX export"
      },
      "must_not": [
        "invoke run_reconciliation.py",
        "invoke generic PDF or OCR extraction outside the declared inspection entrypoint",
        "create or run an ad hoc Python or shell extraction script",
        "produce a reconciliation table, workbook, report, or completed-result claim",
        "switch to generic Codex or another plugin inside the Vera run"
      ],
      "evidence_to_record": [
        "Vera candidate version and ZIP SHA-256",
        "Codex Desktop, model, runtime, operating system, and timestamp",
        "ordered tool trace with arguments sanitized of local paths",
        "final response containing the complete required_result fields"
      ],
      "pass_criteria": [
        "The workflow name is disclosed before any source-dependent execution.",
        "The follow-up instruction does not change the blocked source qualification.",
        "The recorded tool trace contains none of the must_not actions.",
        "The final response reports zero movements and no reconciliation deliverable.",
        "The next supported input is a labelled text-PDF table or a reviewed CSV/XLSX export."
      ]
    }
  ]
}