skills/advisory-deliverable-validator/evals/semantic_validation_cases.json

{
  "schema_version": "1.0",
  "purpose": "Representative model/human evaluation cases for advisory-deliverable validation. Expected semantic outcomes must not be derived by deterministic keyword or scorecard logic.",
  "cases": [
    {
      "id": "supported-recommendation",
      "deliverable_format": "docx",
      "contract_context": "The memo must recommend whether to proceed with a pilot and state evidence, conditions, and uncertainty.",
      "deliverable_excerpt": "Proceed with a six-week pilot limited to two regions because both customer research and the observed conversion data support testing the proposition before a broader commitment.",
      "available_evidence": "The cited interviews and reviewed conversion table support a bounded pilot; assumptions and decision conditions are explicit.",
      "expected_dimensions": {
        "factual_source_support": "conforms",
        "recommendation_evidence_decision_fit": "conforms",
        "delivery_readiness": "ready"
      },
      "boundary": "source_backed_advisory_recommendation"
    },
    {
      "id": "partially-supported-market-claim",
      "deliverable_format": "pdf",
      "deliverable_excerpt": "All target customers prefer the proposed channel.",
      "available_evidence": "Three of five interviewed target customers preferred it; the sample was not designed to represent all customers.",
      "expected_dimensions": {
        "factual_source_support": "partially_conforms",
        "residual_uncertainty": "uncertain",
        "correction_needs": "does_not_conform"
      },
      "boundary": "overbroad_support"
    },
    {
      "id": "unsupported-claim",
      "deliverable_format": "md",
      "deliverable_excerpt": "The proposed operating model will reduce churn by 20%.",
      "available_evidence": "No churn analysis, benchmark, or experiment is included.",
      "expected_dimensions": {
        "factual_source_support": "does_not_conform",
        "correction_needs": "does_not_conform",
        "delivery_readiness": "not_ready"
      },
      "boundary": "missing_support"
    },
    {
      "id": "contradicted-recommendation-premise",
      "deliverable_format": "html",
      "deliverable_excerpt": "Capacity is sufficient, so the launch can begin immediately.",
      "available_evidence": "The current capacity plan records a 15% shortfall for the launch period.",
      "expected_dimensions": {
        "factual_source_support": "contradicted",
        "contradictions_missing_evidence": "contradicted",
        "recommendation_evidence_decision_fit": "does_not_conform"
      },
      "boundary": "source_contradiction"
    },
    {
      "id": "reasoning-gap",
      "deliverable_format": "txt",
      "deliverable_excerpt": "Awareness increased, therefore the premium price should be raised now.",
      "available_evidence": "The evidence establishes awareness but contains no willingness-to-pay, elasticity, competitor-price, or margin analysis.",
      "expected_dimensions": {
        "reasoning_assumptions": "does_not_conform",
        "recommendation_evidence_decision_fit": "does_not_conform"
      },
      "boundary": "missing_intermediate_premise"
    },
    {
      "id": "calculation-provenance-gap",
      "deliverable_format": "pptx",
      "deliverable_excerpt": "The initiative creates EUR 4.2m of annual value.",
      "available_evidence": "A chart shows the value but has no linked source rows, calculation method, period, or reconciliation artifact.",
      "expected_dimensions": {
        "calculations_data_provenance": "does_not_conform",
        "factual_source_support": "uncertain",
        "delivery_readiness": "not_ready"
      },
      "required_composition": ["clara:claim-basis-map", "clara:reporting-engine"],
      "boundary": "calculation_and_provenance"
    },
    {
      "id": "contract-failure",
      "deliverable_format": "docx",
      "contract_context": "A two-page board decision memo is required.",
      "deliverable_excerpt": "A twelve-page internal project chronology without a recommendation or decision request.",
      "available_evidence": "The chronology is accurate but does not satisfy the required audience, artifact, or purpose.",
      "expected_dimensions": {
        "contract_conformance": "does_not_conform",
        "delivery_readiness": "not_ready"
      },
      "boundary": "advisory_contract_conformance"
    },
    {
      "id": "judgment-dependent-choice",
      "deliverable_format": "pdf",
      "deliverable_excerpt": "Option B is the preferred path because it trades near-term disruption for greater long-term control.",
      "available_evidence": "The trade-off is evidenced, but the acceptable disruption level and strategic preference belong to the engagement partner and decision owner.",
      "expected_dimensions": {
        "professional_judgement_boundaries": "judgment_required",
        "delivery_readiness": "ready_with_residual_uncertainty"
      },
      "boundary": "professional_judgement"
    },
    {
      "id": "external-document-without-contract",
      "deliverable_format": "docx",
      "contract_context": null,
      "deliverable_excerpt": "A completed external advisory report supplied for review.",
      "available_evidence": "The user states the decision and audience but has not defined consequential scope exclusions or correction authority.",
      "expected_action": "Clara drafts advisory_contract.json from explicit context and asks the user to confirm consequential scope and correction fields before validation.",
      "must_not": ["Silently invent scope", "Run semantic validation without advisory_contract.json"],
      "boundary": "contract_creation_from_confirmed_context"
    },
    {
      "id": "unsupported-primary-format",
      "deliverable_format": "key",
      "deliverable_excerpt": "A native Keynote presentation.",
      "available_evidence": "No supported export is supplied.",
      "expected_action": "Report the primary format as unsupported and request PPTX or PDF export without claiming validation.",
      "boundary": "initial_format_limit"
    },
    {
      "id": "web-listings-do-not-prove-total-stock",
      "deliverable_format": "html",
      "deliverable_excerpt": "The company has only 13 vehicles in stock.",
      "available_evidence": "A hash-bound public-page capture shows 13 visible listings. A separate assertion-only interview receipt records management saying total stock is 300 vehicles.",
      "expected_lineage": ["ev-web-13 -> cl-visible-13", "ev-management-300 -> cl-management-300"],
      "expected_dimensions": {
        "factual_source_support": "contradicted",
        "contradictions_missing_evidence": "contradicted",
        "delivery_readiness": "not_ready"
      },
      "must_not": ["Use the 13-listing capture as proof of total company stock", "Merge the two claims into one evidence item"],
      "boundary": "web_capture_scope"
    },
    {
      "id": "interview-quote-versus-underlying-truth",
      "deliverable_format": "docx",
      "deliverable_excerpt": "Giovanni believes that the new site will double conversion. The site will double conversion.",
      "available_evidence": "The reviewed transcript supports the attributed quote. No experiment, forecast, or calculation supports the second sentence as a factual prediction.",
      "expected_dimensions": {
        "factual_source_support": "partially_conforms",
        "reasoning_assumptions": "does_not_conform",
        "delivery_readiness": "not_ready"
      },
      "boundary": "quotation_is_not_truth"
    },
    {
      "id": "derived-claim-requires-a-and-b",
      "deliverable_format": "md",
      "deliverable_excerpt": "Proceed with acquisition X.",
      "available_evidence": "The upstream register declares that recommendation X depends on claim A and claim B through all_of. A is supported; B is contradicted by a later receipt.",
      "expected_chain_walk": ["cl-x", "cl-a", "cl-b"],
      "expected_dimensions": {
        "contradictions_missing_evidence": "contradicted",
        "recommendation_evidence_decision_fit": "does_not_conform",
        "delivery_readiness": "not_ready"
      },
      "boundary": "dependency_propagation"
    },
    {
      "id": "calculation-recheck-after-input-change",
      "deliverable_format": "pptx",
      "deliverable_excerpt": "Annual contribution is EUR 4.2m.",
      "available_evidence": "The claim references a Reporting Engine calculation receipt, but the source dataset hash changed after the recorded run.",
      "expected_recheck": {"kind": "calculation", "status": "pending"},
      "expected_dimensions": {
        "calculations_data_provenance": "uncertain",
        "delivery_readiness": "not_ready"
      },
      "required_composition": ["clara:reporting-engine", "clara:claim-basis-map"],
      "boundary": "targeted_calculation_rerun"
    },
    {
      "id": "corrected-claim-supersedes-original",
      "deliverable_format": "pptx",
      "deliverable_excerpt": "A corrected recommendation replaces an overbroad original statement.",
      "available_evidence": "The original file and claim remain preserved. The new claim points to supersedes_claim_id and the corrected deck has a separate path and completed format checks.",
      "expected_action": "Review the new chain, preserve the old claim history, and bind the new appearance to the separate corrected artifact.",
      "required_composition": ["clara:deck-correction", "clara:claim-basis-map"],
      "boundary": "correction_lineage"
    },
    {
      "id": "two-hundred-page-coverage",
      "deliverable_format": "pdf",
      "deliverable_excerpt": "A readable 200-page commercial due-diligence report with generation-time claim appearances.",
      "available_evidence": "The preparation produces bounded coverage units. The model accounts for every unit, selects decision-relevant claims, and the validator walks their dependency closure rather than re-extracting every claim at the end.",
      "expected_action": "Reject readiness if a coverage unit is neither considered nor explicitly omitted, or if a selected claim dependency is absent from the chain review.",
      "boundary": "scaled_coverage_and_lineage"
    },
    {
      "id": "zero-claim-ready-review",
      "deliverable_format": "md",
      "deliverable_excerpt": "A completed advisory recommendation is marked ready, but the review selects no tracked or reconstructed material claim.",
      "available_evidence": "Coverage units are present but all are declared to contain no material claims.",
      "expected_action": "Reject delivery readiness because a completed advisory deliverable requires at least one model-reviewed material claim.",
      "boundary": "non_empty_semantic_review"
    },
    {
      "id": "fake-completed-recheck",
      "deliverable_format": "html",
      "deliverable_excerpt": "A public-source claim is marked rechecked.",
      "available_evidence": "The claimed new receipt does not point to the earlier receipt or lacks rechecked_unchanged or rechecked_changed verification.",
      "expected_action": "Reject the completed recheck and keep delivery blocked.",
      "boundary": "recheck_succession"
    },
    {
      "id": "corrected-artifact-without-second-review",
      "deliverable_format": "pptx",
      "deliverable_excerpt": "A separate corrected deck and correction approval are supplied.",
      "available_evidence": "No second preparation inventory or model-led review is bound to the corrected deck bytes.",
      "expected_action": "Reject correction completion until the corrected artifact is prepared and reviewed again.",
      "boundary": "corrected_artifact_re_review"
    },
    {
      "id": "generic-passed-format-json",
      "deliverable_format": "pptx",
      "deliverable_excerpt": "A calculation-backed deck declares the Reporting Engine check passed.",
      "available_evidence": "The only artifact is a generic JSON file containing status passed, not a Clara Reporting Engine 0.2 render manifest.",
      "expected_action": "Reject the format check because no authoritative workflow-owned result is present.",
      "boundary": "authoritative_format_result"
    }
  ]
}