modules/deep-research-validator/evals/semantic_validation_cases.json

{
  "schema_version": "1.0",
  "purpose": "Human/model evaluation cases for semantic support, source identity, reasoning, and professional-judgment boundaries. Fixed code must not derive these labels from lexical overlap.",
  "cases": [
    {
      "id": "taxonomy-entailment",
      "claim": "This is a dog.",
      "source_passage": "This is a terrier.",
      "expected_support": "supported",
      "expected_reasoning": "sound",
      "boundary": "semantic_entailment",
      "rationale": "A terrier is a dog even though the strings differ."
    },
    {
      "id": "negation-contradiction",
      "claim": "This is a dog.",
      "source_passage": "This is not a dog.",
      "expected_support": "contradicted",
      "expected_reasoning": "unsound",
      "boundary": "negation",
      "rationale": "The source reverses the claim despite high lexical overlap."
    },
    {
      "id": "future-modality",
      "claim": "This is a dog.",
      "source_passage": "This will be a dog.",
      "expected_support": "not_supported",
      "expected_reasoning": "unsound",
      "boundary": "time_and_modality",
      "rationale": "A future proposition does not establish the present claim."
    },
    {
      "id": "past-time",
      "claim": "This is a dog.",
      "source_passage": "This was a dog.",
      "expected_support": "not_supported",
      "expected_reasoning": "unsound",
      "boundary": "time_and_modality",
      "rationale": "A past proposition does not establish the present claim."
    },
    {
      "id": "qualification-loss",
      "claim": "The taxpayer qualifies for the exemption.",
      "source_passage": "The taxpayer may qualify if the independent-activity test is met.",
      "expected_support": "partially_supported",
      "expected_reasoning": "unsound",
      "boundary": "qualification_and_missing_premise",
      "rationale": "The answer removes a condition and certainty qualifier."
    },
    {
      "id": "right-proposition-wrong-authority",
      "claim": "The appeal period is 30 days.",
      "source_passage": "The appeal period is 30 days.",
      "expected_support": "uncertain",
      "expected_reasoning": "uncertain",
      "expected_source_identity": "different_source",
      "boundary": "source_identity",
      "rationale": "Matching words from a different jurisdiction or version do not validate the cited authority."
    },
    {
      "id": "judgment-dependent-application",
      "claim": "The conduct was reasonable in the circumstances.",
      "source_passage": "Reasonableness depends on all relevant circumstances.",
      "expected_support": "partially_supported",
      "expected_reasoning": "uncertain",
      "expected_judgment": "professional_judgment_required",
      "boundary": "professional_judgment",
      "rationale": "The legal standard is sourced, but applying it requires fact-sensitive professional judgment."
    },
    {
      "id": "historical-decision-outside-rolling-public-window",
      "claim": "The historical decision states the cited legal principle.",
      "source_passage": "A non-institutional reproduction contains the decision and the stated principle; independent references corroborate the decision identifier and substance, while the official historical archive is gated and the free portal has a documented rolling window.",
      "expected_support": "supported",
      "expected_reasoning": "sound",
      "expected_source_identity": "matches_cited_source",
      "expected_authority_relation": "non_institutional_reproduction",
      "expected_official_text_access": "restricted_or_gated_archive",
      "expected_text_fidelity": "corroborated_not_text_verified",
      "expected_limitation": "Fidelity to the official full text was not verified.",
      "boundary": "archive_access_vs_substantive_support",
      "rationale": "A rolling or gated official access surface limits text-fidelity verification; it does not establish nonexistence or automatically defeat corroborated substantive support."
    },
    {
      "id": "no-result-does-not-prove-nonexistence",
      "claim": "The cited decision does not exist.",
      "source_passage": "The decision was not returned by a public portal whose completeness for the relevant period was not established.",
      "expected_support": "not_supported",
      "expected_reasoning": "unsound",
      "expected_official_text_access": "not_assessed",
      "boundary": "archive_coverage_inference",
      "rationale": "A no-result finding is not proof of nonexistence unless the searched archive is shown to be complete for the relevant decision class and period."
    }
  ]
}