skills/evaluate-ai-release/scripts/vendor/ragops/schemas/statistical-report-0.1.schema.json

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://ragops.dev/schemas/statistical-report-0.1.schema.json",
  "title": "RAGOps statistical comparison report 0.1",
  "type": "object",
  "additionalProperties": false,
  "required": [
    "report_version", "scenario_id", "passed", "confidence", "case_count",
    "baseline_observations", "candidate_observations", "method", "resamples",
    "seed", "failed_gates", "metrics", "provenance"
  ],
  "properties": {
    "report_version": {"const": "0.1"},
    "scenario_id": {"type": "string", "minLength": 1},
    "passed": {"type": "boolean"},
    "confidence": {"type": "number", "exclusiveMinimum": 0.5, "exclusiveMaximum": 1},
    "case_count": {"type": "integer", "minimum": 1},
    "baseline_observations": {"type": "integer", "minimum": 1},
    "candidate_observations": {"type": "integer", "minimum": 1},
    "method": {"const": "paired_hierarchical_bootstrap_one_sided"},
    "resamples": {"type": "integer", "minimum": 100},
    "seed": {"type": "integer"},
    "failed_gates": {"type": "array", "items": {"type": "string"}},
    "metrics": {
      "type": "object",
      "minProperties": 1,
      "additionalProperties": {
        "type": "object",
        "additionalProperties": false,
        "required": [
          "direction", "baseline_mean", "candidate_mean", "delta",
          "candidate_bound", "regression_bound", "absolute_threshold",
          "max_regression", "passed", "failed_gates"
        ],
        "properties": {
          "direction": {"enum": ["higher", "lower"]},
          "baseline_mean": {"type": "number"},
          "candidate_mean": {"type": "number"},
          "delta": {"type": "number"},
          "candidate_bound": {"type": ["number", "null"]},
          "regression_bound": {"type": ["number", "null"]},
          "absolute_threshold": {"type": "number"},
          "max_regression": {"type": "number", "minimum": 0},
          "passed": {"type": "boolean"},
          "failed_gates": {"type": "array", "items": {"type": "string"}}
        }
      }
    },
    "provenance": {
      "type": "object",
      "additionalProperties": false,
      "required": ["scenario_digest", "changed_axes", "baseline", "candidate"],
      "properties": {
        "scenario_digest": {"type": "string"},
        "changed_axes": {"type": "array", "items": {"type": "string"}},
        "baseline": {"$ref": "#/$defs/provenance"},
        "candidate": {"$ref": "#/$defs/provenance"}
      }
    }
  },
  "$defs": {
    "provenance": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "dataset", "evidence", "evaluator", "application", "model", "model_config", "environment"
      ],
      "properties": {
        "dataset": {"type": "string"},
        "evidence": {"type": "string"},
        "evaluator": {"type": "string"},
        "application": {"type": "string"},
        "model": {"type": "string"},
        "model_config": {"type": "string"},
        "environment": {"type": "string"}
      }
    }
  }
}