evals/html_deck_benchmark_runs.schema.json

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://mparanza.com/schemas/clara/html-deck-benchmark-runs-v1.json",
  "title": "Clara HTML deck benchmark candidate runs",
  "type": "object",
  "additionalProperties": false,
  "required": [
    "schema_version",
    "suite_id",
    "protocol_evidence",
    "runs",
    "semantic_reviews"
  ],
  "properties": {
    "schema_version": {
      "const": "clara.html_deck_benchmark_runs.v1"
    },
    "suite_id": {
      "type": "string",
      "minLength": 1
    },
    "protocol_evidence": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "producer",
        "suite_fingerprint_sha256",
        "codex_cli_version",
        "recorded_at",
        "candidate_skill_identity",
        "candidate_skill_paths",
        "baseline_evidence_manifests",
        "review_packet_mappings"
      ],
      "properties": {
        "producer": {
          "const": "run_clara_deck_benchmark.py"
        },
        "suite_fingerprint_sha256": {
          "$ref": "#/$defs/sha256"
        },
        "codex_cli_version": {
          "type": "string",
          "minLength": 1
        },
        "recorded_at": {
          "type": "string",
          "format": "date-time"
        },
        "candidate_skill_identity": {"type": "object"},
        "candidate_skill_paths": {"type": "object"},
        "baseline_evidence_manifests": {"type": "object"},
        "review_packet_mappings": {
          "type": "array",
          "minItems": 4,
          "items": {"$ref": "#/$defs/review_packet_mapping"}
        }
        }
      },
    "runs": {
      "type": "array",
      "minItems": 4,
      "items": {
        "$ref": "#/$defs/run"
      }
    },
    "semantic_reviews": {
      "type": "array",
      "minItems": 4,
      "items": {
        "$ref": "#/$defs/semantic_review"
      }
    }
  },
  "$defs": {
    "sha256": {
      "type": "string",
      "pattern": "^[0-9a-f]{64}$"
    },
    "positive_integer": {
      "type": "integer",
      "minimum": 1
    },
    "non_negative_integer": {
      "type": "integer",
      "minimum": 0
    },
    "review_packet_mapping": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "packet_id",
        "case_id",
        "format",
        "candidate_label",
        "baseline_label",
        "prompt_sha256",
        "source_requirements_sha256",
        "candidate_render_set_sha256",
        "baseline_render_set_sha256"
      ],
      "properties": {
        "packet_id": {"type": "string", "minLength": 1},
        "case_id": {"type": "string", "minLength": 1},
        "format": {"enum": ["HTML", "PPTX"]},
        "candidate_label": {"enum": ["A", "B"]},
        "baseline_label": {"enum": ["A", "B"]},
        "prompt_sha256": {"$ref": "#/$defs/sha256"},
        "source_requirements_sha256": {"$ref": "#/$defs/sha256"},
        "candidate_render_set_sha256": {"$ref": "#/$defs/sha256"},
        "baseline_render_set_sha256": {"$ref": "#/$defs/sha256"}
      }
    },
    "run": {
      "type": "object",
      "required": [
        "case_id",
        "format",
        "duration_ms",
        "total_tokens",
        "input_tokens",
        "cached_input_tokens",
        "output_tokens",
        "noncached_input_plus_output_tokens",
        "tool_calls",
        "process_exit_code",
        "execution_identity",
        "protocol",
        "artifact",
        "checks"
      ],
      "properties": {
        "case_id": {
          "type": "string",
          "minLength": 1
        },
        "format": {
          "enum": [
            "HTML",
            "PPTX"
          ]
        },
        "duration_ms": {
          "$ref": "#/$defs/positive_integer"
        },
        "total_tokens": {
          "$ref": "#/$defs/positive_integer"
        },
        "input_tokens": {
          "$ref": "#/$defs/positive_integer"
        },
        "cached_input_tokens": {
          "$ref": "#/$defs/non_negative_integer"
        },
        "output_tokens": {
          "$ref": "#/$defs/positive_integer"
        },
        "noncached_input_plus_output_tokens": {
          "$ref": "#/$defs/positive_integer"
        },
        "tool_calls": {
          "$ref": "#/$defs/positive_integer"
        },
        "process_exit_code": {
          "type": "integer"
        },
        "execution_identity": {
          "type": "object",
          "required": [
            "model",
            "reasoning_effort",
            "enforced_by",
            "skill_identity"
          ],
          "properties": {
            "model": {
              "type": "string",
              "minLength": 1
            },
            "reasoning_effort": {
              "type": "string",
              "minLength": 1
            },
            "enforced_by": {
              "const": "codex_cli_explicit_override"
            },
            "skill_identity": {
              "type": "object"
            }
          }
        },
        "protocol": {
          "type": "object",
          "required": [
            "ephemeral",
            "isolated_workdir",
            "output_root",
            "prompt_sha256",
            "normalized_prompt_sha256",
            "source_manifest_sha256",
            "source_manifest_verified",
            "sealed_task_manifest_sha256",
            "sealed_task_verified_before_after",
            "event_log_sha256",
            "thread_id",
            "read_audit",
            "started_at",
            "completed_at"
          ],
          "properties": {
            "ephemeral": {
              "const": true
            },
            "isolated_workdir": {
              "type": "string",
              "minLength": 1
            },
            "output_root": {
              "type": "string",
              "minLength": 1
            },
            "prompt_sha256": {
              "$ref": "#/$defs/sha256"
            },
            "normalized_prompt_sha256": {
              "$ref": "#/$defs/sha256"
            },
            "source_manifest_sha256": {
              "$ref": "#/$defs/sha256"
            },
            "source_manifest_verified": {
              "const": true
            },
            "sealed_task_manifest_sha256": {
              "$ref": "#/$defs/sha256"
            },
            "sealed_task_verified_before_after": {
              "const": true
            },
            "event_log_sha256": {
              "$ref": "#/$defs/sha256"
            },
            "thread_id": {
              "type": "string",
              "minLength": 1
            },
            "read_audit": {
              "enum": [
                "pass",
                "fail"
              ]
            },
            "started_at": {
              "type": "string",
              "format": "date-time"
            },
            "completed_at": {
              "type": "string",
              "format": "date-time"
            }
          }
        },
        "artifact": {
          "type": "object",
          "required": [
            "path",
            "sha256",
            "bytes",
            "render_set_sha256",
            "rendered_slides"
          ],
          "properties": {
            "path": {
              "type": "string",
              "minLength": 1
            },
            "sha256": {
              "$ref": "#/$defs/sha256"
            },
            "bytes": {
              "$ref": "#/$defs/positive_integer"
            },
            "render_set_sha256": {
              "$ref": "#/$defs/sha256"
            },
            "rendered_slides": {
              "type": "array",
              "minItems": 1,
              "items": {
                "type": "object",
                "required": [
                  "path",
                  "sha256",
                  "bytes",
                  "width",
                  "height"
                  ,"renderer"
                ],
                "properties": {
                  "path": {
                    "type": "string",
                    "minLength": 1
                  },
                  "sha256": {
                    "$ref": "#/$defs/sha256"
                  },
                  "bytes": {
                    "$ref": "#/$defs/positive_integer"
                  },
                  "width": {
                    "$ref": "#/$defs/positive_integer"
                  },
                  "height": {
                    "$ref": "#/$defs/positive_integer"
                  },
                  "renderer": {
                    "type": "string",
                    "minLength": 1
                  }
                }
              }
            }
          }
        },
        "checks": {
          "type": "object",
          "minProperties": 1,
          "additionalProperties": {
            "type": "boolean"
          }
        }
      }
    },
    "semantic_review": {
      "type": "object",
      "required": [
        "case_id",
        "format",
        "reviewer",
        "review_packet_id",
        "review_prompt_sha256",
        "source_requirements_sha256",
        "candidate_artifact_sha256",
        "baseline_artifact_sha256",
        "candidate_render_set_sha256",
        "baseline_render_set_sha256",
        "scores_by_label",
        "overall_rationale"
      ],
      "properties": {
        "case_id": {
          "type": "string",
          "minLength": 1
        },
        "format": {
          "enum": [
            "HTML",
            "PPTX"
          ]
        },
        "reviewer": {
          "type": "object",
          "required": [
            "type",
            "id",
            "model"
            ,"thread_id"
          ],
          "properties": {
            "type": {
              "enum": [
                "model",
                "human"
              ]
            },
            "id": {
              "type": "string",
              "minLength": 1
            },
            "model": {
              "type": [
                "string",
                "null"
              ]
            },
            "thread_id": {
              "type": "string",
              "minLength": 1
            }
          }
        },
        "review_packet_id": {
          "type": "string",
          "minLength": 1
        },
        "review_prompt_sha256": {
          "$ref": "#/$defs/sha256"
        },
        "source_requirements_sha256": {
          "$ref": "#/$defs/sha256"
        },
        "candidate_artifact_sha256": {
          "$ref": "#/$defs/sha256"
        },
        "baseline_artifact_sha256": {
          "$ref": "#/$defs/sha256"
        },
        "candidate_render_set_sha256": {
          "$ref": "#/$defs/sha256"
        },
        "baseline_render_set_sha256": {
          "$ref": "#/$defs/sha256"
        },
        "scores_by_label": {
          "type": "object",
          "additionalProperties": false,
          "required": ["A", "B"],
          "properties": {
            "A": {"$ref": "#/$defs/label_scores"},
            "B": {"$ref": "#/$defs/label_scores"}
          }
        },
        "overall_rationale": {
          "type": "string",
          "minLength": 1
        }
      }
    },
    "label_scores": {
      "type": "object",
      "minProperties": 4,
      "additionalProperties": {
        "type": "object",
        "additionalProperties": false,
        "required": ["score", "pass", "rationale"],
        "properties": {
          "score": {
            "type": "number",
            "minimum": 1,
            "maximum": 5
          },
          "pass": {
            "type": "boolean"
          },
          "rationale": {
            "type": "string",
            "minLength": 1
          }
        }
      }
    }
  }
}