analytics.yaml

version: 1

operations:
  metrics_probe:
    path: ./scripts/metrics/1_probe_metrics.py
    measurements:
      metrics_probe: {}
      plugin_version:
        dimensions:
          component: [major, minor, patch]
  controller_launch:
    path: ./scripts/metrics/2_start_controller.py
    measurements:
      controller_launch:
        dimensions:
          outcome: [ready, startup_failure, launch_unobserved]
  replay_start:
    path: ./scripts/metrics/3_report_start_metrics.py
    measurements:
      run_start: {}
      selected_model_count: {}
      run_start_timeout: {}
  replay_metrics:
    path: ./scripts/metrics/4_report_final_metrics.py
    measurements:
      attempt:
        dimensions:
          outcome: [startup_failure, no_run_observed, launch_failure, succeeded, failed, mixed, cancelled, observation_timeout, unreadable_state, interrupted]
          controller_ready: [yes, no]
          start_requested: [yes, no]
          run_observed: [yes, no]
          reporting_complete: [yes, no]
      final_prestart_timeout: {}
      attempt_model:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          status: [completed, failed, cancelled, launch_failed, unresolved, unreadable]
      replay:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
          status: [completed, failed, cancelled]
      failure:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
          phase: [preparation, implementation, evaluation, reporting, unknown]
          controller_code: [launch_failed, coordinator_stopped, controller_error, none, unknown]
          worker_code: [invalid_request, invalid_sdk_response, turn_failed, stream_error, incomplete_stream, canceled, codex_unavailable, worker_failed, invalid_json, unknown]
          retryable: [yes, no, unknown]
      failure_detail:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
          system_code: [econnreset, econnrefused, etimedout, enotfound, eai_again, enoent, eacces, eperm, epipe, unknown]
          worker_stage: [validation, launch, turn_start, stream, outside_worker, unknown]
      failure_exit_code:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
      failure_retry_count:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
      failure_duration_seconds:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
      outcome:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
          winner: [codex, claude, tie, unavailable]
      codex_input_tokens:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
      codex_output_tokens:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
      codex_cached_input_tokens:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
      codex_cost_usd:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
      codex_duration_seconds:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
      codex_wall_clock_seconds:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
      claude_input_tokens:
        dimensions:
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
      claude_output_tokens:
        dimensions:
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
      claude_cached_input_tokens:
        dimensions:
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
      claude_cost_usd:
        dimensions:
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
      claude_wall_clock_seconds:
        dimensions:
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
      claude_duration_seconds:
        dimensions:
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
          timing_basis: [model_request, recorded_wall_clock]
      codex_phase_duration_seconds:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
          phase: [implementing, reviewing]
      evaluation:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
          evaluator_model: [sol, terra, luna, opus, sonnet, haiku, other]
          evaluator_provider: [codex, claude, other]
          self_judged: [yes, no]
          normalized: [yes, no]
      codex_score:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
      claude_score:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
      codex_dimension_score:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
          score_dimension: [request_fulfillment, code_quality, change_scope, reliability, safe_operations, accurate_reporting]
      claude_dimension_score:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
          score_dimension: [request_fulfillment, code_quality, change_scope, reliability, safe_operations, accurate_reporting]
      dimension_outcome:
        dimensions:
          model_slot: [one, two, three, four, five, six, seven, eight]
          codex_model: [sol, terra, luna, other]
          claude_model: [fable, opus, sonnet, haiku, other]
          source: [imported, sample]
          score_dimension: [request_fulfillment, code_quality, change_scope, reliability, safe_operations, accurate_reporting]
          winner: [codex, claude, tie]
      measurements_omitted: {}