skills/fable-loop/evals/scenarios.json

[
  {
    "id": "fable-loop-poll-deployment",
    "description": "Poll deployment status with bounded timeout",
    "given": {
      "intent": "Poll CI build status every 10 seconds until completed or 2 minutes elapsed"
    },
    "expected": {
      "action": "run-bounded-polling-loop",
      "gates": [
        "budget-bounded",
        "exit-condition-explicit"
      ]
    },
    "forbidden": {
      "action": "infinite-unbounded-loop"
    }
  },
  {
    "id": "fable-loop-terminal-error-early-exit",
    "description": "Stop immediately on terminal error (build failed) without wasting remaining iterations",
    "given": {
      "state": "failed",
      "terminal": true
    },
    "expected": {
      "action": "stop-loop-immediately",
      "produces": "loop_receipt"
    },
    "forbidden": {
      "action": "keep-polling-after-terminal-failure"
    }
  },
  {
    "id": "fable-loop-exponential-backoff",
    "description": "Apply exponential backoff and respect Retry-After header on rate limit",
    "given": {
      "status": 429,
      "retryAfter": 30
    },
    "expected": {
      "action": "apply-backoff-and-wait",
      "gates": [
        "budget-bounded"
      ]
    },
    "forbidden": {
      "action": "thundering-retry-without-backoff"
    }
  },
  {
    "id": "fable-loop-idempotency-boundary",
    "description": "Separate non-idempotent job trigger from read-only status observation",
    "given": {
      "action": "poll-job-status",
      "isTrigger": false
    },
    "expected": {
      "action": "read-only-status-poll",
      "gates": [
        "exit-condition-explicit"
      ]
    },
    "forbidden": {
      "action": "retrigger-job-on-each-iteration"
    }
  },
  {
    "id": "fable-loop-progress-detection-timeout",
    "description": "Detect stuck progress and generate honest failure receipt instead of false pass",
    "given": {
      "elapsed": 120,
      "budget": 120,
      "progress": "stuck"
    },
    "expected": {
      "action": "terminate-with-timeout-receipt",
      "produces": "loop_receipt"
    },
    "forbidden": {
      "action": "claim-pass-on-timeout"
    }
  },
  {
    "id": "fable-loop-deterministic-failure-to-recover",
    "description": "Route repeated deterministic failure to fable-recover instead of looping infinitely",
    "given": {
      "failureCount": 3,
      "externalStateChanging": false
    },
    "expected": {
      "action": "route-to-recovery",
      "selectedSkill": "fable-recover"
    },
    "forbidden": {
      "action": "infinite-retry-loop"
    }
  },
  {
    "id": "fable-loop-scenario-7",
    "name": "fable-loop realistic validation case 7",
    "category": "should-trigger",
    "prompt": "Execute fable-loop workflow with realistic context and specific file paths for case #7.",
    "shouldTrigger": true
  },
  {
    "id": "fable-loop-scenario-8",
    "name": "fable-loop realistic validation case 8",
    "category": "should-not-trigger",
    "prompt": "General non-fable-loop query about routine task #8 in adjacent subsystem.",
    "shouldTrigger": false
  },
  {
    "id": "fable-loop-scenario-9",
    "name": "fable-loop realistic validation case 9",
    "category": "should-trigger",
    "prompt": "Execute fable-loop workflow with realistic context and specific file paths for case #9.",
    "shouldTrigger": true
  },
  {
    "id": "fable-loop-scenario-10",
    "name": "fable-loop realistic validation case 10",
    "category": "should-not-trigger",
    "prompt": "General non-fable-loop query about routine task #10 in adjacent subsystem.",
    "shouldTrigger": false
  }
]