skills/fable-spark/evals/scenarios.json

[
  {
    "id": "spark-code-mutated-verification-stale",
    "description": "Code mutated, previous verification is stale; predict running tests",
    "given": {
      "state": {
        "phase": "executing",
        "mutationGeneration": 2,
        "verifiedGeneration": 1
      }
    },
    "expected": {
      "suggestion": "run the affected tests",
      "reasonCode": "verification-stale-after-mutation",
      "silent": false
    },
    "forbidden": {
      "silent": true,
      "suggestion": "route the task"
    }
  },
  {
    "id": "spark-same-failure-repeated",
    "description": "Same failure repeated; predict diagnosing failure before retry",
    "given": {
      "state": {
        "phase": "executing",
        "failureStreak": 2
      }
    },
    "expected": {
      "suggestion": "diagnose the repeated failure",
      "reasonCode": "failure-loop-diagnose-required",
      "silent": false
    },
    "forbidden": {
      "suggestion": "run the affected tests"
    }
  },
  {
    "id": "spark-idle-no-intent-silence",
    "description": "No pending work or intent while idle; predict silence",
    "given": {
      "state": {
        "phase": "idle"
      }
    },
    "expected": {
      "suggestion": null,
      "reasonCode": "idle-no-intent-silent",
      "silent": true
    },
    "forbidden": {
      "silent": false
    }
  },
  {
    "id": "spark-scope-complete-silence",
    "description": "Phase complete; predict silence",
    "given": {
      "state": {
        "phase": "complete"
      }
    },
    "expected": {
      "suggestion": null,
      "reasonCode": "scope-complete-silent",
      "silent": true
    },
    "forbidden": {
      "silent": false
    }
  },
  {
    "id": "spark-security-evidence-functional-bug",
    "description": "Security evidence passes but bug fix requires functional verification",
    "given": {
      "state": {
        "currentSkill": "fable-security",
        "mutationGeneration": 1,
        "verifiedGeneration": 0,
        "evidence": [
          {
            "kind": "security",
            "result": "pass"
          }
        ]
      },
      "userIntent": "Fix the regression bug in login"
    },
    "expected": {
      "suggestion": "verify the repaired behavior",
      "reasonCode": "security-does-not-prove-functional-repair"
    },
    "forbidden": {
      "suggestion": "check release readiness"
    }
  },
  {
    "id": "spark-no-macro-plan",
    "description": "Several dependent future stages remain, but Spark must not return a multi-step plan",
    "given": {
      "state": {
        "phase": "planning"
      },
      "candidateSuggestion": "research, implement, test and publish"
    },
    "expected": {
      "action": "reduce-to-one-atomic-action"
    },
    "forbidden": {
      "action": "return-multi-stage-plan"
    }
  },
  {
    "id": "spark-specialist-obvious-action-silence",
    "description": "Active specialist already owns an obvious immediate command and no gate changed",
    "given": {
      "state": {
        "phase": "executing",
        "currentSkill": "fable-execute"
      },
      "obviousNextAction": true
    },
    "expected": {
      "action": "prefer-silence-over-narration"
    },
    "forbidden": {
      "action": "narrate-obvious-step"
    }
  },
  {
    "id": "spark-unknown-external-fact-no-mutation",
    "description": "Implementation is blocked on an unknown current API contract",
    "given": {
      "state": {
        "phase": "executing"
      },
      "externalContractKnown": false
    },
    "expected": {
      "action": "point-to-research-not-implementation"
    },
    "forbidden": {
      "action": "suggest-code-mutation-from-unknown-contract"
    }
  },
  {
    "id": "fable-spark-scenario-9",
    "name": "fable-spark realistic validation case 9",
    "category": "should-trigger",
    "prompt": "Execute fable-spark workflow with realistic context and specific file paths for case #9.",
    "shouldTrigger": true
  },
  {
    "id": "fable-spark-scenario-10",
    "name": "fable-spark realistic validation case 10",
    "category": "should-not-trigger",
    "prompt": "General non-fable-spark query about routine task #10 in adjacent subsystem.",
    "shouldTrigger": false
  }
]