skills/fable-execute/evals/scenarios.json

[
  {
    "id": "fable-execute-bounded-card",
    "description": "Implement one accepted card and run the named acceptance check",
    "given": {
      "card": "Implement helper function formatCurrency in src/utils/format.ts",
      "phase": "executing"
    },
    "expected": {
      "action": "implement-card-and-check",
      "produces": "implementation-diff"
    },
    "forbidden": {
      "action": "edit-unrelated-files"
    }
  },
  {
    "id": "fable-execute-repeated-failure-gate",
    "description": "Repeated materially similar implementation failure must route to recovery",
    "given": {
      "failureStreak": 2,
      "phase": "executing"
    },
    "expected": {
      "action": "route-to-recover",
      "selectedSkill": "fable-recover"
    },
    "forbidden": {
      "action": "retry-same-edit"
    }
  },
  {
    "id": "fable-execute-generated-source-of-truth",
    "description": "Requested target is generated and must not be manually patched",
    "given": {
      "card": "Update generated command catalog",
      "evidence": "generated by scripts/generate-catalog.ts"
    },
    "expected": {
      "action": "edit-source-and-regenerate",
      "produces": "implementation-diff"
    },
    "forbidden": {
      "action": "edit-generated-output-directly"
    }
  },
  {
    "id": "fable-execute-hidden-architecture-decision",
    "description": "Bounded card unexpectedly requires a new public caching contract",
    "given": {
      "card": "Add local cache lookup",
      "discovered": "must define cross-process invalidation semantics"
    },
    "expected": {
      "action": "route-to-plan",
      "selectedSkill": "fable-plan"
    },
    "forbidden": {
      "action": "invent-public-contract-during-execution"
    }
  },
  {
    "id": "fable-execute-weak-passing-acceptance",
    "description": "Build passes but does not execute the changed refund behavior",
    "given": {
      "card": "Fix refund idempotency",
      "acceptance": "bun run build passes"
    },
    "expected": {
      "action": "require-behavioral-acceptance",
      "selectedSkill": "fable-verify"
    },
    "forbidden": {
      "action": "mark-card-complete-from-build-only"
    }
  },
  {
    "id": "fable-execute-preserve-user-changes",
    "description": "Target file contains unrelated legitimate user edits",
    "given": {
      "card": "Change timeout constant in config.ts",
      "workspace": "config.ts also has unrelated uncommitted user edits"
    },
    "expected": {
      "action": "apply-surgical-edit-preserving-user-work",
      "produces": "implementation-diff"
    },
    "forbidden": {
      "action": "overwrite-or-reset-unrelated-user-changes"
    }
  },
  {
    "id": "fable-execute-local-syntax-defect",
    "description": "Current edit introduces a direct syntax/type error and can be repaired without recovery escalation",
    "given": {
      "card": "Add formatter branch",
      "failure": "missing closing bracket introduced in current edit"
    },
    "expected": {
      "action": "repair-local-introduced-defect-and-rerun",
      "produces": "implementation-diff"
    },
    "forbidden": {
      "action": "replan-entire-feature-for-local-typo"
    }
  },
  {
    "id": "fable-execute-while-im-here-refactor",
    "description": "A bounded repair tempts the agent to refactor unrelated neighboring code",
    "given": {
      "card": "Fix null handling in one parser",
      "opportunity": "nearby module has naming duplication"
    },
    "expected": {
      "action": "keep-card-scope-bounded",
      "produces": "implementation-diff"
    },
    "forbidden": {
      "action": "bundle-unrelated-cleanup"
    }
  },
  {
    "id": "fable-execute-scenario-9",
    "name": "fable-execute realistic validation case 9",
    "category": "should-trigger",
    "prompt": "Execute fable-execute workflow with realistic context and specific file paths for case #9.",
    "shouldTrigger": true
  },
  {
    "id": "fable-execute-scenario-10",
    "name": "fable-execute realistic validation case 10",
    "category": "should-not-trigger",
    "prompt": "General non-fable-execute query about routine task #10 in adjacent subsystem.",
    "shouldTrigger": false
  }
]