skills/fable-discover/evals/scenarios.json

[
  {
    "id": "fable-discover-unknown-architecture",
    "description": "Unknown feature implementation requires tracing a proven execution path",
    "given": {
      "intent": "How does the plugin registration mechanism work in this repo?",
      "phase": "discovering"
    },
    "expected": {
      "action": "trace-execution-paths",
      "produces": "repository-evidence"
    },
    "forbidden": {
      "action": "modify-source-files"
    }
  },
  {
    "id": "fable-discover-boundary-trivial-edit",
    "description": "Known typo in a known file should bypass broad discovery",
    "given": {
      "intent": "Fix the typo 'recieve' in README.md line 12"
    },
    "expected": {
      "action": "defer-to-execute",
      "selectedSkill": "fable-execute"
    },
    "forbidden": {
      "action": "broad-codebase-search"
    }
  },
  {
    "id": "fable-discover-dynamic-plugin-loading",
    "description": "No direct import exists because plugins are discovered from a manifest at runtime",
    "given": {
      "intent": "Find why payment-provider-x loads even though grep shows no imports",
      "evidence": "plugin manifest and runtime loader exist"
    },
    "expected": {
      "action": "trace-execution-paths",
      "produces": "repository-evidence"
    },
    "forbidden": {
      "action": "conclude-code-is-unused-from-no-import"
    }
  },
  {
    "id": "fable-discover-generated-artifact",
    "description": "Behavior is visible in a generated catalog and the agent must locate its generator before proposing edits",
    "given": {
      "intent": "Change the command list in src/generated/catalog.ts",
      "evidence": "file header says generated by scripts/generate-catalog.ts"
    },
    "expected": {
      "action": "trace-execution-paths",
      "produces": "repository-evidence"
    },
    "forbidden": {
      "action": "edit-generated-output-as-source"
    }
  },
  {
    "id": "fable-discover-config-precedence",
    "description": "Checked-in defaults conflict with an environment override, so active behavior cannot be inferred from the default file",
    "given": {
      "intent": "Why is timeout 30s in production when config.ts says 5s?",
      "evidence": "TIMEOUT_MS environment variable is present"
    },
    "expected": {
      "action": "trace-execution-paths",
      "produces": "repository-evidence"
    },
    "forbidden": {
      "action": "report-default-as-runtime-fact"
    }
  },
  {
    "id": "fable-discover-test-runtime-divergence",
    "description": "Tests execute source directly while production runs built dist output",
    "given": {
      "intent": "Tests show the fix but the CLI still behaves like the old code",
      "evidence": "test imports src; bin imports dist"
    },
    "expected": {
      "action": "trace-execution-paths",
      "produces": "repository-evidence"
    },
    "forbidden": {
      "action": "assume-passing-test-proves-runtime-path"
    }
  },
  {
    "id": "fable-discover-stop-when-load-bearing-answered",
    "description": "All decisions are safe but several nice-to-know questions remain",
    "given": {
      "intent": "Keep exploring every helper before planning",
      "loadBearingUnknowns": 0,
      "niceToKnowUnknowns": 4
    },
    "expected": {
      "action": "handoff-with-repository-evidence"
    },
    "forbidden": {
      "action": "continue-unbounded-exploration"
    }
  },
  {
    "id": "fable-discover-scenario-8",
    "name": "fable-discover realistic validation case 8",
    "category": "should-not-trigger",
    "prompt": "General non-fable-discover query about routine task #8 in adjacent subsystem.",
    "shouldTrigger": false
  },
  {
    "id": "fable-discover-scenario-9",
    "name": "fable-discover realistic validation case 9",
    "category": "should-trigger",
    "prompt": "Execute fable-discover workflow with realistic context and specific file paths for case #9.",
    "shouldTrigger": true
  },
  {
    "id": "fable-discover-scenario-10",
    "name": "fable-discover realistic validation case 10",
    "category": "should-not-trigger",
    "prompt": "General non-fable-discover query about routine task #10 in adjacent subsystem.",
    "shouldTrigger": false
  }
]