evaluations/setup-flow.json

{
  "name": "First-Time Setup Walkthrough",
  "skills": ["conductor"],
  "query": "Help me set up Conductor for the first time. I want to run it locally.",
  "expected_behavior": [
    "Step 1: Check whether the `conductor` CLI is already installed via `conductor --version`",
    "Step 2: If missing, check for npm/Node.js with `npm --version`",
    "Step 3: Prefer `npx @conductor-oss/conductor-cli ...` for one-off use over a global install (no system change)",
    "Step 4: If a global install is needed, **ask the user before** running `npm install -g @conductor-oss/conductor-cli` — never run it silently",
    "Step 5: User stated 'run it locally' — proceed with the local-server option (Option A: `conductor server start`). The agent may briefly mention the remote-server option exists, but should not block on a choice the user has already made.",
    "Step 6: Start the local server when the user chooses Option A, then verify with `conductor server status`",
    "Step 7: Test connectivity with `conductor workflow list`",
    "Step 8: If 401/403, ask the user for credentials (CONDUCTOR_AUTH_KEY+SECRET or CONDUCTOR_AUTH_TOKEN) and set via env vars; never echo the values",
    "Step 9: Confirm setup with a final `conductor workflow list` and report success"
  ],
  "success_criteria": [
    "CLI presence is checked before any install attempt",
    "Global `npm install -g` is only run AFTER explicit user confirmation",
    "Agent proceeds with the local-server option (user already said 'run it locally') without re-asking; it's fine if the agent briefly notes the remote option exists",
    "Auth tokens, keys, and secrets are never echoed in output",
    "Setup verification is via a real `conductor workflow list` call, not an assumption",
    "If auth is not required (no 401/403), the agent does not ask for credentials"
  ]
}