evaluations/orkes-secrets.json

{
  "name": "Save and Reference an Orkes Secret",
  "skills": ["conductor"],
  "query": "Save my Stripe API key as a secret called STRIPE_KEY and use it in my charge_card workflow",
  "expected_behavior": [
    "Step 1: Recognize that secrets are an Orkes-only feature (per references/orkes.md) and confirm the user is connected to an Orkes server",
    "Step 2: Ask the user for the secret value privately — do NOT echo the value in chat, do NOT put it in shell history-visible commands without warning",
    "Step 3: Save the secret with `conductor secret put STRIPE_KEY <value>` — but represent the value via an env var or prompt, never inline in the visible command",
    "Step 4: Confirm the secret was saved by NAME only (e.g. via `conductor secret list`), NEVER by echoing the value back",
    "Step 5: Show how to reference the secret in a workflow task via `${workflow.secrets.STRIPE_KEY}` (e.g. in an HTTP task's headers or an LLM task's auth)",
    "Step 6: Mention that secret references are resolved server-side at task execution time — the secret value never appears in the workflow definition or execution view"
  ],
  "success_criteria": [
    "Agent recognizes secrets as Orkes-only (and would fail on plain OSS) — does not claim this works on local OSS Conductor",
    "Secret value is NEVER echoed in any output, log, or shell command construction",
    "Confirmation after `secret put` is by name only, not value",
    "Workflow reference syntax `${workflow.secrets.STRIPE_KEY}` is used correctly",
    "Agent explains the value is resolved server-side, not by the agent or the CLI",
    "If the user passes the secret value in the prompt, the agent does not repeat it back in its response"
  ]
}