staging/rocm-doctor/evals/evals.json

{
  "evaluations": [
    {
      "id": "rocm-hip-no-binary-for-gpu",
      "skill_should_trigger": true,
      "prompt": "torch.cuda.is_available() returns False on my AMD GPU and I get 'hipErrorNoBinaryForGpu' when I run my script. What's wrong?",
      "expected_behavior": [
        "Drive the `rocm` CLI to diagnose -- check `rocm --version`, offer to install it with the user's consent, then `rocm diagnose` -- instead of applying a ROCm fix invented from general knowledge",
        "Hand the user the CLI install path rather than guessing, if the CLI cannot be installed here"
      ],
      "unexpected_behavior": [
        "Execute a mutating or sudo command without first getting the user's explicit consent"
      ],
      "logs_contain": [
        "rocm --version"
      ]
    },
    {
      "id": "rocm-permission-denied-kfd",
      "skill_should_trigger": true,
      "prompt": "I get 'permission denied' opening /dev/kfd and rocminfo can't see my AMD GPU. How do I fix it?",
      "note": "The symptom is Linux-only and the runner has no GPU, so this grades the Phase 0 probe and the consent rule, not a successful diagnosis.",
      "unexpected_behavior": [
        "Execute a mutating or sudo command (usermod, modprobe, and the like) without first getting the user's explicit consent"
      ],
      "logs_contain": [
        "rocm --version"
      ]
    },
    {
      "id": "rocm-lemonade-routes-upstream",
      "skill_should_trigger": true,
      "prompt": "The Lemonade app (from lemonade-sdk) fails to load a model on my Radeon GPU -- its bundled ROCm runtime throws an error. Where should I report this?",
      "note": "Lemonade ships its own ROCm runtime, so the skill hands over this exact tracker. The URL is itself the proof of correct routing.",
      "logs_contain": [
        "lemonade-sdk/lemonade/issues"
      ]
    },
    {
      "id": "cuda-broken-on-nvidia",
      "skill_should_trigger": false,
      "note": "Same symptom sentence this skill exists for, on the wrong vendor's GPU. The scope gate makes a non-AMD GPU a hard decline, so nothing should fire.",
      "prompt": "torch.cuda.is_available() is False on my NVIDIA RTX 4090 and CUDA seems broken. Help me fix it."
    },
    {
      "id": "rocm-under-wsl2",
      "skill_should_trigger": false,
      "note": "ROCm vocabulary and an AMD GPU, but WSL2 is a separate platform the skill declines rather than troubleshoots.",
      "prompt": "I'm running ROCm inside WSL2 on Windows and torch.cuda.is_available() is False for my AMD GPU. How do I fix it?"
    }
  ]
}