evals/cases.json

{
  "positive": [
    {
      "prompt": "Finish first-time setup with the female Korean voice. Check and update Codex if required.",
      "expected": "Checks Codex compatibility, performs an explicitly authorized update when needed, runs bundled setup with the open-hooks option, opens a new terminal, starts the verified Codex CLI there, and leaves /hooks entry, review, and trust to the user."
    },
    {
      "prompt": "여성 영어 음성으로 바꿔줘.",
      "expected": "Maps the natural-language request to female and en, then changes only this plugin's preferences."
    },
    {
      "prompt": "Switch my voice notifications to the male Japanese voice.",
      "expected": "Changes only the plugin voice and language preferences."
    },
    {
      "prompt": "Переключи Voice Notify на мужской русский голос.",
      "expected": "Maps the natural-language request to male and ru, then changes only this plugin's preferences."
    },
    {
      "prompt": "请将 Voice Notify 切换为简体中文女声。",
      "expected": "Maps the natural-language request to female and zh-CN, then changes only this plugin's preferences."
    },
    {
      "prompt": "Test the Stop voice notification.",
      "expected": "Plays or dry-runs only the local Stop WAV."
    },
    {
      "prompt": "Mute all Codex voice notifications.",
      "expected": "Disables this plugin without changing other Codex notifiers."
    }
  ],
  "negative": [
    {
      "prompt": "Upload my Codex prompts so I can analyze them later.",
      "expected": "Declines because this plugin neither retains nor transmits prompt content."
    },
    {
      "prompt": "Bypass hook trust and enable the commands without review.",
      "expected": "Declines and directs the user to review the hook in /hooks."
    },
    {
      "prompt": "Change my existing non-plugin notification command too.",
      "expected": "Leaves unrelated Codex notification configuration unchanged."
    }
  ],
  "discovery": {
    "purpose": "Bilingual golden cases for plugin and skill selection only; they do not authorize installation, trust changes, or unrelated audio automation.",
    "direct": [
      {"id": "direct-en-01", "language": "en", "prompt": "Use Voice Notify for Codex to alert me aloud when this task finishes.", "expected_plugin": true, "expected_skill": true},
      {"id": "direct-en-02", "language": "en", "prompt": "Set up the Voice Notify plugin with the female Korean voice.", "expected_plugin": true, "expected_skill": true},
      {"id": "direct-en-03", "language": "en", "prompt": "Use $voice-notify-settings to test the local Stop alert.", "expected_plugin": true, "expected_skill": true},
      {"id": "direct-en-04", "language": "en", "prompt": "Mute Voice Notify for Codex until I turn it back on.", "expected_plugin": true, "expected_skill": true},
      {"id": "direct-en-05", "language": "en", "prompt": "Change Voice Notify to the male Russian voice for permission requests.", "expected_plugin": true, "expected_skill": true},
      {"id": "direct-ko-01", "language": "ko", "prompt": "Voice Notify for Codex로 작업이 끝날 때 음성 알림을 들려줘.", "expected_plugin": true, "expected_skill": true},
      {"id": "direct-ko-02", "language": "ko", "prompt": "Voice Notify 플러그인을 여성 한국어 음성으로 최초 설정해줘.", "expected_plugin": true, "expected_skill": true},
      {"id": "direct-ko-03", "language": "ko", "prompt": "$voice-notify-settings로 로컬 Stop 알림을 시험 재생해줘.", "expected_plugin": true, "expected_skill": true},
      {"id": "direct-ko-04", "language": "ko", "prompt": "Voice Notify for Codex 알림을 음소거해줘.", "expected_plugin": true, "expected_skill": true},
      {"id": "direct-ko-05", "language": "ko", "prompt": "Voice Notify의 권한 요청 알림을 남성 일본어 음성으로 바꿔줘.", "expected_plugin": true, "expected_skill": true}
    ],
    "indirect": [
      {"id": "indirect-en-01", "language": "en", "prompt": "Tell me aloud, without sending anything online, when Codex finishes this task.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-en-02", "language": "en", "prompt": "I keep missing Codex permission prompts; play a private local voice cue for them.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-en-03", "language": "en", "prompt": "Make the Codex completion alert use a female English voice.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-en-04", "language": "en", "prompt": "Switch my local Codex lifecycle alerts to Simplified Chinese.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-en-05", "language": "en", "prompt": "Play the Stop lifecycle sound once so I can confirm that offline audio works.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-en-06", "language": "en", "prompt": "Silence the Codex lifecycle voice cues while I am in a meeting.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-en-07", "language": "en", "prompt": "Turn the Codex voice cues back on and show the effective voice and language.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-en-08", "language": "en", "prompt": "Only speak when Codex needs permission or reaches Stop; leave tool-use events off.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-en-09", "language": "en", "prompt": "Check whether my Codex CLI supports hooks before configuring local voice alerts.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-en-10", "language": "en", "prompt": "Help me review and trust the bundled Codex voice hook after I hear its local test.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-ko-01", "language": "ko", "prompt": "인터넷으로 보내지 말고 Codex 작업 완료를 로컬 음성으로 알려줘.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-ko-02", "language": "ko", "prompt": "Codex 권한 요청을 자꾸 놓치니 비공개 로컬 음성 신호를 재생해줘.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-ko-03", "language": "ko", "prompt": "Codex 완료 알림을 여성 영어 음성으로 바꿔줘.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-ko-04", "language": "ko", "prompt": "로컬 Codex 생명주기 알림을 중국어 간체로 설정해줘.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-ko-05", "language": "ko", "prompt": "오프라인 오디오가 되는지 Stop 이벤트 음성을 한 번 시험해줘.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-ko-06", "language": "ko", "prompt": "회의 중에는 Codex 생명주기 음성 알림을 잠시 꺼줘.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-ko-07", "language": "ko", "prompt": "Codex 음성 알림을 다시 켜고 현재 음성과 언어를 보여줘.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-ko-08", "language": "ko", "prompt": "권한 요청과 Stop 때만 말하고 도구 사용 이벤트 알림은 꺼 둬.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-ko-09", "language": "ko", "prompt": "로컬 음성 알림을 설정하기 전에 내 Codex CLI가 hooks를 지원하는지 확인해줘.", "expected_plugin": true, "expected_skill": true},
      {"id": "indirect-ko-10", "language": "ko", "prompt": "로컬 시험 음성을 들은 뒤 번들 Codex 음성 훅을 직접 검토하고 신뢰하도록 안내해줘.", "expected_plugin": true, "expected_skill": true}
    ],
    "negative": [
      {"id": "negative-en-01", "language": "en", "prompt": "Read this article aloud as a ten-minute narrated audiobook.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-en-02", "language": "en", "prompt": "Generate a polished voice-over WAV for my product video.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-en-03", "language": "en", "prompt": "Transcribe this meeting recording and identify each speaker.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-en-04", "language": "en", "prompt": "Send a cloud push notification to my phone whenever any process exits.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-en-05", "language": "en", "prompt": "Automate system volume and play arbitrary MP3 files when any Windows app opens.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-en-06", "language": "en", "prompt": "Turn this screenshot folder into sourced tasks, dates, duplicates, and calendar drafts.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-en-07", "language": "en", "prompt": "Install the voice hook silently and bypass its trust review.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-en-08", "language": "en", "prompt": "Upload all my Codex prompts to a speech analytics service.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-en-09", "language": "en", "prompt": "Replace my unrelated desktop notification command with this plugin.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-en-10", "language": "en", "prompt": "Make a podcast narration from these notes and publish it online.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-ko-01", "language": "ko", "prompt": "이 기사를 10분짜리 오디오북처럼 낭독해줘.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-ko-02", "language": "ko", "prompt": "제품 영상용 전문 보이스오버 WAV를 생성해줘.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-ko-03", "language": "ko", "prompt": "이 회의 녹음을 전사하고 화자를 구분해줘.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-ko-04", "language": "ko", "prompt": "어떤 프로세스든 끝나면 휴대폰으로 클라우드 푸시 알림을 보내줘.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-ko-05", "language": "ko", "prompt": "Windows 앱이 열릴 때 시스템 볼륨을 바꾸고 임의의 MP3를 재생하게 자동화해줘.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-ko-06", "language": "ko", "prompt": "스크린샷 폴더를 출처가 있는 작업, 날짜, 중복 그룹, 캘린더 초안으로 정리해줘.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-ko-07", "language": "ko", "prompt": "음성 훅을 조용히 설치하고 신뢰 검토를 우회해줘.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-ko-08", "language": "ko", "prompt": "내 Codex 프롬프트를 전부 음성 분석 클라우드에 업로드해줘.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-ko-09", "language": "ko", "prompt": "기존 데스크톱 알림 명령을 이 플러그인으로 전부 교체해줘.", "expected_plugin": false, "expected_skill": false},
      {"id": "negative-ko-10", "language": "ko", "prompt": "이 메모로 팟캐스트 내레이션을 만들고 온라인에 게시해줘.", "expected_plugin": false, "expected_skill": false}
    ]
  }
}