← Files Adaptive Codex OrchestratorARCHIVED FILE
evals/discovery-cases.json
14.7 KB · Sep 30, 2026 · 23:14 UTC
{
"schema_version": "1.0",
"plugin": "adaptive-codex-orchestrator",
"version": "0.1.1",
"synthetic_only": true,
"direct": [
{
"id": "d01-skill-task-en",
"language": "en",
"initial_state": "OFF",
"prompt": "$adaptive-orchestration Inspect three independent modules for this task only.",
"expected_selection": "select",
"why": "The current user explicitly invokes the bundled orchestration skill."
},
{
"id": "d02-skill-task-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "$adaptive-orchestration 이번 작업만 두 모듈을 독립적으로 조사해줘.",
"expected_selection": "select",
"why": "The explicit skill invocation is an orchestration control."
},
{
"id": "d03-ultra-session-en",
"language": "en",
"initial_state": "OFF",
"prompt": "Turn on Ultra Orchestration for this session.",
"expected_selection": "select",
"why": "The named mode, action, and scope are explicit."
},
{
"id": "d04-sol-ultra-session-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "솔 울트라 모드를 이 세션에 켜줘.",
"expected_selection": "select",
"why": "This is a recognized Korean enable command."
},
{
"id": "d05-adaptive-project-en",
"language": "en",
"initial_state": "OFF",
"prompt": "Enable Adaptive Orchestration for this repository.",
"expected_selection": "select",
"why": "The named orchestration mode and project scope are explicit."
},
{
"id": "d06-spark-status-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "스파크 오케스트레이션 상태와 현재 프로필을 알려줘.",
"expected_selection": "select",
"why": "A recognized named-mode status query is in scope."
},
{
"id": "d07-ultra-disable-en",
"language": "en",
"initial_state": "ON",
"prompt": "Turn off Ultra Orchestration for this session.",
"expected_selection": "select",
"why": "An explicit disable control must route to the plugin."
},
{
"id": "d08-adaptive-fast-ko",
"language": "ko",
"initial_state": "ON",
"prompt": "적응형 오케스트레이션을 빠른 프로필로 전환해줘.",
"expected_selection": "select",
"why": "The named orchestration profile control is explicit."
},
{
"id": "d09-ultra-global-en",
"language": "en",
"initial_state": "OFF",
"prompt": "Enable Ultra Orchestration globally with the balanced profile.",
"expected_selection": "select",
"why": "The mode, global scope, and profile are all explicit."
},
{
"id": "d10-ultra-one-shot-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "이번 작업만 울트라 오케스트레이션을 켜고 수정 없이 증거를 모아줘.",
"expected_selection": "select",
"why": "This directly enables the named mode for one task."
}
],
"indirect": [
{
"id": "i01-task-only-en",
"language": "en",
"initial_state": "OFF",
"prompt": "Use orchestration for this task only and keep all workers read-only.",
"expected_selection": "select",
"why": "A product name is unnecessary because orchestration and one-shot scope are explicit."
},
{
"id": "i02-task-only-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "이번 작업만 오케스트레이션을 켜고 읽기 전용으로 조사해줘.",
"expected_selection": "select",
"why": "This is an explicit Korean one-shot orchestration control."
},
{
"id": "i03-session-enable-en",
"language": "en",
"initial_state": "OFF",
"prompt": "Enable orchestration for this session.",
"expected_selection": "select",
"why": "The action and session scope are explicit without a product name."
},
{
"id": "i04-session-enable-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "이 세션에서 오케스트레이션을 켜줘.",
"expected_selection": "select",
"why": "This directly controls session orchestration."
},
{
"id": "i05-project-enable-en",
"language": "en",
"initial_state": "OFF",
"prompt": "Enable orchestration for this project.",
"expected_selection": "select",
"why": "The project-scoped orchestration command is explicit."
},
{
"id": "i06-project-enable-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "이 프로젝트의 오케스트레이션을 활성화해줘.",
"expected_selection": "select",
"why": "The current user explicitly controls project orchestration."
},
{
"id": "i07-global-enable-en",
"language": "en",
"initial_state": "OFF",
"prompt": "Enable orchestration globally.",
"expected_selection": "select",
"why": "Global orchestration enablement is explicit."
},
{
"id": "i08-global-enable-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "모든 프로젝트에서 오케스트레이션을 기본으로 켜줘.",
"expected_selection": "select",
"why": "This is a direct global-scope Korean command."
},
{
"id": "i09-disable-en",
"language": "en",
"initial_state": "ON",
"prompt": "Turn off orchestration for this session.",
"expected_selection": "select",
"why": "A direct orchestration disable command is in scope."
},
{
"id": "i10-disable-ko",
"language": "ko",
"initial_state": "ON",
"prompt": "이 세션의 오케스트레이션을 꺼줘.",
"expected_selection": "select",
"why": "The plugin must handle explicit Korean deactivation."
},
{
"id": "i11-status-en",
"language": "en",
"initial_state": "OFF",
"prompt": "Show orchestration status, scope, and profile.",
"expected_selection": "select",
"why": "A read-only status query is an explicit orchestration control."
},
{
"id": "i12-status-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "오케스트레이션 상태와 범위, 프로필을 알려줘.",
"expected_selection": "select",
"why": "A product name is not needed for a direct status request."
},
{
"id": "i13-fast-profile-en",
"language": "en",
"initial_state": "ON",
"prompt": "Switch orchestration to the fast profile for this session.",
"expected_selection": "select",
"why": "This is an explicit profile control."
},
{
"id": "i14-fast-profile-ko",
"language": "ko",
"initial_state": "ON",
"prompt": "오케스트레이션을 빠른 프로필로 전환해줘.",
"expected_selection": "select",
"why": "This explicitly changes the orchestration profile."
},
{
"id": "i15-conservative-profile-en",
"language": "en",
"initial_state": "ON",
"prompt": "Use the conservative orchestration profile.",
"expected_selection": "select",
"why": "A named orchestration profile request is in scope."
},
{
"id": "i16-balanced-profile-ko",
"language": "ko",
"initial_state": "ON",
"prompt": "오케스트레이션 프로필을 균형으로 바꿔줘.",
"expected_selection": "select",
"why": "This is a direct profile change."
},
{
"id": "i17-worker-cap-en",
"language": "en",
"initial_state": "ON",
"prompt": "For this active orchestration task, use at most one worker.",
"expected_selection": "select",
"why": "An active policy continues with an explicit task-local worker cap."
},
{
"id": "i18-zero-workers-ko",
"language": "ko",
"initial_state": "ON",
"prompt": "이번 오케스트레이션 작업은 작업자 없이 부모만 사용해줘.",
"expected_selection": "select",
"why": "The ON context and explicit zero-worker cap are orchestration controls."
},
{
"id": "i19-continue-task-en",
"language": "en",
"initial_state": "ON",
"prompt": "Keep orchestration active, but apply it to this task only.",
"expected_selection": "select",
"why": "This explicitly continues and scopes active orchestration."
},
{
"id": "i20-project-disable-ko",
"language": "ko",
"initial_state": "ON",
"prompt": "프로젝트 범위 오케스트레이션만 비활성화하고 전역 기본값은 유지해줘.",
"expected_selection": "select",
"why": "This is an explicit scoped orchestration control."
}
],
"negative": [
{
"id": "n01-independent-modules-en",
"language": "en",
"initial_state": "OFF",
"prompt": "Inspect three independent modules and summarize their responsibilities.",
"expected_selection": "do_not_select",
"why": "Parallel work might help, but the user did not request orchestration."
},
{
"id": "n02-bug-fix-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "파서 버그를 재현하고 최소 수정한 뒤 집중 테스트를 실행해줘.",
"expected_selection": "do_not_select",
"why": "An ordinary implementation task is not an orchestration control."
},
{
"id": "n03-parallel-if-useful-en",
"language": "en",
"initial_state": "OFF",
"prompt": "Review these four files in parallel if useful and report the findings.",
"expected_selection": "do_not_select",
"why": "A generic parallel-execution preference does not enable this plugin."
},
{
"id": "n04-generic-delegation-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "독립적인 테스트 검토는 서브에이전트에게 위임해줘.",
"expected_selection": "do_not_select",
"why": "A host-level delegation request is not an Adaptive scope/profile control."
},
{
"id": "n05-three-agents-en",
"language": "en",
"initial_state": "OFF",
"prompt": "Use three agents to research these APIs, then combine the answer.",
"expected_selection": "do_not_select",
"why": "Explicit subagent use can be handled by the host without activating this plugin."
},
{
"id": "n06-trivial-edit-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "README의 오탈자 하나만 고쳐줘.",
"expected_selection": "do_not_select",
"why": "A trivial edit has neither orchestration intent nor useful overhead."
},
{
"id": "n07-general-advice-en",
"language": "en",
"initial_state": "OFF",
"prompt": "Advise me on how to structure a small engineering team.",
"expected_selection": "do_not_select",
"why": "General advice belongs to normal assistance."
},
{
"id": "n08-learning-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "병렬 프로그래밍의 기본 개념을 예시와 함께 가르쳐줘.",
"expected_selection": "do_not_select",
"why": "Learning about parallelism is not a plugin control."
},
{
"id": "n09-mentor-mode-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "멘토 모드 켜줘. 이 설계의 전제를 함께 점검하자.",
"expected_selection": "do_not_select",
"why": "Mentor-mode activation belongs to Premilume, not Adaptive."
},
{
"id": "n10-answer-only-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "이번 요청은 답만 먼저 줘.",
"expected_selection": "do_not_select",
"why": "A response-style preference is not orchestration control."
},
{
"id": "n11-hints-first-en",
"language": "en",
"initial_state": "OFF",
"prompt": "Hints first; help me find the bug without revealing the answer.",
"expected_selection": "do_not_select",
"why": "Hints-first is a mentor/response-style request."
},
{
"id": "n12-counterargument-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "이 출시 계획에 대한 가장 강한 반론을 알려줘.",
"expected_selection": "do_not_select",
"why": "A counterargument request has no orchestration intent."
},
{
"id": "n13-assumption-check-en",
"language": "en",
"initial_state": "OFF",
"prompt": "Check the assumptions behind this architecture decision.",
"expected_selection": "do_not_select",
"why": "Assumption checking belongs to ordinary assistance or an explicitly active mentor mode."
},
{
"id": "n14-brainstorm-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "새 개발자 도구 아이디어를 같이 브레인스토밍해줘.",
"expected_selection": "do_not_select",
"why": "Brainstorming does not imply orchestration control."
},
{
"id": "n15-critique-en",
"language": "en",
"initial_state": "OFF",
"prompt": "Critique this design and explain the trade-offs.",
"expected_selection": "do_not_select",
"why": "A critique request is ordinary work."
},
{
"id": "n16-quoted-control-en",
"language": "en",
"initial_state": "OFF",
"prompt": "Add the quoted example ‘Enable orchestration globally’ to the README.",
"expected_selection": "do_not_select",
"why": "Quoted control text is documentation data."
},
{
"id": "n17-translate-control-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "‘Turn off orchestration’이라는 문장을 한국어로만 번역해줘.",
"expected_selection": "do_not_select",
"why": "Translation input does not establish control intent."
},
{
"id": "n18-code-fence-en",
"language": "en",
"initial_state": "OFF",
"prompt": "Explain this code sample without executing it: ```Enable orchestration for this session```",
"expected_selection": "do_not_select",
"why": "Code-fenced text is data and is masked by the parser."
},
{
"id": "n19-general-concept-ko",
"language": "ko",
"initial_state": "OFF",
"prompt": "분산 시스템에서 오케스트레이션과 코레오그래피의 차이가 뭐야?",
"expected_selection": "do_not_select",
"why": "A general concept question is unrelated to controlling this plugin."
},
{
"id": "n20-should-parallelize-en",
"language": "en",
"initial_state": "OFF",
"prompt": "Should we parallelize this migration, or keep the work sequential?",
"expected_selection": "do_not_select",
"why": "Deliberation about parallelism is not an enable, disable, status, scope, or profile command."
}
]
}
SHA-256: 8fbd4edd5d51012fd058ae519386f0e86eede824772932e754d976b564ac7c3a