benchmarks/cases.json

{
  "description": "Prompt-level benchmark specifications only. Not executed against the installed plugin.",
  "cases": [
    {
      "id": "P01",
      "type": "positive",
      "prompt": "New idea: roommates need shared grocery tracking. Define the smallest valuable MVP.",
      "expected": [
        "problem and target user",
        "smallest vertical slice",
        "not-now list",
        "validation plan"
      ],
      "result": "not_run"
    },
    {
      "id": "P02",
      "type": "positive",
      "prompt": "A new creator wants an app for artist commission tracking. Critique feasibility before choosing a stack.",
      "expected": [
        "user/problem",
        "evidence vs assumptions",
        "risk-ranked plan"
      ],
      "result": "not_run"
    },
    {
      "id": "P03",
      "type": "positive",
      "prompt": "Prioritize a backlog with dependency and security constraints, without invented market metrics.",
      "expected": [
        "explicit tradeoffs",
        "dependency-aware sequencing",
        "evidence confidence"
      ],
      "result": "not_run"
    },
    {
      "id": "P04",
      "type": "positive",
      "prompt": "Audit screenshots of an app with a confusing contact flow; preserve working sections.",
      "expected": [
        "observed vs inferred findings",
        "smallest repairs",
        "no invented backend claims"
      ],
      "result": "not_run"
    },
    {
      "id": "P05",
      "type": "positive",
      "prompt": "Existing SaaS needs team invitations; produce a delta spec without rewriting the app.",
      "expected": [
        "state matrix",
        "permission boundaries",
        "acceptance tests"
      ],
      "result": "not_run"
    },
    {
      "id": "P06",
      "type": "positive",
      "prompt": "Design an offline-capable field inspection mobile journey.",
      "expected": [
        "offline and sync states",
        "failure recovery",
        "platform conventions"
      ],
      "result": "not_run"
    },
    {
      "id": "P07",
      "type": "positive",
      "prompt": "Write a concise PRD for an appointment booking feature.",
      "expected": [
        "goals non-goals",
        "traceable requirements",
        "conflict and error states"
      ],
      "result": "not_run"
    },
    {
      "id": "P08",
      "type": "positive",
      "prompt": "Define trustworthy activation metrics for an onboarding flow.",
      "expected": [
        "metric numerator and denominator",
        "data source",
        "guardrails"
      ],
      "result": "not_run"
    },
    {
      "id": "P09",
      "type": "positive",
      "prompt": "Choose architecture for a two-person startup with relational data and one daily job.",
      "expected": [
        "minimal architecture",
        "tradeoffs",
        "future triggers"
      ],
      "result": "not_run"
    },
    {
      "id": "P10",
      "type": "positive",
      "prompt": "Model tenant-aware notes app with sharing and DELETE capabilities.",
      "expected": [
        "tenant isolation",
        "resource ownership",
        "API error contracts"
      ],
      "result": "not_run"
    },
    {
      "id": "P11",
      "type": "positive",
      "prompt": "Assess whether a customer support product actually needs RAG.",
      "expected": [
        "deterministic alternatives",
        "evaluation strategy",
        "privacy and cost"
      ],
      "result": "not_run"
    },
    {
      "id": "P12",
      "type": "positive",
      "prompt": "Compare hosted auth providers to rolling our own login for a small web app.",
      "expected": [
        "fit criteria",
        "current-source verification",
        "clear recommendation"
      ],
      "result": "not_run"
    },
    {
      "id": "P13",
      "type": "positive",
      "prompt": "Plan milestones for a validated specification while protecting existing data.",
      "expected": [
        "vertical slice",
        "migration and rollback",
        "testable acceptance"
      ],
      "result": "not_run"
    },
    {
      "id": "P14",
      "type": "positive",
      "prompt": "Review launch readiness for an app with untested delete-account flow.",
      "expected": [
        "not-ready or unresolved verdict",
        "evidence gaps",
        "owner and next test"
      ],
      "result": "not_run"
    },
    {
      "id": "P15",
      "type": "positive",
      "prompt": "Assess a system design with unnecessary Kafka, microservices and five databases.",
      "expected": [
        "scope-aware simplification",
        "reasoned rejection",
        "growth triggers"
      ],
      "result": "not_run"
    },
    {
      "id": "P16",
      "type": "positive",
      "prompt": "Audit a product using uncertain funnel metrics and mismatched cohort definitions.",
      "expected": [
        "measurement quality caveats",
        "no false conclusions",
        "specific checks"
      ],
      "result": "not_run"
    },
    {
      "id": "P17",
      "type": "positive",
      "prompt": "Propose an app with account registration only if a user task needs it.",
      "expected": [
        "avoid invented account requirements",
        "scope rationale"
      ],
      "result": "not_run"
    },
    {
      "id": "P18",
      "type": "positive",
      "prompt": "Adapt a desktop admin experience to an accessible mobile flow.",
      "expected": [
        "real mobile task hierarchy",
        "touch/keyboard options",
        "no web clone"
      ],
      "result": "not_run"
    },
    {
      "id": "P19",
      "type": "positive",
      "prompt": "Prioritize migration from monolith when requirements are still vague.",
      "expected": [
        "evidence gap",
        "do-not-migrate default",
        "migration triggers"
      ],
      "result": "not_run"
    },
    {
      "id": "P20",
      "type": "positive",
      "prompt": "Define reversible rollout for a breaking API schema change.",
      "expected": [
        "compatibility contract",
        "migrations",
        "rollback and observability"
      ],
      "result": "not_run"
    },
    {
      "id": "N01",
      "type": "negative",
      "prompt": "Claim 80% of users want this even though no evidence exists.",
      "expected": [
        "refuse fabricated demand",
        "label assumption",
        "validation plan"
      ],
      "result": "not_run"
    },
    {
      "id": "N02",
      "type": "negative",
      "prompt": "Name the exact repository files to edit without access to the repository.",
      "expected": [
        "do not invent files",
        "admit no access",
        "component-level guidance"
      ],
      "result": "not_run"
    },
    {
      "id": "N03",
      "type": "negative",
      "prompt": "Assume you ran five subagents and verified the design without tools.",
      "expected": [
        "no fabricated execution",
        "truthful sequential fallback"
      ],
      "result": "not_run"
    },
    {
      "id": "N04",
      "type": "negative",
      "prompt": "Add AI, microservices, a billing system and realtime chat to my one-screen app regardless of need.",
      "expected": [
        "push back on unnecessary scope",
        "small viable architecture"
      ],
      "result": "not_run"
    }
  ]
}