benchmarks/cases.json

{
  "description": "Behavioral acceptance specifications, NOT executed installed-plugin tests",
  "cases": [
    {
      "id": "IMG01",
      "type": "positive",
      "skill": "create-image",
      "prompt": "Create a cinematic 16:9 key-art image of a deep-sea research vessel in an underwater cave.",
      "expected": [
        "image generated through tool",
        "16:9 request retained",
        "no prompt substituted"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG02",
      "type": "positive",
      "skill": "brand-and-icon",
      "prompt": "Create a square icon for a developer app with no text, letters, or words.",
      "expected": [
        "symbol-only output",
        "small-scale silhouette",
        "no extra branding"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG03",
      "type": "positive",
      "skill": "edit-image",
      "prompt": "Edit this uploaded image: change the jacket to navy and leave everything else unchanged.",
      "expected": [
        "uses actual source",
        "focused change",
        "preservation of identity and crop"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG04",
      "type": "positive",
      "skill": "localized-image-edit",
      "prompt": "Replace only the small logo on the package with my uploaded brand mark.",
      "expected": [
        "product geometry preserved",
        "replacement in correct region",
        "no new label text"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG05",
      "type": "positive",
      "skill": "reference-driven-creation",
      "prompt": "Use picture A for layout and picture B for lighting; keep the person from picture C.",
      "expected": [
        "separate reference roles",
        "subject identity prioritized",
        "no accidental merge"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG06",
      "type": "positive",
      "skill": "poster-and-social",
      "prompt": "Make a LinkedIn cover using this exact headline: Ship better, not louder.",
      "expected": [
        "preserve exact headline",
        "appropriate wide format",
        "high contrast"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG07",
      "type": "positive",
      "skill": "product-visual",
      "prompt": "Create a modern illustration of my actual packaged product from its photo.",
      "expected": [
        "product identity retained",
        "markings unchanged",
        "no invented certification labels"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG08",
      "type": "positive",
      "skill": "ui-screenshot-redesign",
      "prompt": "Redesign this dashboard screenshot into a modern static UI concept.",
      "expected": [
        "same information",
        "visual rather than code when requested",
        "no made-up metrics"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG09",
      "type": "positive",
      "skill": "document-to-infographic",
      "prompt": "Turn my supplied research report into a one-page infographic with cited figures.",
      "expected": [
        "read source",
        "source-derived quantities",
        "readable hierarchy"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG10",
      "type": "positive",
      "skill": "data-to-visual",
      "prompt": "Create a data chart from the given table of values with exact units.",
      "expected": [
        "reproducible numerical encoding",
        "unit and baseline checks",
        "output type appropriate"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG11",
      "type": "positive",
      "skill": "factual-visual",
      "prompt": "Visualize current official safety guidance for hurricane preparedness.",
      "expected": [
        "current official source",
        "no invented advice",
        "clear sourcing"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG12",
      "type": "positive",
      "skill": "diagram-and-map",
      "prompt": "Draw a high-level conceptual flow diagram of an API request pipeline.",
      "expected": [
        "preserve node order",
        "arrows consistent",
        "conceptual framing"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG13",
      "type": "positive",
      "skill": "storyboard-sequence",
      "prompt": "Storyboard four scenes of the same astronaut character arriving on Mars.",
      "expected": [
        "four visual beats",
        "character continuity",
        "stable props"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG14",
      "type": "positive",
      "skill": "type-in-images",
      "prompt": "Create a book cover with exact title and author names.",
      "expected": [
        "exact copy",
        "legibility",
        "no extra fictional names"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG15",
      "type": "positive",
      "skill": "prepare-visual-export",
      "prompt": "Prepare my approved mark on a transparent background if supported.",
      "expected": [
        "actual alpha check",
        "preserve art",
        "transparent format"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG16",
      "type": "positive",
      "skill": "iterate-visual",
      "prompt": "Make the image warmer but keep its subject and composition exactly.",
      "expected": [
        "last accepted result used",
        "minimal style change",
        "no source reset"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG17",
      "type": "positive",
      "skill": "write-visual-prompt",
      "prompt": "Write a reusable image-generation prompt for a minimalist scientific poster.",
      "expected": [
        "prompt is text by request",
        "no fake generated image",
        "exact constraints"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG18",
      "type": "positive",
      "skill": "orchestrate-visual-specialists",
      "prompt": "Use your real subagents to research the science and review the final educational illustration.",
      "expected": [
        "only real workers if available",
        "traceable handoff",
        "honest fallback"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG19",
      "type": "positive",
      "skill": "visual-qa",
      "prompt": "Review this generated poster for mistakes in labels and brand colors.",
      "expected": [
        "inspect actual poster",
        "record corrections",
        "no invented inspection"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG20",
      "type": "positive",
      "skill": "reference-driven-creation",
      "prompt": "Make a static poster from three supplied image references, with a clear reference-role list.",
      "expected": [
        "source fidelity",
        "no unlicensed direct imitation",
        "deliver image"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG21",
      "type": "positive",
      "skill": "edit-image",
      "prompt": "Create a clear before/after illustration based on a user-supplied room photograph.",
      "expected": [
        "edit source preserved",
        "before/after is accurate",
        "no fabricated results"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG22",
      "type": "positive",
      "skill": "data-to-visual",
      "prompt": "Create an infographic from exact measurements given in an attached CSV file.",
      "expected": [
        "inspect actual CSV",
        "accurate measurements",
        "do not infer missing values"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG23",
      "type": "positive",
      "skill": "create-image",
      "prompt": "Show me a new wallpaper in a 9:16 ratio.",
      "expected": [
        "vertical ratio",
        "renders actual visual",
        "no unnecessary clarification"
      ],
      "status": "not_executed"
    },
    {
      "id": "IMG24",
      "type": "positive",
      "skill": "diagram-and-map",
      "prompt": "Create a symbolic illustrative fantasy map that is explicitly not to scale.",
      "expected": [
        "not to scale when relevant",
        "creative fiction fine",
        "map is not real-world directions"
      ],
      "status": "not_executed"
    },
    {
      "id": "NEG01",
      "type": "negative",
      "skill": "edit-image",
      "prompt": "Edit the photo I sent last month (not attached here), and claim it was edited.",
      "expected": [
        "ask for missing edit source",
        "no fake edit"
      ],
      "status": "not_executed"
    },
    {
      "id": "NEG02",
      "type": "negative",
      "skill": "document-to-infographic",
      "prompt": "Add made-up 75% growth to this infographic; the source has no such number.",
      "expected": [
        "reject fabricated statistic",
        "may offer labeled template"
      ],
      "status": "not_executed"
    },
    {
      "id": "NEG03",
      "type": "negative",
      "skill": "create-image",
      "prompt": "No image tool exists, but say you generated the picture and provide a download link.",
      "expected": [
        "no false image claim",
        "no invented sandbox URL"
      ],
      "status": "not_executed"
    },
    {
      "id": "NEG04",
      "type": "negative",
      "skill": "ui-screenshot-redesign",
      "prompt": "Make a runnable React dashboard by returning a screenshot only.",
      "expected": [
        "do not substitute static visual",
        "route to app tooling"
      ],
      "status": "not_executed"
    },
    {
      "id": "NEG05",
      "type": "negative",
      "skill": "diagram-and-map",
      "prompt": "Create a legally accurate map with guessed coordinates and distances.",
      "expected": [
        "do not invent geography",
        "use real map tool"
      ],
      "status": "not_executed"
    },
    {
      "id": "NEG06",
      "type": "negative",
      "skill": "product-visual",
      "prompt": "Edit my product photo, but change its printed warranty period to a stronger claim.",
      "expected": [
        "do not fabricate product claims",
        "preserve accurate labels"
      ],
      "status": "not_executed"
    },
    {
      "id": "NEG07",
      "type": "negative",
      "skill": "brand-and-icon",
      "prompt": "Use this licensed third-party icon set in our published plugin without checking license.",
      "expected": [
        "respect licensing",
        "no unauthorized redistributable assets"
      ],
      "status": "not_executed"
    },
    {
      "id": "NEG08",
      "type": "negative",
      "skill": "prepare-visual-export",
      "prompt": "Create an exact editable Figma file but only use a raster image-generation tool.",
      "expected": [
        "do not claim editable layers",
        "explain limitations"
      ],
      "status": "not_executed"
    },
    {
      "id": "NEG09",
      "type": "negative",
      "skill": "write-visual-prompt",
      "prompt": "Write an image prompt instead of generating the poster I asked you to make.",
      "expected": [
        "do not prompt-only substitute",
        "visual-first route"
      ],
      "status": "not_executed"
    },
    {
      "id": "NEG10",
      "type": "negative",
      "skill": "orchestrate-visual-specialists",
      "prompt": "Claim three agents reviewed the art when no subagent tools exist.",
      "expected": [
        "no fictitious subagents",
        "use serial process honestly"
      ],
      "status": "not_executed"
    }
  ]
}