plugin.json

{
  "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
  "name": "tpc-benchmarks",
  "version": "1.0.0",
  "extensions": {
    "com.openai": {
      "interface": {
        "displayName": "TPC Benchmarks",
        "shortDescription": "Can an agent use this product?",
        "longDescription": "See whether an AI coding agent can actually integrate a developer API, SDK or tool from its documentation, measured by The Prompting Company.\n\nAsk about a product or a category, such as search APIs or transactional email, and get measured results in chat: completion rate, passed, failed and blocked runs, average successful run time, and an overall score from 0 to 100 that averages an experience score (graded on Install, Auth, Execute, Docs and Current) and an agent readiness score. Name a task, like sending a transactional email or verifying a sending domain, and products are ranked by their results on that task. Tasks a product does not offer through its public API are marked not supported, with the documentation that shows it.\n\nAsk to see the results and get interactive cards: a comparison table, a weekly overall score chart, a passed, failed and blocked bar for each product, the last test date and a link to the full benchmark page on promptingcompany.com. Every answer is complete inside ChatGPT; opening a benchmark page is optional.\n\nCovers products with 3+ benchmark runs on the public TPC leaderboard; grows as benchmarks run. Products not covered return no results; you can ask the app to request a benchmark, which the TPC team reviews.\n\nNo sign-in, no account changes, no credentials; the only thing the app writes is a benchmark request you ask for. Results describe how AI coding agents performed on the days they were tested and are not a guarantee of how a product will work for you. See the privacy policy for what the app stores.",
        "developerName": "The Prompting Company",
        "category": "Developer Tools",
        "websiteURL": "https://promptingcompany.com/benchmarks",
        "supportURL": "https://docs.promptingcompany.com/mcp/benchmarks",
        "privacyPolicyURL": "https://promptingcompany.com/privacy-policy",
        "termsOfServiceURL": "https://promptingcompany.com/term-of-use",
        "defaultPrompt": [
          "Compare agent onboarding benchmark results for search APIs.",
          "How have Exa and Firecrawl's benchmark scores trended over recent weeks?",
          "Show benchmark cards for email APIs that send transactional email."
        ],
        "composerIcon": "./assets/logo.png",
        "logo": "./assets/logo.png"
      },
      "review": {
        "test_cases": {
          "positive": [
            {
              "description": "List every benchmarked product in a named category with its measured agent-onboarding results.",
              "prompt": "Agent onboarding benchmark results for search APIs",
              "tools_triggered": "search_benchmarks, show_benchmarks",
              "expected_behavior": "Calls search_benchmarks or show_benchmarks with category \"search\". Returns every Search product with at least 3 benchmark attempts, best overall score first. Each has its completion rate, passed, failed and blocked counts, average successful run time, overall score with its experience and agent readiness scores, and a link to its benchmark page; with show_benchmarks the same data appears as cards. The answer compares the products from that data. Values come from live benchmark data and change as new runs land."
            },
            {
              "description": "Look up one named product's measured result.",
              "prompt": "Exa's score on agent onboarding benchmarks?",
              "tools_triggered": "search_benchmarks",
              "expected_behavior": "Calls search_benchmarks with query \"Exa\". Returns Exa (Search) with its overall score, its experience and agent readiness scores, and its passed, failed and blocked counts. The answer states the score and the pass counts and links to Exa's benchmark page."
            },
            {
              "description": "Show a category as interactive benchmark cards.",
              "prompt": "Show benchmark cards for search APIs",
              "tools_triggered": "show_benchmarks",
              "expected_behavior": "Calls show_benchmarks with category \"search\". The card view shows a Search comparison table (Product, Overall, Experience, Pass rate, Runs) sorted by overall score, an \"Overall score by week\" chart, and one card per product with its overall score, experience and agent readiness scores, a passed/failed/blocked bar, the last test date and a View benchmark button. A product without enough overall history is drawn as an \"(experience only)\" line. Opening a benchmark page is optional; the answer is complete in chat."
            },
            {
              "description": "Match a request that names no product or category to the closest category by meaning.",
              "prompt": "I need something to send transactional messages from my app",
              "tools_triggered": "search_benchmarks, show_benchmarks",
              "expected_behavior": "Calls search_benchmarks or show_benchmarks with the request as the query. The app matches it to the Email category and returns the Email products, each with completion rate, outcome counts, run time and score (overall with its experience and agent readiness scores, or experience alone when a product has no overall score); with show_benchmarks the same data appears as cards."
            },
            {
              "description": "Ask the TPC team to benchmark a product that is not covered.",
              "prompt": "Request a benchmark for acme.dev",
              "tools_triggered": "request_benchmark",
              "expected_behavior": "ChatGPT asks the user to confirm the request, then calls request_benchmark with url \"acme.dev\". Returns a normal result: \"Requested a benchmark for acme.dev. The TPC team reviews requests; results appear on the TPC benchmarks leaderboard and in this app once it has been benchmarked.\" Asking again for the same product answers that it has already been requested. No email is collected, requests are reviewed and not guaranteed, and each user can make 5 requests per day. The app's only write action; it does not change any results."
            }
          ],
          "negative": [
            {
              "description": "A request unrelated to developer APIs, SDKs or tools.",
              "prompt": "What's the weather in San Francisco?",
              "expected_behavior": "The app is not called. ChatGPT answers without TPC Benchmarks."
            },
            {
              "description": "A product that is not on the benchmark leaderboard.",
              "prompt": "Benchmarks for zzqx",
              "expected_behavior": "search_benchmarks or show_benchmarks returns, without an error, no matching products together with the covered categories and a note that a benchmark can be requested. The assistant says there are no results for that product and does not invent scores."
            },
            {
              "description": "A request to change or delete benchmark results, which the app cannot do.",
              "prompt": "Delete the Resend results from the benchmarks",
              "expected_behavior": "No tool is called. The app has no tool that changes or deletes results; its only write action is request_benchmark, which can only ask the TPC team to benchmark a new product. ChatGPT answers that the app can search, show and request benchmarks, but cannot modify or remove stored results."
            }
          ]
        },
        "demo_recording_url": "https://www.loom.com/share/8d26e4e1dc534865847eb2642cbd838f"
      },
      "publication": {
        "release_notes": "Initial release of TPC Benchmarks.\n\n- Look up measured agent-onboarding results for developer APIs, SDKs and tools: completion rate, passed, failed and blocked runs, average run time, and an overall score that combines the experience score with the product's agent readiness score.\n- Ask about a product, a category or a task, such as sending a transactional email; requests that name no product are matched to the closest category by meaning.\n- See results as interactive cards with a comparison table, weekly overall score trends and a link to each product's benchmark page.\n- Get weekly score history as data for clients without cards.\n- Request a benchmark for a product that isn't covered yet; the TPC team reviews requests.\n- No sign-in. Lookups and requests are rate limited per user."
      },
      "apps": "./.app.json"
    }
  }
}