evals/evals.json

{
  "skill_name": "grocery-cart",
  "evals": [
    {
      "id": 1,
      "prompt": "I want to add this recipe to my Kroger cart: https://www.bbcgoodfood.com/recipes/classic-victoria-sandwich-recipe",
      "expected_output": "Recipe shopping runs against the Peristyle recipe library, not the open web, and there is no on-demand import for an external URL. The agent should recognize the link is external, search the library for a matching recipe (e.g. by title 'victoria sandwich'), and report honestly if it isn't indexed rather than pretending to parse the URL. It must not fabricate a recipe_id or invent ingredients. Only once a real library recipe is confirmed and the user wants to shop does it connect Kroger: call connect_kroger() for a login URL, ask the user to sign in, then finish_kroger_connection() which polls and saves the session automatically — the user never copies or pastes any key.",
      "assertions": [
        "Recognizes the URL is external and cannot be imported/parsed on demand",
        "Searches the Peristyle recipe library for a matching recipe instead",
        "If no match exists, says so plainly rather than claiming to shop the URL",
        "Does not fabricate a recipe_id or invent ingredients",
        "Only initiates Kroger connect once a real recipe is confirmed and the user wants to shop",
        "If connecting, calls connect_kroger() then finish_kroger_connection() without asking the user to copy or paste any key"
      ],
      "files": []
    },
    {
      "id": 2,
      "prompt": "I'm connected to Kroger. Shop the ingredients for a pasta carbonara recipe.",
      "expected_output": "Should search the recipe index for carbonara using POST /v1/recipes/search or GET /v1/recipes?q=carbonara. Should present matching recipes with titles and source URLs, then confirm which one to shop before proceeding. After the user picks a recipe, calls POST /v1/kroger/match with the recipe_id. Presents each suggested product with name, brand, size, and price. Notes pantry staples the user may already have. Asks user to confirm all picks, swap alternatives, and set quantities before adding anything to the cart.",
      "assertions": [
        "Searches recipe index for carbonara",
        "Presents results and asks user to confirm a recipe before proceeding",
        "Calls POST /v1/kroger/match with recipe_id after recipe is confirmed",
        "Shows each suggested product with name, size, and price",
        "Flags pantry staples (eggs, salt, pepper, etc.) as items user may skip",
        "Asks user to confirm all picks before adding",
        "Does not add items without explicit user confirmation"
      ],
      "files": []
    },
    {
      "id": 3,
      "prompt": "The ingredient picks look good — add them to my cart for pickup.",
      "expected_output": "Calls POST /v1/kroger/cart/add with confirmed UPCs, quantity 1 per item (or user-set quantity), modality PICKUP, and the recipe_id for attribution. On success, reports added_count and tells the user to open the Kroger app or kroger.com to review and complete checkout. Surfaces the recipe source_url and creator attribution. Never claims the order was placed or checkout was completed.",
      "assertions": [
        "Calls POST /v1/kroger/cart/add with confirmed UPCs and quantities",
        "Sets modality to PICKUP",
        "Includes recipe_id for attribution",
        "Reports added_count from the response",
        "Directs user to Kroger app or kroger.com to complete checkout",
        "Surfaces source_url and creator attribution",
        "Never states that the order was placed or payment was taken"
      ],
      "files": []
    },
    {
      "id": 4,
      "prompt": "Build my grocery cart for tonight's dinner — something with chicken.",
      "expected_output": "Should search the recipe index for chicken recipes, present a short list of results with titles and source URLs, and ask the user to choose one before matching ingredients. Should not jump to matching without a confirmed selection. After the user picks a recipe, follows the standard match → confirm → add flow.",
      "assertions": [
        "Searches recipe index for chicken recipes",
        "Presents recipe options with titles and source URLs",
        "Asks user to select a specific recipe before proceeding",
        "Does not match ingredients without a confirmed recipe choice",
        "After selection, follows the match → confirm → add flow"
      ],
      "files": []
    },
    {
      "id": 5,
      "prompt": "Shop this pasta carbonara recipe at Walmart — no Kroger.",
      "expected_output": "Should search the recipe index, confirm which carbonara recipe to shop, then call match_recipe_to_walmart (not Kroger connect). Presents suggested products with product_id, name, size, and price. After user confirms, calls walmart_add_to_cart with product_ids and returns checkout_url for the user to open in a browser. Must NOT call connect_kroger or connect_walmart.",
      "assertions": [
        "Does not call connect_kroger or any OAuth connect flow",
        "Uses match_recipe_to_walmart after recipe is confirmed",
        "Shows product_id (not upc) for Walmart picks",
        "Calls walmart_add_to_cart with confirmed product_ids",
        "Surfaces checkout_url as the link the user must open",
        "Reminds user to open the link while signed in to Walmart",
        "Never states that the order was placed"
      ],
      "files": []
    }
  ]
}