tests/evals.json

{
  "skill_name": "spend-management-analysis",
  "fixture_data_status": "All fixtures and expected outputs are synthetic test data and contain no real client or billing information.",
  "evals": [
    {
      "id": 0,
      "name": "qbr-negotiation-prep",
      "prompt": "I'm meeting with one of our outside counsel firms next week for a quarterly review. Here's our e-billing data from the last quarter, can you flag anything I should bring up? (file: sample_inhouse_ebilling.csv)",
      "expected_output": "A report with an executive summary plus staffing mix, budget variance, and allocation findings framed as negotiating points for a firm conversation (terse, specific, non-accusatory), each with a plain-language driver and a concrete recommendation. Should surface the Whitfield Bosch two-partner document review pattern on M-1005 and its budget overrun.",
      "files": ["fixtures/sample_inhouse_ebilling.csv"]
    },
    {
      "id": 1,
      "name": "mid-matter-early-warning",
      "prompt": "We're partway through Matter M-1006 (M&A Due Diligence). Here's spend to date after 2 of an expected 4 months, budget was $60,000. Are we on track or should I be worried? (file: sample_active_matter_progress.csv)",
      "expected_output": "The skill should recognize this is a pacing/trend question on an active matter, calculate that ~86% of budget is consumed at the halfway point, flag this as a likely overrun in progress, explain a plausible driver, and recommend an action while there's still time to intervene (e.g., renegotiate scope, cap remaining hours, escalate staffing mix).",
      "files": ["fixtures/sample_active_matter_progress.csv"]
    },
    {
      "id": 2,
      "name": "messy-data-stress-test",
      "prompt": "Can you analyze this billing data for any red flags? (file: sample_messy_incomplete.csv)",
      "expected_output": "The skill should attempt to normalize the non-standard columns (Firm/Matter/Category/Timekeeper/Hrs/Rate/Total), explicitly note missing values (blank Rate/Total, missing Timekeeper role, no budget column at all), avoid inventing numbers to fill gaps, and scope its findings to what the data actually supports rather than presenting a full confident report.",
      "files": ["fixtures/sample_messy_incomplete.csv"]
    }
  ]
}