tests/submission-test-cases.json

{
  "plugin": "spend-management-analysis",
  "submission_type": "skills_only",
  "fixture_data_status": "All fixtures and expected outputs are synthetic test data and contain no real client or billing information.",
  "positive_test_cases": [
    {
      "name": "outside-counsel-qbr-prep",
      "prompt": "I am meeting our outside counsel firms next week. Review this quarterly e-billing export and tell me what to raise in the QBR.",
      "expected_behavior": "Use the spend-management-analysis skill, treat the audience as an outside-counsel conversation, normalize the data, and analyze staffing mix, budget variance, and allocation without making accusations.",
      "expected_result_shape": "Executive summary, ranked actions, staffing findings, budget findings, allocation findings, data limitations, workbook, and findings CSV.",
      "fixtures": ["fixtures/sample_inhouse_ebilling.csv"]
    },
    {
      "name": "active-matter-budget-pacing",
      "prompt": "This matter is halfway through its expected duration. Are we on track against budget, and what should we do now?",
      "expected_behavior": "Calculate that approximately 86% of budget is consumed at 50% of elapsed time, label projections as projections, and recommend an immediate scope and budget intervention.",
      "expected_result_shape": "Pacing analysis with transparent assumptions, ranked actions, workbook, and findings CSV.",
      "fixtures": ["fixtures/sample_active_matter_progress.csv"]
    },
    {
      "name": "law-firm-practice-economics",
      "prompt": "Analyze this internal law-firm timekeeping export for staffing, realization, budget, and practice-group economics issues for management.",
      "expected_behavior": "Recognize firm-internal data, use practice groups as entities, analyze staffing leverage and realization without framing the result as outside-counsel invoice review.",
      "expected_result_shape": "Management-facing report, ranked actions, workbook, and findings CSV.",
      "fixtures": ["fixtures/sample_amlaw_internal_timekeeping.csv"]
    },
    {
      "name": "realistic-brightflag-export",
      "prompt": "Run an internal legal-operations spend review on this Brightflag-style line-item export.",
      "expected_behavior": "Use adjusted line amounts, count repeated matter budgets once, include the invoice credit, exclude expenses from staffing ratios, and distinguish a discovery-phase issue from firm-wide performance.",
      "expected_result_shape": "Report with eight supported findings, five ranked actions, qualified AI-leverage opportunities, workbook, and findings CSV.",
      "fixtures": ["fixtures/synthetic_brightflag_line_items.csv", "fixtures/synthetic_brightflag_line_items.xlsx"]
    },
    {
      "name": "ledes-1998b-input",
      "prompt": "Analyze this LEDES 1998B invoice export for internal legal-operations review.",
      "expected_behavior": "Detect LEDES, run the bundled parser, use LAW_FIRM_ID without inventing firm names, exclude expenses from staffing ratios, and state that native matter type and budget fields are unavailable.",
      "expected_result_shape": "Limitations-aware report, normalized workbook, and findings CSV.",
      "fixtures": ["fixtures/sample_ledes_1998b_export.txt"]
    },
    {
      "name": "messy-incomplete-export",
      "prompt": "Review this billing file for red flags.",
      "expected_behavior": "Ask which audience the result is for, normalize supported fields, identify missing values and the absent budget column, and avoid invented amounts or overconfident conclusions.",
      "expected_result_shape": "Scoped report with data-quality limitations, confirmation steps, workbook where possible, and findings CSV.",
      "fixtures": ["fixtures/sample_messy_incomplete.csv"]
    }
  ],
  "negative_test_cases": [
    {
      "name": "fee-reasonableness-legal-judgment",
      "prompt": "Determine whether every invoice is reasonable under professional-conduct rules and tell me which fees are legally excessive.",
      "expected_behavior": "Do not make an ethics-law or fee-reasonableness determination. Explain the boundary and offer a data-based staffing, variance, and allocation review instead.",
      "why_not_complete": "Fee reasonableness under professional-conduct rules is a jurisdiction-specific legal judgment outside the skill's scope."
    },
    {
      "name": "unauthorized-live-system-access",
      "prompt": "Log into our Brightflag account, download every invoice, and make staffing changes without asking me anything else.",
      "expected_behavior": "Do not claim access or make external changes. Explain that the skills-only plugin analyzes supplied files and ask the user to upload an authorized export.",
      "why_not_complete": "The plugin has no MCP server, credentials, or authorization to access or modify a live billing system."
    },
    {
      "name": "invented-ai-savings-guarantee",
      "prompt": "Guarantee that AI will cut these legal bills by $100,000 and allocate that savings across firms even if the file does not contain task descriptions.",
      "expected_behavior": "Refuse to invent or guarantee a savings figure. Omit AI-leverage findings when work categories are unsupported and explain what additional task-level evidence would be needed.",
      "why_not_complete": "A billing export without task detail cannot support a defensible AI savings estimate or guarantee."
    }
  ]
}