skills/rating-analysis/evals/evals.json

{
  "skill_name": "rating-analysis",
  "evals": [
    {
      "id": 1,
      "prompt": "Create a rating pitch report for Boeing Company in the Aerospace/Defense sector",
      "expected_output": "Two output files: (a) rating_pitch.html — a single self-contained HTML report built from scripts/build_html.py containing all 26 sections (cover, agenda, 3 part dividers, 21 content sections, sources, thank-you, disclaimer) with inline Chart.js charts, styled tables, and bullet lists using the Moody's brand palette; (b) payload.json — the fully resolved data payload. No PPTX is produced. The HTML file opens directly in a browser with no additional dependencies beyond an internet connection for the Chart.js CDN.",
      "files": [],
      "assertions": [
        "Both output files exist: rating_pitch.html and payload.json",
        "rating_pitch.html is a valid standalone HTML document (starts with <!DOCTYPE html> and ends with </html>)",
        "rating_pitch.html contains >= 20 <section> elements",
        "rating_pitch.html contains all 3 part divider slides (Sector Analysis, Company Credit Overview, Company Positioning vs. Peers)",
        "rating_pitch.html contains >= 14 <canvas> elements (one per Chart.js chart)",
        "rating_pitch.html contains >= 14 new Chart( calls in <script> blocks",
        "rating_pitch.html contains >= 6 <table> elements covering GDP, rating actions, peer summary, peer financials, scorecard, ESG",
        "SWOT section present with 4 quadrants (Strengths, Weaknesses, Opportunities, Threats) each containing >= 2 <li> bullets",
        "Peer summary table has >= 3 company columns",
        "Peer financials table has >= 3 company columns and >= 5 financial metric rows",
        "Macro GDP table has >= 3 country rows",
        "Rating history line chart present with >= 2 data points",
        "Scorecard table has >= 3 factor rows",
        "ESG table has >= 1 data row",
        "Sources section present with >= 5 listed references",
        "payload.json matches the skill schema (report_date, target_company, sector, currency, companies, sources, sections)",
        "Peer debt and profitability bar charts include period labels on each bar (e.g. FY2024) and a value-axis title with unit"
      ]
    },
    {
      "id": 2,
      "prompt": "I need a rating pitch for Tesla Inc. — sector is Automotive. Use USD and compare against 6 peers",
      "expected_output": "Same structural requirements as eval-1, with 6 peers resolved via getEntityPeers, USD honoured in payload.currency, and peer tables/charts reflecting 6 comparators. Tests natural-language phrasing and parameter handling (currency, peer count). Output is rating_pitch.html + payload.json only — no PPTX.",
      "files": [],
      "assertions": [
        "Both output files exist: rating_pitch.html and payload.json",
        "rating_pitch.html is a valid standalone HTML document (starts with <!DOCTYPE html> and ends with </html>)",
        "rating_pitch.html contains >= 20 <section> elements",
        "rating_pitch.html contains all 3 part divider slides (Sector Analysis, Company Credit Overview, Company Positioning vs. Peers)",
        "rating_pitch.html contains >= 14 <canvas> elements (one per Chart.js chart)",
        "rating_pitch.html contains >= 14 new Chart( calls in <script> blocks",
        "rating_pitch.html contains >= 6 <table> elements",
        "SWOT section present with 4 quadrants (Strengths, Weaknesses, Opportunities, Threats) each containing >= 2 <li> bullets",
        "Peer summary table has >= 6 company columns (6 peers requested)",
        "Peer financials table has >= 6 company columns and >= 5 financial metric rows",
        "Macro GDP table has >= 3 country rows",
        "Rating history line chart present with >= 2 data points",
        "Scorecard table has >= 3 factor rows",
        "ESG table has >= 1 data row",
        "Sources section present with >= 5 listed references",
        "payload.json currency field equals USD",
        "payload.json companies array contains >= 6 peer entries",
        "payload.json matches the skill schema (report_date, target_company, sector, currency, companies, sources, sections)",
        "Peer debt and profitability bar charts include period labels on each bar and a value-axis title with unit"
      ]
    },
    {
      "id": 3,
      "prompt": "Generate a credit pitch report for JPMorgan Chase, Banking sector",
      "expected_output": "Same structural requirements as eval-1, applied to a financial-institution target where bank-specific credit metrics (deposits, capital ratios, NIM) may substitute for generic Debt/EBITDA where appropriate. Tests sector flexibility (Banking vs A&D/Auto). Output is rating_pitch.html + payload.json only — no PPTX.",
      "files": [],
      "assertions": [
        "Both output files exist: rating_pitch.html and payload.json",
        "rating_pitch.html is a valid standalone HTML document (starts with <!DOCTYPE html> and ends with </html>)",
        "rating_pitch.html contains >= 20 <section> elements",
        "rating_pitch.html contains all 3 part divider slides (Sector Analysis, Company Credit Overview, Company Positioning vs. Peers)",
        "rating_pitch.html contains >= 14 <canvas> elements (one per Chart.js chart)",
        "rating_pitch.html contains >= 14 new Chart( calls in <script> blocks",
        "rating_pitch.html contains >= 6 <table> elements",
        "SWOT section present with 4 quadrants (Strengths, Weaknesses, Opportunities, Threats) each containing >= 2 <li> bullets",
        "Peer summary table has >= 3 company columns",
        "Peer financials table has >= 3 company columns and >= 5 financial metric rows",
        "Macro GDP table has >= 3 country rows",
        "Rating history line chart present with >= 2 data points",
        "Scorecard table has >= 3 factor rows",
        "ESG table has >= 1 data row",
        "Sources section present with >= 5 listed references",
        "payload.json target_company field contains JPMorgan and sector field contains Banking",
        "payload.json matches the skill schema (report_date, target_company, sector, currency, companies, sources, sections)",
        "Peer debt and profitability bar charts include period labels on each bar and a value-axis title with unit"
      ]
    }
  ]
}