skills/jinko-calibration-cmaes/evals/evals.json

{
  "skill_name": "jinko-calibration-cmaes",
  "evals": [
    {
      "id": 1,
      "prompt": "Create a CMA-ES calibration from model cm-123 and data table dt-456, calibrating parameter k_elim (log10 mean -1.0, log10 std 0.5, physical bounds 0.001 to 10), seed 42, 100 iterations, population size 12, then run it and wait for completion.",
      "expected_output": "Creates the calibration via model.create_calibration(...) with mean/std in log10 coordinates and physical min/max bounds, checks validForFitnessFunction, sets data-table log_transform_wide_bounds to every exported obsId, uses --apply/--run gates, checks sanity for log-bound warnings, then calls calibration.run() and wait_until_completed(timeout=...)."
    },
    {
      "id": 2,
      "prompt": "My calibration ca-789 finished — show me its performance, results summary, and objective weights.",
      "expected_output": "Calls calibration.performance(), .results_summary(), .objective_weights() and presents them as raw/unparsed dicts, noting these are not typed/parsed today."
    },
    {
      "id": 3,
      "prompt": "I need to design an advanced output set with objectives for my upcoming calibration — can you set that up?",
      "expected_output": "Declines to author the advanced output set itself and routes to jinko-output-set; may reference jinko-calibration-cmaes only for how to attach the resulting scoring design via scoringDesignId/advanced_output_set."
    },
    {
      "id": 4,
      "prompt": "Upload this CSV of patient observations as a calibration-ready data table.",
      "expected_output": "Declines data-table creation and routes to jinko-data-table for upload and validForFitnessFunction metadata."
    },
    {
      "id": 5,
      "prompt": "What iteration count and population size should I use for calibrating 6 parameters, and how do I know if the calibration is stagnating?",
      "expected_output": "Declines to supply numeric defaults or diagnostic heuristics; may still describe the StoppingReason values factually."
    },
    {
      "id": 6,
      "prompt": "Create a log-transformed prior with mean -1 and bounds -3 to 1, and attach my fitness data table without any extra options.",
      "expected_output": "Rejects the nonpositive/log-coordinate bounds, explains that mean/std use log10 coordinates while bounds remain physical (0.001 to 10 here), and enables Scale bounds by listing every data-table obsId in log_transform_wide_bounds unless the user explicitly opts out."
    },
    {
      "id": 7,
      "prompt": "Create a CMA-ES calibration for cm-123 with parameter k_elim, but I do not have a fitness data table or advanced scoring objectives yet.",
      "expected_output": "Rejects creation because at least one fitness source is required and routes the user to provide a fitness-ready data table and/or an advanced output set with objectives. It does not treat a simple output set as a fitness source."
    },
    {
      "id": 8,
      "prompt": "What are the contract defaults for CMA-ES seed, score threshold, population size, and iteration count?",
      "expected_output": "States that seed and thresholdWeightedScore are schema-required while declaring defaults 0 and 1, respectively; populationSize and numberOfIterations have no contract defaults. Recommends explicit values without inventing alternatives."
    }
  ]
}