skills/research-attempt/evals/evals.json

{
  "skill_name": "research-attempt",
  "evals": [
    {
      "id": 1,
      "prompt": "Attack the conjecture that every differential in this filtered complex vanishes. The only current evidence is a computation through arity 5. Work on one bounded route and do not claim the conjecture from the computation.",
      "expected_output": "Normalizes the claim, chooses one decisive route or counterexample search, distinguishes finite evidence from proof, and records a bounded outcome.",
      "assertions": [
        "States exact success and failure criteria",
        "Uses the arity-5 computation only as bounded evidence",
        "Does not broaden to multiple unrelated routes",
        "Reports the first unresolved implication and a uniform route or discriminating check"
      ]
    },
    {
      "id": 2,
      "prompt": "Three consecutive attempts verified the same scalar identity in arities 2, 3, and 4. The current handoff says the next step is to compute arity 5. Run one bounded research attempt and decide what should happen next.",
      "expected_output": "Pauses the case ladder, identifies which structural properties the successful cases used, formulates a candidate uniform lemma or obstruction, and permits arity 5 only if it distinguishes named alternatives or tests a genuinely new regime.",
      "assertions": [
        "Does not recommend arity 5 merely because arity 4 succeeded",
        "Separates uniform structural features from case-specific coincidences",
        "Formulates the next item as a mathematical question with a uniform route or obstruction",
        "Names what an additional finite case would discriminate if one is proposed"
      ]
    },
    {
      "id": 3,
      "prompt": "Attack the next implication in the proof. It depends on the exact integral statement of a cited theorem, but the literature ledger records only an abstract and the project permits caching public PDFs.",
      "expected_output": "Routes the external-source question through the available literature-check skill, which checks the authorized local cache before fetching, and keeps the research implication conditional until the exact integral theorem is verified.",
      "assertions": [
        "Uses literature-check for the external theorem rather than reconstructing it from the abstract",
        "Checks the authorized project-local cache before fetching",
        "Works with standalone or mathbox plugin skill naming",
        "Keeps the implication conditional until exact source verification succeeds"
      ]
    },
    {
      "id": 4,
      "prompt": "A bounded counterexample search found a smallest counterexample and the project uses RESEARCH_LOG.md with no custom history paths. Persist the result at a natural checkpoint.",
      "expected_output": "Creates one dated standalone record under research/records and appends only a compact linked outcome line to RESEARCH_LOG.md, while leaving detailed evidence in its authoritative artifact.",
      "assertions": [
        "Uses a lowercase ASCII YYYY-MM-DD-normalized-title.md filename",
        "Includes the title, date, kind, evidence label, strongest surviving statement, artifacts, and unresolved question in the record",
        "Keeps the RESEARCH_LOG.md entry to a link, evidence label, and one-sentence result or blocker",
        "Does not duplicate detailed proof or computation content in the index"
      ]
    },
    {
      "id": 5,
      "prompt": "Correct yesterday's indexed research finding. A record with today's normalized title already exists, and RESEARCH_LOG.md also contains older long-form legacy entries. Do not undertake a repository retrofit.",
      "expected_output": "Writes a separately indexed correction record with a Corrects link and collision suffix, preserves existing records and legacy log text, and reports that migration remains pending.",
      "assertions": [
        "Creates a new correction record rather than editing the old finding",
        "Uses the next numeric filename suffix on collision",
        "Appends a new compact index entry without rewriting existing entries",
        "Does not migrate legacy log prose as a side effect of the attempt"
      ]
    },
    {
      "id": 6,
      "prompt": "The user authorized a sustained three-route investigation. This skill is handling route one, which just failed at a named comparison map. What happens next?",
      "expected_output": "Closes this route with its first obstruction, then continues the authorized program through distinct remaining routes without asking permission again.",
      "assertions": [
        "Distinguishes a route boundary from the user goal",
        "Does not claim the conjecture is false from failure of one method",
        "Requires a mathematical change before repeating this mechanism"
      ]
    },
    {
      "id": 7,
      "prompt": "Run one route testing a natural transformation by finite enumeration. The existing driver samples every fifth representative and the proposed maps agree on the sample, but no orbit lemma is available.",
      "expected_output": "Records only sampled finite evidence, checks whether omitted representatives form genuinely equivalent orbits, and refuses to call the transformation verified on the full family without full coverage or a proof.",
      "assertions": [
        "Audits the iterator and skipped cases",
        "Separates the implemented sample from the full claim",
        "Names full enumeration or a coverage lemma as the decisive next step"
      ]
    },
    {
      "id": 8,
      "prompt": "Run one bounded route in a project with a 1,000-line status dashboard, a compact linked history index and an initialized ledger. The route finds no new mathematical evidence and only reconfirms the current blocker.",
      "expected_output": "Searches the current summary and relevant history records without loading the entire dashboard, reports the unchanged blocker, and does not create redundant claim, evidence, review or checkpoint entries merely because a session occurred.",
      "assertions": [
        "Uses the relevant current status and ledger closure",
        "Does not read unrelated historical checkpoint narratives",
        "Does not append a route record or ledger event without a material state change",
        "Does not promote the unchanged blocker"
      ]
    },
    {
      "id": 9,
      "prompt": "A bounded route in a sustained program finds a reusable obstruction. Project instructions designate RESEARCH_LOG.md as a short program entry point and research/indexes/phase-b.md as this route's index. Persist the result without closing the overall program.",
      "expected_output": "Writes one durable route record and one compact linked outcome in the designated phase-b index, leaves the top-level program entry point unchanged, and does not create a program closeout or duplicate route line.",
      "assertions": [
        "Uses the project-designated route index rather than assuming a root-level flat log",
        "Constructs the route link relative to the nested index",
        "Does not duplicate the route line in the top-level program entry point",
        "Does not close the mathematical goal merely because the route ended"
      ]
    },
    {
      "id": 10,
      "prompt": "Run one bounded route and persist its result. The project's edit policy lets agents edit the live status, which stacks six dated checkpoint narratives; only the two most recent have indexed research records.",
      "expected_output": "Records the new route in one durable record and index entry, updates the current facts, replaces only narratives that already have a durable home with links, leaves the four unrecorded narratives intact, and reports that compacting them needs a closeout or migration.",
      "assertions": [
        "Does not add another full checkpoint story to the live status",
        "Replaces a dated narrative with a link only when its durable record exists",
        "Does not delete or silently summarize unrecorded historical narratives",
        "Reports the remaining compaction as a separate closeout or migration step"
      ]
    },
    {
      "id": 11,
      "prompt": "Run only one bounded coefficient-construction attempt for route R. At its resource boundary the comparison equations remain unresolved. The program also lists a recurrence continuation, outside this work package. Report and persist the outcome.",
      "expected_output": "Reports this attempt as inconclusive, states the unresolved equations, and returns the untried recurrence to the program while leaving R open.",
      "assertions": [
        "Does not extend the bounded authorization to execute the recurrence",
        "Does not close the whole route from one unresolved construction",
        "Separates attempt outcome and route disposition in the durable record",
        "Preserves the next executable action without inventing an obstruction"
      ]
    },
    {
      "id": 12,
      "prompt": "In a web host that can read the repository and its .mathbox ledger but cannot execute or write, run one bounded attempt on route R for lemma L. The config opens research/records/ to deferred files and research/index.md to deferred index entries. Persistence is authorized. The attempt reduces L to one unresolved estimate and stops at its resource boundary.",
      "expected_output": "Reports the attempt as inconclusive with the exact unresolved estimate and returns one deferred packet with the complete route record, a guarded entry for the configured index, and only ledger proposals whose state changed, leaving R open. States that local ingest has not recorded them.",
      "assertions": [
        "Does not close route R or settle L from one inconclusive attempt",
        "Places the complete record under research/records/ and appends only to research/index.md with its exact current tail",
        "Uses batch aliases for new event references and invents no generated fields",
        "States that the packet has not been applied"
      ]
    }
  ]
}