skills/research-program/evals/evals.json

{
  "skill_name": "research-program",
  "evals": [
    {
      "id": 1,
      "prompt": "Investigate the universal target through three distinct routes. Two approaches fail at the same naturality lemma and finite computations pass to arity 6. Continue the investigation.",
      "expected_output": "Treats the shared naturality obstruction as one mechanism, executes a materially distinct next route and preserves the universal target and bounded evidence.",
      "assertions": [
        "Executes mathematics rather than only proposing routes",
        "Does not stop after the first portfolio fails",
        "Distinguishes method failure, finite success and the original goal",
        "Requires a new input before reopening a failed route"
      ]
    },
    {
      "id": 2,
      "prompt": "Pursue a theorem through several routes. The first route proves a useful special case but leaves two named comparison lemmas open; the original request says to continue while plausible routes remain.",
      "expected_output": "Records the special case as partial progress, keeps the original theorem contract active, and executes a distinct route against one of the remaining comparison lemmas.",
      "assertions": [
        "Does not present the special case as completion",
        "Names the unresolved implications",
        "Continues without asking again for already granted authority",
        "Preserves a checkpoint before changing routes"
      ]
    },
    {
      "id": 3,
      "prompt": "Two delegated routes return in an unreliable display order. Both started from checkpoint P17 and both propose a different replacement for notes/result.md. Reconcile them and continue the program.",
      "expected_output": "Uses the declared common base rather than arrival order, preserves both results, exposes the output collision, and serializes any shared-state update only after comparing the actual artifacts.",
      "assertions": [
        "Does not infer chronology from response order",
        "Uses base checkpoint and artifact identity",
        "Does not silently overwrite a colliding result",
        "Keeps mathematical disagreement separate from file reconciliation"
      ]
    },
    {
      "id": 4,
      "prompt": "A theorem needs two named lemmas. There are three proposed mechanisms for the first lemma, and one mechanism is recorded under the parent theorem because it also informs the second lemma. Continue the research program and prepare its handoff.",
      "expected_output": "Keeps the two lemmas as jointly required claim dependencies, treats the three mechanisms as alternatives, records which obligation the parent-owned route directly resolves, and preserves route-only prerequisites as context rather than theorem dependencies.",
      "assertions": [
        "Does not count stages of one mechanism as independent routes",
        "Distinguishes alternative routes from jointly required obligations",
        "Names the sub-obligation resolved by the parent-owned route",
        "Does not alter theorem dependencies to make the handoff include route context"
      ]
    },
    {
      "id": 5,
      "prompt": "Continue a multi-route research program whose live dashboard has grown through twenty dated checkpoint sections. The ledger has a goal-scoped handoff, and the next mathematical move depends on one conditional review.",
      "expected_output": "Uses a brief goal handoff to locate the live blocker, opens the exact conditional review and source artifacts, continues a distinct route, and keeps the live dashboard current by linking prior checkpoints rather than appending another full narrative.",
      "assertions": [
        "Does not load or reproduce all twenty historical dashboard sections",
        "Inspects the conditional review before treating evidence as settled",
        "Preserves the original theorem target across route changes",
        "Keeps historical checkpoint records discoverable"
      ]
    },
    {
      "id": 6,
      "prompt": "Close out this authorized multi-route research program without starting new research. Its goal remains open, the live status contains fifteen dated checkpoint narratives, and a flat history index lists many route records. The ledger already records the latest claim and review states.",
      "expected_output": "Checks the exact recorded evidence and base, writes one linked program synthesis without promoting the open goal, replaces superseded live narratives with a compact current decision view, and proposes or uses a program/route index hierarchy while preserving historical entries and ledger events.",
      "assertions": [
        "Does not execute a new mathematical route in a closeout-only request",
        "Separates the still-open goal from strongest partial results and failed mechanisms",
        "Links exact proofs, reviews and route records instead of copying them into live status",
        "Does not add ledger events solely because a closeout occurred",
        "Keeps old flat index entries and links reachable before changing the default entry point"
      ]
    },
    {
      "id": 7,
      "prompt": "Write a retrospective closeout for a 2024 program. Its last recorded route was inconclusive; a valid proof and independent review appeared in 2026. The exact 2024 ledger head is available, but one mission-relevant route cannot be assigned confidently to this or the next program.",
      "expected_output": "States the outcome supported at the 2024 cutoff separately from the 2026 assessment, cites both checkpoints, leaves the old route outside the program verdict in a linked unclassified inventory, and does not fabricate a 2024 proof or review.",
      "assertions": [
        "Records both historical cutoff and later assessment dates and revisions",
        "Does not backdate the 2026 proof or independent review",
        "Keeps the inconclusive 2024 outcome distinct from current status",
        "Preserves the uncertain route with its source and reason outside the program outcome",
        "Treats unclassified membership as an indexing question, not a mathematical evidence label"
      ]
    },
    {
      "id": 8,
      "prompt": "Prepare a retrospective closeout for an old route collection whose dates conflict and whose cutoff revision cannot be recovered. A proof is available now, but there is no evidence that it existed during the old program.",
      "expected_output": "Marks the historical cutoff and its outcome indeterminate, records the current assessment checkpoint and proof separately, and preserves the old routes without inventing a completion date.",
      "assertions": [
        "Marks cutoff provenance unavailable rather than picking a convenient date",
        "Does not backdate the current proof",
        "Keeps current proof status separate from the indeterminate historical outcome"
      ]
    },
    {
      "id": 9,
      "prompt": "Continue an authorized program. The brief ledger handoff shows one live run constructing a lift for route R from checkpoint P17. Start a parallel obstruction search for R from the same checkpoint; it has a separate owner and disjoint output files.",
      "expected_output": "Checks the live run's work scope, starts the distinct obstruction search as its own run from P17 with a separate owner and disjoint write scope, and leaves the lift run active for later reconciliation.",
      "assertions": [
        "Inspects the existing run before assigning the new one",
        "Does not treat a live run as a blanket block on distinct parallel work for the same route",
        "Records separate run identities, owners, base checkpoints and write scopes",
        "Does not close or abandon the lift run merely to launch the obstruction search"
      ]
    },
    {
      "id": 10,
      "prompt": "Continue an authorized program. Route R has two proposed continuations: a coefficient construction and a recurrence argument. One agent could not solve the coefficient equations and returned an inconclusive attempt. The recurrence has not been tried, and no obstruction to it is known. Decide the route disposition and continue the mathematics.",
      "expected_output": "Records the coefficient attempt as inconclusive with its unresolved implication, keeps R open, and executes the recurrence continuation within the authorized resources.",
      "assertions": [
        "Does not infer impossibility or route exhaustion from the agent getting stuck",
        "Accounts for both proposed continuations and preserves the untried one",
        "Executes the remaining continuation rather than stopping at a suggestion",
        "Does not require new mathematical input to resume unfinished work"
      ]
    },
    {
      "id": 11,
      "prompt": "The authorized time budget has ended. A route tried a direct construction without settling its comparison lemma; a second construction remains untried. Preserve the handoff and stop work at the budget boundary.",
      "expected_output": "Stops at the resource limit, records the inconclusive attempt separately, and leaves the route open with the second construction deferred and its next step and resumption condition visible.",
      "assertions": [
        "Honors the resource boundary without executing more research",
        "Does not classify untried work as exhausted or mathematically obstructed",
        "Preserves an executable deferred continuation without requiring a new theorem"
      ]
    },
    {
      "id": 12,
      "prompt": "A program route requires a nonzero equivariant linear map from the one-dimensional sign representation of C2 over Q to the trivial representation. Two proposed constructions are averaging and choosing a basis. The first attempt proves that equivariance and linearity force 2f(v)=0 for every v. The basis construction remains untried. Decide this route’s disposition.",
      "expected_output": "Checks the obstruction over Q, explains that it rules out every nonzero equivariant linear map and hence both proposed constructions, and closes this scoped route as failed without needing to execute the basis construction.",
      "assertions": [
        "Checks that the obstruction covers the whole mechanism under its stated hypotheses",
        "Accounts for the untried construction as ruled out by evidence",
        "Does not require trying a continuation already excluded by a valid obstruction",
        "Does not infer that the broader program goal is false solely because this route fails"
      ]
    },
    {
      "id": 13,
      "prompt": "Continue an authorized program. Route R's coefficient continuation was deferred after stalling on one product simplification. A later attempt resumed it with the same expansion and stalled at the same simplification. R's recurrence continuation is untried, no obstruction is known, and resources remain. Decide the next work package.",
      "expected_output": "Records the repeated stall as R's bottleneck without calling it an obstruction, and neither reruns the same expansion nor closes R; it executes a changed approach to that step or the untried recurrence.",
      "assertions": [
        "Does not rerun the stalled simplification with the same method and resources",
        "Names what the next work package changes, or switches to the untried continuation",
        "Does not classify the repeated stall as an obstruction or close the route",
        "Executes mathematics rather than only proposing the change"
      ]
    },
    {
      "id": 14,
      "prompt": "Continue program P for goal G in a read-only web host with no execution. The .mathbox config opens proofs/ and research/records/ to deferred files. Route R1 yields a lemma under an extra hypothesis; route R2 remains untried. Persistence is authorized.",
      "expected_output": "Pursues the routes, returns a single complete deferred packet pinned to the inspected head rather than one per route, keeps R2 open with its next step, and does not treat the conditional lemma as reaching G. States that ingest remains pending.",
      "assertions": [
        "Returns one packet, since every packet must pin the same inspected ledger head",
        "States the lemma's extra hypothesis and does not declare G reached",
        "Keeps the untried R2 in the handoff rather than closing it",
        "States that ingest remains pending"
      ]
    }
  ]
}