← Files Intuitive Software DesignARCHIVED FILE
tests/test_plugin.py
59.9 KB · Nov 20, 2026 · 06:43 UTC
#!/usr/bin/env python3
"""Deterministic checks for the plugin structure and bundled tooling."""
from __future__ import annotations
import base64
import json
import re
import subprocess
import sys
from pathlib import Path
from tempfile import TemporaryDirectory
ROOT = Path(__file__).resolve().parents[1]
SKILL = ROOT / "skills" / "intuitive-software-design"
REDESIGN_SKILL = ROOT / "skills" / "purpose-first-redesign"
CONNECTED_SKILL = ROOT / "skills" / "connected-experience-design"
REFERENCES = SKILL / "references"
SCENARIOS = ROOT / "tests" / "scenarios"
def require(condition: bool, message: str) -> None:
if not condition:
raise AssertionError(message)
def text(path: Path) -> str:
require(path.is_file(), f"Missing required file: {path.relative_to(ROOT)}")
return path.read_text(encoding="utf-8")
def grader_input_match(body: str) -> re.Pattern[str]:
match = re.search(r"(?m)^input_match: '(.*)'$", body)
require(match is not None, "Routing grader lacks input_match")
return re.compile(match.group(1))
# Grader keys documented for `claude plugin eval`; an unknown key is ignored at run time, so a typo would
# silently change what a paid run measures.
GRADER_KEYS = {
"regex": {"pattern", "flags", "match", "target"},
"tool_used": {"tool", "input_match", "min", "max"},
"tool_order": {"before", "after"},
"file_exists": {"path", "exists"},
"llm": {"criteria", "focus"},
"baseline": {"baseline_file", "criteria"},
}
def grader_frontmatter(body: str) -> dict[str, str]:
match = re.match(r"---\n(.*?)\n---\n", body, re.DOTALL)
require(match is not None, "Eval grader lacks frontmatter")
return dict(re.findall(r"(?m)^([a-z_]+):\s*(.*)$", match.group(1)))
def grader_is_scored(fields: dict[str, str]) -> bool:
"""Scored in a two-arm run: not an unscored plugin-fired indicator (see the plugin-evals docs)."""
skill_indicator = fields.get("type") == "tool_used" and fields.get("tool") == "Skill"
return fields.get("arm") != "with-only" and (not skill_indicator or fields.get("arm") == "both")
def test_manifest() -> None:
manifest_path = ROOT / ".codex-plugin" / "plugin.json"
manifest = json.loads(text(manifest_path))
claude = json.loads(text(ROOT / ".claude-plugin" / "plugin.json"))
require(manifest["name"] == claude["name"] == "intuitive-software-design", "Plugin names disagree")
require(re.fullmatch(r"\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?", manifest["version"]) is not None, "Version is not semver")
require(manifest["version"] == claude["version"], "Codex and Claude versions disagree")
require(f"version-{manifest['version']}-" in text(ROOT / "README.md"), "README version badge disagrees")
require(manifest.get("skills") == "./skills/", "Manifest must expose ./skills/")
require("mcpServers" not in manifest and "apps" not in manifest and "hooks" not in manifest, "Plugin must remain self-contained")
interface = manifest["interface"]
require(interface["displayName"] == "Intuitive Software Design", "Display name is not discoverable")
# Match the OpenAI directory upload contract, which rejects the old "Design" label.
require(interface.get("category") in {
"Productivity", "Creativity", "Developer Tools", "Business & Operations", "Data & Analytics",
"Communication", "Education & Research", "Security", "Finance", "Healthcare", "Travel",
"Entertainment", "Other",
}, "Codex category is not accepted by the OpenAI plugin directory")
prompts = interface.get("defaultPrompt", [])
require(isinstance(prompts, list) and 1 <= len(prompts) <= 3, "defaultPrompt must contain one to three prompts")
require(all(isinstance(prompt, str) and len(prompt) <= 128 for prompt in prompts), "A default prompt exceeds 128 characters")
artwork_keys = ("composerIcon", "logo")
has_artwork = any(key in interface for key in artwork_keys)
require(all(key in interface for key in artwork_keys) or not has_artwork,
"Codex artwork paths must be both present or both absent")
if has_artwork:
for key in artwork_keys:
require((ROOT / interface[key]).is_file(), f"Codex {key} image is missing")
require("excluded" in text(ROOT / "assets" / "LICENSE").lower(),
"Artwork license must state the Apache exclusion")
else:
require(not (ROOT / "assets" / "logo.png").exists(),
"Unbranded forks must remove the reserved artwork")
require('<img src="assets/logo.png"' not in text(ROOT / "README.md"),
"Unbranded forks must remove the README logo reference")
marketplace = json.loads(text(ROOT / ".claude-plugin" / "marketplace.json"))
require(any(entry.get("name") == manifest["name"] and entry.get("source") in {".", "./"}
for entry in marketplace.get("plugins", [])), "Claude marketplace must expose the root plugin")
def test_skill_frontmatter() -> None:
for path in (ROOT / "skills").glob("*/SKILL.md"):
body = text(path)
match = re.match(r"\A---\s*\n(.*?)\n---\s*\n", body, re.DOTALL)
require(match is not None, f"Missing YAML frontmatter: {path.parent.name}")
fields = dict(re.findall(r"^([a-z][a-z-]*):\s*(.+)$", match.group(1), re.MULTILINE))
require(fields.get("name") == path.parent.name, f"Wrong skill name: {path.parent.name}")
require(bool(fields.get("description", "").strip()), f"Missing description: {path.parent.name}")
def test_skill_contract() -> None:
skill = text(SKILL / "SKILL.md")
for token in ("DESIGN", "REVIEW", "AUDIT", "IMPROVE", "Not Evaluated", "Critical Failure"):
require(token in skill, f"Skill routing/guardrail missing: {token}")
require("Do not produce a screen redesign by default" in skill, "Improve guardrail is missing")
require("assign numeric scores unless the user asks" in skill, "Review scoring guardrail is missing")
require("They do not prove responsiveness" in skill, "Screenshot evidence guardrail is missing")
require("../purpose-first-redesign/SKILL.md" in skill and "broad creative freedom" in skill,
"Broad redesign route is missing")
require("Loop Break:" in skill and "Validation:" in skill, "Finding shape lacks loop or validation")
# Eval transcripts showed routed runs acting on SKILL.md alone, so the contract summary must live here
# and point back to the standard, which controls.
require("## Recovery and validation contract" in skill
and "references/intuitive-software-design-standard.md#recovery-and-validation-contract" in skill,
"Main skill must summarize and link the standard's recovery and validation contract")
require("Behavior Confidence from one observed success path is `NE`" in skill,
"AUDIT guidance must keep Behavior Confidence NE without representative coverage")
require("UI Clarity, Flow Intuition, and Behavior Confidence—as `NE`" in skill
and "do not offer provisional or partial dimension percentages" in skill,
"AUDIT guidance must keep every product dimension NE without representative coverage")
require("that the evidence does not establish, even in proposed interface copy" in skill,
"Unestablished business rules must not be stated as fact")
# Eval traces showed a blanket "read the standard for every mode" was rarely followed, while a named file
# tied to the AUDIT mode was read every time; each mode therefore names the sections it requires.
for mode, anchor in {"DESIGN": "#39-design-process", "REVIEW": "#40-review-process",
"AUDIT": "#38-audit-process", "IMPROVE": "#41-improve-process"}.items():
line = re.search(rf"(?m)^- `{mode}`: read .*$", skill)
require(line is not None and f"intuitive-software-design-standard.md{anchor}" in line.group(0),
f"{mode} guidance must link the standard section it requires: {anchor}")
require("for every mode, then use the relevant sections" not in skill,
"Blanket every-mode reading requirement returned; name sections per mode instead")
openai = text(SKILL / "agents" / "openai.yaml")
require("$intuitive-software-design" in openai, "Skill default prompt must name the skill")
def test_purpose_first_redesign_contract() -> None:
skill = text(REDESIGN_SKILL / "SKILL.md")
required = (
"GitNexus",
"Route and entry points",
"Must preserve",
"Free to rethink",
"Orient -> Recognize -> Predict -> Act -> Confirm -> Continue",
"Treat the current layout as evidence",
"Do not edit source",
"discussion-ready brief",
"intuitive-software-design-standard.md#recovery-and-validation-contract",
"check with intended users that states their goal without naming the control",
"that the evidence does not establish, even in proposed copy",
"intuitive-software-design-standard.md#39-design-process",
)
for token in required:
require(token in skill, f"Purpose-first redesign contract missing: {token}")
openai = text(REDESIGN_SKILL / "agents" / "openai.yaml")
require("$purpose-first-redesign" in openai, "Redesign default prompt must name the skill")
def test_connected_experience_contract() -> None:
skill = text(CONNECTED_SKILL / "SKILL.md")
required = (
"Shared identity",
"Shared data",
"Task handoff",
"Platform capability",
"Connected action",
"actual current and durable state",
"universal synchronization",
"marketing statement as a promise to investigate",
"privacy and authority",
"Source can show what code is intended to do",
)
for token in required:
require(token.lower() in skill.lower(), f"Connected-experience contract missing: {token}")
openai = text(CONNECTED_SKILL / "agents" / "openai.yaml")
require("$connected-experience-design" in openai, "Connected-experience default prompt must name the skill")
def test_authoritative_standard() -> None:
standard = text(REFERENCES / "intuitive-software-design-standard.md")
required = (
"the intended user can correctly infer what to do next",
"UI, Flow, and Feel",
"ORIENT",
"RECOGNIZE",
"PREDICT",
"CONFIRM",
"CONTINUE",
"Technical Architecture ≠ User Workflow",
"System-backed experience",
"actual saved, shared, or external state",
"universal synchronization",
"Prediction Gap",
"Decision Density",
"Navigation Friction",
"Interpretation Friction",
"Decision Friction",
"Interaction Friction",
"Memory Friction",
"Feedback Friction",
"Recovery Friction",
"Process Friction",
"Intuition Critical Failures",
"Minimal UI does not automatically mean intuitive UI",
"The goal is not to eliminate thinking",
"### Recovery and validation contract",
"One observed success path is never representative",
"Do not present provisional, partial, or estimated dimension percentages",
"including in proposed interface copy",
)
for token in required:
require(token in standard, f"Authoritative concept missing: {token}")
finding_format = standard.split("## 37. Finding format", 1)[1].split("## 38. Audit process", 1)[0]
require("Loop Break:" in finding_format and "Validation:" in finding_format,
"Standard finding format lacks loop or validation")
require(standard.count("```mermaid") >= 6, "Standard must include at least six Mermaid diagrams")
def test_scoring_contract() -> None:
scoring = text(REFERENCES / "scoring-reference.md")
standard = text(REFERENCES / "intuitive-software-design-standard.md")
for score in ("0 | Broken", "1 | Difficult", "2 | Understandable", "3 | Intuitive", "4 | Effortless"):
require(score in scoring, f"Universal score missing: {score}")
require("NE | Not Evaluated" in scoring, "Insufficient-evidence state is missing")
screen = (
"Orientation", "Visual hierarchy", "Action clarity", "Control recognition",
"Information clarity", "Feedback and state visibility", "Cognitive load", "Consistency",
)
workflow = (
"Entry-point clarity", "Next-action clarity", "Sequence matches user task",
"Context preservation", "Decision load", "Backtracking required", "Progress visibility",
"Completion confidence", "Error recovery", "Prediction consistency",
)
for criterion in screen + workflow:
require(f"## {criterion}" in scoring, f"Detailed scoring rubric missing: {criterion}")
require(scoring.count("**Warning signs:**") == 22, "Every scored criterion needs warning signs")
require(scoring.count("**Test:**") == 22, "Every scored criterion needs a test method")
for criterion in ("Outcome predictability", "State feedback", "Cross-context consistency", "Failure recovery"):
require(criterion in scoring, f"Behavior Confidence input missing: {criterion}")
require("Do not copy screen or workflow scores" in scoring, "Behavior Confidence must not reuse other scores")
require("Do not reuse UI or Flow criterion scores" in standard, "Standard does not define distinct Behavior Confidence inputs")
audit = text(REFERENCES / "audit-template.md")
require("| Expected outcome and affected object | Observed action, result, and state |" in audit
and "Behavior Confidence coverage:" in audit,
"Audit template lacks interaction-level scoring evidence")
require("One observed success path is never representative" in scoring
and "Behavior Confidence: __ / 100 or NE" in scoring
and "Representative: yes / no (if no, Behavior Confidence is NE)" in audit,
"Behavior Confidence must be NE without representative interaction coverage")
require("Do not report provisional or partial dimension percentages" in scoring
and "do not enter provisional percentages" in audit,
"Product dimensions without representative coverage must not get provisional scores")
def test_references_and_links() -> None:
required = (
"intuitive-software-design-standard.md",
"scoring-reference.md",
"examples.md",
"audit-template.md",
"ai-review-template.md",
)
for name in required:
text(REFERENCES / name)
for markdown in (ROOT / "skills").rglob("*.md"):
body = text(markdown)
for target in re.findall(r"\[[^\]]+\]\(([^)]+)\)", body):
if target.startswith(("http://", "https://")):
continue
relative, _, fragment = target.partition("#")
linked = (markdown.parent / relative).resolve() if relative else markdown
require(linked.is_file(), f"Broken link in {markdown.name}: {target}")
if fragment:
require(fragment in heading_slugs(text(linked)), f"Broken anchor in {markdown.name}: {target}")
def test_journey_renderer_cli() -> None:
renderer = ROOT / "skills" / "user-task-walkthrough" / "scripts" / "render_journey.py"
step = {
"id": "check-result",
"title": "Check the result",
"surface": "Draft view",
"cue": "The draft shows its name and save status.",
"expectation": "The name persists after reopening.",
"action": "Reopen the draft.",
"result": "Confirm the saved name.",
"continuation": "Return to the draft list.",
"basis": "proposed",
"sources": [],
"next": None,
}
journey = {
"version": 1,
"title": "Review a saved draft",
"goal": "Confirm the literal label </script> remains text.",
"role": "Draft author",
"entry": "A named draft",
"success": "The saved name is visible.",
"coverage": "Proposed behavior.",
"start": step["id"],
"sources": [],
"steps": [step],
}
with TemporaryDirectory(prefix="journey map ") as directory:
record = Path(directory) / "journey.json"
record.write_text(json.dumps(journey), encoding="utf-8")
command = [sys.executable, str(renderer), str(record)]
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode == 0, result.stderr)
output = record.with_name("index.html")
rendered = output.read_text(encoding="utf-8")
data = re.search(r'<script id="journey-data" type="application/json">(.*?)</script>', rendered)
require(data is not None and json.loads(data[1]) == journey,
"The standalone page must preserve the record inside its inert JSON element")
manifest = json.loads(text(ROOT / ".codex-plugin" / "plugin.json"))
logo_path = manifest["interface"].get("logo")
embedded_logo = re.search(r'<img class="masthead-logo" src="data:image/png;base64,([^"]+)"', rendered)
if logo_path:
require(embedded_logo is not None and base64.b64decode(embedded_logo[1]) == (ROOT / logo_path).read_bytes(),
"The standalone page must embed the original plugin logo")
else:
require(embedded_logo is None, "The page should render without reserved artwork")
source_root = Path(directory) / "evidence"
source_root.mkdir()
source_file = source_root / "source file.txt"
source_file.write_bytes(b"First line\nSecond line")
git = ["git", "-c", "core.autocrlf=false", "-C", str(source_root)]
subprocess.run([*git, "init", "-q"], check=True)
subprocess.run([*git, "add", "--", source_file.name], check=True)
subprocess.run([*git, "-c", "user.name=Journey Test", "-c", "user.email=test@example.invalid",
"-c", f"core.hooksPath={source_root / 'no-hooks'}", "-c", "commit.gpgsign=false",
"commit", "-qm", "Pin two evidence lines"], check=True)
revision = subprocess.check_output([*git, "rev-parse", "HEAD"], text=True).strip()
source_file.write_bytes(b"First line\r\nSecond line\r\nOnly in the working copy\r\n")
cited = {"id": "pinned-file", "label": "Saved evidence", "note": "The pinned file has two lines."}
journey["sources"] = [cited]
with_root = [*command, "--source-root", str(source_root)]
for location, invocation, expected in (
(f"source file.txt:1, 2–3 @ {revision}", with_root, "ends at line 3, but the cited file has 2 lines"),
(f"source file.txt:2–1 @ {revision}", with_root, "positive, ordered line range"),
(f"source file.txt:1–2 @ {revision}", command, "needs --source-root"),
):
cited["location"] = location
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(invocation, capture_output=True, text=True, check=False)
require(result.returncode != 0 and expected in result.stderr, result.stderr)
require(output.read_text(encoding="utf-8") == rendered,
"A failed citation check must preserve the existing page")
cited["location"] = f"source file.txt:1–2 @ {revision}"
journey["sources"].append({
"id": "local-file", "label": "Working source", "location": "source file.txt:2-3",
"note": "The unversioned file has an additional line.",
})
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(with_root, capture_output=True, text=True, check=False)
require(result.returncode == 0, result.stderr)
rendered = output.read_text(encoding="utf-8")
journey["sources"] = []
step["branches"] = [{"label": "Save failed", "to": "missing-step"}]
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0 and "unknown id: 'missing-step'" in result.stderr,
"A missing branch destination should produce an actionable error")
require(output.read_text(encoding="utf-8") == rendered,
"Rendering an invalid record should leave the previous page intact")
review = {**step, "id": "read-inbox", "title": "Find the submitted draft", "actor": "reviewer"}
del review["branches"]
journey["people"] = [
{"id": "author", "label": "Author", "responsibility": "Prepare and send the draft."},
{"id": "reviewer", "label": "Reviewer", "responsibility": "Open submitted drafts."},
]
journey["sources"] = [
{"id": "fixture", "label": "Synthetic decisions", "location": "test fixture", "note": "First choices."},
{"id": "draft-spec", "label": "Draft workflow", "location": "test specification",
"note": "Saved drafts can be reopened from the original list."},
]
step.update({
"actor": "author",
"assignments": [
{"role": "reviewer", "work": "Review this draft and return a decision."},
],
"next": review["id"],
"nextLabel": "Send",
"nextCondition": "The draft is ready to send.",
"nextHandoff": {
"work": "Submitted draft",
"entry": "Review inbox",
"acknowledgment": "The reviewer sees the item in their inbox.",
},
"branches": [{"id": "keep-editing", "label": "Keep editing", "to": step["id"]}],
"measurements": [{
"basis": "fixture", "source": "fixture", "cohort": "Draft authors",
"period": "Synthetic sample", "window": "First choice within one day",
"unit": "drafts", "eligible": 10, "counts": {"next": 7, "keep-editing": 2},
}],
})
journey["steps"].append(review)
gap = {
"step": review["id"],
"question": "When is the draft visible to the reviewer?",
"impact": "Determines whether the author needs a receipt confirmation.",
"check": "Open the review inbox in a test account after submission.",
}
journey["scenarios"] = [{
"label": "A reviewer receives a draft",
"steps": [step["id"], review["id"]],
"gaps": [gap],
}]
draft_list = {
"label": "Use the draft list",
"kind": "baseline",
"change": "Find the draft in the original list.",
"effect": "The author uses the established workflow.",
"tradeoff": "Finding one draft may require searching.",
"support": "The fixture specifies reopening from the list.",
"steps": [step["id"]],
"sources": ["draft-spec"],
}
draft_link = {
"label": "Link to the submitted draft",
"kind": "proposed",
"change": "Expose a saved draft link on the confirmation.",
"effect": "The author may find the submitted draft with less effort.",
"tradeoff": "The link depends on the draft retaining its access rules.",
"support": "The existing record shows that the draft can be saved.",
"openQuestion": "Can authors use the link after leaving the submission view?",
"steps": [step["id"]],
"sources": ["draft-spec"],
"preview": {
"start": "confirmation",
"screens": [
{"id": "confirmation", "title": "Draft saved", "actor": "Author",
"body": "The sample draft remains available after confirmation.",
"actions": [{"label": "Open the draft", "to": "draft"}]},
{"id": "draft", "title": "Sample draft", "actor": "Author",
"body": "The saved draft opens with its original values."},
],
},
}
journey["comparisons"] = [{
"question": "How should an author find a submitted draft?",
"goal": "Let the author reopen the saved draft.",
"finding": {
"step": step["id"], "basis": "specified", "sources": ["draft-spec"],
"summary": "The author needs to locate the saved draft in the original list.",
},
"options": [draft_list, draft_link],
"check": "Ask authors to reopen a submitted draft using each option.",
"decisionRule": "Choose the link if it helps authors find the draft and preserves access.",
}]
completion = {
"kind": "completion",
"definition": "First submitted drafts with a recorded review decision.",
"eligible": 10, "measured": 8, "count": 6,
"steps": [review["id"]],
}
wait = {
"kind": "waiting",
"definition": "From first submission to first reviewer entry for drafts with paired times.",
"measured": 4, "statistic": "median", "seconds": 52.5,
"steps": [step["id"], review["id"]],
}
journey["outcomes"] = [{
"basis": "fixture", "source": "fixture", "cohort": "Draft authors",
"period": "Synthetic follow-up sample", "window": "Within two days of first submission",
"unit": "drafts", "metrics": [completion, wait, {
"kind": "rework", "definition": "First submitted drafts that were corrected and resubmitted.",
"eligible": 10, "measured": 8, "count": 0,
}, {
"kind": "recovery", "definition": "Interrupted draft reviews that resumed.",
"eligible": 3, "measured": 2, "count": 1,
}],
}]
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode == 0, result.stderr)
rendered = output.read_text(encoding="utf-8")
data = re.search(r'<script id="journey-data" type="application/json">(.*?)</script>', rendered)
require(data is not None and json.loads(data[1]) == journey,
"The generated HTML must preserve handoffs, measurements, gaps, comparisons, and outcomes")
journey["version"] = 2
del step["surface"], review["surface"]
step.update(statusLocation="Draft list", actionLocation="Draft editor", sources=["fixture"])
review.update(statusLocation="Review inbox", actionLocation=None)
journey["sources"].extend([
{"id": "send-contract", "label": "Sending rules", "location": "test sending rules",
"note": "A ready draft can be delivered to review. A draft can also remain with the author."},
{"id": "review-authority", "label": "Reviewer access", "location": "test reviewer policy",
"note": "The assigned reviewer can open submitted drafts."},
])
step["nextSources"] = ["send-contract"]
step["branches"][0].update(actionLocation=None, sources=["send-contract"])
review["actorSources"] = ["review-authority"]
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode == 0, result.stderr)
rendered = output.read_text(encoding="utf-8")
data = re.search(r'<script id="journey-data" type="application/json">(.*?)</script>', rendered)
require(data is not None and json.loads(data[1]) == journey,
"The generated HTML must preserve separate step and branch locations, including explicit unknowns")
require("Where they see the status" in rendered and "Where they take the action" in rendered,
"New maps must give status and action locations different labels")
audit = subprocess.run([*command, "--review-routes"], capture_output=True, text=True, check=False)
require(audit.returncode == 0, audit.stderr)
delivered, _, retry = audit.stdout.partition("check-result -> check-result | Keep editing")
require("check-result -> read-inbox | Send" in delivered
and "Condition: The draft is ready to send." in delivered
and "Actor: Author -> Reviewer" in delivered
and "Receiving action: Reopen the draft. | Location: Unspecified" in delivered
and "Review inbox" not in delivered,
"The route review must identify the receiving actor and leave an unknown action location explicit")
require("Branch action location: Unspecified" in retry and "Location: Draft editor" in retry,
"The route review must report the branch location separately from the destination location")
focused = subprocess.run([*command, "--review-routes", step["id"]],
capture_output=True, text=True, check=False)
require(focused.returncode == 0 and "Origin sources: test fixture" in focused.stdout
and "Destination sources: Unspecified" in focused.stdout
and "Transition sources: test sending rules" in focused.stdout
and "Receiving actor sources: test reviewer policy" in focused.stdout,
focused.stderr or focused.stdout)
require(output.read_text(encoding="utf-8") == rendered,
"Reviewing the saved routes must leave the existing page intact")
step["branches"][0]["sources"] = ["missing-transition-source"]
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0 and "branches[0].sources refers to an unknown id" in result.stderr,
"A branch must cite a registered source for its own transition")
require(output.read_text(encoding="utf-8") == rendered,
"A rejected transition citation must leave the previous page intact")
step["branches"][0]["sources"] = ["send-contract"]
scenario = journey["scenarios"][0]
scenario["path"] = True
scenario["steps"].reverse()
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0 and "has no transition from 'read-inbox' to 'check-result'" in result.stderr,
"A declared route must fail if one intended continuation is absent from the graph")
require(output.read_text(encoding="utf-8") == rendered,
"A disconnected declared route must leave the previous page intact")
review["branches"] = [{"label": "Return draft", "to": step["id"]}]
scenario["steps"].append(review["id"])
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode == 0, result.stderr)
route_html = output.read_text(encoding="utf-8")
route_data = re.search(r'<script id="journey-data" type="application/json">(.*?)</script>', route_html)
require(route_data is not None and json.loads(route_data[1])["scenarios"][0] == scenario,
"Coverage must preserve an ordered return route that revisits a checkpoint")
scenario["steps"].pop()
del review["branches"], scenario["path"]
scenario["steps"].reverse()
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode == 0 and output.read_text(encoding="utf-8") == rendered, result.stderr)
step.update(record="Original draft", nextCreates="Review request")
review.update(record="Review request", continuationUnknown=True)
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode == 0, result.stderr)
accepted_record_html = output.read_text(encoding="utf-8")
outline = {
"steps": [
{"id": step["id"], "source": "Draft policy, section 1", "assignees": ["reviewer"]},
{"id": review["id"], "source": "Review policy, section 2", "actor": "reviewer",
"record": "Review request", "statusLocation": "Review inbox", "actionLocation": None},
],
"routes": [{"from": step["id"], "to": review["id"], "source": "Draft policy, section 3",
"creates": "Review request", "condition": "The draft is ready to send.", "handoff": True}],
}
outline_path = Path(directory) / "source-outline.json"
outline_path.write_text(json.dumps(outline), encoding="utf-8")
check = [*command, "--check-outline", str(outline_path)]
result = subprocess.run(check, capture_output=True, text=True, check=False)
require(result.returncode == 0 and "matches the supplied source outline" in result.stdout, result.stderr)
outline["steps"][0]["routesComplete"] = True
outline["routes"].append({"from": step["id"], "to": step["id"], "source": "Draft policy, section 3"})
outline_path.write_text(json.dumps(outline), encoding="utf-8")
result = subprocess.run(check, capture_output=True, text=True, check=False)
require(result.returncode == 0, result.stderr)
reviewed_render = [*command, "--render-reviewed", str(outline_path)]
result = subprocess.run(reviewed_render, capture_output=True, text=True, check=False)
require(result.returncode != 0 and "Checkpoint 'read-inbox' needs routesComplete=true" in result.stderr
and "Checkpoint 'check-result' needs an assertion for actionLocation" in result.stderr,
"Reviewed rendering must check the receiving routes and each named location")
require(output.read_text(encoding="utf-8") == accepted_record_html,
"An uncovered new-record boundary must leave the previous page intact")
outline["steps"][1]["routesComplete"] = True
outline["steps"][0].update(statusLocation="Draft list", actionLocation="Draft editor")
outline_path.write_text(json.dumps(outline), encoding="utf-8")
result = subprocess.run(reviewed_render, capture_output=True, text=True, check=False)
require(result.returncode == 0 and output.read_text(encoding="utf-8") == accepted_record_html,
result.stderr or "Reviewed rendering changed the output of a valid map")
reached = {"label": "Find the already-saved request", "to": review["id"], "sources": ["send-contract"],
"condition": "The previous send committed.", "existingRecord": "Review request"}
step["branches"].append(reached)
outline["routes"].append({"from": step["id"], "to": review["id"], "source": "Draft policy, section 3",
"condition": reached["condition"], "existingRecord": "Review request"})
record.write_text(json.dumps(journey), encoding="utf-8")
outline_path.write_text(json.dumps(outline), encoding="utf-8")
result = subprocess.run(check, capture_output=True, text=True, check=False)
require(result.returncode == 0, result.stderr)
routes = subprocess.run([*command, "--review-routes"], capture_output=True, text=True, check=False)
resumed = routes.stdout.partition("Find the already-saved request")[2]
require(routes.returncode == 0 and "Existing record: Review request" in resumed and "New record:" not in resumed,
"An already-saved record must have its own transition effect")
reached["creates"] = reached.pop("existingRecord")
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(reviewed_render, capture_output=True, text=True, check=False)
require(result.returncode != 0 and "existingRecord" in result.stderr
and output.read_text(encoding="utf-8") == accepted_record_html,
"A route that finds a saved record cannot satisfy a reviewed outline by marking it as created")
reached["existingRecord"] = "Review request"
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0 and "cannot both create and continue on an existing record" in result.stderr,
"Record transition effects are mutually exclusive")
step["branches"].pop()
outline["routes"].pop()
outline_path.write_text(json.dumps(outline), encoding="utf-8")
retry_branch = step["branches"][0]
retry_branch["actionLocation"] = "Draft editor"
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(reviewed_render, capture_output=True, text=True, check=False)
require(result.returncode != 0 and "needs a matching actionLocation assertion" in result.stderr
and output.read_text(encoding="utf-8") == accepted_record_html,
"A known step location does not establish the location of an alternate action")
outline["routes"][-1]["actionLocation"] = "Draft editor"
outline_path.write_text(json.dumps(outline), encoding="utf-8")
result = subprocess.run(check, capture_output=True, text=True, check=False)
require(result.returncode == 0, result.stderr)
del outline["routes"][-1]["actionLocation"]
outline_path.write_text(json.dumps(outline), encoding="utf-8")
retry_branch["actionLocation"] = None
step["branches"].append({
"label": "Restart a locked draft", "to": step["id"], "condition": "The draft is locked."
})
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(reviewed_render, capture_output=True, text=True, check=False)
require(result.returncode != 0 and "Unexpected transition 'check-result' -> 'check-result'" in result.stderr
and "The draft is locked." in result.stderr
and output.read_text(encoding="utf-8") == accepted_record_html,
"A complete route outline must reject an extra choice even when it goes to an allowed checkpoint")
step["branches"].pop()
handoff = step.pop("nextHandoff")
record.write_text(json.dumps(journey), encoding="utf-8")
outline["steps"][1]["actor"] = "author"
outline["steps"][0]["assignees"] = ["author", "reviewer"]
outline["routes"].append({"from": review["id"], "to": step["id"], "source": "Review policy, section 3"})
outline_path.write_text(json.dumps(outline), encoding="utf-8")
result = subprocess.run(check, capture_output=True, text=True, check=False)
require(result.returncode != 0 and "expects actor='author'" in result.stderr
and "expects assignees=['author', 'reviewer']" in result.stderr
and "'handoff': True" in result.stderr
and "Missing transition 'read-inbox' -> 'check-result'" in result.stderr
and "Review policy, section 3" in result.stderr,
"The independent outline must detect an incorrect actor, missing route, and missing handoff")
require(output.read_text(encoding="utf-8") == accepted_record_html,
"A source-outline check must leave the generated page intact")
step["nextHandoff"] = handoff
branch = {"label": "Resume the original draft", "to": review["id"]}
step["branches"].append(branch)
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0 and "without a record transition" in result.stderr,
"An original-record route must fail if it shares the new record's first step")
step["branches"].remove(branch)
actor_sources = review.pop("actorSources")
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0 and "actorSources must support the actor on the new record" in result.stderr,
"A receiving actor for new work must have its own evidence")
review["actorSources"] = actor_sources
review["branches"] = [{"label": "Try again", "to": step["id"]}]
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0 and "requires a local evidence gap and no outgoing routes" in result.stderr,
"An unverified next action cannot appear as a navigable retry")
require(output.read_text(encoding="utf-8") == accepted_record_html,
"Rejected routes must leave the previous valid page intact")
del step["record"], step["nextCreates"], review["record"], review["branches"], review["continuationUnknown"]
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode == 0, result.stderr)
rendered = output.read_text(encoding="utf-8")
restored = re.search(r'<script id="journey-data" type="application/json">(.*?)</script>', rendered)
require(restored is not None and json.loads(restored[1]) == journey,
"The renderer must preserve the restored journey after record effects are removed")
location = step.pop("actionLocation")
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0 and ".actionLocation must be a location or null" in result.stderr,
"A version 2 step must supply an action location or an explicit unknown")
step["actionLocation"] = location
step["surface"] = "Ambiguous location"
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0 and "Use statusLocation and actionLocation" in result.stderr,
"A version 2 step must reject the shared legacy location")
require(output.read_text(encoding="utf-8") == rendered,
"A rejected location record must leave the valid map intact")
del step["surface"]
assignment = step["assignments"][0]
assignment["role"] = "missing-reviewer"
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0
and "assignments[0].role refers to an unknown id: 'missing-reviewer'" in result.stderr,
"An assigned role must appear in the journey's role inventory")
require(output.read_text(encoding="utf-8") == rendered,
"A rejected assignment must leave the valid map intact")
assignment["role"] = "reviewer"
for metric, key, value, message in (
(completion, "eligible", 7, ".eligible must be an integer from measured"),
(completion, "count", 9, ".count must be an integer from 0 through measured"),
(wait, "seconds", float("inf"), ".seconds must be a finite nonnegative duration"),
):
original = metric[key]
metric[key] = value
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0 and message in result.stderr,
"The CLI must reject inconsistent outcome evidence")
require(output.read_text(encoding="utf-8") == rendered,
"An invalid outcome measurement must leave the valid map intact")
metric[key] = original
draft_link["sources"] = ["missing-source"]
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0
and "comparisons[0].options[1].sources refers to an unknown id: 'missing-source'" in result.stderr,
"A comparison must reference known evidence")
require(output.read_text(encoding="utf-8") == rendered,
"A rejected comparison must leave the valid map intact")
draft_link["sources"] = ["draft-spec"]
del draft_list["sources"]
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0
and "comparisons[0].options[0].sources needs at least one supporting source for a baseline"
in result.stderr, "A baseline must cite supporting evidence")
require(output.read_text(encoding="utf-8") == rendered,
"An unsupported baseline record must leave the valid map intact")
draft_list["sources"] = ["draft-spec"]
finding = journey["comparisons"][0]["finding"]
finding["step"] = "missing-step"
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0 and "comparisons[0].finding.step refers to an unknown id" in result.stderr,
"A friction finding must link to a mapped checkpoint")
finding["step"] = step["id"]
finding.pop("sources")
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0 and "finding.sources needs supporting evidence" in result.stderr,
"A source-backed friction finding needs evidence")
require(output.read_text(encoding="utf-8") == rendered,
"A rejected friction finding must leave the valid map intact")
finding["sources"] = ["draft-spec"]
preview_action = draft_link["preview"]["screens"][0]["actions"][0]
preview_action["to"] = "missing-screen"
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0 and "preview.screens.confirmation.actions[0].to refers to an unknown id"
in result.stderr, "A proposed flow must reject broken action links")
require(output.read_text(encoding="utf-8") == rendered,
"A rejected preview must leave the valid map intact")
preview_action["to"] = "draft"
journey["scenarios"][0]["steps"].remove(review["id"])
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0
and "scenarios[0].gaps[0].step must be listed in scenarios[0].steps" in result.stderr,
"An evidence gap must link to a step on its scenario")
require(output.read_text(encoding="utf-8") == rendered,
"A rejected evidence gap must leave the valid map intact")
journey["scenarios"][0]["steps"].append(review["id"])
del gap["step"]
gap["steps"] = [step["id"], review["id"]]
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode == 0, result.stderr)
rendered = output.read_text(encoding="utf-8")
data = re.search(r'<script id="journey-data" type="application/json">(.*?)</script>', rendered)
require(data is not None and json.loads(data[1]) == journey,
"The standalone page must retain both checkpoints for one evidence gap")
journey["scenarios"][0]["steps"].remove(review["id"])
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0
and "scenarios[0].gaps[0].steps must all be listed in scenarios[0].steps" in result.stderr,
"Every checkpoint for a shared evidence gap must be part of the scenario")
require(output.read_text(encoding="utf-8") == rendered,
"A rejected shared evidence gap must leave the valid map intact")
journey["scenarios"][0]["steps"].append(review["id"])
journey["gaps"] = journey["scenarios"][0].pop("gaps")
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0 and "Nest evidence gaps in scenarios[].gaps" in result.stderr,
"The renderer must reject gaps it would otherwise silently omit")
require(output.read_text(encoding="utf-8") == rendered,
"Misplaced evidence gaps must leave the valid map intact")
journey["scenarios"][0]["gaps"] = journey.pop("gaps")
step["measurements"][0]["counts"]["keep-editing"] = 4
record.write_text(json.dumps(journey), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True, check=False)
require(result.returncode != 0 and "counts exceed the eligible denominator" in result.stderr,
"The CLI must reject overlapping or inconsistent decision counts")
require(output.read_text(encoding="utf-8") == rendered,
"A rejected measurement must leave the valid map intact")
def heading_slugs(markdown: str) -> set[str]:
"""GitHub-style heading anchors, ignoring headings inside fenced code blocks."""
prose = re.sub(r"(?ms)^```.*?^```", "", markdown)
return {re.sub(r"[^\w\- ]", "", heading.lower()).replace(" ", "-")
for heading in re.findall(r"(?m)^#{1,6}\s+(.+?)\s*$", prose)}
def test_scenarios() -> None:
expected = {
"01-screen-review.md": ("UI", "Flow", "Feel"),
"02-workflow-audit.md": ("AUDIT", "Decision Density", "Intuitive Software Loop"),
"03-new-design.md": ("DESIGN", "mental model", "hypothesis"),
"04-critical-failure.md": ("Critical Failure", "independently"),
"05-expert-software.md": ("expert", "density"),
"06-minimal-unclear.md": ("minimal", "cognitive"),
"07-purpose-first-redesign.md": ("GitNexus", "current layout", "stops for discussion"),
"08-website-task-walkthrough.md": ("knowledge", "conversion"),
"09-draft-task-walkthrough.md": ("persistence", "human"),
"10-incomplete-walkthrough-evidence.md": ("unknown", "proposed"),
}
found = {path.name for path in SCENARIOS.glob("*.md")}
require(found == set(expected), f"Scenario set mismatch: {sorted(found)}")
for name, tokens in expected.items():
body = text(SCENARIOS / name)
require("## Prompt" in body and "## Acceptance gates" in body, f"Malformed scenario: {name}")
gates = body.split("## Acceptance gates", 1)[1]
require(len(re.findall(r"(?m)^- .+", gates)) >= 4, f"Scenario {name} needs substantive acceptance gates")
for token in tokens:
flags = 0 if token in {"DESIGN", "REVIEW", "AUDIT", "IMPROVE"} else re.IGNORECASE
require(re.search(rf"(?<!\w){re.escape(token)}(?!\w)", gates, flags) is not None,
f"Scenario {name} lacks acceptance gate: {token}")
def test_no_placeholders() -> None:
marker = "[" + "TODO:"
for path in ROOT.rglob("*"):
if path.is_file() and path.suffix.lower() in {".md", ".json", ".yaml", ".py", ".ps1"}:
body = path.read_text(encoding="utf-8")
require(marker not in body, f"Unfinished placeholder: {path.relative_to(ROOT)}")
def test_bundled_skill_discovery() -> None:
expected = {
"intuitive-software-design", "purpose-first-redesign", "user-task-walkthrough",
"product-mental-model", "decision-support-design", "multi-role-workflow",
"behavioral-evaluation", "connected-experience-design",
"journey-analytics",
}
found = {path.parent.name for path in (ROOT / "skills").glob("*/SKILL.md")}
require(found == expected, f"Bundled skill catalog mismatch: {sorted(found)}")
for name in expected:
folder = ROOT / "skills" / name
metadata = text(folder / "agents" / "openai.yaml")
require(f"${name}" in metadata, f"Default invocation targets wrong skill: {name}")
for name in ("findability.md", "ui-language.md", "experience-progression.md"):
text(REFERENCES / name)
def test_evaluation_kit_integrity() -> None:
kit = ROOT / "skills" / "behavioral-evaluation" / "references"
index = text(kit / "case-index.md")
packets = set((kit / "fixtures").glob("*.md"))
linked = {
(kit / target).resolve()
for target in re.findall(r"\[[^\]]+\]\((fixtures/[^)]+)\)", index)
}
require(len(packets) == 11 and {p.resolve() for p in packets} == linked,
"Every evaluation packet must be indexed exactly once within the eleven-case set")
text(kit / "assessor-rubric.md")
text(kit / "comparison-record.md")
for packet in packets:
links = re.findall(r"\[[^\]]+\]\(([^)]+)\)", text(packet))
require(not any("assessor-rubric" in target for target in links),
f"Raw execution packet leaks assessor instructions: {packet.name}")
require("Cases 01–06 closely follow worked examples" in index, "Example-aligned regression cases must be labeled")
cases = sorted((ROOT / "evals").glob("*/prompt.md"))
require(len(cases) >= 3, "Held-out eval suite needs diverse cases")
for prompt in cases:
body = text(prompt)
require(body.startswith("---\n") and "allowed_tools:" in body, f"Malformed eval prompt: {prompt}")
if re.search(r"-v\d+$", prompt.parent.name):
description = re.search(r"(?m)^description:\s*(.+)$", body.split("---", 2)[1])
require(description is not None and "not comparable" in description.group(1),
f"Versioned eval case must say its scores are not comparable with earlier versions: {prompt.parent.name}")
graders = sorted((prompt.parent / "graders").glob("*.md"))
require(all(text(grader).startswith("---\n") for grader in graders),
f"Eval grader lacks frontmatter: {prompt.parent.name}")
scored = 0
for grader in graders:
body = text(grader)
fields = grader_frontmatter(body)
kind = fields.get("type", "")
require(kind in GRADER_KEYS, f"Unknown grader type in {prompt.parent.name}/{grader.name}")
unknown = set(fields) - {"type", "weight", "arm"} - GRADER_KEYS[kind]
require(not unknown, f"Undocumented grader keys {sorted(unknown)} in {prompt.parent.name}/{grader.name}")
require(fields.get("arm", "with-only") in {"with-only", "both"},
f"Unknown grader arm in {prompt.parent.name}/{grader.name}")
if kind == "regex":
pattern = re.match(r"'(.*)'$", fields.get("pattern", ""))
require(pattern is not None, f"Regex grader needs a single-quoted pattern: {grader.name}")
re.compile(pattern.group(1), re.IGNORECASE if "i" in fields.get("flags", "") else 0)
if grader_is_scored(fields):
scored += 1
if kind == "llm":
require("PASS if" in body and "FAIL if" in body,
f"Scored rubric needs explicit PASS and FAIL conditions: {prompt.parent.name}/{grader.name}")
# One conjunctive rubric can only score a run 0 or 1, which hides partial improvements; a case still
# passes only when every scored criterion passes.
require(scored >= 3, f"Eval case needs at least three independently scored criteria: {prompt.parent.name}")
invoked = text(prompt.parent / "graders" / "skill-invoked.md")
read = text(prompt.parent / "graders" / "skill-read.md")
invoke_pattern = grader_input_match(invoked)
target_match = re.search(r'([a-z][a-z0-9-]+)"$', invoke_pattern.pattern)
require(target_match is not None, f"Skill indicator has no target: {prompt.parent.name}")
skill_name = target_match.group(1)
require((ROOT / "skills" / skill_name / "SKILL.md").is_file(),
f"Skill indicator targets an unbundled skill: {prompt.parent.name}")
require("type: tool_used" in invoked and "tool: Skill" in invoked and "arm: with-only" in invoked
and skill_name in invoked and '"skill"' in invoked,
f"Skill routing indicator must match an exact invocation: {prompt.parent.name}")
require("type: tool_used" in read and "tool: Read" in read and "arm: with-only" in read
and skill_name in read and r"SKILL\.md" in read,
f"Read routing indicator must match the skill file: {prompt.parent.name}")
other_skill = next(path.parent.name for path in sorted((ROOT / "skills").glob("*/SKILL.md"))
if path.parent.name != skill_name)
read_pattern = grader_input_match(read)
require(invoke_pattern.search(json.dumps({"skill": f"intuitive-software-design:{skill_name}"})) is not None
and invoke_pattern.search(json.dumps({"skill": f"intuitive-software-design:{other_skill}"})) is None,
f"Skill indicator matches the wrong skill: {prompt.parent.name}")
for separator in ("/", "\\"):
skill_root = f"intuitive-software-design{separator}skills{separator}"
correct_paths = (
f"{skill_root}{skill_name}{separator}SKILL.md",
f"{skill_root}{other_skill}{separator}..{separator}{skill_name}{separator}SKILL.md",
)
wrong_paths = (
f"{skill_root}{other_skill}{separator}SKILL.md",
f"{skill_root}{skill_name}{separator}..{separator}{other_skill}{separator}SKILL.md",
)
require(all(read_pattern.search(json.dumps({"file_path": path})) is not None for path in correct_paths)
and all(read_pattern.search(json.dumps({"file_path": path})) is None for path in wrong_paths),
f"Read indicator matches the wrong skill file: {prompt.parent.name}")
standard_read = text(prompt.parent / "graders" / "standard-read.md")
require("type: tool_used" in standard_read and "tool: Read" in standard_read and "arm: with-only" in standard_read,
f"Standard-read indicator must be an unscored Read check: {prompt.parent.name}")
standard_pattern = grader_input_match(standard_read)
references = f"intuitive-software-design{{0}}skills{{0}}intuitive-software-design{{0}}references{{0}}"
for separator in ("/", "\\"):
folder = references.format(separator)
require(standard_pattern.search(json.dumps({"file_path": folder + "intuitive-software-design-standard.md"}))
and not standard_pattern.search(json.dumps({"file_path": folder + "scoring-reference.md"})),
f"Standard-read indicator must match only the standard: {prompt.parent.name}")
confidence = ROOT / "evals" / "behavior-confidence-heldout" / "graders"
require("arm: with-only" in text(confidence / "vocabulary.md"),
"Plugin-specific vocabulary must not inflate the baseline comparison")
require("--ablation with-without" in text(ROOT / "README.md"),
"Eval instructions must use the supported two-arm comparison")
def main() -> None:
tests = (
test_manifest,
test_skill_frontmatter,
test_skill_contract,
test_purpose_first_redesign_contract,
test_connected_experience_contract,
test_authoritative_standard,
test_scoring_contract,
test_references_and_links,
test_journey_renderer_cli,
test_scenarios,
test_no_placeholders,
test_bundled_skill_discovery,
test_evaluation_kit_integrity,
)
for test in tests:
test()
print(f"PASS {test.__name__}")
print(f"PASS all {len(tests)} plugin contract tests")
if __name__ == "__main__":
main()
SHA-256: eb4958b62a4281c37b7d68ee25bd45dfed2ffe929824136bed19b3ecf8b43133