← Files AICtrlARCHIVED FILE
skills/ux-testing/scripts/simulate.cjs
29.4 KB · Nov 19, 2026 · 00:22 UTC
// Simulated first-time users ("System One"): fast, cheap usability estimates before running agent testers.
//
// Each simulated user walks one path through the page: code perceives the screen, a System One model judges
// what this user would do next (probabilities over the visible controls), one action is sampled from those
// probabilities, and the page is checked after every step against the task model's `success` and `must_not`
// expressions.
//
// Usage (needs Playwright with Chromium; js-yaml for a YAML task model):
// node simulate.cjs --config sim.config.json [--tasks T1,T3] [--n 16] [--no-load] # score tasks
// node simulate.cjs --config sim.config.json --hyp H1.json [--n 16] # A/B hypothesis
// Options: --out <dir> (default: a new temp dir), --cache <file> (default: <out>/.sim-cache.json with --out, else a
// per-config file under ~/.cache/ux-sim, so re-runs stay cached), --workers <n>, --seed <n> (same seed and a warm
// cache repeat a run exactly, for any --workers).
//
// Needs a simulator backend: TYPESAFE_API_KEY (TypeSafe Jev), or UX_SIM_ENDPOINT + UX_SIM_API_KEY +
// UX_SIM_MODEL for a compatible endpoint. Without one it exits with code 2 and says how to set one up.
// Keys are read from the environment only and never logged or written.
//
// Exit codes: 0 the run finished. 2 a usage or config error, found before a browser starts: a bad option, no
// simulator backend, an unreadable config, task model, --hyp or observed file, or a setup module that is missing
// or does not export a function (also 2: no task matches --tasks). 3 the run started and stopped: Chromium would not
// start, a setup module threw, a check expression threw, the backend failed, a task cannot be graded, or too
// many walks ended in harness errors. A stopped run is not scored; hypothesis rows already written are kept.
// See reference/simulator.md for the config, the request/response contract, conditions and limits.
const fs = require('fs');
const os = require('os');
const path = require('path');
const crypto = require('crypto');
const { loadPlaywright, loadModel, loadSetup, exitOnConfigError, readJson, redactSecrets, requireSimBackend, options, rng, seedOf } = require('./common.cjs');
const args = process.argv.slice(2);
const O = options(args);
const opt = O.str;
const flag = O.has;
const N = O.int('n', 16, 1);
const WORKERS = O.int('workers', 4, 1);
const SEED = O.int('seed', 42, 0);
if (!opt('config')) { console.error('Usage: node simulate.cjs --config sim.config.json [--tasks T1,T3] [--hyp H.json] [--n 16] [--out dir]'); process.exit(2); }
const BACKEND = requireSimBackend(); // exits with the not-configured message when there is no backend
const { chromium } = loadPlaywright();
const CONFIG_PATH = path.resolve(opt('config'));
const CONFIG_DIR = path.dirname(CONFIG_PATH);
// The files a run is configured by are read up front: an unreadable one is a config error (exit 2).
const CFG = exitOnConfigError(() => readJson(CONFIG_PATH));
// Relative paths in the config resolve against the config's directory; in a hypothesis file, against its own.
// An http(s) or file URL is used as it is.
const fileUrl = (p, base = CONFIG_DIR) => (/^(https?|file):/.test(p) ? p : `file://${path.resolve(base, p)}`);
// The setup module run after every page opens (e.g. signing in to a real app): the config's `setup`, unless a
// variant or hypothesis arm sets its own (`null` turns it off there). A bad setup module the run needs (the
// config's, a hypothesis arm's or a selected variant's) is a config error: exit 2 before a browser starts or
// <out> is written.
const CFG_SETUP = exitOnConfigError(() => loadSetup(CFG.setup, CONFIG_DIR));
const setupOf = (v, base) => (v && 'setup' in v ? loadSetup(v.setup, base) : CFG_SETUP);
const MODEL = exitOnConfigError(() => loadModel(path.resolve(CONFIG_DIR, CFG.model)));
const TASKS = Object.fromEntries(MODEL.tasks.map((t) => [t.id, t]));
// The page's state hook: named by the config or the model, else window.__state, falling back to window.__mock
// (the name older task models used) so they keep working without a state_hook.
const STATE_HOOK = CFG.stateHook || (MODEL.meta && MODEL.meta.state_hook) || null;
// "needs-mock" is the older spelling of "needs-target"; both are skipped.
const SKIPPED = new Set(['needs-target', 'needs-mock']);
const oldStatus = MODEL.tasks.filter((t) => t.status === 'needs-mock').map((t) => t.id);
if (oldStatus.length) console.warn(`note: ${oldStatus.join(', ')} ${oldStatus.length > 1 ? 'still say' : 'still says'} "status: needs-mock"; skipped as needs-target (the new name).`);
// The hypothesis file and both of its arms' setup modules load up front, before a browser starts.
const HYP = opt('hyp') ? exitOnConfigError(() => { const hypPath = path.resolve(opt('hyp')); return { H: readJson(hypPath), dir: path.dirname(hypPath) }; }) : null;
const HYP_SETUPS = HYP ? exitOnConfigError(() => ({ a: setupOf(HYP.H.a, HYP.dir), b: setupOf(HYP.H.b, HYP.dir) })) : null;
const ONLY = opt('tasks') ? opt('tasks').split(',') : null;
const CASES = HYP ? [] : [
...MODEL.tasks.filter((t) => !SKIPPED.has(t.status)).map((t) => ({ id: t.id, task: t.id, url: fileUrl(CFG.url), setup: CFG_SETUP })),
...(CFG.variants || []).map((v) => ({ ...v, url: fileUrl(v.url || CFG.url), _setupSrc: v })),
].filter((c) => !ONLY || ONLY.includes(c.id) || ONLY.includes(c.task));
// Variants load their setup only once selected, so --tasks can skip a variant whose setup is broken.
for (const c of CASES) if (c._setupSrc) { const v = c._setupSrc; delete c._setupSrc; c.setup = exitOnConfigError(() => setupOf(v, CONFIG_DIR)); }
// A --tasks list that matches nothing is a usage error, reported before a browser starts or <out> is written.
if (!HYP && !CASES.length) exitOnConfigError(() => { throw new Error(ONLY ? `no task or variant matches --tasks ${ONLY.join(',')}` : 'the task model has no runnable tasks'); });
const observedPath = CFG.observed && path.resolve(CONFIG_DIR, CFG.observed);
const OBSERVED = observedPath && fs.existsSync(observedPath) ? exitOnConfigError(() => readJson(observedPath)) : {};
const OUT = path.resolve(opt('out') || fs.mkdtempSync(path.join(os.tmpdir(), 'ux-sim-')));
fs.mkdirSync(OUT, { recursive: true });
const MAX_STEPS = 16; // a path longer than this counts as a failure ("too-long")
const MAX_CHATS = 3; // a user who types into the chat more than this gives up
const MAX_ERROR_SHARE = 0.1; // a run whose walks end in harness errors more often than this (and more than once) stops, unscored
const STOP_MIN = 0.3; // a user only considers stopping when the model's stop probability reaches this
const SETTLE_MS = 1100;
const LAPTOP = { width: 1366, height: 768 };
const DESKTOP = { width: 1440, height: 900 };
// A chat region is marked with data-ux-chat; a plain <aside> (the older default) still counts.
const REGIONS = { main: 'main', chat: '[data-ux-chat], aside', menu: 'nav', topbar: 'header', ...(CFG.regions || {}) };
// Text boxes inside the chat region; each selector in a region list is scoped on its own.
const chatBoxes = (page, extra = '') => page.locator(REGIONS.chat).locator(`textarea${extra}`);
const HIDE = CFG.hideCss || '';
const HIDE_SEL = HIDE ? HIDE.split('{')[0] : '';
// Answers depend only on what the user sees, so they are cached per screen (and per backend); re-runs cost nothing.
// The cache holds screen text and probabilities, never keys. Without --cache or --out it lives outside the repo,
// one file per config, so a re-run of the same config stays free even though each run writes to a new temp dir.
const cacheHome = process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache');
const CACHE_F = path.resolve(opt('cache') || (opt('out')
? path.join(OUT, '.sim-cache.json')
: path.join(cacheHome, 'ux-sim', `${crypto.createHash('sha256').update(CONFIG_PATH).digest('hex').slice(0, 16)}.json`)));
fs.mkdirSync(path.dirname(CACHE_F), { recursive: true });
const cache = fs.existsSync(CACHE_F) ? JSON.parse(fs.readFileSync(CACHE_F, 'utf8')) : {};
let calls = 0; let dirty = 0;
const saveCache = () => { if (dirty) { fs.writeFileSync(CACHE_F, JSON.stringify(cache)); dirty = 0; } };
// Every walk gets its own generator, seeded from --seed and the walk's identity (case, profile, condition,
// run index), so a re-run with the same --seed and a warm cache repeats exactly, whatever the worker count.
// Probabilities may sum to 1 ± 0.02 (the contract's tolerance); renormalise so no leftover mass falls on one option.
const sample = (rnd, dist) => {
const total = dist.reduce((t, [, v]) => t + v, 0);
const r = rnd() * total; let acc = 0;
for (const [k, v] of dist) { acc += v; if (r < acc) return k; }
return dist.find(([, v]) => v > 0)[0]; // floating-point edge: the most likely option, never the least
};
const SEL = 'button, a[href], [role="button"], [role="tab"], [role="switch"], input[type="radio"], input[type="checkbox"], summary, select';
// Perception is code, not the model. A "scanner" sees only what is on screen without scrolling (the 14 most
// prominent page controls, icons as symbols); a "reader" gets every control's accessible name and all page text.
async function perceive(page, profile) {
// Each perceived control is tagged with this screen's generation, so act() clicks exactly that element, or
// fails with "control gone" if a re-render replaced it, instead of clicking whatever now sits at its index.
const gen = crypto.randomBytes(4).toString('hex');
const g = await page.evaluate(({ profile, SEL, R, HIDE_SEL, ROW, ROW_TITLE, gen }) => {
const vh = innerHeight;
const hidden = (el) => HIDE_SEL && el.closest(HIDE_SEL);
const within = (el, sel) => { try { return sel && el.closest(sel); } catch { return false; } };
const regionOf = (el) => (within(el, R.menu) ? 'menu' : within(el, R.chat) ? 'chat' : within(el, R.topbar) ? 'topbar' : within(el, R.main) ? 'main' : 'side');
const mainEl = document.querySelector(R.main) || document.body;
const mainBox = mainEl.getBoundingClientRect();
const els = [...document.querySelectorAll(SEL)].filter((el) => !hidden(el));
const out = []; let below = 0; let above = 0;
els.forEach((el, idx) => {
el.setAttribute('data-ux-sim', `${gen}:${idx}`);
const r = el.getBoundingClientRect();
if (r.width === 0 || r.height === 0 || getComputedStyle(el).visibility === 'hidden') return;
const reg = regionOf(el);
if (profile === 'scanner') {
// The fold: the viewport, or the main region's own box when it scrolls inside a fixed layout.
const inPage = reg === 'main' || reg === 'side';
const box = reg === 'main' ? mainBox : { top: 0, bottom: vh };
if (r.top >= Math.min(vh, box.bottom) - 4) { if (inPage) below++; return; }
if (r.bottom <= Math.max(0, box.top)) { if (inPage) above++; return; }
}
const lbl = el.labels && el.labels[0] ? el.labels[0].innerText : '';
const raw = (el.tagName === 'INPUT' ? lbl : el.tagName === 'SELECT' ? `${lbl} ${(el.selectedOptions[0] || {}).text || ''}` : el.innerText || '').trim().replace(/\s+/g, ' ');
const aria = (el.getAttribute('aria-label') || raw).trim().replace(/\s+/g, ' ');
const icon = !/[\p{L}\p{N}]/u.test(raw);
const host = ROW ? el.closest(ROW) : null;
const hostTitle = host && ROW_TITLE ? ((host.querySelector(ROW_TITLE) || {}).innerText || '').trim().replace(/\s+/g, ' ').slice(0, 50) : '';
const cls = typeof el.className === 'string' ? el.className : '';
const role = el.getAttribute('role');
const style = /\b(primary|cta)\b|-primary\b/i.test(cls) ? 'primary button'
: /\b(danger|destructive)\b|-danger\b/i.test(cls) ? 'red button'
: role === 'switch' ? 'switch'
: el.tagName === 'BUTTON' || role === 'button' || /\bbtn\b/.test(cls) ? 'button'
: role === 'tab' ? 'tab'
: el.type === 'radio' ? 'radio option' : el.type === 'checkbox' ? 'checkbox'
: el.tagName === 'SELECT' ? 'dropdown' : el.tagName === 'A' ? 'link' : 'clickable text';
// visual weight: area × emphasis, discounted the further down the screen it sits
const emphasis = { 'primary button': 3, 'red button': 2, button: 1.6, tab: 1.4, 'radio option': 1.4, checkbox: 1.2, switch: 1.2, dropdown: 1.2, link: 1, 'clickable text': 0.9 }[style];
const weight = (r.width * r.height) * emphasis / (1 + Math.max(0, r.top) / vh);
const label = profile === 'reader' ? (aria || '(unnamed control)') : icon ? `(icon ${raw || 'symbol'}, no label)` : raw;
out.push({ idx, reg, label: label.slice(0, 90), style, hostTitle, weight, top: Math.round(r.top) });
});
let controls;
if (profile === 'scanner') {
const main = out.filter((c) => c.reg === 'main' || c.reg === 'side').sort((a, b) => b.weight - a.weight).slice(0, 14).sort((a, b) => a.top - b.top);
controls = [...main, ...out.filter((c) => c.reg !== 'main' && c.reg !== 'side')];
} else controls = out.slice(0, 120);
let text = '';
if (profile === 'reader') text = document.body.innerText.replace(/\s+/g, ' ').slice(0, 3000);
else {
const w = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT);
while (w.nextNode() && text.length < 1600) {
const n = w.currentNode; const s = n.textContent.trim(); if (!s) continue;
const p = n.parentElement; if (!p || hidden(p) || p.closest('script,style,noscript')) continue;
const r = p.getBoundingClientRect(); if (r.height === 0 || r.bottom < 0 || r.top > vh) continue;
text += s + ' | ';
}
}
const chatEl = R.chat ? document.querySelector(R.chat) : null;
const chat = chatEl ? chatEl.innerText.replace(/\s+/g, ' ').slice(-900) : '';
return { controls, below, above, text, chat };
}, { profile, SEL, R: REGIONS, HIDE_SEL, ROW: CFG.rowSelector || '', ROW_TITLE: CFG.rowTitleSelector || '', gen });
g.gen = gen;
return g;
}
async function judge(task, g, profile, cond, step) {
const facts = task.answer || [];
const criteria = {};
g.controls.forEach((c, i) => { criteria['c' + i] = `${c.style} ${c.label.startsWith('(icon') ? c.label : `"${c.label}"`}${c.hostTitle && !c.label.includes(c.hostTitle) ? ` on "${c.hostTitle}"` : ''} (${c.reg === 'main' || c.reg === 'side' ? 'page' : c.reg})`; });
if (g.below) criteria.__scroll_down = `Scroll the page down to see more (${g.below} more controls below, not visible yet)`;
if (g.above) criteria.__scroll_up = 'Scroll the page back up';
criteria.__wait = 'Wait a few seconds for something that is in progress to update';
if (g.hasChat) criteria.__type_chat = 'Type a question or request into the chat box';
criteria.__give_up = 'Give up: nothing here seems to lead to the goal';
const user = profile === 'scanner'
? 'A busy first-time user of this web app. They glance at the screen and act on the first control that seems to lead to their goal; they do not read everything.'
: 'A careful first-time user of this web app who reads the whole page before acting.';
const interrupted = cond === 'interrupt' && step >= 2;
const goal = cond === 'paraphrase' ? (task.paraphrase || task.prompt) : task.prompt;
const state = {
user: interrupted ? `${user} They were interrupted and are coming back to this screen, remembering only their goal.` : user,
task: goal,
screen: { [profile === 'scanner' ? 'readable_now' : 'page_text']: g.text, chat_panel: interrupted ? '' : g.chat },
};
const questions = {
next: { type: 'choice', instructions: 'Which one thing would this user do next to make progress on the `task`, judging only from what they can see on the `screen`?', criteria },
believes_done: { type: 'noul', instructions: 'Looking at the `screen`, would this user believe their `task` is already complete and stop here?', criteria: { true: 'The screen looks like the task has been done', false: 'The task clearly still needs action' } },
can_answer: { type: 'noul', instructions: 'Can the user answer the question in the `task` from what is readable on the `screen` right now?' },
};
if (facts.length) questions.answer_correct = { type: 'noul', instructions: { expected_answer_contains: facts, question: 'If the user answered the `task` from what is readable on the `screen`, would their answer contain the information in `expected_answer_contains` (different wording is fine)?' } };
// Hash the full questions: their instructions carry the task's answer facts, so editing them re-asks.
const k = crypto.createHash('sha1').update(JSON.stringify([BACKEND.endpoint, BACKEND.model, state, questions])).digest('hex');
if (cache[k]) return cache[k];
const j = await BACKEND.ask(state, questions);
calls++;
const out = { next: j.answers.next.probabilities, done: j.answers.believes_done.noul, answer: j.answers.can_answer.noul, correct: j.answers.answer_correct ? j.answers.answer_correct.noul : null };
cache[k] = out; dirty++; if (dirty > 20) saveCache();
return out;
}
async function act(page, g, a, prompt) {
if (a === '__scroll_down' || a === '__scroll_up') {
await page.evaluate(({ d, main }) => {
const m = document.querySelector(main);
const scroller = m && m.scrollHeight > m.clientHeight + 4 && getComputedStyle(m).overflowY !== 'visible' ? m : document.scrollingElement;
scroller.scrollBy(0, d * Math.min(scroller.clientHeight, innerHeight) * 0.75);
}, { d: a === '__scroll_down' ? 1 : -1, main: REGIONS.main });
await page.waitForTimeout(250); return;
}
if (a === '__wait') { await page.waitForTimeout(3500); return; }
if (a === '__type_chat') {
// A user who turns to chat asks for the goal in their own words; the task text stands in for that.
const box = chatBoxes(page, ':visible').last();
await box.fill(prompt, { timeout: 5000 }); await box.press('Enter'); await page.waitForTimeout(SETTLE_MS + 600); return;
}
const c = g.controls[Number(a.slice(1))];
await page.evaluate(({ tag }) => {
const el = document.querySelector(`[data-ux-sim="${tag}"]`);
if (!el) throw new Error('control gone');
if (el.tagName === 'SELECT') { el.selectedIndex = (el.selectedIndex + 1) % el.options.length; el.dispatchEvent(new Event('change', { bubbles: true })); } else el.click();
}, { tag: `${g.gen}:${c.idx}` });
await page.waitForTimeout(SETTLE_MS);
}
// Evaluates the task model's own success / must_not expressions in the page. `S` is the page's read-only
// state hook (window[stateHook]; without one, window.__state, else window.__mock) when it has one; expressions
// may also query the DOM.
// Test harness only: the expressions come from the author's own task model file.
// An expression that throws is a broken check, not a failed user: the run stops and names it, like verify.cjs.
async function check(page, task) {
const r = await page.evaluate(([s, n, meta, hook]) => {
const S = hook ? window[hook] : (window.__state !== undefined ? window.__state : window.__mock); const META = meta; // eslint-disable-line no-unused-vars
const ev = (e) => { if (!e) return { v: null }; try { return { v: !!eval(e) }; } catch (x) { return { err: String(x && x.message || x) }; } }; // eslint-disable-line no-eval
return { success: ev(s), harm: ev(n) };
}, [task.success, task.must_not, MODEL.meta || {}, STATE_HOOK]);
for (const [field, key] of [['success', 'success'], ['must_not', 'harm']]) {
if (r[key].err) throw new CheckError(`task ${task.id}: ${field} expression threw "${r[key].err}" (${task[field]}). Make it return true or false in every page state.`);
}
return { success: r.success.v, harm: r.harm.v };
}
class CheckError extends Error {}
class RunError extends Error {}
// A setup module that throws (a wrong test password, a drifted sign-in selector, the auth endpoint down) fails
// every walk the same way: it is neither a usability result nor a harness error, so the run stops at once (exit 3).
class SetupError extends Error {}
function viewportFor(task, cond) {
if (cond === 'laptop') return LAPTOP;
const v = task.start && task.start.viewport;
return v ? { width: v[0], height: v[1] } : DESKTOP;
}
async function walkOne(browser, c, task, profile, cond, rnd) {
const page = await browser.newPage({ viewport: viewportFor(task, cond) });
try {
await page.goto(c.url);
if (c.setup) {
try { await c.setup.run(page, page.context()); } catch (e) {
throw new SetupError(`task ${task.id}: setup module ${c.setup.file} failed: ${redactSecrets(String(e && e.message || e).split('\n')[0]).replace(/\.$/, '')}. Nothing from this run is scored; fix the setup module, the test account or the app and run again.`);
}
}
const scenario = c.scenario || (task.start && task.start.scenario);
if (CFG.scenarioSelect && scenario && scenario !== 'cold') await page.selectOption(CFG.scenarioSelect, scenario);
if (HIDE) await page.addStyleTag({ content: HIDE });
await page.waitForTimeout((CFG.slowScenarios || {})[scenario] || 500);
if (c.mutate) await page.evaluate(c.mutate);
const hasChat = (await chatBoxes(page).count()) > 0;
const isQuestion = !task.success;
let chats = 0; const trail = [];
for (let step = 0; step <= MAX_STEPS; step++) {
if (c.mutate) await page.evaluate(c.mutate); // re-apply after re-renders
const st = await check(page, task);
if (st.harm) return { end: 'harm', steps: step, trail };
if (!isQuestion && st.success) return { end: 'success', steps: step, trail };
if (step === MAX_STEPS) return { end: 'too-long', steps: step, trail };
const g = await perceive(page, profile); g.hasChat = hasChat;
const J = await judge(task, g, profile, cond, step);
const stopP = isQuestion ? J.answer : J.done;
if (stopP >= STOP_MIN && rnd() < stopP) {
if (!isQuestion) return { end: 'premature-stop', steps: step, trail };
const right = rnd() < J.correct; // question tasks always have answer facts (checked at start-up)
return { end: right ? 'success' : 'wrong-answer', steps: step, trail };
}
const a = sample(rnd, Object.entries(J.next).sort((x, y) => y[1] - x[1]));
if (a === '__give_up') return { end: 'give-up', steps: step, trail };
if (a === '__type_chat' && ++chats > MAX_CHATS) return { end: 'give-up', steps: step, trail };
trail.push(a.startsWith('__') ? a.slice(2) : g.controls[Number(a.slice(1))].label.slice(0, 40));
await act(page, g, a, task.prompt);
}
} catch (e) {
// A backend failure or a broken check is not a usability result: stop the run rather than score it.
if (e instanceof CheckError || e instanceof SetupError) throw e;
if (/^simulator /.test(e.message)) throw new RunError(`task ${task.id}: a walk failed from a simulator backend error (${e.message}). Nothing from this run is scored; check the backend and run again.`);
// Any other failure (page crash, navigation, a control replaced mid-click) is the harness's, not the user's:
// it ends as `error`, which run() leaves out of success and harm and reports separately.
return { end: 'error', steps: 0, trail: [String(e.message).split('\n')[0].slice(0, 120)] };
} finally { await page.close().catch(() => {}); }
}
// A task needs a success check, or (a question task) the facts its answer must contain; otherwise every walk
// would "succeed" with nothing checked.
function assertGradeable(ids) {
for (const id of ids) {
const t = TASKS[id];
if (!t) throw new Error(`Task ${id} is not in the model`);
if (!t.success && !(Array.isArray(t.answer) && t.answer.length)) throw new Error(`Task ${id} has neither a success check nor answer facts, so it cannot be graded. Add \`success\` or \`answer: [...]\`.`);
}
}
async function run(browser, c, profile, cond = 'focused') {
const task = TASKS[c.task];
if (!task) throw new Error(`Task ${c.task} is not in the model`);
const results = new Array(N); let started = 0;
const worker = async () => {
while (started < N) {
const i = started++;
results[i] = await walkOne(browser, c, task, profile, cond, rng(seedOf(SEED, c.id, c.url, c.mutate || '', profile, cond, i)));
}
};
await Promise.all(Array.from({ length: Math.min(WORKERS, N) }, worker));
saveCache();
// Walks that ended in a harness error say nothing about the user: they are left out of success, harm and the
// bootstrap, and counted in `errors`. Too many of them and the run is not trustworthy, so it stops. One error
// is tolerated at any N (so a small smoke run survives one transient failure), unless no walk was scored.
const errors = results.filter((r) => r.end === 'error');
if (errors.length === N || errors.length > Math.max(1, N * MAX_ERROR_SHARE)) {
throw new RunError(`task ${task.id} (${profile}, ${cond}): ${errors.length} of ${N} walks failed from harness errors (the limit is one, or ${MAX_ERROR_SHARE * 100}% of the walks), e.g. "${errors[0].trail[0]}". Nothing from this run is scored; fix the page or the config and run again.`);
}
const scored = results.filter((r) => r.end !== 'error');
const count = (e) => scored.filter((r) => r.end === e).length;
const ok = scored.filter((r) => r.end === 'success');
const ends = {}; results.forEach((r) => { ends[r.end] = (ends[r.end] || 0) + 1; });
const fails = {}; scored.filter((r) => r.end !== 'success').forEach((r) => { const k = `${r.end}: ${r.trail.slice(0, 4).join(' → ') || '(start)'}`; fails[k] = (fails[k] || 0) + 1; });
return {
raw: scored.map((r) => ({ ok: r.end === 'success' ? 1 : 0, harm: r.end === 'harm' ? 1 : 0, steps: r.steps })),
n: scored.length,
errors: errors.length,
success: +(count('success') / scored.length).toFixed(2),
harm: +(count('harm') / scored.length).toFixed(2),
steps: ok.length ? +(ok.reduce((s, r) => s + r.steps, 0) / ok.length).toFixed(1) : null,
ends,
topFail: Object.entries(fails).sort((a, b) => b[1] - a[1]).slice(0, 1).map(([k, v]) => `${v}× ${k}`)[0] || '',
};
}
// 90% bootstrap interval for the difference B − A.
function boot(a, b, key, iters = 2000) {
const rnd = rng(seedOf(SEED, 'bootstrap', key, a.length, b.length));
const mean = (xs) => xs.reduce((s, x) => s + x[key], 0) / xs.length; const ds = [];
for (let i = 0; i < iters; i++) {
const ra = a.map(() => a[Math.floor(rnd() * a.length)]); const rb = b.map(() => b[Math.floor(rnd() * b.length)]);
ds.push(mean(rb) - mean(ra));
}
ds.sort((x, y) => x - y);
return { delta: +(mean(b) - mean(a)).toFixed(2), lo: +ds[Math.floor(iters * 0.05)].toFixed(2), hi: +ds[Math.floor(iters * 0.95)].toFixed(2) };
}
async function hypothesis(browser, H, hypDir, setups) {
assertGradeable(H.tasks);
const rows = [];
// Written after every row, so a later failure keeps the rows already computed.
const file = path.join(OUT, `hyp-${H.id}.json`);
const save = (extra) => fs.writeFileSync(file, JSON.stringify({ H, n: N, rows, ...extra }, null, 2));
const variant = (v, arm, tid) => ({ id: tid, task: tid, url: v && v.url ? fileUrl(v.url, hypDir) : fileUrl(CFG.url), mutate: v && v.mutate, setup: setups[arm] });
const stepsOf = (R) => { const ok = R.raw.filter((r) => r.ok); return ok.length ? +(ok.reduce((t, r) => t + r.steps, 0) / ok.length).toFixed(1) : null; };
const verdict = (x) => (x.lo > 0 ? 'B higher' : x.hi < 0 ? 'B lower' : 'no clear difference');
for (const tid of H.tasks) {
for (const cond of H.conds || ['focused']) {
const A = await run(browser, variant(H.a, 'a', tid), 'scanner', cond);
const B = await run(browser, variant(H.b, 'b', tid), 'scanner', cond);
const s = boot(A.raw, B.raw, 'ok'); const h = boot(A.raw, B.raw, 'harm');
const row = { task: tid, cond, A: A.success, B: B.success, nA: A.n, nB: B.n, errorsA: A.errors, errorsB: B.errors, stepsA: stepsOf(A), stepsB: stepsOf(B), success: { ...s, verdict: verdict(s) }, harmA: A.harm, harmB: B.harm, harm: { ...h, verdict: verdict(h) }, failA: A.topFail, failB: B.topFail };
rows.push(row); console.log(JSON.stringify(row)); save();
}
}
return { rows, file };
}
(async () => {
console.log(`simulator: ${BACKEND.label} · output ${OUT} · cache ${CACHE_F}`);
let browser;
try {
browser = await chromium.launch();
} catch (e) {
console.error(`ERROR: could not start Chromium (${String(e.message).split('\n')[0]}). Run \`npx playwright install chromium\`.`);
process.exit(3);
}
try {
if (HYP) {
const { file } = await hypothesis(browser, HYP.H, HYP.dir, HYP_SETUPS);
console.log(`wrote ${file} · model calls ${calls}`);
return;
}
const cases = CASES;
assertGradeable(cases.map((c) => c.task));
const rows = [];
for (const c of cases) {
const t0 = Date.now();
const scanner = await run(browser, c, 'scanner');
const reader = await run(browser, c, 'reader');
const load = {};
// Load conditions: a laptop screen (the fold hides more), an interruption mid-task, and the goal in the user's own words.
if (!flag('no-load')) for (const cond of ['laptop', 'interrupt', 'paraphrase']) load[cond] = await run(browser, c, 'scanner', cond);
const loadMean = flag('no-load') ? null : +(Object.values(load).reduce((s, r) => s + r.success, 0) / 3).toFixed(2);
const row = { id: c.id, scanner, reader, load, robust: loadMean, fragility: loadMean == null ? null : +(scanner.success - loadMean).toFixed(2), observed: c.observed || OBSERVED[c.id] || null, secs: Math.round((Date.now() - t0) / 1000) };
for (const r of [row.scanner, row.reader, ...Object.values(row.load)]) delete r.raw;
rows.push(row); console.log(JSON.stringify(row));
fs.writeFileSync(path.join(OUT, 'sim-results.json'), JSON.stringify(rows, null, 2));
}
console.log(`wrote ${path.join(OUT, 'sim-results.json')} · model calls ${calls}`);
} catch (e) {
console.error(`ERROR: ${e.message}`);
process.exitCode = 3;
} finally {
await browser.close(); saveCache();
}
})();
SHA-256: 9a26803b577a57143c995ba9c95a1f1e60fc9e549127f8c346ab25fb3f91a50a