--- before
+++ after
@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""Run the behaviour suite twice and compare: the full plugin against a
+baseline that keeps the same mocked FactIQ tools but has no skill and no
+scripts.
+
+Why not `claude plugin eval --ablation with-without`? That mode removes the
+whole plugin from the second arm, including its MCP server. The second arm
+then has no FactIQ tools at all, so the comparison is "plugin with data"
+against "no data", and every data case fails without the plugin for a
+reason that has nothing to do with the skill. This script builds a stripped
+copy of the plugin (manifest plus MCP configuration, no `skills/`, no
+`scripts/`) and runs both plugins under `--ablation none` with the same
+mocks, the same model, the same judge, and the same tool grants. The
+difference between the two scores is then what the skill and the bundled
+scripts add on top of the tools.
+
+Usage (from the plugin root):
+
+    python3 evals/run_ablation.py --model opus --judge-model sonnet
+    python3 evals/run_ablation.py --model opus --case 'yoy-*' --runs 1
+    python3 evals/run_ablation.py --model opus --tag filings --tag charts
+    python3 evals/run_ablation.py --model opus --skip-baseline
+
+Results land in `evals/results/<timestamp>/{with,baseline}/` with a
+`comparison.md` beside them. The exit code is 1 when any with-arm case
+scores below `--threshold` (default 0.8; one disagreeing judge vote in a
+three-run case is a score of about 0.9, so 1.0 fails on judge noise).
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import json
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+PLUGIN_ROOT = Path(__file__).resolve().parent.parent
+EVAL_DIR = PLUGIN_ROOT / "evals"
+
+
+def build_baseline_plugin(dest: Path) -> None:
+    """Copy the manifest, the MCP configuration and the eval suite, and
+    nothing else. The baseline plugin keeps the plugin's name so the tool
+    names inside the sandbox (`mcp__plugin_factiq_factiq__<tool>`) are the
+    same in both arms and every `tool_used` grader compares like with like."""
+    manifest = json.loads((PLUGIN_ROOT / ".claude-plugin" / "plugin.json").read_text())
+    manifest.pop("skills", None)
+    manifest.pop("hooks", None)
+    manifest.pop("commands", None)
+    manifest.pop("agents", None)
+    manifest["description"] = "Baseline for the FactIQ behaviour suite: MCP tools only, no skill."
+    (dest / ".claude-plugin").mkdir(parents=True)
+    (dest / ".claude-plugin" / "plugin.json").write_text(json.dumps(manifest, indent=2) + "\n")
+    shutil.copy(PLUGIN_ROOT / ".mcp.json", dest / ".mcp.json")
+    shutil.copytree(
+        EVAL_DIR,
+        dest / "evals",
+        ignore=shutil.ignore_patterns("results", "__pycache__", "*.py"),
+    )
+
+
+def run_eval(plugin_dir: Path, out_dir: Path, args: argparse.Namespace, label: str) -> dict:
+    out_dir.mkdir(parents=True, exist_ok=True)
+    json_path = out_dir / "aggregate-result.json"
+    cmd = [
+        "claude", "plugin", "eval", str(plugin_dir),
+        "--ablation", "none",
+        "--trust-plugin",
+        "--no-publish",
+        "--threshold", "0",
+        "--model", args.model,
+        "--judge-model", args.judge_model,
+        "--json", str(json_path),
+        "--output-dir", str(out_dir),
+        "--concurrency", str(args.concurrency),
+    ]
+    if args.allow_tools:
+        cmd += ["--allow-tools", *args.allow_tools.split(",")]
+    if args.runs:
+        cmd += ["--runs", str(args.runs)]
+    if args.case:
+        cmd += ["--case", args.case]
+    for tag in args.tag:
+        cmd += ["--tag", tag]
+    if args.max_cost_usd:
+        cmd += ["--max-cost-usd", str(args.max_cost_usd)]
+    if args.keep_temp:
+        cmd.append("--keep-temp")
+    print(f"\n=== {label}: {' '.join(cmd)}\n", flush=True)
+    proc = subprocess.run(cmd, cwd=plugin_dir)
+    if not json_path.exists():
+        sys.exit(f"{label}: no result file was written (exit {proc.returncode})")
+    return json.loads(json_path.read_text())
+
+
+def case_table(result: dict) -> dict[str, dict]:
+    table: dict[str, dict] = {}
+    for case in result["cases"]:
+        runs = case["arms"]["with"]
+        scores = [r["score"] for r in runs]
+        graders: dict[str, list[bool]] = {}
+        for run in runs:
+            for g in run.get("graders", []):
+                graders.setdefault(g["name"], []).append(bool(g.get("passed")))
+        table[case["name"]] = {
+            "score": sum(scores) / len(scores) if scores else 0.0,
+            "runs": len(runs),
+            "errors": sum(1 for r in runs if r.get("error") or r.get("aborted")),
+            "cost": sum(r.get("costUsd", 0) + r.get("judgeCostUsd", 0) for r in runs),
+            "graders": {k: sum(v) / len(v) for k, v in graders.items()},
+        }
+    return table
+
+
+def fmt(x: float | None) -> str:
+    return "  -  " if x is None else f"{x:.2f}"
+
+
+def comparison(with_res: dict, base_res: dict | None, args: argparse.Namespace) -> str:
+    w = case_table(with_res)
+    b = case_table(base_res) if base_res else {}
+    lines = [
+        "# Plugin against baseline",
+        "",
+        f"Model `{args.model}`, judge `{args.judge_model}`, Claude Code "
+        f"{with_res.get('claudeVersion', '?')}, {with_res.get('startedAt', '')[:10]}.",
+        "",
+        "The baseline arm has the same mocked FactIQ tools and no skill and no scripts.",
+        "",
+        "| Case | With plugin | Baseline | Delta | Runs | Aborted or errored |",
+        "| --- | ---: | ---: | ---: | ---: | ---: |",
+    ]
+    for name in sorted(w):
+        ws = w[name]["score"]
+        bs = b[name]["score"] if name in b else None
+        delta = None if bs is None else ws - bs
+        err = f"{w[name]['errors']}" + (f" / {b[name]['errors']}" if name in b else "")
+        lines.append(f"| `{name}` | {fmt(ws)} | {fmt(bs)} | {fmt(delta)} | {w[name]['runs']} | {err} |")
+    wcost = with_res.get("costUsd", 0.0)
+    bcost = base_res.get("costUsd", 0.0) if base_res else 0.0
+    lines += ["", f"Cost: with plugin ${wcost:.2f}" + (f", baseline ${bcost:.2f}" if base_res else "") + ".", ""]
+    lines += ["## Graders (fraction of runs passed)", ""]
+    for name in sorted(w):
+        lines += [f"### `{name}`", "", "| Grader | With plugin | Baseline |", "| --- | ---: | ---: |"]
+        for g in sorted(w[name]["graders"]):
+            bg = b[name]["graders"].get(g) if name in b else None
+            lines.append(f"| `{g}` | {fmt(w[name]['graders'][g])} | {fmt(bg)} |")
+        lines.append("")
+    return "\n".join(lines)
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--model", required=True, help="model for the agent under test, for example opus or claude-opus-5")
+    p.add_argument("--judge-model", default="sonnet")
+    p.add_argument("--runs", type=int, default=None, help="runs per case; default is each case's own setting")
+    p.add_argument("--case", default=None, help="case name glob, for example 'yoy-*' (one value; use --tag to pick several unrelated cases)")
+    p.add_argument("--tag", action="append", default=[], help="case tag; repeatable")
+    p.add_argument("--allow-tools", default="Write,Edit,Bash", help="comma-separated operator grant")
+    p.add_argument("--concurrency", "-j", type=int, default=2)
+    p.add_argument("--threshold", type=float, default=0.8, help="exit 1 if a with-arm case scores below this")
+    p.add_argument("--max-cost-usd", type=float, default=None, help="per-arm cost ceiling passed through")
+    p.add_argument("--skip-baseline", action="store_true", help="run only the plugin arm")
+    p.add_argument("--keep-temp", action="store_true", help="keep each run's sandbox for inspection")
+    p.add_argument("--output-dir", default=None, help="default: evals/results/<timestamp>")
+    args = p.parse_args()
+
+    stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ")
+    # Absolute: the baseline arm runs inside a temporary plugin directory that
+    # is deleted afterwards, so a relative path would be lost with it.
+    out_root = (Path(args.output_dir) if args.output_dir else EVAL_DIR / "results" / stamp).resolve()
+    out_root.mkdir(parents=True, exist_ok=True)
+
+    with_res = run_eval(PLUGIN_ROOT, out_root / "with", args, "with plugin")
+
+    base_res = None
+    if not args.skip_baseline:
+        tmp = Path(tempfile.mkdtemp(prefix="factiq-baseline-"))
+        try:
+            build_baseline_plugin(tmp)
+            base_res = run_eval(tmp, out_root / "baseline", args, "baseline (tools only)")
+        finally:
+            shutil.rmtree(tmp, ignore_errors=True)
+
+    report = comparison(with_res, base_res, args)
+    (out_root / "comparison.md").write_text(report + "\n")
+    print("\n" + report)
+    print(f"\nWritten to {out_root}")
+
+    below = [n for n, c in case_table(with_res).items() if c["score"] < args.threshold]
+    if below:
+        print(f"\nBelow threshold {args.threshold}: {', '.join(sorted(below))}")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

Full technical diff · 104 changed fields

removed `/files/.app.json`

BEFORE

{
  "sha256": "34235df105b1fcd3e30dacd9793ed81848078adc18bed0df6fd01837e0473b42",
  "size": 127
}

AFTER

Field is absent

added `/files/.claude-plugin~1marketplace.json`

BEFORE

Field was absent

AFTER

{
  "sha256": "49fb6b53e79d5715aa9a9a093a0bf8d7f3dbd8b9a9328bb9ae21b336aa88cf49",
  "size": 610
}

added `/files/.claude-plugin~1plugin.json`

BEFORE

Field was absent

AFTER

{
  "sha256": "dec61fe1e1790fd335e220e1c54c16120e2e7632e16e044fe32f010d58d9c36e",
  "size": 927
}

changed `/files/.codex-plugin~1plugin.json/sha256`

BEFORE

"40992ee953af3848c3a0f65115d97dea99dcb6e0cd84aad7ae5539b6173ef4b3"

AFTER

"add8506ef8caab254eb8e3518555c6a398af5aea31b7e1089d38ff02d4ea2cf9"

changed `/files/.codex-plugin~1plugin.json/size`

BEFORE

AFTER

added `/files/.gitignore`

BEFORE

Field was absent

AFTER

{
  "sha256": "566702bcfdb6180828438f56a6f35b82c75d7221aa69a8c6aa7621adf8df5f00",
  "size": 126
}

added `/files/.mcp.json`

BEFORE

Field was absent

AFTER

{
  "sha256": "bd6807e5f73d473c49f99a29cd0eb178ba1e25349cd3db388d4f980963acaec9",
  "size": 112
}

added `/files/LICENSE`

BEFORE

Field was absent

AFTER

{
  "sha256": "ffe02ec2d2f9b23b61b38602bfd44d0fa777600a1cb1e8765da0626d09bd7114",
  "size": 1068
}

added `/files/README.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "ff859c70ed0a7e6e53f2f077afd79125a1f6ac0d5e93e6764a17f49096f704ab",
  "size": 16746
}

added `/files/SETUP.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "4a6e0b5db6c5948e362c9442ee2d9c35dd9980bafd6c01a056a7d123be5a286e",
  "size": 2958
}

added `/files/assets~1factiq-logo.png`

BEFORE

Field was absent

AFTER

{
  "sha256": "3b7bc0447b0612514780bc363f5a205710ef66de15daed4524ac4624998161e3",
  "size": 17317
}

added `/files/evals~1README.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "976b22bcc20d210ac3187e45154be00bfbeb68ffc17412de2d8eea9417f6a163",
  "size": 8941
}

added `/files/evals~1bilateral-trade-sql~1graders~1one-hs-level-only.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "1858daf1ab5d4774154d77458e7aa079da270af35e3476e904752180e6b82726",
  "size": 471
}

added `/files/evals~1bilateral-trade-sql~1graders~1partner-code-is-numeric.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "ebfb683108dca135318753256062de5ababbd417c3fa3eaa3a1b9c712cd92f17",
  "size": 523
}

added `/files/evals~1bilateral-trade-sql~1graders~1used-the-bundled-generator.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "8d5dfaa90973849c823636a1cf001b537fe4346f3f31e4467179557b45d90536",
  "size": 121
}

added `/files/evals~1bilateral-trade-sql~1graders~1value-series-only.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "83258957a67431b6ddaaa98e01cd4694533f814ba4ecf97b14b0ae981eef9bec",
  "size": 466
}

added `/files/evals~1bilateral-trade-sql~1prompt.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "6e0aeeaef20feb8ad194bfac0fb77fd63816c42fc8b1fc24914a1072e88e6b29",
  "size": 842
}

added `/files/evals~1filings-duplicate-line~1graders~1figure-is-not-doubled.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "7f7444985429c85502638acdd809f0bc96b3b7a676d6f9b6ac246bfd88505ddf",
  "size": 123
}

added `/files/evals~1filings-duplicate-line~1graders~1figure-is-the-first-line.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "325edda2814c36316d071e690539068175a5d9334eaf9bd555e84c5a38d2667d",
  "size": 808
}

added `/files/evals~1filings-duplicate-line~1graders~1no-invented-filing-links.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "46043047d26759ce2ed60643464fd5a377f2d75a46937b1f17ee5f53f420f15b",
  "size": 550
}

added `/files/evals~1filings-duplicate-line~1graders~1source-link-is-the-filing.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "30cf2586d074446b24b07333cf4860bc267b42d3efd36ed9f319f9aafcb11c41",
  "size": 163
}

added `/files/evals~1filings-duplicate-line~1prompt.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "4cd0b9906584614ca73ddec45d3681a3573bbc68ce076274ae89826e228ac055",
  "size": 813
}

added `/files/evals~1latest-value-direct-answer~1graders~1answers-in-prose.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "04588e346003462f88adf6a2fe6b85b5dd62f8c17ded2212b2bc9d068e5929a7",
  "size": 669
}

added `/files/evals~1latest-value-direct-answer~1graders~1no-chart-file-written.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "a06cd9171ce44db1511d189c1467b55dd76e0ddd3025bf9c63ffd6df1eaada46",
  "size": 60
}

added `/files/evals~1latest-value-direct-answer~1prompt.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "3a957c231947743012ecb886cd3c5fffa6e528444826b487df1b180f59d2d40e",
  "size": 577
}

added `/files/evals~1mocks~1factiq~1_tools.json`

BEFORE

Field was absent

AFTER

{
  "sha256": "b867d3bc2218c923d117350a8d19391578e31aaba251d806b766f711c4083373",
  "size": 58563
}

added `/files/evals~1mocks~1factiq~1describe_dataset.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "aef6ae3bad4de381194fd3533f5ac2bf291f5c94cbe2a4c840eb09ad6fa91558",
  "size": 2132
}

added `/files/evals~1mocks~1factiq~1fixtures~1LNS14000000.json`

BEFORE

Field was absent

AFTER

{
  "sha256": "3432a266cfbd4d7c32b11cec8d8f7305db7e97b4f7a91930c166b13719071b93",
  "size": 8750
}

added `/files/evals~1mocks~1factiq~1fixtures~1PKG.json`

BEFORE

Field was absent

AFTER

{
  "sha256": "236b9ba9dd79f5e160faaa32e6c5991a909c4162011ec539f11fba0ee682215f",
  "size": 15247
}

added `/files/evals~1mocks~1factiq~1get_data_catalog.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "840dc24cf2f2eb91611464c95d3414152c404df96600fae160fd0d1c07a777bb",
  "size": 6858
}

added `/files/evals~1mocks~1factiq~1get_series.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "8be660b04874d51a9f4476049e8623e07eca076fb71dce5dcf7fccb9817c9d5b",
  "size": 41
}

added `/files/evals~1mocks~1factiq~1get_style_guides.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "6363e4370e733e924f18a2c71a611509a1926a781e68ead99646b76659256f5a",
  "size": 880
}

added `/files/evals~1mocks~1factiq~1run_sql.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "77b6cee5f1133990bc93147fe8e2956f9b8b0b4a5fc14783dbcc9971d91130d5",
  "size": 2181
}

added `/files/evals~1mocks~1factiq~1search_company_filings.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "d7fa2f3b3a67aef362b459b0eba2362f2f3d12c242dded32a3f1402f95f3f268",
  "size": 39
}

added `/files/evals~1mocks~1factiq~1search_datasets.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "98f364b4a4189012c607031b595c3e1bd7cb8e3ba95e10f59dd7aaf78883418f",
  "size": 1034
}

added `/files/evals~1mocks~1factiq~1search_earnings_transcripts.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "6f0187121c6c4e5f998be00612e9d5bff30c6d5b6ad36ffccd447e21ba1650d5",
  "size": 6189
}

added `/files/evals~1mocks~1factiq~1search_series.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "4a78565f723fcf708bcdc08f7df142f3568ad1652db43b87796d091716e9c41b",
  "size": 1302
}

added `/files/evals~1quick-chart-of-a-trend~1graders~1answer-states-gap-and-source.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "b2dd20ccd3a717585bdfe2b5bfe101042ca3933734ab6cb273d5ac4bff90561f",
  "size": 492
}

added `/files/evals~1quick-chart-of-a-trend~1graders~1chart-file-written.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "53a654575c25d75251de7e79e8d40d1837b12f186d31060f3a2a7819dfcaefcc",
  "size": 53
}

added `/files/evals~1quick-chart-of-a-trend~1graders~1missing-month-is-not-filled-in.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "4012378ca9d63de3ebee5f3fdb39b78ff5095bd861f97716323d8564a3c8d1b2",
  "size": 528
}

added `/files/evals~1quick-chart-of-a-trend~1graders~1spec-is-a-valid-chartspec.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "72ea853a887ba0bd91a97f49ae52cdbd66cb295eb4888c1bcb47719892bad82c",
  "size": 946
}

added `/files/evals~1quick-chart-of-a-trend~1graders~1title-makes-a-claim.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "2d9c5e1e0ff470e04d170bd88140d874324a7db4086feb53adf19216d371a4d0",
  "size": 656
}

added `/files/evals~1quick-chart-of-a-trend~1prompt.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "956fc00fd943f70ae85de3bd22b6b60147846dd67a3297ecbec36d64eff723d2",
  "size": 690
}

added `/files/evals~1quote-and-source-discipline~1graders~1analyst-framing-is-not-management.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "db4672fe197c1bbc858969604d0fbdfece844c4a95bb4cbffb2a94da904b0553",
  "size": 608
}

added `/files/evals~1quote-and-source-discipline~1graders~1link-fy2025q2-is-the-supplied-one.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "e5723ee4f90d36c669c66dfa247d5674dd4e71f6bb8b5290f4d1c7d65a4b89e4",
  "size": 216
}

added `/files/evals~1quote-and-source-discipline~1graders~1link-fy2026q4-is-the-supplied-one.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "f0615b5f82e65e997603fd0dce33d331e0dc5aa86738521ea7e0720d9ffa484c",
  "size": 215
}

added `/files/evals~1quote-and-source-discipline~1graders~1link-fy2027q1-is-the-supplied-one.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "da50e2f5694fdaf824130515c6cad1edd6581cc0068054c6eead1f03546fed87",
  "size": 215
}

added `/files/evals~1quote-and-source-discipline~1graders~1missing-link-is-declared.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "3286a7f25db8fc1c059b14977cc35cd72cbb6bba67720b7b58137a47ad201177",
  "size": 486
}

added `/files/evals~1quote-and-source-discipline~1graders~1no-invented-source-links.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "3418802ca26fd0b78017ccdab87a0a243cce30966c5e3482a87d1708a6e98ee0",
  "size": 167
}

added `/files/evals~1quote-and-source-discipline~1graders~1quotes-are-verbatim.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "af6f44f2a953d8de18d8592486114447a94e8f621f0ea0a2e52d6d284d74ed14",
  "size": 2001
}

added `/files/evals~1quote-and-source-discipline~1prompt.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "1287da04c18787fa65b4cb016cef62fe7d208b360f0d7d7f6750ffb68db35f78",
  "size": 781
}

added `/files/evals~1run_ablation.py`

BEFORE

Field was absent

AFTER

{
  "sha256": "550702da90fbe53889c67ef30f6cb7254e31fa1009719ec869f313b3aed9f1d3",
  "size": 9041
}

added `/files/evals~1unrelated-coding-request~1graders~1answers-the-python-question.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "4db03d4fcb5e4bb6a04162e0dad4b5db3fef97d145103d64bfb041b50946f12a",
  "size": 407
}

added `/files/evals~1unrelated-coding-request~1graders~1no-catalog-lookup.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "60bfc77bae34c51bdb2a2b7a3b8c72719e5bd55d81a281a71e9026595d426260",
  "size": 98
}

added `/files/evals~1unrelated-coding-request~1graders~1no-factiq-skill-invoked.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "22813294a356e15059d59459cbe5da9ec0a4a18c760d9521ddc35ae886dcd7c5",
  "size": 80
}

added `/files/evals~1unrelated-coding-request~1graders~1no-series-fetch.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "ede84d92b6c6e73e1e42d8968ddd3c8c0c8f9d7560bd169131987ca1db3e23f9",
  "size": 92
}

added `/files/evals~1unrelated-coding-request~1graders~1no-sql-call.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "3e5d7fdb2a6fb1566d4ba420eb12550281f72f0deec9cb9e613378ceb76cd7ae",
  "size": 89
}

added `/files/evals~1unrelated-coding-request~1prompt.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "40eb7a0228cca78e53786e361c687eb96238814f69911fc9d998f501b6575250",
  "size": 492
}

added `/files/evals~1yoy-across-a-missing-month~1graders~1missing-month-is-stated.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "560524f7356fb5ceedf5314bfb511ae0b70c63c6b92bace819c52cc0cce26ee3",
  "size": 376
}

added `/files/evals~1yoy-across-a-missing-month~1graders~1yoy-values-are-date-matched.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "3d6dfbc6957296d246e5e767d23ce83e134b92aa179d824cb6988100f3ba80cd",
  "size": 860
}

added `/files/evals~1yoy-across-a-missing-month~1prompt.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "fcdbe2f5cc33e6a2d0e27d88143ea5d5fd57c59cd1ccbb889c58b9cbbe7aa380",
  "size": 678
}

added `/files/evals~1yoy-without-a-server-hint~1graders~1missing-month-is-stated.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "560524f7356fb5ceedf5314bfb511ae0b70c63c6b92bace819c52cc0cce26ee3",
  "size": 376
}

added `/files/evals~1yoy-without-a-server-hint~1graders~1yoy-values-are-date-matched.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "3d6dfbc6957296d246e5e767d23ce83e134b92aa179d824cb6988100f3ba80cd",
  "size": 860
}

added `/files/evals~1yoy-without-a-server-hint~1mocks~1factiq~1fixtures~1LNS14000000.json`

BEFORE

Field was absent

AFTER

{
  "sha256": "9827dc5a5e9f5390e7ebf352502d29ad95a7a0654355622ab0a53aad40c6bd6a",
  "size": 7971
}

added `/files/evals~1yoy-without-a-server-hint~1mocks~1factiq~1get_series.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "8be660b04874d51a9f4476049e8623e07eca076fb71dce5dcf7fccb9817c9d5b",
  "size": 41
}

added `/files/evals~1yoy-without-a-server-hint~1mocks~1factiq~1run_sql.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "8236cf759b4a8f1f91c7725c7f54d69679490766f110aa29bd846c6a7537e98a",
  "size": 2146
}

added `/files/evals~1yoy-without-a-server-hint~1prompt.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "678326246a74ba8f44c43dad47b172248ff9e030df00146f766acd52ec88467b",
  "size": 786
}

added `/files/references~1data~1satellite.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "2d310c84485c86f91b8a864dc69147988c07f5d9a06a96e0c42961014406b882",
  "size": 13687
}

added `/files/references~1data~1schemas.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "51fd1bd6e71b5a48dd8b15cc923e16d608c2d2638edfda5c0fff1a1d56ca47b8",
  "size": 22132
}

added `/files/references~1data~1sql-guide.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "4924bd8ae696ef3ee8aaa5c5dbb4ba53356cefa82653d65b3040df407e800027",
  "size": 21312
}

added `/files/references~1output~1chart-spec.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "2fb1b622fa6b037e8860c3bfba20bfd97b744192f34d36df16fec82b6f4c30dc",
  "size": 8704
}

added `/files/references~1output~1report-spec.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "5e16ae46b0fc2083a35dc262e4483d87e88cc9c6fb8f1412eebd7b2b0570250c",
  "size": 14050
}

added `/files/references~1report-patterns~1README.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "674c9df1c81c8cb5d4beb07243ba837217da86912e5e2a59c63641a9531166b9",
  "size": 11027
}

added `/files/references~1report-patterns~1bilateral-economic-policy.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "48c89a71f875aa92d7a7ddd11e3c134b108788585f48a8f13af65b2e323e4305",
  "size": 9123
}

added `/files/references~1report-patterns~1bilateral-trade.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "98cc223764dd709ffb64ddcb13d3e549669e28331736140e769bae7319d76dac",
  "size": 19502
}

added `/files/references~1report-patterns~1business-formation-statistics.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "324ea681d9b3ad7eceb9cc5371608ac01fd86743b3c3581236398053184cbcda",
  "size": 5440
}

added `/files/references~1report-patterns~1earnings-intelligence.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "9e491f60843c1fa4d9b0e4f5ddd831974ce2f3eddbc202ad77f64549f33c0634",
  "size": 17082
}

added `/files/references~1report-patterns~1fiscal-policy-revenue.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "7525d09c34a0b990e0eb5cf73bcf1e7077bd373e0be8be238706cde1b0cbc670",
  "size": 7628
}

added `/files/references~1report-patterns~1interview-step.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "b5edb929dfcc6257c1b25fc407ae29878383c0dca9cf974041e437f0b70e0682",
  "size": 3806
}

added `/files/references~1report-patterns~1media-intelligence.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "a9f31dc9760757af5c6462333ca176bfa724980814f3fc55e77ba107b50f0f22",
  "size": 17242
}

added `/files/references~1report-patterns~1monetary-policy.md`

BEFORE

Field was absent

AFTER

{
  "sha256": "6439f63e7c77ab5d5d424a09bc112edea12f37630615c96315e80490939ffa38",
  "size": 11190
}

added `/files/scripts~1pycache~1term_chart.cpython-312.pyc`

BEFORE

Field was absent

AFTER

{
  "sha256": "9b87da15116997e594cba9e058b9ff8e015cd91be7322c1c84d1d0bf6599d14b",
  "size": 32120
}

added `/files/scripts~1comext_sql.py`

BEFORE

Field was absent

AFTER

{
  "sha256": "8a8bcc07156d5f1527540dae466ca6ee6ae0c6a743dadeb3d1c8cf94fb87d187",
  "size": 19633
}

added `/files/scripts~1data~1hs_names.json.gz`

BEFORE

Field was absent

AFTER

{
  "sha256": "3f5def41a331d3ad3a4ebb70c49e442e6e19e898673c577c04fa8be1ea9e017b",
  "size": 155694
}

added `/files/scripts~1data~1trade_partners.json`

BEFORE

Field was absent

AFTER

{
  "sha256": "7445d1193623a87480dbe68107a007bcd23c73aa330cec3c30e04ba3b662bc67",
  "size": 38397
}

added `/files/scripts~1hs_codes.py`

BEFORE

Field was absent

AFTER

{
  "sha256": "38230174536c64a18adfb130b486d81a9dc7e6c69ca866f08a1fd6ecca071f98",
  "size": 7191
}

added `/files/scripts~1series_math.py`

BEFORE

Field was absent

AFTER

{
  "sha256": "5ee047d8b9d00c934b3e1bd1f25c1a25e6a545c09371f8f5a93ad46adce6cdf1",
  "size": 22168
}

added `/files/scripts~1trade_sql.py`

BEFORE

Field was absent

AFTER

{
  "sha256": "246f4a7bdf972f3d30374398d50d7701ae3aa8f00e86403175e82df6f062b6a7",
  "size": 23894
}

changed `/files/skills~1factiq~1SKILL.md/sha256`

BEFORE

"46d0e8dcc2169a508f3ba93b5a00cc9d44d8b8fdbf5d7109f2a6d3938c13e9ab"

AFTER

"b1583be415b4c87002379f6737819536a9556cc0b55ca47b5ee1af33dee8b4ad"

changed `/files/skills~1factiq~1SKILL.md/size`

BEFORE

AFTER

added `/files/tests~1pycache~1test_directory_submission_readiness.cpython-311-pytest-9.1.1.pyc`

BEFORE

Field was absent

AFTER

{
  "sha256": "7cd7a02f7438f3ccb26d9887b3434e9de712e1a1ba8b31e02188fe7ed259e5b4",
  "size": 9731
}

added `/files/tests~1pycache~1test_earnings_docs.cpython-311-pytest-9.1.1.pyc`

BEFORE

Field was absent

AFTER

{
  "sha256": "cce9b51a6d22b7085758a56649b264874da358e8d661ab19901e96dd9c749b23",
  "size": 8276
}

added `/files/tests~1pycache~1test_earnings_docs.cpython-312-pytest-8.4.0.pyc`

BEFORE

Field was absent

AFTER

{
  "sha256": "6ea5eab90eab2f70dde65e6870c0b4c0604a5126b884d20d3cb9541167d81df7",
  "size": 6506
}

added `/files/tests~1pycache~1test_media_docs.cpython-311-pytest-9.1.1.pyc`

BEFORE

Field was absent

AFTER

{
  "sha256": "a9c11af354f7918893faf1dc53bd478e8b622f44f8435072dab99d7394a28e29",
  "size": 9927
}

added `/files/tests~1pycache~1test_media_docs.cpython-312-pytest-8.4.0.pyc`

BEFORE

Field was absent

AFTER

{
  "sha256": "77c4092fa0c7e91219fca81739eec2094b70779623b518b2a464ef97dc1f466b",
  "size": 8624
}

added `/files/tests~1pycache~1test_retired_publishing_tools.cpython-311-pytest-9.1.1.pyc`

BEFORE

Field was absent

AFTER

{
  "sha256": "092c2a078d0d34ec21dd9cf5e0ca61665cba3080d44c80f2a338c72c40fd7720",
  "size": 2790
}

added `/files/tests~1pycache~1test_trade_scripts.cpython-311-pytest-9.1.1.pyc`

BEFORE

Field was absent

AFTER

{
  "sha256": "d341dcc6f3409f2eb1a6fef285d8ee450cdc9af181aa4b1bf44753dce27be131",
  "size": 39627
}

added `/files/tests~1pycache~1test_trade_scripts.cpython-312-pytest-8.4.0.pyc`

BEFORE

Field was absent

AFTER

{
  "sha256": "a7915418895584c253832c1d221e17fe97606e39b0e2e8f041cbf1c64f425a55",
  "size": 37264
}

added `/files/tests~1test_directory_submission_readiness.py`

BEFORE

Field was absent

AFTER

{
  "sha256": "996daa950b56bd3368d7e45221c75b8fa9ce7a1a7a23d91734f8ddc286f81f3a",
  "size": 6918
}

added `/files/tests~1test_earnings_docs.py`

BEFORE

Field was absent

AFTER

{
  "sha256": "6e596f81295baaefb70ae514dee2f9e6175c34d146afba1ac27a01412040304a",
  "size": 4884
}

added `/files/tests~1test_media_docs.py`

BEFORE

Field was absent

AFTER

{
  "sha256": "214149d3ddb196a85639600ce7ae75a25add42196a5923d8b95123c36dd54974",
  "size": 6290
}

added `/files/tests~1test_retired_publishing_tools.py`

BEFORE

Field was absent

AFTER

{
  "sha256": "395261b5067573c050ecda46f594530d91d67fc2ec07484540c9adeaa0a2306a",
  "size": 1442
}

added `/files/tests~1test_sql_docs.py`

BEFORE

Field was absent

AFTER

{
  "sha256": "18b2581835f82ce007d78e538e5359d2ad30fa13f37652bea06aba5da21c6872",
  "size": 12090
}

added `/files/tests~1test_trade_scripts.py`

BEFORE

Field was absent

AFTER

{
  "sha256": "d024d95711d2da25f2407626d81422357ded2ec8b3791dd9934dab1140123d04",
  "size": 27782
}

Full snapshot data

{
  "files": {
    ".claude-plugin/marketplace.json": {
      "sha256": "49fb6b53e79d5715aa9a9a093a0bf8d7f3dbd8b9a9328bb9ae21b336aa88cf49",
      "size": 610
    },
    ".claude-plugin/plugin.json": {
      "sha256": "dec61fe1e1790fd335e220e1c54c16120e2e7632e16e044fe32f010d58d9c36e",
      "size": 927
    },
    ".codex-plugin/plugin.json": {
      "sha256": "add8506ef8caab254eb8e3518555c6a398af5aea31b7e1089d38ff02d4ea2cf9",
      "size": 2536
    },
    ".gitignore": {
      "sha256": "566702bcfdb6180828438f56a6f35b82c75d7221aa69a8c6aa7621adf8df5f00",
      "size": 126
    },
    ".mcp.json": {
      "sha256": "bd6807e5f73d473c49f99a29cd0eb178ba1e25349cd3db388d4f980963acaec9",
      "size": 112
    },
    "LICENSE": {
      "sha256": "ffe02ec2d2f9b23b61b38602bfd44d0fa777600a1cb1e8765da0626d09bd7114",
      "size": 1068
    },
    "README.md": {
      "sha256": "ff859c70ed0a7e6e53f2f077afd79125a1f6ac0d5e93e6764a17f49096f704ab",
      "size": 16746
    },
    "SETUP.md": {
      "sha256": "4a6e0b5db6c5948e362c9442ee2d9c35dd9980bafd6c01a056a7d123be5a286e",
      "size": 2958
    },
    "assets/factiq-logo.png": {
      "sha256": "3b7bc0447b0612514780bc363f5a205710ef66de15daed4524ac4624998161e3",
      "size": 17317
    },
    "evals/README.md": {
      "sha256": "976b22bcc20d210ac3187e45154be00bfbeb68ffc17412de2d8eea9417f6a163",
      "size": 8941
    },
    "evals/bilateral-trade-sql/graders/one-hs-level-only.md": {
      "sha256": "1858daf1ab5d4774154d77458e7aa079da270af35e3476e904752180e6b82726",
      "size": 471
    },
    "evals/bilateral-trade-sql/graders/partner-code-is-numeric.md": {
      "sha256": "ebfb683108dca135318753256062de5ababbd417c3fa3eaa3a1b9c712cd92f17",
      "size": 523
    },
    "evals/bilateral-trade-sql/graders/used-the-bundled-generator.md": {
      "sha256": "8d5dfaa90973849c823636a1cf001b537fe4346f3f31e4467179557b45d90536",
      "size": 121
    },
    "evals/bilateral-trade-sql/graders/value-series-only.md": {
      "sha256": "83258957a67431b6ddaaa98e01cd4694533f814ba4ecf97b14b0ae981eef9bec",
      "size": 466
    },
    "evals/bilateral-trade-sql/prompt.md": {
      "sha256": "6e0aeeaef20feb8ad194bfac0fb77fd63816c42fc8b1fc24914a1072e88e6b29",
      "size": 842
    },
    "evals/filings-duplicate-line/graders/figure-is-not-doubled.md": {
      "sha256": "7f7444985429c85502638acdd809f0bc96b3b7a676d6f9b6ac246bfd88505ddf",
      "size": 123
    },
    "evals/filings-duplicate-line/graders/figure-is-the-first-line.md": {
      "sha256": "325edda2814c36316d071e690539068175a5d9334eaf9bd555e84c5a38d2667d",
      "size": 808
    },
    "evals/filings-duplicate-line/graders/no-invented-filing-links.md": {
      "sha256": "46043047d26759ce2ed60643464fd5a377f2d75a46937b1f17ee5f53f420f15b",
      "size": 550
    },
    "evals/filings-duplicate-line/graders/source-link-is-the-filing.md": {
      "sha256": "30cf2586d074446b24b07333cf4860bc267b42d3efd36ed9f319f9aafcb11c41",
      "size": 163
    },
    "evals/filings-duplicate-line/prompt.md": {
      "sha256": "4cd0b9906584614ca73ddec45d3681a3573bbc68ce076274ae89826e228ac055",
      "size": 813
    },
    "evals/latest-value-direct-answer/graders/answers-in-prose.md": {
      "sha256": "04588e346003462f88adf6a2fe6b85b5dd62f8c17ded2212b2bc9d068e5929a7",
      "size": 669
    },
    "evals/latest-value-direct-answer/graders/no-chart-file-written.md": {
      "sha256": "a06cd9171ce44db1511d189c1467b55dd76e0ddd3025bf9c63ffd6df1eaada46",
      "size": 60
    },
    "evals/latest-value-direct-answer/prompt.md": {
      "sha256": "3a957c231947743012ecb886cd3c5fffa6e528444826b487df1b180f59d2d40e",
      "size": 577
    },
    "evals/mocks/factiq/_tools.json": {
      "sha256": "b867d3bc2218c923d117350a8d19391578e31aaba251d806b766f711c4083373",
      "size": 58563
    },
    "evals/mocks/factiq/describe_dataset.md": {
      "sha256": "aef6ae3bad4de381194fd3533f5ac2bf291f5c94cbe2a4c840eb09ad6fa91558",
      "size": 2132
    },
    "evals/mocks/factiq/fixtures/LNS14000000.json": {
      "sha256": "3432a266cfbd4d7c32b11cec8d8f7305db7e97b4f7a91930c166b13719071b93",
      "size": 8750
    },
    "evals/mocks/factiq/fixtures/PKG.json": {
      "sha256": "236b9ba9dd79f5e160faaa32e6c5991a909c4162011ec539f11fba0ee682215f",
      "size": 15247
    },
    "evals/mocks/factiq/get_data_catalog.md": {
      "sha256": "840dc24cf2f2eb91611464c95d3414152c404df96600fae160fd0d1c07a777bb",
      "size": 6858
    },
    "evals/mocks/factiq/get_series.md": {
      "sha256": "8be660b04874d51a9f4476049e8623e07eca076fb71dce5dcf7fccb9817c9d5b",
      "size": 41
    },
    "evals/mocks/factiq/get_style_guides.md": {
      "sha256": "6363e4370e733e924f18a2c71a611509a1926a781e68ead99646b76659256f5a",
      "size": 880
    },
    "evals/mocks/factiq/run_sql.md": {
      "sha256": "77b6cee5f1133990bc93147fe8e2956f9b8b0b4a5fc14783dbcc9971d91130d5",
      "size": 2181
    },
    "evals/mocks/factiq/search_company_filings.md": {
      "sha256": "d7fa2f3b3a67aef362b459b0eba2362f2f3d12c242dded32a3f1402f95f3f268",
      "size": 39
    },
    "evals/mocks/factiq/search_datasets.md": {
      "sha256": "98f364b4a4189012c607031b595c3e1bd7cb8e3ba95e10f59dd7aaf78883418f",
      "size": 1034
    },
    "evals/mocks/factiq/search_earnings_transcripts.md": {
      "sha256": "6f0187121c6c4e5f998be00612e9d5bff30c6d5b6ad36ffccd447e21ba1650d5",
      "size": 6189
    },
    "evals/mocks/factiq/search_series.md": {
      "sha256": "4a78565f723fcf708bcdc08f7df142f3568ad1652db43b87796d091716e9c41b",
      "size": 1302
    },
    "evals/quick-chart-of-a-trend/graders/answer-states-gap-and-source.md": {
      "sha256": "b2dd20ccd3a717585bdfe2b5bfe101042ca3933734ab6cb273d5ac4bff90561f",
      "size": 492
    },
    "evals/quick-chart-of-a-trend/graders/chart-file-written.md": {
      "sha256": "53a654575c25d75251de7e79e8d40d1837b12f186d31060f3a2a7819dfcaefcc",
      "size": 53
    },
    "evals/quick-chart-of-a-trend/graders/missing-month-is-not-filled-in.md": {
      "sha256": "4012378ca9d63de3ebee5f3fdb39b78ff5095bd861f97716323d8564a3c8d1b2",
      "size": 528
    },
    "evals/quick-chart-of-a-trend/graders/spec-is-a-valid-chartspec.md": {
      "sha256": "72ea853a887ba0bd91a97f49ae52cdbd66cb295eb4888c1bcb47719892bad82c",
      "size": 946
    },
    "evals/quick-chart-of-a-trend/graders/title-makes-a-claim.md": {
      "sha256": "2d9c5e1e0ff470e04d170bd88140d874324a7db4086feb53adf19216d371a4d0",
      "size": 656
    },
    "evals/quick-chart-of-a-trend/prompt.md": {
      "sha256": "956fc00fd943f70ae85de3bd22b6b60147846dd67a3297ecbec36d64eff723d2",
      "size": 690
    },
    "evals/quote-and-source-discipline/graders/analyst-framing-is-not-management.md": {
      "sha256": "db4672fe197c1bbc858969604d0fbdfece844c4a95bb4cbffb2a94da904b0553",
      "size": 608
    },
    "evals/quote-and-source-discipline/graders/link-fy2025q2-is-the-supplied-one.md": {
      "sha256": "e5723ee4f90d36c669c66dfa247d5674dd4e71f6bb8b5290f4d1c7d65a4b89e4",
      "size": 216
    },
    "evals/quote-and-source-discipline/graders/link-fy2026q4-is-the-supplied-one.md": {
      "sha256": "f0615b5f82e65e997603fd0dce33d331e0dc5aa86738521ea7e0720d9ffa484c",
      "size": 215
    },
    "evals/quote-and-source-discipline/graders/link-fy2027q1-is-the-supplied-one.md": {
      "sha256": "da50e2f5694fdaf824130515c6cad1edd6581cc0068054c6eead1f03546fed87",
      "size": 215
    },
    "evals/quote-and-source-discipline/graders/missing-link-is-declared.md": {
      "sha256": "3286a7f25db8fc1c059b14977cc35cd72cbb6bba67720b7b58137a47ad201177",
      "size": 486
    },
    "evals/quote-and-source-discipline/graders/no-invented-source-links.md": {
      "sha256": "3418802ca26fd0b78017ccdab87a0a243cce30966c5e3482a87d1708a6e98ee0",
      "size": 167
    },
    "evals/quote-and-source-discipline/graders/quotes-are-verbatim.md": {
      "sha256": "af6f44f2a953d8de18d8592486114447a94e8f621f0ea0a2e52d6d284d74ed14",
      "size": 2001
    },
    "evals/quote-and-source-discipline/prompt.md": {
      "sha256": "1287da04c18787fa65b4cb016cef62fe7d208b360f0d7d7f6750ffb68db35f78",
      "size": 781
    },
    "evals/run_ablation.py": {
      "sha256": "550702da90fbe53889c67ef30f6cb7254e31fa1009719ec869f313b3aed9f1d3",
      "size": 9041
    },
    "evals/unrelated-coding-request/graders/answers-the-python-question.md": {
      "sha256": "4db03d4fcb5e4bb6a04162e0dad4b5db3fef97d145103d64bfb041b50946f12a",
      "size": 407
    },
    "evals/unrelated-coding-request/graders/no-catalog-lookup.md": {
      "sha256": "60bfc77bae34c51bdb2a2b7a3b8c72719e5bd55d81a281a71e9026595d426260",
      "size": 98
    },
    "evals/unrelated-coding-request/graders/no-factiq-skill-invoked.md": {
      "sha256": "22813294a356e15059d59459cbe5da9ec0a4a18c760d9521ddc35ae886dcd7c5",
      "size": 80
    },
    "evals/unrelated-coding-request/graders/no-series-fetch.md": {
      "sha256": "ede84d92b6c6e73e1e42d8968ddd3c8c0c8f9d7560bd169131987ca1db3e23f9",
      "size": 92
    },
    "evals/unrelated-coding-request/graders/no-sql-call.md": {
      "sha256": "3e5d7fdb2a6fb1566d4ba420eb12550281f72f0deec9cb9e613378ceb76cd7ae",
      "size": 89
    },
    "evals/unrelated-coding-request/prompt.md": {
      "sha256": "40eb7a0228cca78e53786e361c687eb96238814f69911fc9d998f501b6575250",
      "size": 492
    },
    "evals/yoy-across-a-missing-month/graders/missing-month-is-stated.md": {
      "sha256": "560524f7356fb5ceedf5314bfb511ae0b70c63c6b92bace819c52cc0cce26ee3",
      "size": 376
    },
    "evals/yoy-across-a-missing-month/graders/yoy-values-are-date-matched.md": {
      "sha256": "3d6dfbc6957296d246e5e767d23ce83e134b92aa179d824cb6988100f3ba80cd",
      "size": 860
    },
    "evals/yoy-across-a-missing-month/prompt.md": {
      "sha256": "fcdbe2f5cc33e6a2d0e27d88143ea5d5fd57c59cd1ccbb889c58b9cbbe7aa380",
      "size": 678
    },
    "evals/yoy-without-a-server-hint/graders/missing-month-is-stated.md": {
      "sha256": "560524f7356fb5ceedf5314bfb511ae0b70c63c6b92bace819c52cc0cce26ee3",
      "size": 376
    },
    "evals/yoy-without-a-server-hint/graders/yoy-values-are-date-matched.md": {
      "sha256": "3d6dfbc6957296d246e5e767d23ce83e134b92aa179d824cb6988100f3ba80cd",
      "size": 860
    },
    "evals/yoy-without-a-server-hint/mocks/factiq/fixtures/LNS14000000.json": {
      "sha256": "9827dc5a5e9f5390e7ebf352502d29ad95a7a0654355622ab0a53aad40c6bd6a",
      "size": 7971
    },
    "evals/yoy-without-a-server-hint/mocks/factiq/get_series.md": {
      "sha256": "8be660b04874d51a9f4476049e8623e07eca076fb71dce5dcf7fccb9817c9d5b",
      "size": 41
    },
    "evals/yoy-without-a-server-hint/mocks/factiq/run_sql.md": {
      "sha256": "8236cf759b4a8f1f91c7725c7f54d69679490766f110aa29bd846c6a7537e98a",
      "size": 2146
    },
    "evals/yoy-without-a-server-hint/prompt.md": {
      "sha256": "678326246a74ba8f44c43dad47b172248ff9e030df00146f766acd52ec88467b",
      "size": 786
    },
    "references/data/satellite.md": {
      "sha256": "2d310c84485c86f91b8a864dc69147988c07f5d9a06a96e0c42961014406b882",
      "size": 13687
    },
    "references/data/schemas.md": {
      "sha256": "51fd1bd6e71b5a48dd8b15cc923e16d608c2d2638edfda5c0fff1a1d56ca47b8",
      "size": 22132
    },
    "references/data/sql-guide.md": {
      "sha256": "4924bd8ae696ef3ee8aaa5c5dbb4ba53356cefa82653d65b3040df407e800027",
      "size": 21312
    },
    "references/output/chart-spec.md": {
      "sha256": "2fb1b622fa6b037e8860c3bfba20bfd97b744192f34d36df16fec82b6f4c30dc",
      "size": 8704
    },
    "references/output/report-spec.md": {
      "sha256": "5e16ae46b0fc2083a35dc262e4483d87e88cc9c6fb8f1412eebd7b2b0570250c",
      "size": 14050
    },
    "references/report-patterns/README.md": {
      "sha256": "674c9df1c81c8cb5d4beb07243ba837217da86912e5e2a59c63641a9531166b9",
      "size": 11027
    },
    "references/report-patterns/bilateral-economic-policy.md": {
      "sha256": "48c89a71f875aa92d7a7ddd11e3c134b108788585f48a8f13af65b2e323e4305",
      "size": 9123
    },
    "references/report-patterns/bilateral-trade.md": {
      "sha256": "98cc223764dd709ffb64ddcb13d3e549669e28331736140e769bae7319d76dac",
      "size": 19502
    },
    "references/report-patterns/business-formation-statistics.md": {
      "sha256": "324ea681d9b3ad7eceb9cc5371608ac01fd86743b3c3581236398053184cbcda",
      "size": 5440
    },
    "references/report-patterns/earnings-intelligence.md": {
      "sha256": "9e491f60843c1fa4d9b0e4f5ddd831974ce2f3eddbc202ad77f64549f33c0634",
      "size": 17082
    },
    "references/report-patterns/fiscal-policy-revenue.md": {
      "sha256": "7525d09c34a0b990e0eb5cf73bcf1e7077bd373e0be8be238706cde1b0cbc670",
      "size": 7628
    },
    "references/report-patterns/interview-step.md": {
      "sha256": "b5edb929dfcc6257c1b25fc407ae29878383c0dca9cf974041e437f0b70e0682",
      "size": 3806
    },
    "references/report-patterns/media-intelligence.md": {
      "sha256": "a9f31dc9760757af5c6462333ca176bfa724980814f3fc55e77ba107b50f0f22",
      "size": 17242
    },
    "references/report-patterns/monetary-policy.md": {
      "sha256": "6439f63e7c77ab5d5d424a09bc112edea12f37630615c96315e80490939ffa38",
      "size": 11190
    },
    "scripts/__pycache__/term_chart.cpython-312.pyc": {
      "sha256": "9b87da15116997e594cba9e058b9ff8e015cd91be7322c1c84d1d0bf6599d14b",
      "size": 32120
    },
    "scripts/comext_sql.py": {
      "sha256": "8a8bcc07156d5f1527540dae466ca6ee6ae0c6a743dadeb3d1c8cf94fb87d187",
      "size": 19633
    },
    "scripts/data/hs_names.json.gz": {
      "sha256": "3f5def41a331d3ad3a4ebb70c49e442e6e19e898673c577c04fa8be1ea9e017b",
      "size": 155694
    },
    "scripts/data/trade_partners.json": {
      "sha256": "7445d1193623a87480dbe68107a007bcd23c73aa330cec3c30e04ba3b662bc67",
      "size": 38397
    },
    "scripts/hs_codes.py": {
      "sha256": "38230174536c64a18adfb130b486d81a9dc7e6c69ca866f08a1fd6ecca071f98",
      "size": 7191
    },
    "scripts/series_math.py": {
      "sha256": "5ee047d8b9d00c934b3e1bd1f25c1a25e6a545c09371f8f5a93ad46adce6cdf1",
      "size": 22168
    },
    "scripts/trade_sql.py": {
      "sha256": "246f4a7bdf972f3d30374398d50d7701ae3aa8f00e86403175e82df6f062b6a7",
      "size": 23894
    },
    "skills/factiq/SKILL.md": {
      "sha256": "b1583be415b4c87002379f6737819536a9556cc0b55ca47b5ee1af33dee8b4ad",
      "size": 52129
    },
    "tests/__pycache__/test_directory_submission_readiness.cpython-311-pytest-9.1.1.pyc": {
      "sha256": "7cd7a02f7438f3ccb26d9887b3434e9de712e1a1ba8b31e02188fe7ed259e5b4",
      "size": 9731
    },
    "tests/__pycache__/test_earnings_docs.cpython-311-pytest-9.1.1.pyc": {
      "sha256": "cce9b51a6d22b7085758a56649b264874da358e8d661ab19901e96dd9c749b23",
      "size": 8276
    },
    "tests/__pycache__/test_earnings_docs.cpython-312-pytest-8.4.0.pyc": {
      "sha256": "6ea5eab90eab2f70dde65e6870c0b4c0604a5126b884d20d3cb9541167d81df7",
      "size": 6506
    },
    "tests/__pycache__/test_media_docs.cpython-311-pytest-9.1.1.pyc": {
      "sha256": "a9c11af354f7918893faf1dc53bd478e8b622f44f8435072dab99d7394a28e29",
      "size": 9927
    },
    "tests/__pycache__/test_media_docs.cpython-312-pytest-8.4.0.pyc": {
      "sha256": "77c4092fa0c7e91219fca81739eec2094b70779623b518b2a464ef97dc1f466b",
      "size": 8624
    },
    "tests/__pycache__/test_retired_publishing_tools.cpython-311-pytest-9.1.1.pyc": {
      "sha256": "092c2a078d0d34ec21dd9cf5e0ca61665cba3080d44c80f2a338c72c40fd7720",
      "size": 2790
    },
    "tests/__pycache__/test_trade_scripts.cpython-311-pytest-9.1.1.pyc": {
      "sha256": "d341dcc6f3409f2eb1a6fef285d8ee450cdc9af181aa4b1bf44753dce27be131",
      "size": 39627
    },
    "tests/__pycache__/test_trade_scripts.cpython-312-pytest-8.4.0.pyc": {
      "sha256": "a7915418895584c253832c1d221e17fe97606e39b0e2e8f041cbf1c64f425a55",
      "size": 37264
    },
    "tests/test_directory_submission_readiness.py": {
      "sha256": "996daa950b56bd3368d7e45221c75b8fa9ce7a1a7a23d91734f8ddc286f81f3a",
      "size": 6918
    },
    "tests/test_earnings_docs.py": {
      "sha256": "6e596f81295baaefb70ae514dee2f9e6175c34d146afba1ac27a01412040304a",
      "size": 4884
    },
    "tests/test_media_docs.py": {
      "sha256": "214149d3ddb196a85639600ce7ae75a25add42196a5923d8b95123c36dd54974",
      "size": 6290
    },
    "tests/test_retired_publishing_tools.py": {
      "sha256": "395261b5067573c050ecda46f594530d91d67fc2ec07484540c9adeaa0a2306a",
      "size": 1442
    },
    "tests/test_sql_docs.py": {
      "sha256": "18b2581835f82ce007d78e538e5359d2ad30fa13f37652bea06aba5da21c6872",
      "size": 12090
    },
    "tests/test_trade_scripts.py": {
      "sha256": "d024d95711d2da25f2407626d81422357ded2ec8b3791dd9934dab1140123d04",
      "size": 27782
    }
  }
}

SHA-256 of public snapshot: 2437a378d6bdbf34db5cb4c2c9fed0392b6d6acd0b48c7fc5d12ab4a5fd75c12

Update to FactIQ

Package or technical metadata updated

Package file

Package file

Package file

Package file

Compare saved observations

Changed files

evals/run_ablation.py

removed /files/.app.json

added /files/.claude-plugin~1marketplace.json

added /files/.claude-plugin~1plugin.json

changed /files/.codex-plugin~1plugin.json/sha256

changed /files/.codex-plugin~1plugin.json/size

added /files/.gitignore

added /files/.mcp.json

added /files/LICENSE

added /files/README.md

added /files/SETUP.md

added /files/assets~1factiq-logo.png

added /files/evals~1README.md

added /files/evals~1bilateral-trade-sql~1graders~1one-hs-level-only.md

added /files/evals~1bilateral-trade-sql~1graders~1partner-code-is-numeric.md

added /files/evals~1bilateral-trade-sql~1graders~1used-the-bundled-generator.md

added /files/evals~1bilateral-trade-sql~1graders~1value-series-only.md

added /files/evals~1bilateral-trade-sql~1prompt.md

added /files/evals~1filings-duplicate-line~1graders~1figure-is-not-doubled.md

added /files/evals~1filings-duplicate-line~1graders~1figure-is-the-first-line.md

added /files/evals~1filings-duplicate-line~1graders~1no-invented-filing-links.md

added /files/evals~1filings-duplicate-line~1graders~1source-link-is-the-filing.md

added /files/evals~1filings-duplicate-line~1prompt.md

added /files/evals~1latest-value-direct-answer~1graders~1answers-in-prose.md

added /files/evals~1latest-value-direct-answer~1graders~1no-chart-file-written.md

added /files/evals~1latest-value-direct-answer~1prompt.md

added /files/evals~1mocks~1factiq~1_tools.json

added /files/evals~1mocks~1factiq~1describe_dataset.md

added /files/evals~1mocks~1factiq~1fixtures~1LNS14000000.json

added /files/evals~1mocks~1factiq~1fixtures~1PKG.json

added /files/evals~1mocks~1factiq~1get_data_catalog.md

added /files/evals~1mocks~1factiq~1get_series.md

added /files/evals~1mocks~1factiq~1get_style_guides.md

added /files/evals~1mocks~1factiq~1run_sql.md

added /files/evals~1mocks~1factiq~1search_company_filings.md

added /files/evals~1mocks~1factiq~1search_datasets.md

added /files/evals~1mocks~1factiq~1search_earnings_transcripts.md

added /files/evals~1mocks~1factiq~1search_series.md

added /files/evals~1quick-chart-of-a-trend~1graders~1answer-states-gap-and-source.md

added /files/evals~1quick-chart-of-a-trend~1graders~1chart-file-written.md

added /files/evals~1quick-chart-of-a-trend~1graders~1missing-month-is-not-filled-in.md

added /files/evals~1quick-chart-of-a-trend~1graders~1spec-is-a-valid-chartspec.md

added /files/evals~1quick-chart-of-a-trend~1graders~1title-makes-a-claim.md

added /files/evals~1quick-chart-of-a-trend~1prompt.md

added /files/evals~1quote-and-source-discipline~1graders~1analyst-framing-is-not-management.md

added /files/evals~1quote-and-source-discipline~1graders~1link-fy2025q2-is-the-supplied-one.md

added /files/evals~1quote-and-source-discipline~1graders~1link-fy2026q4-is-the-supplied-one.md

added /files/evals~1quote-and-source-discipline~1graders~1link-fy2027q1-is-the-supplied-one.md

added /files/evals~1quote-and-source-discipline~1graders~1missing-link-is-declared.md

added /files/evals~1quote-and-source-discipline~1graders~1no-invented-source-links.md

added /files/evals~1quote-and-source-discipline~1graders~1quotes-are-verbatim.md

added /files/evals~1quote-and-source-discipline~1prompt.md

added /files/evals~1run_ablation.py

added /files/evals~1unrelated-coding-request~1graders~1answers-the-python-question.md

added /files/evals~1unrelated-coding-request~1graders~1no-catalog-lookup.md

added /files/evals~1unrelated-coding-request~1graders~1no-factiq-skill-invoked.md

added /files/evals~1unrelated-coding-request~1graders~1no-series-fetch.md

added /files/evals~1unrelated-coding-request~1graders~1no-sql-call.md

added /files/evals~1unrelated-coding-request~1prompt.md

added /files/evals~1yoy-across-a-missing-month~1graders~1missing-month-is-stated.md

added /files/evals~1yoy-across-a-missing-month~1graders~1yoy-values-are-date-matched.md

added /files/evals~1yoy-across-a-missing-month~1prompt.md

added /files/evals~1yoy-without-a-server-hint~1graders~1missing-month-is-stated.md

added /files/evals~1yoy-without-a-server-hint~1graders~1yoy-values-are-date-matched.md

added /files/evals~1yoy-without-a-server-hint~1mocks~1factiq~1fixtures~1LNS14000000.json

added /files/evals~1yoy-without-a-server-hint~1mocks~1factiq~1get_series.md

added /files/evals~1yoy-without-a-server-hint~1mocks~1factiq~1run_sql.md

added /files/evals~1yoy-without-a-server-hint~1prompt.md

added /files/references~1data~1satellite.md

added /files/references~1data~1schemas.md

added /files/references~1data~1sql-guide.md

added /files/references~1output~1chart-spec.md

removed `/files/.app.json`

added `/files/.claude-plugin~1marketplace.json`

added `/files/.claude-plugin~1plugin.json`

changed `/files/.codex-plugin~1plugin.json/sha256`

changed `/files/.codex-plugin~1plugin.json/size`

added `/files/.gitignore`

added `/files/.mcp.json`

added `/files/LICENSE`

added `/files/README.md`

added `/files/SETUP.md`

added `/files/assets~1factiq-logo.png`

added `/files/evals~1README.md`

added `/files/evals~1bilateral-trade-sql~1graders~1one-hs-level-only.md`

added `/files/evals~1bilateral-trade-sql~1graders~1partner-code-is-numeric.md`

added `/files/evals~1bilateral-trade-sql~1graders~1used-the-bundled-generator.md`

added `/files/evals~1bilateral-trade-sql~1graders~1value-series-only.md`

added `/files/evals~1bilateral-trade-sql~1prompt.md`

added `/files/evals~1filings-duplicate-line~1graders~1figure-is-not-doubled.md`

added `/files/evals~1filings-duplicate-line~1graders~1figure-is-the-first-line.md`

added `/files/evals~1filings-duplicate-line~1graders~1no-invented-filing-links.md`

added `/files/evals~1filings-duplicate-line~1graders~1source-link-is-the-filing.md`

added `/files/evals~1filings-duplicate-line~1prompt.md`

added `/files/evals~1latest-value-direct-answer~1graders~1answers-in-prose.md`

added `/files/evals~1latest-value-direct-answer~1graders~1no-chart-file-written.md`

added `/files/evals~1latest-value-direct-answer~1prompt.md`

added `/files/evals~1mocks~1factiq~1_tools.json`

added `/files/evals~1mocks~1factiq~1describe_dataset.md`

added `/files/evals~1mocks~1factiq~1fixtures~1LNS14000000.json`

added `/files/evals~1mocks~1factiq~1fixtures~1PKG.json`

added `/files/evals~1mocks~1factiq~1get_data_catalog.md`

added `/files/evals~1mocks~1factiq~1get_series.md`

added `/files/evals~1mocks~1factiq~1get_style_guides.md`

added `/files/evals~1mocks~1factiq~1run_sql.md`

added `/files/evals~1mocks~1factiq~1search_company_filings.md`

added `/files/evals~1mocks~1factiq~1search_datasets.md`

added `/files/evals~1mocks~1factiq~1search_earnings_transcripts.md`

added `/files/evals~1mocks~1factiq~1search_series.md`

added `/files/evals~1quick-chart-of-a-trend~1graders~1answer-states-gap-and-source.md`

added `/files/evals~1quick-chart-of-a-trend~1graders~1chart-file-written.md`

added `/files/evals~1quick-chart-of-a-trend~1graders~1missing-month-is-not-filled-in.md`

added `/files/evals~1quick-chart-of-a-trend~1graders~1spec-is-a-valid-chartspec.md`

added `/files/evals~1quick-chart-of-a-trend~1graders~1title-makes-a-claim.md`

added `/files/evals~1quick-chart-of-a-trend~1prompt.md`

added `/files/evals~1quote-and-source-discipline~1graders~1analyst-framing-is-not-management.md`

added `/files/evals~1quote-and-source-discipline~1graders~1link-fy2025q2-is-the-supplied-one.md`

added `/files/evals~1quote-and-source-discipline~1graders~1link-fy2026q4-is-the-supplied-one.md`

added `/files/evals~1quote-and-source-discipline~1graders~1link-fy2027q1-is-the-supplied-one.md`

added `/files/evals~1quote-and-source-discipline~1graders~1missing-link-is-declared.md`

added `/files/evals~1quote-and-source-discipline~1graders~1no-invented-source-links.md`

added `/files/evals~1quote-and-source-discipline~1graders~1quotes-are-verbatim.md`

added `/files/evals~1quote-and-source-discipline~1prompt.md`

added `/files/evals~1run_ablation.py`

added `/files/evals~1unrelated-coding-request~1graders~1answers-the-python-question.md`

added `/files/evals~1unrelated-coding-request~1graders~1no-catalog-lookup.md`

added `/files/evals~1unrelated-coding-request~1graders~1no-factiq-skill-invoked.md`

added `/files/evals~1unrelated-coding-request~1graders~1no-series-fetch.md`

added `/files/evals~1unrelated-coding-request~1graders~1no-sql-call.md`

added `/files/evals~1unrelated-coding-request~1prompt.md`

added `/files/evals~1yoy-across-a-missing-month~1graders~1missing-month-is-stated.md`

added `/files/evals~1yoy-across-a-missing-month~1graders~1yoy-values-are-date-matched.md`

added `/files/evals~1yoy-across-a-missing-month~1prompt.md`

added `/files/evals~1yoy-without-a-server-hint~1graders~1missing-month-is-stated.md`

added `/files/evals~1yoy-without-a-server-hint~1graders~1yoy-values-are-date-matched.md`

added `/files/evals~1yoy-without-a-server-hint~1mocks~1factiq~1fixtures~1LNS14000000.json`

added `/files/evals~1yoy-without-a-server-hint~1mocks~1factiq~1get_series.md`

added `/files/evals~1yoy-without-a-server-hint~1mocks~1factiq~1run_sql.md`

added `/files/evals~1yoy-without-a-server-hint~1prompt.md`

added `/files/references~1data~1satellite.md`

added `/files/references~1data~1schemas.md`

added `/files/references~1data~1sql-guide.md`

added `/files/references~1output~1chart-spec.md`

added `/files/references~1output~1report-spec.md`

added `/files/references~1report-patterns~1README.md`

added `/files/references~1report-patterns~1bilateral-economic-policy.md`

added `/files/references~1report-patterns~1bilateral-trade.md`

added `/files/references~1report-patterns~1business-formation-statistics.md`

added `/files/references~1report-patterns~1earnings-intelligence.md`

added `/files/references~1report-patterns~1fiscal-policy-revenue.md`

added `/files/references~1report-patterns~1interview-step.md`

added `/files/references~1report-patterns~1media-intelligence.md`