1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210// End-to-end exercise of prepare -> run -> report against the stub in stub.mjs.
//
// What it is for: this harness will run unattended for days and its output is used to
// make decisions about Iris. The failure mode to protect against is not a crash — it
// is a report that looks plausible and counts the wrong things. So the assertions are
// mostly about arithmetic and provenance: that an oversize PDF becomes the right
// chunks, that a URL is only "covered" when every chunk of it came back, that a failed
// run's tokens are still counted, and that an unpriced model produces no dollars.
//
// The local axe re-lint is checked separately, in lint.test.mjs, against the
// deployment's own linter.
import { test } from "node:test";
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { mkdtempSync, readFileSync, writeFileSync, existsSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { startStub } from "./stub.mjs";
const SRC = join(dirname(fileURLToPath(import.meta.url)), "..", "src");
const run = (script, args, env, cwd) =>
new Promise((resolve) => {
execFile(
process.execPath,
[join(SRC, script), ...args],
{ cwd, env: { ...process.env, ...env }, maxBuffer: 32 * 1024 * 1024 },
(err, stdout, stderr) => resolve({ code: err?.code ?? 0, stdout, stderr }),
);
});
const jsonl = (p) =>
readFileSync(p, "utf8")
.trim()
.split("\n")
.filter(Boolean)
.map((l) => JSON.parse(l));
test("prepare, run and report over a stub deployment", async (t) => {
const { base, origin, server } = await startStub({ pages: 7, failNth: 3 });
t.after(() => server.close());
const dir = mkdtempSync(join(tmpdir(), "equalify-iris-bench-"));
const env = { IRIS_BASE_URL: base, IRIS_TOKEN: "stub-token" };
writeFileSync(
join(dir, "urls.csv"),
[
"title,pdf_url,agency",
`Seven pages,${origin}/a.pdf,Test`,
`Same bytes behind another URL,${origin}/copy.pdf,Test`,
`A sign-in wall served as application/pdf,${origin}/signin,Test`,
`Gone,${origin}/missing,Test`,
`A repeated row,${origin}/a.pdf,Test`,
"",
].join("\n"),
);
// --- prepare ---
const prep = await run("prepare.mjs", ["--csv", "urls.csv", "--concurrency", "2"], env, dir);
assert.equal(prep.code, 0, prep.stderr);
// The column is chosen by name and reported, because silently picking the wrong one
// would produce a perfectly plausible run over the wrong 2000 things.
assert.match(prep.stderr, /reading URLs from column 1 \("pdf_url"\)/);
assert.match(prep.stderr, /1 duplicate URL\(s\) collapsed/);
const prepared = jsonl(join(dir, "prepared.jsonl"));
const klass = (k) => prepared.filter((r) => r.klass === k);
assert.equal(klass("not_pdf").length, 1, "an HTML page served as application/pdf is caught by magic bytes");
assert.equal(klass("download_failed").length, 1, "a 404 is a class, not an exception");
assert.equal(klass("duplicate").length, 1, "byte-identical files behind two URLs run once");
assert.equal(klass("oversize_pages").length, 1);
// 7 pages against a cap of 3 is 3+3+1, and every page is accounted for exactly once.
const corpus = jsonl(join(dir, "corpus.jsonl"));
assert.equal(corpus.length, 3);
assert.deepEqual(
corpus.map((r) => [r.page_from, r.page_to]),
[
[1, 3],
[4, 6],
[7, 7],
],
);
assert.equal(
corpus.reduce((s, r) => s + r.pages, 0),
7,
"no page is dropped or double-counted by the split",
);
assert.equal(klass("oversize_pages")[0].chunks_dropped, 0);
for (const c of corpus) assert.ok(existsSync(c.path), `${c.path} was written`);
// Idempotent: a second pass re-fetches nothing.
const again = await run("prepare.mjs", ["--csv", "urls.csv"], env, dir);
assert.match(again.stderr, /4 already prepared; 0 to fetch/);
// --- run ---
const ran = await run("run.mjs", ["--poll-ms", "50"], env, dir);
assert.equal(ran.code, 0, ran.stderr);
const ledger = jsonl(join(dir, "runs", "ledger.jsonl"));
assert.equal(ledger.length, 3);
assert.equal(ledger.filter((r) => r.outcome === "ready_for_review").length, 2);
assert.equal(ledger.filter((r) => r.outcome === "failed").length, 1);
// The failed run keeps its log and diagnostics — the only account of why it failed,
// and the record of what it spent getting there — but has no delivered output.
const failedId = ledger.find((r) => r.outcome === "failed").id;
const failedDir = join(dir, "runs", failedId);
assert.ok(existsSync(join(failedDir, "log.jsonl")));
assert.ok(existsSync(join(failedDir, "diagnostics.json")));
assert.ok(!existsSync(join(failedDir, "output.html")));
// Resumable: nothing in the ledger is submitted twice.
const resumed = await run("run.mjs", ["--poll-ms", "50"], env, dir);
assert.match(resumed.stderr, /skipping 3 already in the ledger/);
assert.match(resumed.stderr, /running 0 of 3/);
// --- report ---
const rep = await run("report.mjs", [], env, dir);
assert.equal(rep.code, 0, rep.stderr);
const s = JSON.parse(rep.stdout);
assert.equal(s.corpus.urls_prepared, 4);
assert.equal(s.corpus.submitted, 3);
assert.equal(s.corpus.chunks, 3);
assert.equal(s.corpus.pages_delivered, 6, "only the two delivered chunks' pages count");
// Two of three items succeeded, but they were chunks of the SAME document and its
// third chunk failed — so no URL was fully delivered. A report that called this 67%
// end-to-end would be overstating what a caller received.
assert.equal(Math.round(s.success_rate * 1000) / 1000, 0.667);
assert.equal(s.corpus.urls_covered, 0);
assert.equal(s.end_to_end_rate, 0);
// Cost: both delivered runs' tokens, priced through the table, with the Bedrock
// partner caveat flagged on the model that produced them.
assert.equal(s.cost.tokens.input, 16400);
assert.equal(s.cost.tokens.output, 3800);
assert.equal(s.cost.unpriced_documents, 0);
assert.equal(s.cost.rates.length, 1);
assert.equal(s.cost.rates[0].normalized, "claude-sonnet-4-6");
assert.equal(s.cost.rates[0].estimate_only, true, "a Bedrock model id is an estimate, not an invoice");
assert.ok(s.cost.rates[0].checked, "a rate without a date is not a measurement");
assert.ok(s.cost.usd_total > 0 && s.cost.usd_total < 1);
// Accuracy: the fixture document carries defects of every duplicate-id kind, so a
// clean verdict here would mean the ported lint config had drifted.
assert.equal(s.accuracy.lint_checked, 2);
assert.equal(s.accuracy.lint_clean, 0);
const rules = s.accuracy.top_rules.map((r) => r.rule);
for (const id of ["duplicate-id", "duplicate-id-active", "duplicate-id-aria", "image-alt"]) {
assert.ok(rules.includes(id), `${id} is reported (config parity with the deployment)`);
}
assert.equal(s.accuracy.links_dropped, 3, "a link the editor dropped is counted on failed runs too");
// Failures are grouped by shape, so one stalled-stream class does not become two
// hundred distinct errors once page numbers and timeouts are masked.
assert.equal(s.failures.length, 1);
assert.equal(s.failures[0].error, "page <n>: stream stalled (idle) after <n>ms");
// Per-document rows are the thing to query afterwards.
const rows = jsonl(join(dir, "runs", "results.jsonl"));
assert.equal(rows.length, 3);
const failedRow = rows.find((r) => r.outcome === "failed");
assert.equal(failedRow.tokens.input, 8200, "a failed run's spend is recorded, not discarded");
assert.equal(failedRow.is_chunk, true);
assert.match(failedRow.error, /stream stalled/);
});
test("a refused token stops the campaign instead of consuming it", async (t) => {
// The multi-day hazard: a GitHub user token can expire mid-run. Without this, every
// remaining item would be submitted, refused, and ledgered as a failure in about a
// minute — and recovering would mean re-running the whole corpus.
const { base, origin, server } = await startStub({ pages: 7, unauthorizedAfter: 1 });
t.after(() => server.close());
const dir = mkdtempSync(join(tmpdir(), "equalify-iris-bench-auth-"));
const env = { IRIS_BASE_URL: base, IRIS_TOKEN: "expired-token" };
writeFileSync(join(dir, "urls.csv"), `pdf_url\n${origin}/a.pdf\n`);
assert.equal((await run("prepare.mjs", ["--csv", "urls.csv"], env, dir)).code, 0);
assert.equal(jsonl(join(dir, "corpus.jsonl")).length, 3, "three chunks, of which only one is accepted");
const ran = await run("run.mjs", ["--poll-ms", "50", "--concurrency", "1"], env, dir);
assert.equal(ran.code, 1, "a refused token is a non-zero exit, not a quiet finish");
assert.match(ran.stderr, /STOPPED: http_401/);
assert.match(ran.stderr, /not attempted and are not in the ledger/);
assert.match(ran.stderr, /login\.mjs/, "says how to recover");
// Only the item that actually ran is ledgered, so a fresh token resumes at the
// exact point the old one stopped being accepted.
assert.deepEqual(
jsonl(join(dir, "runs", "ledger.jsonl")).map((r) => r.outcome),
["ready_for_review"],
);
});
test("an unpriced model yields no dollars and says so", async () => {
const { costOf } = await import("../src/pricing.mjs");
const tokens = { input: 1000, output: 1000, cache_read: 0, cache_write: 0 };
const unknown = costOf(tokens, "us.anthropic.claude-whatever-9");
assert.equal(unknown.usd, null, "an unknown model costs null, never zero");
assert.equal(unknown.priced, false);
// The four counts bill at four rates; cache reads at a tenth of input, cache writes
// at 1.25x. Iris sends no cache_control today, so this exists to stay correct when
// it does.
const all = costOf({ input: 1e6, output: 1e6, cache_read: 1e6, cache_write: 1e6 }, "claude-sonnet-4-6");
assert.equal(all.usd, 3 + 15 + 0.3 + 3.75);
});