1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627import { JSDOM, VirtualConsole } from "jsdom";
import { mapWithConcurrency } from "../util/concurrency.ts";
import type { AgentSpec } from "../agents/loader.ts";
import type { InputImage, PipelineContext } from "./context.ts";
import { verifyAgentOutput, VERIFY_KINDS, type VerifyKind, type VerifyVerdict } from "./feedback.ts";
// Does the fidelity verifier DISCRIMINATE? (issue #180)
//
// Fidelity is checked exactly once per page, by the Feedback Agent's VERIFY task, and
// everything the pipeline claims about accuracy rests on that verdict. Every measurement of
// it so far puts its rejection rate near four pages in five: 58 of 75 across three 25-page
// runs (#137, cited in `correction.ts`), then 76 of 100 and 74 of 94 in two benchmark rounds
// (#182, cited in `test/verify-kinds.test.ts`). Two explanations fit that number equally
// well — the extraction really does need correcting on most pages, or the verifier is
// calibrated to find something and finds something. The verdict cannot answer that question
// about itself, which `correction.ts` says in as many words.
//
// So this asks it from outside. Take pages the verifier passed, damage one thing in a copy of
// each, and put both copies back to the same verifier against the same image. Two rates come
// out — how often it passes a clean page, and how often it catches a defect it was handed —
// and the per-defect breakdown is the actionable part either way: "it catches dropped tables
// and misses changed numbers" is a sentence about `agents/feedback.md` that no aggregate
// rejection rate can produce.
//
// This file is the measurement, not a gate. Nothing in the pipeline imports it; `src/tools`
// has the CLI that runs it and prints the report.
//
// The damage is done to HTML rather than to the image on purpose: the verifier's job is to
// compare an agent's output against the page, so the injected defect has to be a defect OF
// THE OUTPUT. Every injector below is a change a real extraction failure produces — a row
// that did not survive, a number transcribed wrong, a heading flattened, alt text dropped, a
// page returned in part.
// A defect that can be injected into a page's HTML.
export interface DefectSpec {
id: string;
// What was done, one line, for the report. Read alongside the caught/missed counts.
what: string;
// The `kind` a verifier that actually saw this defect should tag its problem with
// (`VERIFY_KINDS`). More than one where the honest answer is more than one: a dropped
// heading is content that is gone AND structure that changed, and `agents/feedback.md`
// tells the agent the earliest applicable kind wins, so both are correct tags.
//
// This is the weaker of the two signals reported and is treated as such: a verifier that
// rejects the damaged copy has caught it, and one that also tags it the way this list
// predicts has named it. A mismatch here is a labelling disagreement, not a miss.
expects: VerifyKind[];
// The damaged copy, or null where the page has no such structure to damage. Never a
// silent no-op: a returned string is always different from its input, and the report
// counts how many pages each defect could not be applied to.
damage(html: string): string | null;
}
// A page opened for damage, with the string its own untouched parse serializes to.
//
// The baseline is the whole point of returning a pair. The comparison that decides whether
// an injector did anything has to be like for like: jsdom re-serializes a fragment even
// when nothing was mutated — a `<table>` without `<tbody>` gains one, `—` becomes an
// em dash, attribute quoting and case are normalized — so comparing the output against the
// RAW fragment passes any injector that merely parsed the page. Comparing against the
// clean parse's own serialization measures the mutation and nothing else.
interface Opened {
doc: Document;
baseline: string;
}
function parse(html: string): Opened | null {
try {
const doc = new JSDOM(`<body>${html}</body>`, { virtualConsole: new VirtualConsole() }).window.document;
return { doc, baseline: doc.body.innerHTML };
} catch {
return null;
}
}
// The injectors all end here: serialize, and refuse the case where the edit changed
// nothing. An injector that silently returns its input would be counted as a defect the
// verifier missed, which is the one direction of error this whole measurement cannot
// afford — it would read as the verifier failing a test it was never given.
function serialize(open: Opened): string | null {
const out = open.doc.body.innerHTML;
return out.trim() && out !== open.baseline ? out : null;
}
// Elements in document order, as a plain array (a NodeList is live for some queries and
// the injectors mutate as they go).
function all<T extends Element>(doc: Document, selector: string): T[] {
return Array.from(doc.querySelectorAll(selector)) as T[];
}
const HEADINGS = "h1, h2, h3, h4, h5, h6";
// An element's text with whitespace collapsed, for the one injector that has to ask whether
// a reader could tell the difference.
function text(el: Element): string {
return (el.textContent ?? "").replace(/\s+/g, " ").trim();
}
// Rows that carry data rather than headers. A row of nothing but `<th>` is the header row,
// and dropping it is a different defect (structure, not content) that this list does not
// claim to inject.
function bodyRows(table: Element): Element[] {
return Array.from(table.querySelectorAll("tr")).filter((r) => {
if (r.closest("table") !== table) return false; // nested table's row
const cells = Array.from(r.children).filter((c) => /^(td|th)$/i.test(c.tagName));
return cells.length > 0 && !cells.every((c) => c.tagName.toLowerCase() === "th");
});
}
// Rename an element in place, keeping its attributes, its children and its position. Used
// for the heading demotion, where replacing the element is the only way to change its level.
function rename(doc: Document, el: Element, tag: string): Element {
const next = doc.createElement(tag);
for (const attr of Array.from(el.attributes)) next.setAttribute(attr.name, attr.value);
while (el.firstChild) next.appendChild(el.firstChild);
el.parentNode?.replaceChild(next, el);
return next;
}
// The fixed list from the issue. Fixed on purpose: a defect list that grows with what the
// verifier turns out to miss measures the list rather than the verifier.
export const DEFECTS: DefectSpec[] = [
{
id: "drop_table_row",
what: "the last data row of the first table is removed",
// The row's words are gone from the document; nothing about it is merely restructured.
expects: ["content_missing"],
damage(html) {
const open = parse(html);
if (!open) return null;
const { doc } = open;
for (const table of all(doc, "table")) {
const rows = bodyRows(table);
// Two, so the table still reads as a table afterwards: removing the only data row
// leaves a header with nothing under it, which is a different defect and one the
// "drop the whole table" case below already covers better.
if (rows.length < 2) continue;
rows[rows.length - 1].remove();
return serialize(open);
}
return null;
},
},
{
id: "drop_table",
what: "the first table is removed entirely",
expects: ["content_missing"],
damage(html) {
const open = parse(html);
if (!open) return null;
const { doc } = open;
const table = doc.querySelector("table");
if (!table) return null;
table.remove();
return serialize(open);
},
},
{
id: "change_cell_number",
what: "a number in a table cell is changed to a different number",
// The words are all still there and the structure is untouched; one of them is false.
// This is the defect a reader cannot detect from the document alone, and the one that
// matters most in the documents Iris takes as input — a torque figure, a dose, a price.
expects: ["content_wrong"],
damage(html) {
const open = parse(html);
if (!open) return null;
const { doc } = open;
for (const cell of all(doc, "td, th")) {
const text = cell.textContent ?? "";
const m = /\d+/.exec(text);
if (!m) continue;
// The last digit, moved by one, so the change is a plausible transcription error
// rather than a nonsense string: 3 -> 4, 250 -> 251, 2019 -> 2018 (9 wraps down so
// the digit count never changes and no leading zero appears).
const digits = m[0];
const last = Number(digits[digits.length - 1]);
const moved = digits.slice(0, -1) + String(last === 9 ? 8 : last + 1);
if (moved === digits) continue;
// Replaced in the cell's own text nodes, so markup inside the cell survives.
const walk = (node: Node): boolean => {
if (node.nodeType === 3) {
const t = node.textContent ?? "";
const at = t.indexOf(digits);
if (at === -1) return false;
node.textContent = t.slice(0, at) + moved + t.slice(at + digits.length);
return true;
}
for (const child of Array.from(node.childNodes)) if (walk(child)) return true;
return false;
};
if (walk(cell)) return serialize(open);
}
return null;
},
},
{
id: "drop_heading",
what: "a heading is removed, leaving the content that was under it",
// Its words are gone and the section it opened has lost its boundary. Either tag is a
// verifier that saw the defect.
expects: ["content_missing", "structure_wrong"],
damage(html) {
const open = parse(html);
if (!open) return null;
const { doc } = open;
const headings = all(doc, HEADINGS);
if (!headings.length) return null;
// The second where there is one: removing the only heading on a page is also the
// hardest case to attribute, since a page whose title is its first line legitimately
// renders without one.
(headings[1] ?? headings[0]).remove();
return serialize(open);
},
},
{
id: "demote_heading",
what: "a heading is demoted two levels, breaking the nesting order",
// Every word survives; what changes is where a reader navigating by heading is told
// they are. axe reports a skipped level, so this is also the one defect on the list
// that the gate can catch without the verifier — which is worth knowing separately.
expects: ["structure_wrong"],
damage(html) {
const open = parse(html);
if (!open) return null;
const { doc } = open;
for (const h of all(doc, HEADINGS)) {
const level = Number(h.tagName[1]);
if (level > 4) continue; // no room to demote by two
rename(doc, h, `h${level + 2}`);
return serialize(open);
}
return null;
},
},
{
id: "remove_alt",
what: "the alt text of an image is removed",
// `alt_quality` is the kind for alt text that is thin or wrong; an image with no alt
// attribute at all is the accessibility defect, and both are verdicts that saw it.
expects: ["a11y_only", "alt_quality"],
damage(html) {
const open = parse(html);
if (!open) return null;
const { doc } = open;
// A non-empty alt: `alt=""` is correct markup for a decorative image, so removing
// that one is a defect the verifier is right to weigh differently.
const img = all(doc, "img").find((i) => (i.getAttribute("alt") ?? "").trim());
if (!img) return null;
img.removeAttribute("alt");
return serialize(open);
},
},
{
id: "swap_paragraphs",
what: "two neighbouring paragraphs are swapped",
// Reading order, which is the property this pipeline exists to protect and the one a
// word-counting check cannot see: both paragraphs are present, in the wrong order.
expects: ["structure_wrong", "content_wrong"],
damage(html) {
const open = parse(html);
if (!open) return null;
const { doc } = open;
for (const p of all(doc, "p")) {
const next = p.nextElementSibling;
if (!next || next.tagName.toLowerCase() !== "p") continue;
// Two paragraphs that read the same are the one pair whose order carries nothing:
// a page whose columns both end in "(continued)" would be swapped, serialize to a
// different string, and be scored as a reading-order defect the verifier missed —
// for a document that reads identically either way. The verifier would be right,
// and the measurement wrong, which is the error this file cannot afford.
if (!text(p) || !text(next) || text(p) === text(next)) continue;
p.parentNode?.insertBefore(next, p);
return serialize(open);
}
return null;
},
},
{
id: "truncate_tail",
what: "the last third of the page's top-level blocks is dropped",
// The shape a page takes when the model runs out of output tokens, which is a real and
// measured failure of this pipeline (#135, #159) and the one the page prompt asks for a
// [page not fully transcribed] marker about. Here it arrives with no marker.
expects: ["content_missing"],
damage(html) {
const open = parse(html);
if (!open) return null;
const { doc } = open;
// Top level as the model wrote it, and one level in where the page is wrapped in a
// single container (`<article>`, `<main>`, a `<div>`) — otherwise the whole page is
// one child and the third to drop is either nothing or everything.
let blocks = Array.from(doc.body.children);
while (blocks.length === 1 && blocks[0].children.length > 1) blocks = Array.from(blocks[0].children);
if (blocks.length < 3) return null;
const keep = Math.ceil((blocks.length * 2) / 3);
for (const el of blocks.slice(keep)) el.remove();
return serialize(open);
},
},
];
// One page's own output, as the verifier passed it. `html` is the page fragment (a
// fragment's `innerHtml`, which is what `verifyAgentOutput` is given in the pipeline).
export interface CalibrationPage {
image: InputImage;
html: string;
// The contract this page's HTML was actually written to, where it is not the current
// one. VERIFY is handed the agent's whole contract and judges the output against it, so
// a page extracted under an older `agents/page.md` and judged against today's can be
// rejected for breaking a rule that did not exist when it was written — which is the
// verifier being right, and would be counted here as a false positive. The verifier
// itself (`agents/feedback.md`) is always the current one: today's judge is what is
// being measured.
agent?: AgentSpec;
}
// What one VERIFY call said, flattened to what this measurement reads.
export interface Judgement {
ok: boolean;
problems: string[];
kinds: VerifyKind[];
untagged: number;
// The call produced no judgement at all — no Feedback Agent, nothing to verify, a reply
// that could not be parsed, or a call that threw (those are also listed in
// `CalibrationReport.errors`). `verifyAgentOutput` answers ok=true in the first three so
// that verification never breaks a run, which means "passed" and "could not be judged"
// are the same observation at that interface. Counting the second as a pass would
// overstate exactly the number this file exists to measure, so it is carried separately
// and excluded from both rates.
unjudged: boolean;
}
export interface CalibrationRow {
image: string;
// The blob SHA of the contract this page was judged against, or null for one with no
// upstream object. Recorded per row because it can differ per page: a corpus pooled from
// several sessions is a corpus of several contracts.
contract: string | null;
clean: Judgement;
// Absent where no defect on the list applies to this page — a page of prose has no table
// row to drop. Never a silent skip: `skipped` says which pages and why.
defect?: string;
damaged?: Judgement;
skipped?: string;
}
export interface DefectTally {
applied: number;
// The damaged copy was rejected: ok=false with at least one problem named, which is the
// same test `failedCheck` applies before the pipeline spends a correction call.
caught: number;
// Rejected AND tagged with one of the kinds this defect predicts.
named: number;
// The verifier DESCRIBED a problem and still answered faithful/accessible true, so
// `failedCheck` reads the page as having nothing to correct and it ships unquestioned.
//
// Kept apart from both `caught` and the misses because it is neither, and because it is
// the most actionable number this harness produces: a verifier that cannot see a defect
// needs a better prompt about that defect, while one that sees it, writes it down, and
// then ticks "faithful" needs the two-boolean contract fixed. The first run of this found
// three of its five apparent misses were this — including a paragraph-order error the
// verifier described in full, quoting both paragraphs.
describedOnly: number;
unjudged: number;
}
export interface CalibrationReport {
pages: number;
rows: CalibrationRow[];
clean: { passed: number; failed: number; unjudged: number; describedOnly: number };
perDefect: Record<string, DefectTally>;
// Pages no defect applied to, with the reason. Read with `pages`: a report over 20 pages
// where 6 were skipped is a report over 14.
skipped: { image: string; reason: string }[];
// Calls that threw — a throttled request, a timeout, a provider that refused. Each one is
// counted unjudged above and listed here, because a run whose numbers rest on 40 calls of
// which 12 never happened is a different run and only this says so.
errors: { image: string; defect: string | null; message: string }[];
}
// The same test the pipeline applies before it spends a correction call
// (`failedCheck` in extraction.ts): a verdict with no problems in it is not actionable,
// whatever the flag says.
function rejected(j: Judgement): boolean {
return !j.unjudged && !j.ok && j.problems.length > 0;
}
// The verifier wrote a problem down and passed the page anyway.
function describedOnly(j: Judgement): boolean {
return !j.unjudged && j.ok && j.problems.length > 0;
}
function judge(v: VerifyVerdict): Judgement {
return { ok: v.ok, problems: v.problems, kinds: v.kinds, untagged: v.untagged, unjudged: v.unjudged === true };
}
export interface CalibrateOptions {
// How defects are handed out. "rotate" gives each page one defect, cycling through the
// list in order, which is the 2N calls the issue costs out. "all" applies every
// applicable defect to every page — a fuller per-defect breakdown for (1 + defects) calls
// a page, which on eight defects is nine times the bill.
defects?: "rotate" | "all";
// Which defects to consider, by id. Defaults to all of `DEFECTS`.
only?: string[];
// Model calls in flight, as elsewhere in the pipeline.
concurrency?: number;
}
// Run the calibration. Every page costs one verify call for the clean copy plus one per
// damaged copy, and nothing here is cached: the point is a fresh verdict on each.
export async function calibrateVerifier(
ctx: PipelineContext,
agent: AgentSpec,
pages: CalibrationPage[],
opts: CalibrateOptions = {},
): Promise<CalibrationReport> {
const list = opts.only?.length ? DEFECTS.filter((d) => opts.only!.includes(d.id)) : DEFECTS;
const mode = opts.defects ?? "rotate";
const limit = Math.max(1, Math.floor(opts.concurrency ?? ctx.extractionConcurrency) || 1);
// Which defects to try on which page, decided before any call so the plan is reportable
// and so a rotation is a rotation rather than whatever order the calls happened to
// finish in. In "rotate" mode the offset walks with the page index, so a corpus where
// half the pages have no table still spreads the other defects over the pages that do.
const plan = pages.map((page, i) => {
const applicable: { defect: DefectSpec; html: string }[] = [];
for (let k = 0; k < list.length; k++) {
const defect = list[(i + k) % list.length];
const damaged = defect.damage(page.html);
if (!damaged) continue;
applicable.push({ defect, html: damaged });
if (mode === "rotate") break;
}
return { page, applicable };
});
const perDefect: Record<string, DefectTally> = {};
for (const d of list) perDefect[d.id] = { applied: 0, caught: 0, named: 0, describedOnly: 0, unjudged: 0 };
// One unit of work per verify call, so the whole run is bounded by `limit` rather than by
// `limit` pages each issuing several calls at once.
type Call = { pageIndex: number; defect?: DefectSpec; html: string };
const calls: Call[] = [];
plan.forEach((p, pageIndex) => {
calls.push({ pageIndex, html: p.page.html });
for (const a of p.applicable) calls.push({ pageIndex, defect: a.defect, html: a.html });
});
// A call that failed outright, kept so the report can say so. `mapWithConcurrency`
// rejects on the first failing call, and a run of this is paid for call by call: one
// throttled or timed-out request an hour in would otherwise throw away every verdict
// already bought. So each call catches its own error and comes back unjudged, which is
// the honest reading — nothing judged that copy — and is already excluded from both
// rates. Counted separately from an unparseable reply because the fix is different: one
// is a provider to retry, the other is a verifier to fix.
const errors: { image: string; defect: string | null; message: string }[] = [];
const verdicts = await mapWithConcurrency(calls, limit, async (call) => {
const page = plan[call.pageIndex].page;
try {
// The page's own contract where it has one, so a clean copy is judged against the
// rules it was written to and not against rules added since.
const verdict = await verifyAgentOutput(
ctx,
page.agent ?? agent,
page.image,
[{ html: call.html }],
"agent_calibrate",
);
return judge(verdict);
} catch (e) {
const message = e instanceof Error ? e.message : String(e);
const defect = call.defect?.id ?? null;
errors.push({ image: page.image.name, defect, message });
ctx.log.event("calibrate_call_failed", { image: page.image.name, defect, error: message });
return { ok: true, problems: [], kinds: [], untagged: 0, unjudged: true };
}
});
const rows: CalibrationRow[] = [];
const clean = { passed: 0, failed: 0, unjudged: 0, describedOnly: 0 };
const skipped: { image: string; reason: string }[] = [];
plan.forEach((p, pageIndex) => {
const own = calls.map((c, i) => ({ c, j: verdicts[i] })).filter(({ c }) => c.pageIndex === pageIndex);
const cleanJudgement = own.find(({ c }) => !c.defect)!.j;
const contract = (p.page.agent ?? agent).sha;
if (cleanJudgement.unjudged) clean.unjudged += 1;
else if (rejected(cleanJudgement)) clean.failed += 1;
else {
clean.passed += 1;
// Still counted as passed, because that is what the pipeline does with it. Counted
// here as well, because a clean page the verifier wrote complaints about and then
// passed is not the same event as one it had nothing to say about.
if (describedOnly(cleanJudgement)) clean.describedOnly += 1;
}
if (!p.applicable.length) {
const reason = "no defect on the list applies to this page";
skipped.push({ image: p.page.image.name, reason });
rows.push({ image: p.page.image.name, contract, clean: cleanJudgement, skipped: reason });
return;
}
for (const { c, j } of own) {
if (!c.defect) continue;
const tally = perDefect[c.defect.id];
tally.applied += 1;
if (j.unjudged) tally.unjudged += 1;
else if (rejected(j)) {
tally.caught += 1;
if (c.defect.expects.some((k) => j.kinds.includes(k))) tally.named += 1;
} else if (describedOnly(j)) tally.describedOnly += 1;
rows.push({ image: p.page.image.name, contract, clean: cleanJudgement, defect: c.defect.id, damaged: j });
}
});
return { pages: pages.length, rows, clean, perDefect, skipped, errors };
}
const pct = (n: number, of: number): string => (of === 0 ? "n/a" : `${Math.round((n / of) * 100)}%`);
// The report as text. Written to be readable in a terminal and quotable into the issue,
// which is what it is for: the numbers are the deliverable, not a threshold anything
// compares against.
export function formatCalibration(r: CalibrationReport): string {
const judged = r.clean.passed + r.clean.failed;
const out: string[] = [];
out.push(`Pages: ${r.pages} (${judged} judged, ${r.clean.unjudged} unjudged)`);
out.push(
`Clean copies: ${r.clean.passed} passed, ${r.clean.failed} rejected` +
` — false-positive rate ${pct(r.clean.failed, judged)}`,
);
if (r.clean.describedOnly) {
out.push(
` (${r.clean.describedOnly} of the passes named a problem and answered faithful anyway — see below)`,
);
}
out.push("");
out.push("Defect applied caught named said-not-flagged unjudged");
const totals = { applied: 0, caught: 0, named: 0, describedOnly: 0, unjudged: 0 };
for (const d of DEFECTS) {
const t = r.perDefect[d.id];
if (!t) continue;
totals.applied += t.applied;
totals.caught += t.caught;
totals.named += t.named;
totals.describedOnly += t.describedOnly;
totals.unjudged += t.unjudged;
const rate = t.applied - t.unjudged > 0 ? ` (${pct(t.caught, t.applied - t.unjudged)})` : "";
out.push(
`${d.id.padEnd(22)} ${String(t.applied).padStart(7)} ${String(t.caught).padStart(7)}` +
` ${String(t.named).padStart(6)} ${String(t.describedOnly).padStart(17)}` +
` ${String(t.unjudged).padStart(8)}${rate}`,
);
}
const judgedDamaged = totals.applied - totals.unjudged;
out.push("");
out.push(
`Damaged copies: ${totals.caught} of ${judgedDamaged} caught (${pct(totals.caught, judgedDamaged)})` +
`, ${totals.named} tagged with a kind the defect predicts`,
);
// The distinction that decides what to do about a miss, so it is spelled out rather than
// left as a column heading. `failedCheck` (extraction.ts) needs ok=false AND a non-empty
// problem list, so a verdict in this column costs the page nothing: the defect was seen,
// written down, and shipped.
if (totals.describedOnly) {
const seen = totals.caught + totals.describedOnly;
out.push(
`Of the ${judgedDamaged - totals.caught} not caught, ${totals.describedOnly} were DESCRIBED and ` +
`flagged faithful anyway — the verifier saw ${seen} of ${judgedDamaged} ` +
`(${pct(seen, judgedDamaged)}) and only flagged ${totals.caught}. ` +
`failedCheck needs both, so those pages ship unquestioned.`,
);
}
// Said out loud rather than left to be inferred from the counts: a defect that never got
// applied has not been measured, and a corpus that skipped a third of its pages is a
// smaller corpus than the page count above.
//
// "in this run" and not "no page had the structure": in rotate mode a page stops at the
// first defect that applies to it, so a zero here can mean the corpus had nowhere to put
// this defect OR that the rotation never got to it. The tool's dry run separates those
// two; a reader of the report only needs to know the row was not measured.
//
// Only over the defects this run considered, which is what `perDefect` has a key for: a
// `--only drop_table` run listing the other seven as "never applied" would be reporting
// its own argument back as a gap.
const never = DEFECTS.filter((d) => r.perDefect[d.id] && r.perDefect[d.id].applied === 0).map((d) => d.id);
if (never.length) out.push(`Never applied in this run (not measured): ${never.join(", ")}`);
if (r.skipped.length) out.push(`Pages with no applicable defect: ${r.skipped.length} of ${r.pages}`);
if (totals.unjudged || r.clean.unjudged) {
out.push(
`Unjudged calls are excluded from every rate above: ${r.clean.unjudged} clean, ${totals.unjudged} damaged.`,
);
}
// First and worst: a run that lost calls to the provider has smaller denominators than
// its page count suggests, and the message is what says whether re-running would help.
if (r.errors.length) {
const shown = r.errors.slice(0, 3);
out.push(
`${r.errors.length} call${r.errors.length === 1 ? " failed and is" : "s failed and are"} counted unjudged: ` +
shown.map((e) => `${e.image}${e.defect ? `/${e.defect}` : " (clean)"}: ${e.message}`).join("; ") +
(r.errors.length > shown.length ? ` (and ${r.errors.length - shown.length} more)` : ""),
);
}
// Which contract each page was judged against. Not decoration: the clean-copy rate is
// only a false-positive rate if the pages were judged against the rules they were
// written to, and a corpus pooled from several sessions can be a corpus of several
// contracts. A reader comparing two runs of this needs to know it.
const contracts = [...new Set(r.rows.map((row) => row.contract ?? "(no upstream object)"))];
out.push(`Contract judged against: ${contracts.map((c) => c.slice(0, 12)).join(", ")}`);
return out.join("\n");
}
// Exported for the tool's `--defects` argument, so an unknown id fails before any model
// call rather than silently narrowing the run.
export const DEFECT_IDS: string[] = DEFECTS.map((d) => d.id);
// Kept honest against `VERIFY_KINDS`: a defect predicting a kind the verifier's contract
// does not define would never be counted as named, and the report would read as the
// verifier failing to tag rather than as this file naming a kind that does not exist.
for (const d of DEFECTS) {
for (const k of d.expects) {
if (!VERIFY_KINDS.includes(k)) throw new Error(`defect ${d.id} expects unknown verify kind "${k}"`);
}
}