1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173import { existsSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { extractJson } from "../util/json.ts";
import { mapWithConcurrency } from "../util/concurrency.ts";
import { loadAgent, type AgentSpec } from "../agents/loader.ts";
import { ACCESSIBILITY_REQUIREMENTS } from "./accessibility.ts";
import { loadImage, type InputImage, type PipelineContext } from "./context.ts";
import { flatten } from "./flatten.ts";
import { knownPages, pageIndex } from "./pageindex.ts";
import { createAgentUpdateIssue, installHintFor } from "../github/issue.ts";
import { lessonSlug, recordExample, type CorrectionExample, type LessonKind } from "./memory.ts";
import type { FixtureCase } from "./regression.ts";
import type { PipelineStep } from "../providers/index.ts";
// Previously imported from github/contributions.ts, which was removed when the
// contribution model moved from close-time PRs to issues filed during the run.
// The feedback loop still records proposed library-agent updates in this shape
// (in agent-updates.md, gated by regression fixtures) for maintainer review.
export interface AgentUpdateContribution {
agent_name: string; // e.g. "page.md"
summary: string;
diff_preview: string;
content: string; // full updated agent file content
}
const FEEDBACK_AGENT = "feedback";
interface TrainOutput {
changed?: boolean;
summary?: string;
agent_markdown?: string;
}
interface VerifyOutput {
faithful?: boolean;
accessible?: boolean;
// Read as `unknown` and narrowed in `readProblems`: this is model output, and the two
// shapes it arrives in (a list of strings, a list of `{kind, problem}` objects) are both
// valid replies to a contract that has said both things — see `readProblems`.
problems?: unknown;
// `notes` is deliberately NOT here. The verify contract gives the conclusion of a reading the
// agent ruled out a one-line home in a `notes` string precisely so that it lands somewhere
// nothing acts on, and the prompt tells the model as much in so many words: "read by nothing:
// no correction pass, no other agent, no part of the delivered document". #365 narrowed what
// the field asks for — a conclusion, not the working-out, since the same task now carries the
// Reader's "Do the thinking without writing it down" — and narrowed nothing about this: a
// field read by nothing is the point, and one line of it is as unreadable as ten.
// Adding it to this interface is the first half of
// breaking that promise — the reply's prose reached the corrector before, on 14 of 71
// rejections in a 45-page control round, and `problems` is the only thing `correctPage` is
// licensed to change (issue #339). If a future reader wants that text, it is already
// persisted verbatim on the `agent_call` line and can be read there without giving the
// pipeline a path to it. Declaring the field would not itself fail a test — the pin in
// `test/verify-notes-field.test.ts` is behavioural, and fails as soon as anything READS it.
}
interface ClassifyOutput {
kind?: string;
instruction?: string;
before?: string;
after?: string;
}
// What kind of problem VERIFY named, in the order a reader loses by (agents/feedback.md
// defines each one and tells the agent that the earliest applicable kind wins). A closed
// list: `verify_failed` was a count of pages the verifier had an opinion about, in which a
// page that lost three table rows and a page whose alt text was refined from "orange kayak"
// to "orange-yellow kayak" were the same line (issue #182). `correctionEffect` recovers half
// of that after the fact, but it can only classify corrections that HAPPENED — so a page
// flagged and left materially unchanged reads like a page that never needed anything. This
// is the other half: what was wrong going in.
export const VERIFY_KINDS = [
"content_missing",
"content_wrong",
"structure_wrong",
"a11y_only",
"alt_quality",
] as const;
export type VerifyKind = (typeof VERIFY_KINDS)[number];
export interface VerifyVerdict {
ok: boolean;
// The problems as prose, unchanged: this is what the correction pass is given and what
// the regression gate reports, and both want the sentence and not the label.
problems: string[];
// The DISTINCT kinds named, in `VERIFY_KINDS` order. A set and not a per-problem list,
// because that is what a tally over a fleet of runs can be read from — three missing rows
// and one thin alt is a page that lost content, whether the verifier wrote it as two
// problems or five.
kinds: VerifyKind[];
// How many of `problems` carried no kind this code recognizes. The auditability field: a
// split computed over problems where half arrived untagged is a split that lies about
// which half it measured, and nothing else on the line would say so.
untagged: number;
// Nothing was judged: no Feedback Agent, nothing to verify, or a reply that would not
// parse. `ok` is true in all three because verification is non-blocking and must never
// cost a page (see `verifyAgentOutput`) — which means a page that passed and a page that
// could not be looked at arrive here identical, and any rate computed over `ok` counts
// the second as the first.
//
// Additive and optional: every existing reader tests `ok` and `problems`, and an absent
// field leaves all of them saying exactly what they said before. It is set so that a
// measurement OF the verifier can exclude the calls that were not verdicts — the
// difference matters to `src/pipeline/calibration.ts` and to nothing in the run itself.
unjudged?: true;
}
// The verdict for a page nothing looked at. A function and not a shared constant because
// `problems` and `kinds` are arrays: one frozen object would hand every caller the same two
// arrays, and `Object.freeze` is shallow, so a single `push` anywhere would rewrite the default
// for the whole process.
//
// Three callers, which is why it stopped being a literal: `verifyAgentOutput` returns it when
// there is no Feedback Agent and when the reply will not parse, and `extractPage` returns it for
// a page the agent declared blank and no verify call was bought for (#294). Written out twice it
// was already the same shape by coincidence; a fourth reader of `unjudged` would have had to
// trust that.
export function unjudgedVerdict(): VerifyVerdict {
return { ok: true, problems: [], kinds: [], untagged: 0, unjudged: true };
}
// Read VERIFY's `problems` out of a reply that may predate the kinds, may be a session-built
// or trained agent file whose contract still says `["..."]`, or may simply have answered in
// strings anyway. A string entry is a problem with no kind — never a dropped problem, since
// dropping it would turn a page the verifier rejected into a page that passed (`failedCheck`
// needs a non-empty list) and ship the fragment unquestioned. Same rule for an object whose
// text this cannot find: it is stringified rather than lost, which is what the prose list did
// with it before, minus the "[object Object]" a correction prompt used to be handed.
//
// A kind outside `VERIFY_KINDS` is `untagged`, not a sixth bucket. The five are the contract;
// counting invented ones would make the tally a list of the words a model chose.
function readProblems(raw: unknown): Pick<VerifyVerdict, "problems" | "kinds" | "untagged"> {
if (!Array.isArray(raw)) return { problems: [], kinds: [], untagged: 0 };
const problems: string[] = [];
const seen = new Set<VerifyKind>();
let untagged = 0;
for (const entry of raw) {
let text: string;
let kind: VerifyKind | undefined;
// A null entry is not a problem the verifier named, so it is not one here either. That
// can leave a `faithful: false` verdict with an empty list, which `failedCheck` already
// reads as "not actionable" — the rule this code has always applied to an agent that set
// the flag and named nothing.
if (entry === null || entry === undefined) {
continue;
} else if (typeof entry === "string") {
text = entry;
} else if (typeof entry === "object") {
const rec = entry as Record<string, unknown>;
const prose = [rec.problem, rec.text, rec.description].find((v) => typeof v === "string" && v.trim());
text = typeof prose === "string" ? prose : JSON.stringify(entry);
const named = typeof rec.kind === "string" ? rec.kind.trim().toLowerCase().replace(/[\s-]+/g, "_") : "";
kind = VERIFY_KINDS.find((k) => k === named);
} else {
text = String(entry);
}
if (!text.trim()) continue;
problems.push(text.trim());
if (kind) seen.add(kind);
else untagged += 1;
}
return { problems, kinds: VERIFY_KINDS.filter((k) => seen.has(k)), untagged };
}
function loadFeedbackAgent(ctx: PipelineContext): AgentSpec | null {
return loadAgent(FEEDBACK_AGENT, {
agentsDir: ctx.paths.agentsDir,
tmpAgentsDir: ctx.paths.tmpAgentsDir(ctx.sessionId),
});
}
function loadTargetAgent(ctx: PipelineContext, agentFile: string): AgentSpec | null {
return loadAgent(agentFile, {
agentsDir: ctx.paths.agentsDir,
tmpAgentsDir: ctx.paths.tmpAgentsDir(ctx.sessionId),
});
}
// A naive line-level diff (added/removed lines) for the update PR body and as the
// "correction" context handed to the Feedback Agent. Not a true minimal diff —
// just enough to show what changed.
function diffPreview(before: string, after: string, maxLines = 80): string {
const a = new Set(before.split("\n"));
const b = new Set(after.split("\n"));
const removed = before.split("\n").filter((l) => !b.has(l)).map((l) => `- ${l}`);
const added = after.split("\n").filter((l) => !a.has(l)).map((l) => `+ ${l}`);
const lines = [...removed, ...added];
return (
lines.slice(0, maxLines).join("\n") +
(lines.length > maxLines ? `\n… (${lines.length - maxLines} more changed lines)` : "")
);
}
// ---------------------------------------------------------------------------
// Build-time verification (source-fidelity)
// ---------------------------------------------------------------------------
// A model-written log, made safe to interpolate into the verify message.
//
// The message is STRUCTURED, and the verifier reads that structure: `##` headings separate the
// contract from the output from the annotations, and the output arrives inside a ```html fence. So
// a string that can begin a line can forge a heading, and a single backtick opens inline code that
// swallows the punctuation after it. That is the argument `oneLine` makes for the specialist caution
// in extraction.ts, where the caution's own producer flattens it; this one is flattened HERE rather
// than at the call site, so the calibration harness and the regression gate cannot pass an
// unflattened log by taking the other route into this function.
//
// The clip is a ceiling and not a budget, and the difference decides the number. A page log is
// long — over 2,001 enveloped page replies in 67 bench round logs on file the median is 671
// characters, p90 1,147, p99 1,741 and the longest 2,566 — and it is long because `agents/page.md`
// asks for it by name in 26 places. Two of those records exist in the log and nowhere else, and for
// the rest the log is the verifier's evidence that the DOCUMENT owes its half — a page the MODEL
// could not return in full owes the `[page not fully transcribed]` marker, while a page that merely
// ran out where the SHEET did owes the log entry and nothing more, and the log is what tells the
// verifier which of the two it is looking at. Either way the record sits at the
// end of the entry. Cutting one short cuts those off its END, which is precisely the half
// `agents/feedback.md` now relies on to avoid a false "unrecorded" finding, so a clip that bites a
// real log would make this change worse than not making it. 3,000 is reached by 0 of those 2,001:
// it bounds a pathological reply and truncates nothing observed.
function flattenLog(log: string | undefined): string {
const flat = (log ?? "").replace(/`/g, "'").replace(/\s+/g, " ").trim();
return flat.length > 3000 ? `${flat.slice(0, 3000).trimEnd()}…` : flat;
}
// Ask the Feedback Agent (VERIFY task) whether an agent's output faithfully and
// accessibly captures its source image. In the single-pass pipeline this verifies
// the page agent's per-page output; it is also reused by the regression gate.
// Non-blocking: returns ok=true when the Feedback Agent is unavailable or returns
// nothing, so verification never breaks a run.
// `step` is the CALLER's, not this function's, and that is the whole point of passing it: one
// function here serves five jobs that cost differently and are worth telling apart — the
// fidelity check every page buys, the two re-checks of a correction, the calibration harness
// judging a seeded defect, and the regression gate judging a candidate agent file. Narrowed to
// those five rather than left open as `PipelineStep`, so the type says which jobs reach this
// code and a caller cannot file its verify call under something else.
export async function verifyAgentOutput(
ctx: PipelineContext,
agent: AgentSpec,
img: InputImage,
// `caution` is something the agent under test said about its OWN output, carried through to the
// judgement of it. It rides on the block rather than on a sixth parameter so that `step` stays the
// last argument of every call site — which is what `test/step-attribution.test.ts` reads them with,
// and a per-call vocabulary check is worth more than the argument shape it depends on. Optional,
// deduplicated, and absent by default: a judgement with nothing to carry sends exactly the bytes
// it always did.
// `log` is the `"log"` field of the reply that produced this block — the agent's own note about
// what it did with the page. Carried for the reason `caution` is: the contract quoted above
// places obligations there and nowhere else, and a judge asked to confirm the contract was met
// with that field withheld can only ignore the rule or look for its evidence in the HTML, where
// the contract does not put it. Across 311 verify replies in two bench rounds, 35 problems on 26
// replies demanded something of the log, and 26 of those 35 were about a log that existed and was
// not shown (#349). Optional, and absent wherever the reply being judged has no log of its own:
// a correction reply is parsed for `html` alone, so both rechecks send this empty rather than
// sending the first pass's note about a fragment that has since been rewritten.
blocks: { html: string; caution?: string; log?: string }[],
step: Extract<
PipelineStep,
"verify" | "recheck_binding" | "recheck_sampled" | "agent_calibrate" | "agent_regression"
>,
): Promise<VerifyVerdict> {
const fb = loadFeedbackAgent(ctx);
if (!fb || blocks.length === 0) return unjudgedVerdict();
const html = blocks.map((b) => b.html).join("\n\n");
const cautions = [...new Set(blocks.map((b) => b.caution?.trim()).filter((c): c is string => !!c))];
const logs = [...new Set(blocks.map((b) => flattenLog(b.log)).filter((l) => !!l))];
// Everything this task says that is not about the page in front of it: the task marker
// and the whole contract of the agent being judged. It is the same bytes on every page
// of a document — `agents/page.md` is 16 KB of it, re-sent per page and per correction
// recheck — so it is declared as this message's invariant head and gets a cache
// breakpoint after it (providers/promptCache.ts). On a 25-page document that is a
// handful of writes at 1.25x and the rest reads at 0.1x, against 25 full-price copies.
//
// A handful rather than one, because pages are extracted concurrently: the first
// `extraction_concurrency` verify calls are in flight before any of them has written
// the head, so each of those misses. At the default of 5 that is ~5 writes and ~20
// reads, which is most of the saving and not all of it.
//
// The case that does not win is a ONE-PAGE run with no recheck: its single verify call
// pays 1.25x for a head nothing reads back, ~25% on ~4k tokens. That is the same trade
// `promptCache.ts` reasons through for a system prompt, and it is deliberate — a
// screenshot upload is exactly that case, and a quarter of one prompt on it is worth
// the four fifths saved on every document with pages in it.
//
// It stays in the USER message, in the position it was already in, rather than moving
// into the system prompt to ride the breakpoint already there. The system prompt is
// where the Feedback Agent's OWN instructions live, and an agent's contract is quoted
// material to be judged against — `page.md` ends "Your entire reply must be the JSON
// object and nothing else. Do not write any reasoning, preamble, commentary or summary
// before or after it. Everything you have to say about this page goes inside the fields
// the schema above lists", above a schema of `{ "html", "log", "blank",
// "suggested_agent" }`. That is the wrong answer to THIS task and is exactly what
// putting it in the verifier's own role invites. `user` below is still the complete
// message and still starts with this text; the split changes what is billed, not what
// is said.
//
// #365 made that hazard sharper rather than milder, which is worth stating because the
// sentences above were written when `page.md` merely ended in a schema. It now ends in a
// second-person imperative that is a near-copy of the clause the checker's own prompt
// carries, naming a different set of fields — so a model that reads the quoted contract
// as addressed to itself is being instructed, not just shown. The role split is the
// whole mitigation and the guard below is the backstop: a reply that answers `html`
// instead of both decision flags degrades to `unjudgedVerdict()`, so the failure is a
// page nothing judged and never a page falsely passed. Read `pages_unjudged` beside any
// re-count of this change, and note it now has two reasons to move in opposite
// directions — `docs/verifier-calibration.md` names the other one, a longer reply
// stopping mid-object.
const contract =
`TASK: verify\n\n` +
`## Agent under test: ${agent.file}\n\`\`\`markdown\n${agent.content}\n\`\`\`\n\n`;
// The cautions go here, AFTER the cached prefix and after the output they are about. They are
// per-page by construction, so a copy of one inside `contract` would change the invariant head on
// the page that has it and cost every other page in the document its cache read — the saving the
// comment above is written to protect. `agents/feedback.md` says what a caution narrows: the
// verifier may ask for an unsupported reading to be hedged or removed and may not supply one of
// its own.
// The log goes here for the same reason the cautions do — it is per-page, so a copy inside
// `contract` would break the invariant head and cost every other page in the document its cache
// read. It is quoted rather than summarised, and labelled as a claim: it is the transcriber's
// account of its own work, so it is evidence about the page and not a second source image.
// `agents/feedback.md` says what may be done with it, and the short version is that it can only
// ever support a finding about the HTML: a problem naming the log is a problem the correction pass
// cannot resolve, because that pass is parsed for `html` alone and writes no log at all.
const user =
contract +
`## The agent's output for source image "${img.name}"\n\`\`\`html\n${html}\n\`\`\`\n\n` +
(logs.length ? `## What the agent recorded in its own "log" field\n${logs.map((l) => `- ${l}`).join("\n")}\n\n` : "") +
(cautions.length
? `## What the agent said about its own output\n${cautions.map((c) => `- ${c}`).join("\n")}\n\n`
: "") +
`Compare the output against the attached source image.`;
// NO `maxOutputTokens`, and #365 directive 2 asked for one "copying the corrector's". The
// corrector's shape does not transfer, and what stops it is a property of the reply rather than a
// preference: a ceiling cuts the END of a reply, and on this agent the end is the verdict.
// Across every verify and recheck call the Feedback Agent made in every bench round directory on
// disk — 3,908 attempted, 3,902 returned a reply, 3,897 of those readable, five models, a WIDER
// corpus than the 1,342 the both-flags comment below counts — the 19 replies of 8,000 output
// tokens or more ALL parsed to a usable
// verdict, naming 95 problems between them, and the envelope's own text begins at 86.3%–99.8% of
// the reply (median 94.2%). The narration comes first and the answer last. So a cap here does not
// trim the narration the issue is about: it removes the answer and bills for the narration anyway,
// because output is billed per token emitted and not per token allowed (`DEFAULT_MAX_TOKENS`'s own
// comment says so). `correctPage` caps for the opposite reason — its output IS the payload, so a
// cut tail leaves a usable head — which is what `correctionCeiling` bounds and why it is coherent
// there and not here.
//
// Priced on the 2,511 replies whose page's own first pass is in the same log, so a
// page-proportional rule and a flat one are scored on the same members, in verdicts lost /
// problems lost / dollars saved per 100 verify calls:
//
// flat 8,000 17 91 $0.0711
// max(4000, 2x page) 17 83 $0.0565
// flat 12,000 8 58 $0.0419
// max(4000, 3x page) 12 67 $0.0293
//
// The flat rule is the better shape on verdicts and dollars, which is where the decision sits, and
// it is NOT dominant: at the tighter point it names eight more problems lost than `max(4000, 2x)`
// for the same 17 verdicts. It wins on all three columns at the looser point. Either way the shapes
// differ because a runaway is not a big page: the longest reply in the corpus is 4.6x its own
// page's output tokens and the two quantities correlate at r = 0.42, so scaling by the page is
// loosest where the pages are largest and tightest where the narration is. No rule that keeps
// verdict loss under 1% saves more than about 7 cents per 100 verify calls against the $4.50 #365
// §1 measured for checking 100 pages — a flat 4,000 saves the most of any rule measured, 14 cents,
// and loses 53 of 2,511 verdicts, which is buying money with verdicts rather than with narration.
// The narration itself is $1.99 per 100 pages, and the only mechanism that reaches text billed per
// token emitted is not writing it, which is what #424 put beside the verify schema.
//
// The checker is not unbounded, which the issue's opening paragraph implies and its own caveats
// correct: `DEFAULT_MAX_TOKENS` is 32,000, both adapters take the smaller of that and a caller's
// cap, and it has fired three times in this history — one Sonnet reply of 93,072 characters and
// two Qwen3-VL replies of 137,465 and 145,384, each billed in full and each returning no verdict.
// That bound already exists on this call; every value below it trades verdicts for cents. The tail
// is also model-specific and the ranking inverts between the two halves of it: Sonnet's largest
// returned reply is 30,267 tokens and every other arm's is under 4,300, yet two of the three
// ceiling truncations are Qwen3-VL's — 2 of its 127 calls against 1 of Sonnet's 3,120.
//
// What none of this says is what the distribution looks like AFTER #424, since every reply counted
// above was written without that clause. If it works the tail shrinks and a cap has even less to
// cut; if it does not, these figures stand. Either way they are re-derivable for free from the
// `model_call` and `agent_call` events Iris already writes, which is where they came from — this
// needed no new instrument, so a later attempt at the same question does not need a paid round.
const res = await ctx.router.complete(
FEEDBACK_AGENT,
"vision",
[
{ role: "system", content: fb.content },
{ role: "user", content: user, cachedPrefix: contract },
],
{ step, images: [loadImage(img)] },
);
ctx.log.agentCall({ agent: fb, phase: "extraction", image: img.name, output: res.text });
const parsed = extractJson<VerifyOutput>(res.text);
// Both flags, as booleans, or this is not a verdict. The contract asks for both and all 1,342
// readable verify replies in one round set answer both — but the reason to keep the check is not
// that the shape never occurs, because at a wider width it does. Across every verify and recheck
// call in every round directory (3,897 readable, the corpus the ceiling comment above measures)
// EIGHT do not carry both flags, and in all eight both flags are IN THE REPLY TEXT, inside the
// first sixty bytes of the envelope: what breaks is further right — an unescaped `"` where the
// checker quotes the page's own row-group label (3 replies, all Sonnet, the model the reference
// deployment runs this agent on), decode garbage after a closed envelope (4, all Luna), a raw
// newline inside a string (1, Qwen3-VL). **Those eight still cost nothing here**, which is worth
// being exact about: an object carrying neither flag read `ok = undefined !== false && undefined
// !== false` — true — with `readProblems(undefined)` empty, so before this check they were silent
// passes buying no correction and after it they are `unjudgedVerdict()`, also `ok: true` and also
// empty. The page ships uncorrected either way and only the counter changes, to the better one.
// What the eight revise is the REASON, not the price. The direction that does cost something is
// still `faithful: false` without `accessible`, at 0 observed replies, which is what the test
// below means by "not free in one direction". #426 carries the eight and what each class would
// take to recover — and the loss there is the problems those replies named, never this check.
// The three that quote a row-group label are addressed where they are written rather than here:
// `agents/feedback.md` asks the checker to quote the page's words with single quotes and no `"` of
// its own, which costs nothing per call, where repairing that quote in the parser is the wider
// reading `src/util/json.ts` was narrowed away from and wants its own measurement. This check is
// unchanged by that — such a reply still arrives either readable or unjudged, as before.
// What it buys besides is the failure mode #339's `notes` field opens. `extractJson`
// returns the LAST readable object in a reply, and a `notes` string that quotes the contract back
// ends with one: an unescaped `{ "faithful": true, "problems": [] }` inside the prose, which read
// as a confident PASS on a page the verifier had just rejected for a missing table row — `ok`
// true, `problems` empty, no `unjudged` flag, and a plain `page_verify_ok` line over it. The
// whole-reply repair in `src/util/json.ts` reads the envelope correctly when the reply is nothing
// but its JSON, which is what the prompt asks for; this is the half that also holds when the
// model fences it or writes a sentence first, and it degrades to a page nobody judged rather than
// to a page that passed. `pages_unjudged` counts those, and `docs/API.md` says what it is a
// subset of.
if (!parsed || typeof parsed.faithful !== "boolean" || typeof parsed.accessible !== "boolean") {
return unjudgedVerdict();
}
const ok = parsed.faithful !== false && parsed.accessible !== false;
return { ok, ...readProblems(parsed.problems) };
}
// ---------------------------------------------------------------------------
// Feedback routing: does this feedback need the source images?
// ---------------------------------------------------------------------------
export interface FeedbackScope {
target: "extraction" | "document";
// 1-based page orders to re-extract. Only meaningful when target is
// "extraction"; empty means "source-level, but we could not localize it".
pages: number[];
reason: string;
}
// A feedback message that supposedly targets more than this share of a document's
// pages is treated as document-level: re-extracting nearly everything costs about
// as much as a fresh run and is rarely what a targeted correction means.
const MAX_REEXTRACT_FRACTION = 0.5;
// Ask the Feedback Agent (SCOPE task) whether feedback is about what was read off
// the source pages — which the review loop CANNOT fix, because the Reader only ever
// sees the assembled HTML (by design) and so raises no issue for a misreading
// it cannot detect — or about the assembled document.
//
// Non-blocking and biased toward the cheap path: any doubt (agent unavailable,
// unparseable answer, no pages identified) resolves to "document", which is the
// pre-existing behavior. A wrong "document" answer costs a round of review; a
// wrong "extraction" answer costs a full vision pass per page.
export async function scopeFeedback(
ctx: PipelineContext,
fragments: { order: number; innerHtml: string }[],
): Promise<FeedbackScope> {
const feedback = ctx.feedback?.trim();
if (!feedback) return { target: "document", pages: [], reason: "no feedback" };
const fb = loadFeedbackAgent(ctx);
if (!fb || fragments.length === 0) {
return { target: "document", pages: [], reason: "feedback agent unavailable" };
}
const pageList = pageIndex(fragments);
const user =
`TASK: scope\n\n` +
`## User feedback\n${feedback}\n\n` +
`## Pages in this document (extracted HTML, truncated)\n${pageList}`;
const res = await ctx.router.complete(
FEEDBACK_AGENT,
"text",
[
{ role: "system", content: fb.content },
{ role: "user", content: user },
],
{ step: "feedback_scope" },
);
ctx.log.agentCall({ agent: fb, phase: "extraction", output: res.text });
const parsed = extractJson<{ target?: string; pages?: unknown; reason?: string }>(res.text);
const reason = parsed?.reason?.trim() || "no reason given";
if (parsed?.target !== "extraction") {
return { target: "document", pages: [], reason };
}
// Keep only page numbers that actually exist in this document; a hallucinated
// page would otherwise silently re-extract nothing or throw downstream.
const pages = knownPages(parsed.pages, fragments);
// Source-level feedback that could not be localized: re-extracting the whole
// document is too blunt (and too expensive) a response, and the review loop at
// least applies the wording. Fall back rather than guess.
if (pages.length === 0) {
return { target: "document", pages: [], reason: `${reason} (no pages identified)` };
}
if (pages.length > Math.max(1, Math.floor(fragments.length * MAX_REEXTRACT_FRACTION))) {
return {
target: "document",
pages: [],
reason: `${reason} (${pages.length}/${fragments.length} pages — too broad to re-extract)`,
};
}
return { target: "extraction", pages, reason };
}
// ---------------------------------------------------------------------------
// Regression gate: protect existing uses when an agent is updated
// ---------------------------------------------------------------------------
const MAX_GATE_FIXTURES = 3;
// An updated agent must still reproduce at least this fraction of the words in a
// fixture's accepted output (by screen-reader-flattened text); below it, the
// change is treated as a content regression.
export const MIN_CONTENT_COVERAGE = 0.85;
// Skip the coverage check for very short outputs, where one dropped word swings
// the ratio — rely on the model verdict alone there.
const MIN_COVERAGE_WORDS = 8;
// A proposed prompt change may not drop the agent's mean fixture coverage by more
// than this versus the current prompt (the holds-or-improves eval gate, #3).
const EVAL_REGRESSION_EPS = 0.02;
export interface RegressionResult {
passed: boolean;
failures: string[];
meanCoverage: number | null; // mean content coverage of the candidate over fixtures
scores: FixtureScores; // per-fixture, for the paired comparison (see pairedMeans)
}
// What one prompt scored on each fixture, keyed by the fixture's image file (unique
// within an agent's fixtures directory). `null` means "not measurable for this prompt
// on this fixture" — either the fixture is unjudgeable at all (accepted text under
// MIN_COVERAGE_WORDS) or it could not be read.
//
// Keyed rather than averaged because the eval gate is a PAIRED comparison: two means
// are only comparable if they are taken over the same fixtures, and which fixtures a
// prompt can be scored on is partly a property of that prompt (see pairedMeans).
export type FixtureScores = Record<string, number | null>;
/**
* Mean score for each prompt over the fixtures **both** could be scored on.
*
* This is the fix for the wave-through the unpaired version allowed. Scoring is not
* purely a property of the fixture: `fixtureScore` gives a prompt that produced NO
* output a 0 (a failure, not an absence of evidence), while a fixture whose accepted
* text is too short to judge abstains. Those two rules can land on the same fixture
* for different prompts, and then the two means were over different sets:
*
* fixture A (unjudgeable): current flakes to no output -> 0. candidate produces
* something -> abstains, excluded.
* fixture B (judgeable): current 0.98, candidate 0.88 — a real 0.10 regression.
*
* Unpaired, current = (0 + 0.98)/2 = 0.49 and candidate = 0.88, so
* `0.88 < 0.49 - 0.02` is false, 0.88 clears MIN_CONTENT_COVERAGE, and the regression
* ships. One flake on the CURRENT prompt deflated the bar it was supposed to set.
*
* Pairing drops fixture A from both sides — the candidate has no score there, so
* there is nothing to compare — leaving current 0.98 vs candidate 0.88, which blocks.
* The mean now answers "over the fixtures where both prompts are measurable, is the
* candidate worse?", which is the only question the subtraction can honestly ask.
*
* What this deliberately does NOT do is treat the current prompt's flake as evidence
* against the candidate, or as evidence for it. A prompt that produces nothing is a
* problem, but it is a problem with the CURRENT library agent, and silently lowering
* the bar is the one response that hides both it and any regression behind it. It
* stays visible in the `eval_gate` log line's `unpaired` list instead.
*
* Both means can be null (no fixture is measurable on both sides), which the caller
* reads as "nothing to compare" and defers to the regression gate — not as a pass and
* not as a regression.
*/
export function pairedMeans(
current: FixtureScores,
candidate: FixtureScores,
): { current: number | null; candidate: number | null; paired: string[]; unpaired: string[] } {
const paired: string[] = [];
const unpaired: string[] = [];
let currentSum = 0;
let candidateSum = 0;
// Union of both key sets: a fixture one side never even read is as unpaired as one
// it abstained on, and both belong in the log line.
for (const key of new Set([...Object.keys(current), ...Object.keys(candidate)])) {
const a = current[key];
const b = candidate[key];
if (typeof a === "number" && typeof b === "number") {
paired.push(key);
currentSum += a;
candidateSum += b;
} else {
unpaired.push(key);
}
}
paired.sort();
unpaired.sort();
const n = paired.length;
return {
current: n ? currentSum / n : null,
candidate: n ? candidateSum / n : null,
paired,
unpaired,
};
}
// How one fixture contributes to an agent's mean coverage. `null` means "abstain":
// this fixture is not counted in the mean at all.
//
// Both sides of the eval-regression comparison at the end of
// proposeAgentUpdatesFromFeedback MUST score fixtures by this one rule, because the
// comparison is a subtraction between two means. `regressionGate` scores the
// CANDIDATE prompt and `evalAgent` scores the CURRENT one; when they disagreed the
// difference between them was an artifact of the scoring, not of the prompts.
//
// `evalAgent` used to score an unjudgeable fixture (accepted text shorter than
// MIN_COVERAGE_WORDS, so contentCoverage abstains) as a perfect 1, while
// regressionGate excluded it. Since abstention depends only on `accepted_html`, the
// SAME fixture abstained on both sides — so the 1 landed on the current-prompt side
// only and inflated it. With MAX_GATE_FIXTURES = 3 that inflation is large: two
// judgeable fixtures at 0.90 plus one unjudgeable gave current 0.933 vs candidate
// 0.900, a 0.033 gap from padding alone, past EVAL_REGRESSION_EPS = 0.02. The gate
// blocked an update whose measurable coverage was IDENTICAL, and logged it as
// "eval_regression" — a real improvement discarded with a reason that named a
// regression that had not happened.
//
// No output is scored 0 rather than abstaining because producing nothing is a
// FAILURE on the fixture, not an absence of evidence: abstaining would let a prompt
// that returns nothing score exactly as well as one that handles the fixture.
// regressionGate already scores it 0 (and fails the fixture outright), so 0 is also
// the symmetric choice.
//
// It is the one input where "abstain" is not purely a property of the fixture:
// whether a prompt produced blocks is a property of THAT prompt, so a single fixture
// can be scored 0 for one prompt and excluded for the other. That asymmetry used to
// make the comparison unsound in one direction — if the CURRENT prompt flaked to no
// output on a fixture the candidate abstained on, the current mean was deflated and a
// real regression cleared the bar.
//
// It is no longer this function's problem to solve, and that is the resolution: the
// scoring rule here is right (nothing produced IS a failure on that fixture), and what
// was wrong was averaging two sets of per-fixture scores that did not cover the same
// fixtures. `pairedMeans` now compares only fixtures where BOTH prompts have a score,
// so a per-prompt exclusion drops the fixture from both sides instead of moving the
// bar. See `pairedMeans` for the worked example this used to wave through.
function fixtureScore(coverage: number | null, blockCount: number): number | null {
if (blockCount === 0) return 0;
return coverage;
}
// Fraction of the accepted output's distinct words that still appear in the
// candidate output (screen-reader-flattened, punctuation-insensitive). Returns
// null when the accepted text is too short to judge reliably. Structural role
// markers that flatten() injects ([Heading 2], [List item], …) are stripped so a
// structure-only change is not mistaken for dropped content.
export function contentCoverage(acceptedHtml: string, candidateHtml: string): number | null {
const words = (html: string): Set<string> =>
new Set(
flatten(html)
.replace(/\[[^\]]*\]/g, " ")
.toLowerCase()
.replace(/[^a-z0-9\s]/g, " ")
.split(/\s+/)
.filter((w) => w.length > 1),
);
const accepted = words(acceptedHtml);
if (accepted.size < MIN_COVERAGE_WORDS) return null;
const candidate = words(candidateHtml);
let hit = 0;
for (const w of accepted) if (candidate.has(w)) hit++;
return hit / accepted.size;
}
// Re-run an agent (given its current/updated content) on a fixture image, used by
// the regression gate. Accepts either output shape: { html } (whole-page agents
// like page.md) or { no_content, fragments[] } (content agents).
async function reRunAgentOnImage(
ctx: PipelineContext,
agent: AgentSpec,
img: InputImage,
): Promise<{ html: string }[]> {
const system = `${agent.content}\n\n${ACCESSIBILITY_REQUIREMENTS}`;
const user =
`Process source image "${img.name}" exactly as your contract specifies and respond with ONLY ` +
`JSON — either { "html": "<accessible HTML>" } for a whole-page agent, or ` +
`{ "no_content": false, "fragments": [ { "html": "<accessible HTML>" } ] } for a content agent ` +
`({ "no_content": true } if nothing matches).`;
const capability = agent.capabilities.includes("vision") ? "vision" : "text";
const res = await ctx.router.complete(
agent.name,
capability,
[
{ role: "system", content: system },
{ role: "user", content: user },
],
{ step: "agent_regression", images: [loadImage(img)] },
);
ctx.log.agentCall({ agent, phase: "review", image: img.name, output: res.text });
const parsed = extractJson<{ no_content?: boolean; html?: string; fragments?: { html?: string }[] }>(res.text);
if (!parsed || parsed.no_content) return [];
if (parsed.html) return [{ html: parsed.html }];
if (parsed.fragments?.length) return parsed.fragments.filter((f) => f.html).map((f) => ({ html: f.html! }));
return [];
}
// Before an existing agent is updated/merged, re-run the UPDATED agent against its
// stored regression fixtures and verify each still passes. Blocks the change if
// any fixture regresses, so an agent can't be changed in a way that breaks a use
// it already handled. Passes when the agent has no fixtures yet.
export async function regressionGate(
ctx: PipelineContext,
agentFile: string,
updatedContent: string,
): Promise<RegressionResult> {
const dir = ctx.paths.agentFixtures(agentFile);
if (!existsSync(dir)) return { passed: true, failures: [], meanCoverage: null, scores: {} };
const caseFiles = readdirSync(dir)
.filter((f) => f.endsWith(".json"))
.sort()
.reverse()
.slice(0, MAX_GATE_FIXTURES);
if (caseFiles.length === 0) return { passed: true, failures: [], meanCoverage: null, scores: {} };
const file = agentFile.endsWith(".md") ? agentFile : `${agentFile}.md`;
const updatedAgent: AgentSpec = {
name: file.replace(/\.md$/, ""),
file,
content: updatedContent,
capabilities: /\bvision\b/i.test(updatedContent) ? ["vision"] : ["text"],
sha: null,
sessionBuilt: false,
};
// One fixture's verdict. Collected per fixture and folded in fixture order below,
// rather than pushed as each finishes, because these run CONCURRENTLY: `failures` is
// what a maintainer reads to find out what the candidate broke, and a list whose order
// depends on which provider call returned first is a different list every run.
interface FixtureVerdict {
image: string;
score: number | null;
failure: string | null;
}
// Fixtures are independent — a stored image, its accepted output, and a score computed
// from the two — so they are checked together instead of one after another. Serially
// this gate was up to MAX_GATE_FIXTURES x 2 vision calls end to end (a re-run and a
// verification each). The session that triggered it has been `ready_for_review` since before
// training began (#156), so the user is no longer waiting on it — but every upload behind them
// in the run queue still is, since the run holds its `max_concurrent_runs` slot throughout.
//
// The two calls WITHIN a fixture stay sequential, because the second judges the output
// of the first.
//
// It costs tokens, and this is where the trade is stated rather than left in the diff's
// shadow. Both of a fixture's calls carry a cached head built from the CANDIDATE prompt
// — the agent's system prompt on the re-run, the same content again as the verify task's
// contract — and a candidate is new every round, so that entry is always cold. Serially,
// the first fixture wrote it at 1.25x and the other two read it at 0.1x; together, all
// three miss and all three write. That is roughly +2.3 full-price copies of a ~4k-token
// head, on each of two heads, per gate round. It is the same trade already taken for
// page extraction (providers/promptCache.ts), for the same reason: the concurrency is
// worth more than the writes, and here it is worth more still, because what it buys back
// is a user waiting on training work that is not about their document.
//
// Bounded by the same knob as page extraction and the Reader's chunks
// (`defaults.extraction_concurrency`), so a run's in-flight calls stay where the
// operator set them here too. No first-failure guard like the Reader's: at a limit of
// MAX_GATE_FIXTURES or more every fixture is in flight before any of them can reject, so
// there is nothing queued behind a failure to save. Below that limit — an operator who
// lowered it for a rate-limited provider — one fixture can still be issued after another
// has thrown, which is a single vision call on a round that is already failing. The
// Reader guards this because a long document is many more chunks than three, so the
// waste there is unbounded; here it is one call, and a guard would cost more to read
// than it saves.
const limit = Math.max(1, Math.floor(ctx.extractionConcurrency) || 1);
const verdicts = await mapWithConcurrency(caseFiles, limit, async (caseFile): Promise<FixtureVerdict | null> => {
let c: FixtureCase;
try {
c = JSON.parse(readFileSync(join(dir, caseFile), "utf8")) as FixtureCase;
} catch {
return null;
}
const imgPath = join(dir, c.image_file);
if (!existsSync(imgPath)) return null;
const img: InputImage = { name: c.source_image, order: 0, path: imgPath };
const blocks = await reRunAgentOnImage(ctx, updatedAgent, img);
if (blocks.length === 0) {
// fixtureScore's no-output rule, which never abstains — see its comment for
// why producing nothing scores 0 rather than dropping out of the mean.
const zero = fixtureScore(null, 0) as number;
return { image: c.image_file, score: zero, failure: `${c.image_file}: updated agent produced no output` };
}
// Content-preservation check: the updated agent must still reproduce the
// content it produced when this fixture was accepted. Compare the
// screen-reader-flattened text of the new output against the accepted output;
// a large drop means the change regressed a use we already shipped.
const candidateHtml = blocks.map((b) => b.html).join("\n\n");
const coverage = contentCoverage(c.accepted_html, candidateHtml);
const score = fixtureScore(coverage, blocks.length);
if (coverage !== null && coverage < MIN_CONTENT_COVERAGE) {
return {
image: c.image_file,
score,
failure: `${c.image_file}: only ${(coverage * 100).toFixed(0)}% of the accepted content remained`,
};
}
const verdict = await verifyAgentOutput(ctx, updatedAgent, img, blocks, "agent_regression");
return {
image: c.image_file,
score,
failure: verdict.ok ? null : `${c.image_file}: ${verdict.problems.join("; ") || "failed verification"}`,
};
});
const failures: string[] = [];
const coverages: number[] = [];
// Keyed by fixture as well as averaged: the eval gate compares this prompt against
// the current one fixture-by-fixture (see pairedMeans), which a mean alone cannot
// support. `meanCoverage` stays for the log line and for callers that only want a
// single number.
const scores: FixtureScores = {};
for (const v of verdicts) {
if (!v) continue;
scores[v.image] = v.score;
if (v.score !== null) coverages.push(v.score);
if (v.failure) failures.push(v.failure);
}
const passed = failures.length === 0;
const meanCoverage = coverages.length ? coverages.reduce((a, b) => a + b, 0) / coverages.length : null;
ctx.log.event("regression_gate", { agent: file, cases: caseFiles.length, passed, failures: failures.length, meanCoverage });
return { passed, failures, meanCoverage, scores };
}
// Score an agent's content against each of its regression fixtures, and return both
// the per-fixture scores and their mean — a lightweight eval set (#3).
//
// The per-fixture map is what the eval gate actually uses: the caller pairs it against
// regressionGate's `scores` so the comparison is over fixtures BOTH prompts could be
// scored on (see pairedMeans). The mean is retained for the log line and for callers
// that want one number; it is null when no fixture was judgeable, since a mean over
// zero measurements is not a score of zero.
//
// Both sides score with the shared `fixtureScore` rule; see the comment there for why
// any divergence between the two makes the subtraction meaningless.
export async function evalAgentScores(
ctx: PipelineContext,
agentFile: string,
content: string,
): Promise<{ mean: number | null; scores: FixtureScores }> {
const dir = ctx.paths.agentFixtures(agentFile);
if (!existsSync(dir)) return { mean: null, scores: {} };
const caseFiles = readdirSync(dir).filter((f) => f.endsWith(".json")).sort().reverse().slice(0, MAX_GATE_FIXTURES);
if (caseFiles.length === 0) return { mean: null, scores: {} };
const file = agentFile.endsWith(".md") ? agentFile : `${agentFile}.md`;
const agent: AgentSpec = {
name: file.replace(/\.md$/, ""),
file,
content,
capabilities: /\bvision\b/i.test(content) ? ["vision"] : ["text"],
sha: null,
sessionBuilt: false,
};
// Concurrent for the reason regressionGate's are, and it is the same fixtures: this is
// the other half of the same comparison, run right after that gate on the same feedback
// round. One vision call each, and they share nothing.
const limit = Math.max(1, Math.floor(ctx.extractionConcurrency) || 1);
const results = await mapWithConcurrency(
caseFiles,
limit,
async (caseFile): Promise<{ image: string; score: number | null } | null> => {
let c: FixtureCase;
try {
c = JSON.parse(readFileSync(join(dir, caseFile), "utf8")) as FixtureCase;
} catch {
return null;
}
const imgPath = join(dir, c.image_file);
if (!existsSync(imgPath)) return null;
const img: InputImage = { name: c.source_image, order: 0, path: imgPath };
const blocks = await reRunAgentOnImage(ctx, agent, img);
const cov = contentCoverage(c.accepted_html, blocks.map((b) => b.html).join("\n\n"));
return { image: c.image_file, score: fixtureScore(cov, blocks.length) };
},
);
const measured: number[] = [];
const scores: FixtureScores = {};
for (const r of results) {
if (!r) continue;
scores[r.image] = r.score;
if (r.score !== null) measured.push(r.score);
}
return {
mean: measured.length ? measured.reduce((a, b) => a + b, 0) / measured.length : null,
scores,
};
}
// The mean alone, for callers that do not need the paired comparison.
export async function evalAgent(ctx: PipelineContext, agentFile: string, content: string): Promise<number | null> {
return (await evalAgentScores(ctx, agentFile, content)).mean;
}
// ---------------------------------------------------------------------------
// Feedback-driven agent training
// ---------------------------------------------------------------------------
// On a feedback re-run, turn the document-level correction (the prior reviewed
// body vs. this run's reviewed body) into an improved version of the agent that
// produced the document — the page agent in the single-pass pipeline. For a
// library agent the proposal is gated on its regression fixtures and then filed
// as a GitHub issue (the contribution model uses issues, not close-time PRs),
// while also being recorded in agent-updates.md; a session-built agent is trained
// in place so its new-agent contribution carries the fix.
//
// `lesson` is what `learnFromFeedback` recorded for this same correction, when it
// recorded anything. It is what the filed issue is titled and deduped by (see
// `lessonSlug`), so passing it is what keeps one open issue from swallowing every later
// proposal — pass it whenever the caller has it.
export async function proposeAgentUpdatesFromFeedback(
ctx: PipelineContext,
args: {
agentFile: string;
before: string;
after: string;
feedback: string;
lesson?: CorrectionExample | null;
},
): Promise<AgentUpdateContribution[]> {
const feedbackAgent = loadFeedbackAgent(ctx);
if (!feedbackAgent) {
ctx.log.event("feedback_agent_missing", {
note: "agents/feedback.md not found; skipping agent-update proposals",
});
return [];
}
// Nothing changed this run -> no lesson to learn.
if (!args.before.trim() || args.before.trim() === args.after.trim()) return [];
const target = loadTargetAgent(ctx, args.agentFile);
if (!target) {
ctx.log.event("feedback_target_missing", { agent: args.agentFile });
return [];
}
const correction = diffPreview(args.before, args.after);
const user =
`TASK: train\n\n` +
`## Agent to improve: ${target.file}\n\`\`\`markdown\n${target.content}\n\`\`\`\n\n` +
`## User feedback for this run\n${args.feedback}\n\n` +
`## The correction the feedback caused (diff of the document body)\n\`\`\`diff\n${correction}\n\`\`\``;
const res = await ctx.router.complete(
FEEDBACK_AGENT,
"text",
[
{ role: "system", content: feedbackAgent.content },
{ role: "user", content: user },
],
{ step: "agent_update" },
);
ctx.log.agentCall({ agent: feedbackAgent, phase: "review", output: res.text });
const parsed = extractJson<TrainOutput>(res.text);
if (!parsed?.changed || !parsed.agent_markdown) return [];
const updated = parsed.agent_markdown.trim();
if (!updated || updated === target.content.trim()) return [];
if (target.sessionBuilt) {
// Train the session-built agent in place: overwrite its tmp file so the rest of the
// session uses the improved prompt (`loadAgent` prefers tmp over the library).
// Nothing carries it any further than the session — the contribution model files
// issues rather than PRs, and `runContribution` SKIPS a type an agent already exists
// for, tmp included. Unreachable today for the same reason `agent_content` is always
// null (agents/loader.ts): the only writer of that directory is this line, which is
// behind the flag that only a file there can set.
writeFileSync(join(ctx.paths.tmpAgentsDir(ctx.sessionId), target.file), updated);
ctx.log.event("agent_trained", { agent: target.file, scope: "session_built" });
return [];
}
// Existing library agent: gate the proposed update on its regression fixtures —
// never propose a change that breaks a use it already handled.
const gate = await regressionGate(ctx, target.file, updated);
if (!gate.passed) {
ctx.log.event("agent_update_blocked", { agent: target.file, failures: gate.failures });
return [];
}
// Eval gate (#3): the proposed prompt must hold-or-improve the agent's coverage
// over its fixtures versus the current prompt — not just pass the floor.
//
// PAIRED, per fixture. Both sides score with `fixtureScore`, but whether a given
// fixture HAS a score is partly a property of the prompt: one that produced no
// output scores 0, while a fixture too short to judge abstains. Averaging each side
// over whatever it happened to measure therefore compared two different fixture
// sets, and one flake on the CURRENT prompt could deflate the bar enough to let a
// real regression through (worked example on `pairedMeans`). Comparing only the
// fixtures both prompts could be scored on removes that: an exclusion drops the
// fixture from both means instead of moving the threshold.
//
// Either mean being null means no fixture was measurable on both sides, which is not
// evidence of a regression — the update proceeds on the regression gate's verdict
// alone. `unpaired` is logged either way, because a fixture that only one prompt
// could be scored on is worth seeing: on the current side it usually means the
// library agent itself is flaking.
const current = await evalAgentScores(ctx, target.file, target.content);
const means = pairedMeans(current.scores, gate.scores);
ctx.log.event("eval_gate", {
agent: target.file,
current: means.current === null ? null : Number(means.current.toFixed(3)),
candidate: means.candidate === null ? null : Number(means.candidate.toFixed(3)),
paired: means.paired,
unpaired: means.unpaired,
});
if (means.current !== null && means.candidate !== null && means.candidate < means.current - EVAL_REGRESSION_EPS) {
ctx.log.event("agent_update_blocked", {
agent: target.file,
reason: "eval_regression",
current: Number(means.current.toFixed(3)),
candidate: Number(means.candidate.toFixed(3)),
paired: means.paired,
});
return [];
}
const proposal: AgentUpdateContribution = {
agent_name: target.file,
summary: parsed.summary?.trim() || `Improved ${target.name} from user feedback.`,
diff_preview: diffPreview(target.content, updated),
content: updated,
};
// Merge with any existing proposals (dedupe by agent_name; this run wins).
const path = ctx.paths.sessionAgentUpdates(ctx.sessionId);
let existing: AgentUpdateContribution[] = [];
if (existsSync(path)) {
try {
const prior = JSON.parse(readFileSync(path, "utf8"));
if (Array.isArray(prior)) existing = prior as AgentUpdateContribution[];
} catch {
existing = [];
}
}
const merged = new Map<string, AgentUpdateContribution>();
for (const p of existing) merged.set(p.agent_name, p);
merged.set(proposal.agent_name, proposal);
writeFileSync(path, JSON.stringify([...merged.values()], null, 2));
ctx.log.event("agent_updates_proposed", { agents: [proposal.agent_name], count: 1 });
// Surface the proposal where maintainers act on it: file a GitHub issue (the
// contribution model uses issues, not close-time PRs). This is the path that makes
// a user's feedback give back to the shared library — filed under their own GitHub
// identity, which is why authenticating with GitHub is required.
// `github.issue_token` overrides the attribution to a bot account. No-op without
// any token, so local runs still keep the proposal in agent-updates.md.
const usingServiceToken = Boolean(ctx.cfg.github.issue_token);
const token = ctx.cfg.github.issue_token || ctx.githubToken;
if (token) {
// What the issue is titled and therefore deduped by. Prefer the recorded lesson's
// instruction: it is the string the memory bank corroborates across sessions, so it
// is stable, and the same lesson reported twice reaches the same issue. The summary
// is the fallback for a proposal with no recorded lesson (the correction classified
// as a one-off but still trained the prompt) — the model re-words it every run, so
// it discriminates between lessons without reliably matching itself. Both beat the
// bare title, which could only ever match the one issue already open.
const slug = lessonSlug(args.lesson?.instruction || proposal.summary);
try {
const filed = await createAgentUpdateIssue(token, ctx.cfg.github.upstream_repo, ctx.cfg.github.api_base_url, {
agentName: proposal.agent_name,
agentMarkdown: proposal.content,
summary: proposal.summary,
diffPreview: proposal.diff_preview,
sessionId: ctx.sessionId,
lessonSlug: slug || undefined,
lesson: args.lesson
? {
instruction: args.lesson.instruction,
feedback: args.lesson.feedback,
count: args.lesson.count,
}
: undefined,
});
// `action` distinguishes a new issue from a comment on the one that already tracks
// this lesson. Both are successes; the log line said "(duplicate — skipped)" for
// the second case, which was the only trace of a lesson that went nowhere.
ctx.log.event("agent_update_issue", {
agent: proposal.agent_name,
action: filed.action,
url: filed.url,
lesson_slug: slug || null,
});
} catch (e) {
// Same soft failure and the same likely cause as runContribution's filing
// path, so the same diagnosis — an operator debugging a dead update-proposal
// path needs it as much as the suggestion one.
ctx.log.event("agent_update_issue_failed", {
agent: proposal.agent_name,
error: (e as Error)?.message ?? String(e),
...installHintFor(e, { usingServiceToken }),
});
}
} else {
ctx.log.event("agent_update_issue_skipped", { agent: proposal.agent_name, reason: "no github token" });
}
return [proposal];
}
// Primary, low-rot learning path (#1/#2/#4/#5): classify a feedback correction and,
// when it's a generalizable or accessibility lesson (not a one-off specific to this
// document), distill it into a reusable instruction + localized before/after example
// and record it to the agent's example bank (memory.ts). Recorded lessons are
// corroborated across sessions and injected into the agent's prompt at run time —
// the agent file itself stays stable.
//
// Returns the stored lesson so the caller can hand it to
// `proposeAgentUpdatesFromFeedback`, whose issue title is keyed on it (see
// `lessonSlug`); null whenever nothing was recorded — no feedback, nothing changed, or
// the correction classified as a one-off.
export async function learnFromFeedback(
ctx: PipelineContext,
args: { agentFile: string; before: string; after: string; feedback: string },
): Promise<CorrectionExample | null> {
const fb = loadFeedbackAgent(ctx);
if (!fb || !args.feedback.trim()) return null;
if (args.before.trim() === args.after.trim()) return null; // nothing changed this run
const correction = diffPreview(args.before, args.after);
const user =
`TASK: classify\n\n` +
`## User feedback\n${args.feedback}\n\n` +
`## How the document changed this run (diff)\n\`\`\`diff\n${correction}\n\`\`\``;
const res = await ctx.router.complete(
FEEDBACK_AGENT,
"text",
[
{ role: "system", content: fb.content },
{ role: "user", content: user },
],
{ step: "feedback_learn" },
);
ctx.log.agentCall({ agent: fb, phase: "review", output: res.text });
const parsed = extractJson<ClassifyOutput>(res.text);
const raw = parsed?.kind;
if (!parsed || !parsed.instruction?.trim() || (raw !== "generalizable" && raw !== "a11y_policy")) {
ctx.log.event("feedback_classified", { kind: raw ?? "unknown", recorded: false });
return null;
}
const kind: LessonKind = raw;
const entry = recordExample(ctx.paths, {
agent: args.agentFile,
kind,
instruction: parsed.instruction.trim(),
before: (parsed.before ?? "").trim(),
after: (parsed.after ?? "").trim(),
feedback: args.feedback.trim(),
session: ctx.sessionId,
});
ctx.log.event("feedback_learned", { agent: entry.agent, kind: entry.kind, count: entry.count, instruction: entry.instruction });
return entry;
}