1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778// Distills a session's log.jsonl into a machine-readable health/timing summary
// for maintainers — human or AI. The key signal for "is it hung?" is
// `in_flight.waiting_ms`: a model call that started but has not finished.
interface LogEvent {
ts?: string;
type?: string;
phase?: string;
agent?: string;
step?: string;
model?: string;
provider?: string;
capability?: string;
duration_ms?: number;
ok?: boolean;
error?: string;
input_tokens?: number;
output_tokens?: number;
cache_read_input_tokens?: number;
cache_creation_input_tokens?: number;
[k: string]: unknown;
}
// What a set of model calls cost and took, and who answered them. Shared by `by_agent` and
// `by_step` so the two splits are the same seven numbers over the same calls, differing only
// in how they are keyed — a reader comparing them is comparing groupings, never definitions.
export interface CallTotals {
count: number;
total_ms: number;
max_ms: number;
input_tokens: number;
output_tokens: number;
cache_read_input_tokens: number;
cache_creation_input_tokens: number;
// Which model ids answered these calls, sorted and deduplicated.
//
// The seven numbers say what a bucket cost; this says what the cost is a price OF, and
// the two are only useful together on the one knob a deployment actually turns.
// `providers.per_agent` picks a model per agent, and until this field the run's own
// summary could not say whether a swap had taken effect: an override key that names no
// dispatched agent is ignored and the call falls through to the provider's own model
// (`perAgentKeyWarning`, config.ts), so a swap that never happened and a cheaper model
// that saved nothing produced the same run and the same diagnostics. A boot warning
// catches the bad KEY; nothing said which model each agent had ended up on.
//
// Collected over the same calls as the totals, failures included: a model id that is
// valid for one provider and named to another resolves happily (`resolveAgentModel`
// keeps `providers.default` when only `model:` is overridden and does not check that the
// id belongs to it) and then fails on every call, which is exactly the row whose model is
// the thing worth reading.
//
// A list rather than one id, because one agent is not one model. Resolution keys on agent
// AND capability, and three agents call with both: `page` extracts and corrects with
// `vision` and merges a specialist fragment with `text`, `feedback` judges a page with
// `vision` and classifies with `text`, and the copy editor picks its capability from
// whether the section it is editing has images. So a deployment using a provider's
// `per_capability` block runs one agent on two models on purpose, and a row reporting the
// first or the last would be a claim the config does not make. Sorted so the field is
// stable across two runs of the same shape.
models: string[];
}
export interface Diagnostics {
session_id: string;
status: string;
phase: string;
started_at: string | null;
last_event_at: string | null;
elapsed_ms: number;
// Non-null only while a model call is outstanding (likely culprit if hung).
// When extraction runs pages in parallel, several calls can be open at once;
// this reports the longest-waiting one.
in_flight: null | {
agent: string;
// Which job is waiting, since that is what a stuck run is asked about first and the agent
// name does not settle it — `feedback` in flight is a page being checked or a user's
// feedback being routed, and only one of those is on the critical path of a delivered page.
step: string;
model: string;
provider: string;
capability: string;
since: string;
waiting_ms: number;
};
// How many model calls are outstanding (0 unless running). > 1 means pages are
// being extracted in parallel.
in_flight_count: number;
phase_durations_ms: Record<string, number>;
// `total_ms` is the SUM of call durations, which exceeds wall-clock time when
// calls overlap — that is the point of `concurrency_factor`
// (total_ms / elapsed_ms, rounded to 2dp): ~1 means effectively serial, ~N
// means N calls were typically in flight. Use elapsed_ms for wall-clock.
model_calls: {
count: number;
failed: number;
total_ms: number;
avg_ms: number;
max_ms: number;
concurrency_factor: number;
};
// What the run consumed, in tokens. Deliberately not in dollars: the rate depends
// on the provider, the region and the model, all of which are deployment config and
// any of which can change without this file knowing — the same reason the limits
// endpoint publishes sizes without naming the model behind them. Tokens are the
// durable fact; whoever knows the price sheet does the multiplication.
//
// The four counts bill at four different rates, so they are reported separately
// rather than as a total. `calls_reported` is how many of `model_calls.count`
// carried any usage at all: when it is lower, these sums cover only part of the run
// and a cost derived from them is a floor, not an estimate.
tokens: {
input: number;
output: number;
cache_read: number;
cache_write: number;
calls_reported: number;
};
// Per-agent totals are the attribution that matters for both halves of the bill:
// which agent is slow, and which one is expensive. They are not the same agent.
//
// All four token counts, not just input and output: `input_tokens` excludes what was
// read from the cache, so on a deployment that caches, a two-field split understates
// an agent's prompt by exactly its cached share — and understates it worst for the
// agent that caches best, which inverts the answer the split exists to give. Keyed as
// the log line keys them, so the names that cross the adapter/diagnostics seam are the
// same ones in both places.
//
// This is also the row that says whether a per-agent model override took effect, because
// `models` is on it: `providers.per_agent` is keyed by agent, so the split a swap is
// decided from is the split it has to be confirmed from.
by_agent: Record<string, CallTotals>;
// Per-STEP totals: the same seven numbers and the same models, keyed by the job that bought the call rather
// than by the agent file that answered it (`PipelineStep` in providers/types.ts).
//
// This is the split a per-step cost claim has to be read off, and `by_agent` is not it.
// One agent serves several jobs, so an agent's row is a sum over jobs and a job's cost can
// be spread across rows: extraction's per-page fidelity check books to `feedback`, which
// reported the extraction step at 41% of a document when its jobs together are 57.2% (#280),
// and the table-join step's whole bill arrived inside `copy_editor` next to the review
// round's (#243). Both are unrecoverable from `by_agent` at any effort, because the
// information is not in it.
//
// Kept ALONGSIDE `by_agent` rather than replacing it, because the two answer different
// questions and both get asked: `by_agent` is what a deployment reads to decide a
// per-agent model override (`providers.per_agent`, which is keyed by agent), and a step
// cannot be pointed at a model. Read together they also localize a cost: a step that grew
// while its agent's row did not is a step that took work from another one.
by_step: Record<string, CallTotals>;
slowest_calls: { agent: string; step: string; model: string; capability: string; duration_ms: number; ok: boolean }[];
errors: { ts: string | null; type: string; message: string }[];
// What the verify-then-correct loop did, and what it bought.
//
// Every page is checked against its source image and a page that fails is re-rendered
// once, so a run's page-call count is `pages + corrections` and a high failure rate turns
// an optional pass into a mandatory one — 58 of 75 pages across three real runs, with
// verification alone at 24% of one document's bill (issue #137). `corrections` and not
// `verify_failed`, because a page that PASSED its check is re-rendered too when the code
// finds a link the model dropped, and that costs the same page call: see `triggers`.
// None of that was
// visible here: the log recorded the verdicts and said nothing about the corrections,
// so the loop's cost was inferable from arithmetic and its value not at all.
//
// The counts, not the rates: `verify_failed / (pages_verified - pages_unjudged)` is the
// rejection rate — over the pages a verdict was actually read from, see below — and
// `results.identical + results.empty + results.failed` is what was paid for and bought
// nothing — bought, not discarded: an `identical` fragment is still what ships, since what
// the page call failed to buy is a change and not a page. `rejected` is the one that was
// thrown away, and `failed` the one that cost the most, since a correction that hit the
// output ceiling paid for a full ceiling of tokens before failing (issue #171) — leaving it
// out of that sum would hide the most expensive of the three. But a
// consumer that wants a percentage can divide, and a percentage over three pages is not
// a measurement. Summed over every run this session has had, like `model_calls` — a
// feedback round verifies pages again, and both times count.
verification: {
pages_verified: number;
// Of those, the pages nothing actually judged: no Feedback Agent loaded, nothing to
// verify, a reply that would not parse. `verifyAgentOutput` answers ok=true in all three
// so that verification can never break a run (pipeline/feedback.ts), which means "the
// verifier looked and was satisfied" and "nobody looked" arrive at this fold as the same
// event — and a run that lost its Feedback Agent halfway through reads as a run with an
// unusually good pass rate.
//
// A SUBSET of `pages_verified` rather than a deduction from it, deliberately: that field
// is compared across runs and benchmark rounds, and quietly changing what it counts
// would move every published number without saying so. `verify_failed / (pages_verified
// - pages_unjudged)` is the rejection rate over the pages that were actually judged.
//
// Zero on every log written before the flag existed, which is the one thing it cannot
// distinguish: an old run with no Feedback Agent reported the same `page_verify_ok`
// lines as a passing one, and nothing recoverable from the file says which (issue #211,
// and #180 for the measurement that needed it).
pages_unjudged: number;
// Of THOSE, the pages nothing looked at because nothing was bought: a page the agent declared
// blank, whose fragment is empty and which is no longer sent to the Feedback Agent at all
// (issue #294, `page_verify_ok` with `skipped: "blank"`). A subset of `pages_unjudged`, which is
// a subset of `pages_verified`, so neither of those moves and no published rate changes — what
// this adds is the ability to tell a saving from a failure, because until it existed a run that
// skipped 9 calls and a run whose Feedback Agent would not load produced the same two numbers.
//
// It is also the only way to PRICE the skip from a delivered run: this count times the cost of a
// verify call on an empty fragment ($0.0095 against $0.0212 for an average page, measured on the
// bench's 100-page corpus) is what the run did not spend. Zero on every log written before the
// skip, where those pages were verified and counted in `pages_verified` exactly as they are now.
//
// Calls not bought, which is money not spent only where there was a verifier to spend it on: a run
// with no Feedback Agent loaded skips the blank page's call too and saves nothing by it.
// `pages_unjudged == pages_verified` is consistent with that run and does not identify it — the
// same equality comes out of a run whose verifier loaded and whose every reply failed to parse,
// where the calls were bought — so the thing to read is the calls: `by_step.verify.count` below is
// 0 where no verdict was bought at all. The flag deliberately does not depend
// on whether the agent loaded — it would make one field mean two things, and it would put a disk
// check in the extraction path to decide a label.
pages_skipped_blank: number;
// Of those same `pages_unjudged`, the pages whose verify call was bought and THREW: a throttle, a
// stall, or a reply that overran the output ceiling (issue #364, `page_verify_ok` with
// `skipped: "error"`, and `page_verify_error` for the evidence). Nested the same way, so like the
// blank skip it moves nothing above it.
//
// It is the counterweight to the field above and the reason both are needed rather than one
// `skipped` total. A blank skip is a call NOT MADE and is money saved; an error is a call made,
// billed for a full ceiling of output, and answered with nothing — on the measured case that was
// $0.5051 on a single page, more than twice the average page's whole bill. Adding the two into one
// "unjudged for a reason" number would price the most expensive shape of verification failure as a
// saving, which is exactly the reading the blank counter was added to prevent for the other
// direction. Multiply this one by a full-page verify call, not an empty-fragment one.
//
// Zero on every log written before #364, and that zero is not a measurement of anything: before the
// guard the same failure took the PAGE with it, so those runs recorded it as
// `page_extraction_failed` and a page in `pages_failed` — not as an unjudged verify. A run whose
// verifier was being throttled reads, on an older log, as a run whose vision was failing.
pages_verify_error: number;
verify_failed: number;
// `verify_failed` split by what the verifier said was WRONG, counted in pages
// (pipeline/feedback.ts `VERIFY_KINDS`). Two bench rounds rejected 74 of 94 and 76 of
// 100 pages, and no field here could say whether that was content arriving missing or
// descriptions being polished — a page that lost three table rows and a page whose alt
// text went from "orange kayak" to "orange-yellow kayak" were the same line (issue
// #182). `effects` answers the same question from the other end, and only about
// corrections that changed something; this one is about every page that failed.
//
// NOT a partition, for the same reason `effects` is not: a page with a missing row and a
// thin alt counts in `content_missing` and in `alt_quality`, so these sum to at least
// `verify_failed` and usually more. Read each against `verify_failed`, not against the
// total. Pages and not problems, so that one page naming six things cannot outweigh six
// pages naming one each — `verify_failed` is a page count and these have to divide into
// it.
//
// `untagged_pages` is what keeps the rest honest: a verdict from an agent file that
// predates the kinds, or a trained one whose contract was rewritten without them, names
// its problems in prose with no kind at all. Those pages are in `verify_failed` and in no
// kind bucket, so a split read without it beside them is a split of the tagged share
// reported as the whole run. A page can be here AND in a kind bucket, when some of its
// problems were tagged and some were not — which is why the name says `pages`: the count
// on the log line it is folded from is a count of PROBLEMS, and the two are different
// numbers on the same run. `verify_untagged_problems` below is that other unit, because a
// run that lost one tag per page and a run that lost every tag report the same
// `untagged_pages` and only the second makes the split unusable.
verify_kinds: {
content_missing: number;
content_wrong: number;
structure_wrong: number;
a11y_only: number;
alt_quality: number;
untagged_pages: number;
};
verify_untagged_problems: number;
// Pages the verifier PASSED while naming a problem, and what it named. A verdict's `ok` is
// its `faithful`/`accessible` flags, and a correction is bought only when a flag is false
// AND a problem is named (pipeline/extraction.ts `failedCheck`), so a verdict that
// describes a defect with both flags true ships the page and the sentence it wrote is not
// even in the log — `page_verify_ok` carries no `problems`. Calibrating the verifier
// against injected defects found 3 of 30 damaged pages described in full and passed, which
// is most of the gap between what it perceived (28 of 30) and what it flagged (25) — a
// different failure from a verifier that cannot see, and a different repair (issue #210).
//
// `pages` counts them; the five kind fields split them the way `verify_kinds` splits the
// failures, in pages and not a partition. `content_or_structure` is the pricing field:
// pages naming at least ONE of `content_missing`, `content_wrong` or `structure_wrong`,
// which is exactly the population a kind-gated failure rule would newly fail and newly pay
// a correction for. The complement is not a bug to fix — an `alt_quality` suggestion on a
// page that ships is the Feedback Agent doing what it was asked.
//
// `undecided_pages` is the unknown ABOVE that floor: pages where a kind-gated rule has
// nothing to decide on, because a problem arrived with no kind this code knows and no
// content or structure kind was named either. So `content_or_structure` is the least such a
// rule would cost and `content_or_structure + undecided_pages` the most, and the two can be
// added because neither contains the other. That is deliberately NOT the rule beside
// `verify_kinds`, whose `untagged_pages` counts a partly-tagged page too: there the field
// audits a SPLIT, and a page with one tag missing is a page whose split is incomplete;
// here the question is whether a decision can be made, and a page already naming
// `content_missing` is decided whatever else it left untagged.
//
// Nothing in the run reads any of this: the event decides nothing, and these counts exist
// so the rule can be priced over a fleet before it changes what pages cost. Zero on every
// log written before the event, which cannot be distinguished from a run where it never
// happened.
verify_inconsistent: {
pages: number;
content_missing: number;
content_wrong: number;
structure_wrong: number;
a11y_only: number;
alt_quality: number;
content_or_structure: number;
undecided_pages: number;
};
corrections: number;
// How each correction pass ended: `kept` CHANGED the delivered document, `rejected` was
// discarded in favour of the fragment it was meant to improve — either because it came
// back at a fraction of that fragment's size, on any trigger, or because the links path's
// re-verification found the rewrite had lost something — `identical` changed nothing about
// the page, `empty` returned nothing usable, `failed` never answered at all — the model
// call threw (a truncation, a stall, a throttle) and the page kept the version it had.
// The last three are calls that bought nothing — `identical` on the effect and not on
// string identity, so a model that re-typed its own page to no purpose is counted here
// rather than inflating `kept`, which is the number these fields exist to make honest.
//
// `failed` is apart from `empty` because it is the expensive one: a correction that hit
// the output ceiling has paid for a full ceiling of tokens before failing, where an
// `empty` one usually answered briefly and said nothing. A run whose `failed` count is
// not zero has a `providers.*.max_tokens` to raise or a page too large to correct in one
// reply, and neither is visible if the two are summed (issue #171).
//
// How to read `rejected: 0`, since two bench rounds produced it over 145 corrections and
// it was reported as a gate that accepts everything (issue #166). It was not a gate. Until
// the shrink floor landed, `rejected` was reachable on the LINKS trigger alone — a page
// that had passed its check, was re-rendered for a link, and lost something — so a round
// whose corrections were all verify-driven could not produce a rejection at any rate of
// badness, and the zero measured the absence of a rejection path rather than the absence
// of bad corrections. `CORRECTION_SHRINK_FLOOR` (pipeline/correction.ts) is the first one
// that applies on every trigger. It is deliberately a floor and not a judgement, so
// `rejected: 0` is still the expected reading of a healthy round: it counts corrections
// that came back at a fraction of the page they were given, which is a parser or ceiling
// failure and not a bad rewrite. A correction that is merely WRONG is kept, and
// `rechecks.sampled_problems_*` is where that shows up — see extraction.ts on why
// discarding it would ship the fragment that already failed the same verifier.
results: { kept: number; rejected: number; identical: number; empty: number; failed: number };
// Why each correction ran: `verify` is a page the Feedback Agent rejected, `links` is a
// page that passed and lost a link the code found in the PDF, `alt` is a page that passed
// and described an image with a placeholder instead of a description (pipeline/alt.ts,
// #290), `ids` is a page that passed and used one id on two elements (pipeline/anchors.ts,
// #373), `words` is a page that passed and wrote one word two ways (pipeline/hyphens.ts,
// #334), and `both` is one with more than one of those. These are the split that makes
// `corrections` readable as a bill — a `links`, `alt`, `ids` or `words` correction is a page
// call with no verify failure behind it, so a consumer reading `verify_failed` as the number of
// extra page calls undercounts by all four.
//
// `alt` and `ids` are both expected to be 0 on a healthy run, and that is the point of
// counting them: the alt rule flags nothing in Iris's own output (0 of 1,064 alts across the
// bench corpus) and the id rule flags 2 of 1,501 page replies, none of them from the model
// deployed today, so a non-zero is either a page agent that has started writing placeholders
// or reusing ids, or a regression in one of the rules. Neither is a cost line at that rate.
//
// `words` is the one of the four that is expected to be NON-zero, and it is therefore the one
// with a cost line. On #334's 100-page census the shipped model wrote one word two ways on 4
// pages of 91 — and no arm was clean, where the other three arms are clean on `alt` and on the
// soft hyphen — so this trigger buys a page call at a rate somewhere near 4%, on pages that had
// already passed. That is a rate to watch rather than a number to fear at these volumes, but it
// is the trigger a reader should look at first when `corrections` grows without `verify_failed`.
triggers: { verify: number; links: number; alt: number; ids: number; words: number; both: number };
// What the corrector said it would NOT do, and why that is a number worth publishing rather
// than a log line worth grepping (#373 directive 4). Before it, the corrector's only legal move
// was compliance: a checker's claim that a page's ids are duplicated when they are not is
// answered by editing a page that was right, and the whole event is invisible from inside the
// run — #373 could only find it by re-reading raw replies off disk. The licence is narrow (a
// claim about the HTML the corrector was shown, refuted by that HTML) and it gates nothing, so
// these counts are the only trace it leaves.
//
// Two rates, and both denominators are on this object because neither count can be read alone:
// `pages` against `corrections`, and `problems` against `problems_offered` — the whole bill
// every correction was given, summed from the same `page_corrected` lines. 2 declined of 2 is a
// correction refused outright; 2 of 40 is the pass doing what it was built for.
//
// `code_checked` is the misuse, and it is the field to watch rather than the total: a declined
// `links`, `alt` or `ids` problem is a refusal of something Iris checked against the source file
// or the parsed fragment, so it is wrong by construction, where a declined `verify` problem is a
// disagreement with a reading and may well be right. A run with a non-zero here has a corrector
// reading the licence wider than it is written, which is the failure #373 warns about in
// advance — and it is countable now rather than arguable later.
//
// `words` is the fifth source (#334 part B) and is deliberately NOT part of `code_checked`,
// though it is checked in code. A split-word problem tells the model that the page writes one
// word two ways and asks the image which spelling is right, so "the page really prints both" is
// an answer that problem invites — the one code-checked class where a decline may be correct. It
// is counted because a rate matters in the other direction: this is the check that can raise a
// false problem, and `words` against the `page_split_words` lines is what says how often it does.
//
// `unattributed` is a decline that cited no problem number, or cited one the request never
// listed. Kept apart from all of them: it is not evidence about a code-checked fact, and it is not
// nothing either — it is a disagreement whose subject cannot be recovered, which is a fact about
// the reply's shape and the first thing to look at if `problems` is large and unreadable.
declined: {
pages: number;
problems: number;
problems_offered: number;
code_checked: number;
words: number;
unattributed: number;
};
// What the corrections that DID change something changed, as observed on the two
// fragments rather than claimed by the verdict (pipeline/correction.ts). Not a
// partition: a re-render that rebuilds a table counts under both `text` and
// `structure`. `alt_only` is the one that stands alone, and it is the interesting one
// — a run whose corrections only ever refine alt text is paying a page call per page
// for image descriptions. `attrs` is every attribute but alt, which is where the
// cheapest real fixes live: an `href` the model re-typed, a `<th scope>`, an
// `aria-describedby` — a correction that moves no word and matters.
//
// `text_grew` and `text_shrank` split `text` by DIRECTION, on the size of the prose a
// reader receives rather than of the fragment: how many corrections added words, how
// many removed them, and — on a log where every line carries the sizes — by subtraction
// how many rewrote the same quantity of prose in place. A line from before the sizes
// existed still counts under `text` and lands in neither direction, so that subtraction
// absorbs it as an equal-length rewrite; a session's log is append-only across rounds and
// this sums all of them, so a session that takes a feedback round across the upgrade has
// exactly that mixed log. `text_grew + text_shrank` against `text` is the honest reading
// there. That is what makes a high `verify_failed` rate readable. Two bench rounds put
// it at 71% and 74% of pages, with `attrs` and `structure` touched on nearly every
// correction and `text` on fewer — which reads either as most pages arriving with
// content missing, or as most pages arriving fine and being polished, and the counts
// could not tell the two apart (issue #166). A round whose corrections cluster in
// `text_grew` is recovering content the vision pass dropped; one that barely leaves
// `attrs` and `structure` is buying markup on pages that were already readable, and the
// cheaper remedy for that is the page prompt, not a call per page.
//
// No threshold: a correction that adds one character counts as `text_grew`, because any
// band that called that "cosmetic" would be a number picked rather than measured. The
// magnitudes are on each `page_corrected` line (`text_chars_before`, `text_chars_after`)
// for a consumer with a corpus to calibrate one on.
effects: {
alt_only: number;
text: number;
attrs: number;
structure: number;
text_grew: number;
text_shrank: number;
};
// Second verdicts on a corrected page, kept apart by whether the verdict was allowed to
// decide anything, because the two answer different questions and a single ok-rate over
// both answers neither.
//
// `sampled` is the measurement-only sample — `defaults.recheck_sample_size` pages per
// batch, one by default (correction.ts `recheckSampler`) — taken on a page that FAILED its check and was re-rendered, so
// `sampled_ok / sampled` is whether correction converges: the number that says whether
// the loop is worth its 24%. It accumulates one or two per run, so it is a fleet
// measurement and not a per-document one.
//
// `binding` is the links path's own re-verification, which keeps or discards the
// rewrite. Those pages had already PASSED verification and were re-rendered only to
// recover a link, so their ok-rate is "did a rewrite of a good page stay good" — a
// different question, and on a link-heavy PDF there is one per page, which would swamp
// the sample if the two were summed.
//
// `sampled_problems_before` and `sampled_problems_after` are how many FIDELITY problems
// those sampled pages were sent to be corrected with, and how many the second verdict
// named, summed over the sample. `sampled_ok` on its own read as a pass/fail on a
// single-shot pass that was never expected to reach zero — four samples, four not-ok, and
// no way to see whether the corrections had fixed most of what was flagged or none of it
// (issue #166). 11 problems in and 3 out is a loop that mostly works; 11 and 11 is one
// that does not, and both are `sampled_ok: 0`.
//
// Fidelity problems and not the correction's whole bill, because the two sides have to be
// comparable: a correction is also given the links the code found missing, and the second
// verdict judges the fragment against the IMAGE, where a link target does not appear — so
// a link counted going in could never be counted coming out, and a page with one verdict
// problem and three missing links would report four-in-one-out for a correction that fixed
// nothing the verifier named. The event carries the link share as `links_before`, and
// `page_corrected`'s `problems` is the whole bill.
//
// Sums over pages, so a single page with many problems moves them more than several with
// one each — read them as a ratio and not as a per-page average, and against the samples
// they were summed over, which is `sampled` less the unjudged ones and less any line too
// old to carry both counts (see the field below).
// And `sampled_problems_after: 0` does not mean the sample passed: the verdict's `ok` is
// its `faithful`/`accessible` flags (pipeline/feedback.ts), which an agent can set false
// while naming nothing, so `sampled_ok` remains the answer to whether it passed.
//
// The binding population has no such pair, for the reason it is counted apart: those pages
// had PASSED their check, so their `problems_before` is 0 by construction and the question
// their verdict answers is whether the rewrite lost something, not how far it got.
//
// `*_unjudged` is `pages_unjudged`'s argument one fold down, and the binding one is where
// it bites: a recheck's `ok` is also what an unavailable Feedback Agent looks like, and
// with none loaded every page passes its first check, so every corrected page's recheck is
// the BINDING one and every one of them is a "the rewrite was checked and stayed good"
// line for a page nobody looked at. Subsets of the counts above rather than deductions
// from them, so those totals keep counting what they always counted. Zero on every log
// written before the flag.
//
// The judged-only rate is `(binding_ok - binding_unjudged) / (binding - binding_unjudged)`,
// and the same shape for sampled — BOTH sides, which is where this differs from the fold
// two levels up. There, subtracting from the denominator alone is exact, because
// `verify_failed` can only come from `page_verify_failed` and an unjudged verdict cannot
// produce one, so the numerator and `pages_unjudged` are disjoint. Here the numerator is a
// PASS count and an unjudged recheck is a pass by construction — every one of them is
// inside `binding_ok` — so denominator-only subtraction reports a rate above 100%.
rechecks: {
sampled: number;
sampled_ok: number;
sampled_unjudged: number;
// Summed over the JUDGED samples only — an unjudged recheck contributes neither, even
// though its line carries a real `problems_before`. Its `problems_after` is 0 because
// nothing was named, not because nothing was left, and pairing a true before-count with
// a non-verdict after-count would report that page as a correction that fixed
// everything it was given. Unlike `*_ok` there is no field to back that out of, and no
// published number moves by leaving it out: only a log carrying the flag can be
// affected, and the flag is newer than every round measured so far.
sampled_problems_before: number;
sampled_problems_after: number;
binding: number;
binding_ok: number;
binding_unjudged: number;
// Binding rechecks that were BOUGHT and threw, so they produced no verdict at all
// (`page_verify_error` with `step: "recheck_binding"`, issue #364). NOT a subset of
// `binding` and not inside any rate above: the three fields above are fed from
// `page_correction_recheck`, which does not fire when there is no verdict to report, so
// this population is disjoint from all of them and the judged-only rate in the comment
// above is unaffected by it.
//
// It is here rather than in `pages_verify_error` two levels up, and the distinction is the
// one that field is nested for: that count is a subset of `pages_unjudged`, and a page whose
// BINDING recheck threw is not unjudged — it has a real first verdict and it PASSED. Putting
// it there would put a judged page inside the unjudged count and move a published rate.
//
// Without it this failure reaches no number anywhere, which is the wrong silence to leave:
// it is the more expensive of the two shapes, because the page had already been rendered,
// verified AND corrected, so two calls' work is discarded rather than one. Its only other
// trace in this fold is `page_corrected` `result: "rejected"`, pooled with the shrink floor
// and with a rewrite a second verdict actually refused — and those two are a correction
// judged and found wanting, while this one was never judged. Before #364 the page was
// counted, in `pages_failed`, which was wrong about the cause but not silent.
//
// The sampled recheck's own failure (`page_correction_recheck_failed`, #171) still has no
// counter, and that gap is deliberately left alone here: it predates this and it is the one
// verify failure that costs nothing, since the sample decides nothing whether it answers or
// not. So the asymmetry in this object is a real difference between the two populations and
// not an oversight — read `binding_error` as "a gate that could not be applied", which is
// the only one of the two that changes what ships.
binding_error: number;
// The failing verdicts themselves — not a count; the counts are `sampled - sampled_ok`
// and `binding - binding_ok`. One entry per recheck that named a problem, carrying the
// prose it named it in, because that prose is the whole answer to "what is still wrong
// with the page that shipped" and nothing else in this file holds it: the counts say a
// correction did not converge and never say what it failed to fix.
//
// Here rather than in `errors`, which used to hold these and rendered every one of them
// `message: "unknown"` — it reads `error`, and this event's diagnosis is `problems`
// (issue #296). Moving it rather than fixing that message is the other half of the same
// issue: a second verdict is a measurement, and `errors` has to be readable as "the run
// is in doubt". So the diagnosis is beside the numbers it explains instead.
//
// BOTH populations, marked by `binding`, because the two failures are worth reading and
// are not the same reading: a sampled failure is a page that shipped still wrong, and a
// binding failure is a rewrite that was refused so the page shipped as it was. `null`
// there is a line that did not say, which the counts above put in neither bucket — kept
// here anyway rather than dropped, since the verdict is a fact about a page whatever the
// line failed to say about its own population.
//
// Failing only, so an `ok: true` recheck adds nothing: `failedCheck` (extraction.ts) is
// `!ok && problems.length > 0`, so a line logged `ok: false` always names at least one
// problem and an entry here can never carry an empty message. Which also means an
// unjudged recheck is absent by construction — it logs `ok: true`.
//
// Bounded, and `verdicts_omitted` is how many the bound left out — see `MAX_VERDICTS`
// for why a cap exists here and nowhere else in this file, and which of the two
// populations can actually reach it. The omitted ones are in log.jsonl in full.
failures: { ts: string | null; page: number | null; binding: boolean | null; message: string }[];
verdicts_omitted: number;
};
};
// What the table-join stage did with the pairs it found, and how much of it was free.
//
// Here because nothing looked at it (#326 ask 1). The events have carried all of this since #278
// put `by` on `table_joined`, and reading it meant parsing log.jsonl by hand — so the one step
// whose cost swings most between runs was the one step with no line in the file an operator
// actually reads. `table_join` was 11.5% of a 100-page bench round's bill and moved $0.9533 →
// $1.6701 → $1.2362 across three rounds with this stage's code, `agents/` and the model all
// byte-identical. Every dollar of that swing is `code_declined` moving 8 → 13 → 11.
//
// Counts, and no share. `joined_in_code / (joined_in_code + code_declined)` is a number anyone can
// divide, but this file will not publish it, because the measurement it looks like — "the free path
// takes half the pairs" — is a draw and not a property: 53%, 24% and 31% on the same corpus with
// nothing changed. A range with its corpus attached is in `joinInCode`'s comment; a point here would
// be read as the rate.
//
// Summed across feedback rounds like everything else in this file. A round re-joins the tables of
// every page it re-extracted, so a document whose second round changed one page has two passes'
// worth of pairs here, and that is the bill.
tables: {
// By `by` on `table_joined`, so these are pairs that were merged AND cleared `verifyJoin`. A line
// whose `by` this build does not recognize is counted in neither, for the reason an unknown
// correction trigger is: a total that is visibly short beats a bucket filled by guesswork. Old
// logs from before #278 have no `by` at all, and read as zero joins rather than as free ones.
joined_in_code: number;
joined_by_editor: number;
// Of the free joins, the ones carrying their halves' bytes — the same bound, on the population a
// loosening must not BREAK rather than the one it means to recover. Both numbers or neither: a
// re-score that can read "N of M declines are replayable" and cannot read the same of the joins can
// only ever measure the upside, which is the reading `table_joined` grew these bytes to prevent.
// Paid joins are not in it and cannot be — their bytes are on the decline line that bought them, and
// those are counted below.
joined_in_code_with_halves: number;
// Pairs the free path stood down on, each of which bought a Copy Editor call. NOT a failure
// count: a decline delivers exactly what the pipeline delivered before the code path existed. The
// per-reason split stays in log.jsonl, because the reasons are an open set — `verify:<reason>`
// among them — and a fixed list here would silently stop summing the day a rule is added.
code_declined: number;
// Of those declines, the ones whose two halves' bytes are ON the line, so a looser rule can be
// scored against them without buying a round (#326). Expected to equal `code_declined` exactly —
// the bound that drops them is 2.5x the largest pair this corpus has produced — and it is here
// BECAUSE of that: a bound nothing reaches is a bound whose biting would otherwise be invisible,
// and the difference between these two numbers is the part of a re-score that has no evidence
// behind it. Old logs from before this field read as zero, which is what they are: not replayable.
code_declined_with_halves: number;
// Of those declines, the ones where both halves actually declared a header block WITH CELLS IN IT,
// and of THOSE the ones whose two signatures differ. Two numbers because the first is the
// denominator and it is not `code_declined`: a continued page that reprinted no header has nothing
// to compare, and counting it as agreement or as disagreement would both be inventions.
// `header_compared: 0` with declines above it therefore means "no pair could show this", not
// "every header was stable".
//
// This is the canary #326 asked for, and it is deliberately not `code_declined` filtered to
// `header_differs`: that reason is one guard's verdict, while the instability behind it was
// observed firing the width check and the id rule too, on pairs that had joined for free a round
// earlier. A pair declined for `id_would_be_lost` whose headers also disagree is evidence of the
// same thing and would not be in that filter.
//
// Two readings of one printed header agreed 48–61% of the time across three rounds, so a nonzero
// count here is the expected state of a healthy run rather than an alarm. What is worth reading is
// the direction across rounds of the same corpus: the code did not change between those three.
header_compared: number;
header_differs: number;
// Pairs left as two tables, from `table_join_failed` — the editor refused or could not be read,
// or the pair could not be located in the source bytes at all. A shortfall in the output: the
// document ships both halves, so a reader meets a table cut in two.
//
// Counted only for the lines of that type that are about a PAIR. `table_join_failed` also carries
// one run-level line, `stage: "body"`, for an assembled body no parser could read — that document
// joined nothing at all, so folding it in here would report `failed: 1` for a run where every pair
// stayed split. It gets its own count, and a `stage` this build does not recognize lands in
// neither, on the same reasoning as an unrecognized `by` above.
body_unreadable: number;
failed: number;
// Pairs that ship as two tables having never been attempted, because the run hit
// `MAX_TABLE_JOINS` first (`table_joins_capped`, summing its `pending`). A second reason a reader
// meets a split table, and NOT part of `failed`, which counts pairs that were tried: the remedy is
// a higher cap, where `failed`'s is a better join. Both belong beside the other, so neither is read
// as the total.
capped_pending: number;
};
// What happened to a Copy Editor round whose reply hit the output ceiling, and how much of it was
// kept (#317).
//
// Here because nothing counted it. The editor is the largest agent in the pipeline — 33.1% of a
// 100-page bench round's model bill — and a window that truncates costs 5.2x one that fits: the
// discarded whole-document attempt is paid for in full, and then the remainder is asked for a
// section at a time. The salvage that #295 and #319 built to recover that money records everything
// about itself on three log lines and nothing read them, so its hit rate meant parsing log.jsonl
// by hand. `editor_truncated_rate` and `editor_truncated_lost_rate` on `/v1/quality` are the only
// other numbers about this, and they are per DOCUMENT across a deployment: they cannot say whether
// a truncation was rescued or refused, which of the reasons refused it, or whether a retreat
// happened at all.
//
// Counts of ROUNDS, not of documents. A document reviewed in three rounds can truncate three
// times, and a session's log spans its feedback rounds, so these sum over every editor round the
// session has had — the same denominator as `model_calls` and the one the money is spent in. The
// per-document reading is the quality endpoint's and stays there.
//
// Deliberately no share, and deliberately no cost. `salvaged / truncated` is a number anyone can
// divide and it is not a rate: across every round on file the salvage has fired twice, declined
// twice on the same reason, and rescued nothing. What the money cost is already here and is not
// restated — `by_step.edit` is the whole-document attempts including the discarded one, and
// `by_step.edit_section` the fallback calls it bought.
editor_ceiling: {
// Rounds whose whole-document editor reply hit the ceiling (`editor_truncated`). Every one of
// them reached the salvage, so it is the denominator for the two counts below — with one
// shortfall that is real and worth naming rather than reconciling away.
//
// `truncated - (salvaged + declined)` is not always 0. The salvage answers nothing at all for a
// truncation that returned no text (`EMPTY_REPLY`: the ceiling was spent before the reply began)
// or for an error that matched by message and lost its prototype on the way (see
// `isTruncatedResponseError`, which is broader than the `instanceof` the salvage requires).
// Neither writes a line, because neither is a reply there is anything to say about. A visible
// shortfall here beats a third bucket filled from the absence of evidence.
truncated: number;
// Rounds where part of the reply was kept and SHIPS (`editor_salvaged`). The prefix of the
// document the reply reached was corrected by the whole-document call — which saw every block
// and every attached page image — and only the remainder was asked for again.
salvaged: number;
// Of those, the ones whose edits list had finished before the cut (`closed: true`): a complete
// patch that hit the ceiling on its way out of the envelope. The cheapest shape this can take,
// and the one worth telling from a partial rescue.
//
// It does NOT by itself mean the round cost one call, and this comment said it did until #423's
// second round. A closed list claims the whole document (`claimed = blocks.length`) but the claim
// is still cut back by a retreat (`reached = lostAt ?? claimed`, pipeline/review.ts), so a closed
// reply carrying a `lost_at` leaves a non-empty remainder and the sections are bought after all.
// Read against `retreated`, which is where that case is written up.
salvaged_closed: number;
// Of those, the retreats: a block before the cut gave content up, so the claim was cut back to
// it and the edits behind it were dropped (`lost_at`). **This is the field #317 asked for**, and
// it is not a cost signal. The retreat knowingly accepts a duplicate — a move carrying content
// backwards across the cut leaves the landing edit applied and the source block untouched, so
// the content ships twice — and a truncated round is the review loop's last round, so nothing
// downstream removes it. The remedy is a feedback re-run, which is a person's action, which is
// why the rate is worth watching at all.
//
// Counted only on `editor_salvaged`. A `loss_before_cut` decline carries `lost_at` too and is
// NOT counted here: nothing was applied, so no duplicate can have shipped. That decline is the
// one shape the salvage has actually taken in every round on file, so folding the two together
// would report a duplicate risk of 2 where the observed risk is 0.
//
// `salvaged_closed` and this are not disjoint, and the combination reads oddly and is real: a
// complete patch, part of it re-asked for anyway.
retreated: number;
// Rounds where the salvage kept nothing and the whole body went to the section fallback
// (`editor_salvage_declined`). Not a failure of the salvage — every one of these is a reply it
// was right to refuse — but it is the count that says the recovery did not happen, and the
// sections were bought at the price they always were.
declined: number;
// Which of the salvage's seven refusals fired, off `reason`. These SUM to `declined`, which is
// why `unrecognized` exists: the reasons are a closed list in the emitter, and a bucket set that
// silently dropped a value this build has not heard of would stop summing without saying so
// (unlike `tables`'s `by`, where a visibly short total is the honest answer because there is no
// total to check it against). A reader can therefore check the split against `declined` and know
// the difference is 0 by construction.
//
// The seven are not interchangeable and the split is the point. `loss_before_cut` and
// `all_refused` are the salvage working — a reply whose corrections cannot be kept — and the
// only two the retreat can reach. `no_complete_edit` is a document holding one block bigger than
// the ceiling, which is the failure mode the section fallback exists for. `no_edits_list` is a
// prompt that was not followed, `out_of_order` a reply not written in one pass through the
// document, and `unknown_block`/`unreadable_edit` a reply that may not be about this document at
// all — three prompt-compliance findings sitting in the same total as two cost findings, and
// pooling them would send a reader to the wrong remedy.
decline_reasons: {
no_edits_list: number;
no_complete_edit: number;
unknown_block: number;
unreadable_edit: number;
out_of_order: number;
all_refused: number;
loss_before_cut: number;
unrecognized: number;
};
};
// Source pages whose own extraction threw, so the delivered document carries a
// failure marker instead of that page's content (pipeline/extraction.ts
// `failedPage`). Its own field because a run that ends `ready_for_review` with a
// page missing is otherwise indistinguishable here from one that delivered the
// whole document: the failed model call underneath shows up in `errors` exactly as
// a retried-and-recovered one does, and `status` says the run succeeded — which it
// did, on 24 of 25 pages.
pages_failed: number[];
// Source pages the agent read and reported empty, so the document has no content for
// them BECAUSE THERE WAS NONE (pipeline/extraction.ts `declaredBlank`). Kept apart from
// `pages_failed` because the remedy is opposite: a failed page is work to redo, and a
// blank page is work already finished. Six pages across three of four bench documents
// were reported as failures before this split, which made a document with a blank verso
// look partial to every client following docs/API.md "Partial documents".
//
// Not subtracted from anything: `images` in `run_start` counts source images, blank
// ones included, so `images - pages_blank.length` is the count that produced markup. The
// two sets are disjoint, and stay disjoint across feedback rounds: a page that failed in
// round 1 and came back blank in round 3 has been answered, so it leaves `pages_failed`
// (as `page_recovered`) and arrives here.
pages_blank: number[];
// Source pages whose fragment came from a reply that was markup rather than the envelope, so it
// carried no `"log"` field for the agent to record anything in (pipeline/extraction.ts
// `bareHtml`, event `page_bare_html`). These pages ordinarily SHIP and are in neither set above:
// the HTML is usable, which is why the rescue exists, and 0 of the 41 in two 100-page bench rounds
// left any other line behind. "Ordinarily" rather than "always", because this event is emitted in
// `renderPage` and the binding verify call that follows it is unwrapped — a provider error there
// reaches `failedPage` through `runExtraction`'s `.catch`, so a bare page CAN also appear in
// `pages_failed`. Nothing has been observed doing it and no rate here is computed from the
// difference, so it is stated rather than engineered around. What is missing is every record
// `agents/page.md` asks for in the log — a page ending mid-sentence, an orphan heading, an unkeyed
// symbol, a placeholder image source, a language change, an irregular table — unmet and unreported
// on about one page in seven (#349). Two of those six exist in the log and nowhere else; the other
// four also oblige the HTML, so on a bare page what is lost is the record, not always the remedy.
//
// Named for the reply shape and not for the consequence, because it is the narrower claim: an
// enveloped reply that simply leaves `"log"` empty ALSO has no log, has a different remedy (a
// prompt-compliance question rather than a parse one), and is not counted here. That sibling is
// not merely uncounted, it is unobserved: across 67 bench round logs on file, 2,320 `page.md`
// replies split into 2,001 with a non-empty log and 319 bare-HTML, with 0 carrying an envelope
// whose log was empty or absent. So today this is the whole population of pages with no log and
// not a lower bound on it — but the name still says which of the two it counts, because the day
// the other shape appears is the day the distinction pays for itself.
//
// Folded across feedback rounds like `pages_blank`: a page re-extracted with a proper envelope
// has a log now and leaves the set, a page bare again stays, and a re-extraction that threw
// keeps the prior fragment and so keeps the page — the field describes the DOCUMENT that
// shipped, not every render that happened. The per-round lines are all in log.jsonl, where
// `reextract` tells them apart.
pages_bare_html: number[];
// Fidelity discrepancies the Copy Editor noticed on a page whose image it had and was not
// asked about (pipeline/review.ts `readFidelityObserved`, issue #183). The first fidelity
// signal in the pipeline that does not come from the check that produced the page: VERIFY
// runs once per page during extraction, and its blind spots are the transcriber's by
// construction — same model family, same image, same failure modes — so an observation here
// on a page VERIFY passed is a measured miss rather than an inferred one.
//
// Read as evidence, NOT as a rate. The editor only ever sees the images for pages the Reader
// attributed an issue to, which skews toward pages that already had something wrong with them
// and is no sample at all of a document the Reader read clean. `observed` over `pages` tells
// you how concentrated the observations were and nothing about the document's other pages.
//
// `unattached` and `unplaced` bound how much of it is checkable: an observation about a page
// whose image was not attached is a guess about a page the model could not see (the prompt
// asks for attached pages only), and one that named no page cannot be traced to a page at all.
// Both are counted rather than dropped, so subtracting them is the reader's choice; `kinds`
// uses the same five as `verification.verify_kinds` on purpose, so the two splits can be read
// against each other.
//
// `pages` is every page an observation named, guesses included, because `pages` is where a
// person should look and a guess that turns out to be right is worth the look. `unattached_pages`
// is the subset of it the editor could NOT see, so the difference is the set backed by an image
// the model had in front of it — that decomposition is why the page list is a union rather than
// two disjoint fields.
fidelity_observed: {
observed: number;
pages: number[];
unattached_pages: number[];
kinds: {
content_missing: number;
content_wrong: number;
structure_wrong: number;
a11y_only: number;
alt_quality: number;
untagged: number;
};
unattached: number;
unplaced: number;
};
}
function parse(logText: string): LogEvent[] {
const out: LogEvent[] = [];
for (const line of logText.split("\n")) {
if (!line.trim()) continue;
try {
out.push(JSON.parse(line) as LogEvent);
} catch {
// skip malformed line
}
}
return out;
}
// How a correction pass can end (pipeline/extraction.ts `page_corrected`). A closed
// list, so a `result` this version does not know counts as a correction and is
// attributed to nothing rather than inventing a bucket for it.
const CORRECTION_RESULTS = ["kept", "rejected", "identical", "empty", "failed"] as const;
// And why it ran. A closed list for the same reason, and read off the same event. `alt` since
// #290, `ids` since #373 and `words` since #334; `both` has always meant more than one source, so
// none of the three additions changes what an old log's `both` counted.
//
// A value missing from this list is worse here than a missing `result` is, and that is why each new
// source has to be added: an unknown `result` leaves one bucket short of `corrections`, while an
// unknown TRIGGER leaves the buckets no longer summing to `corrections` at all — a run whose
// every correction was bought by the id rule would report `corrections: 12` and zeros across the
// board, which reads as a reader that cannot count rather than as a source it has not heard of. The
// guard on line ~1200 finds the value in this list before indexing, so an unlisted trigger is
// silently dropped rather than turned into a NaN, which is what makes the omission hard to see.
const CORRECTION_TRIGGERS = ["verify", "links", "alt", "ids", "words", "both"] as const;
// What the Feedback Agent said was wrong with a page (`page_verify_failed`'s `kinds`).
// Declared here rather than imported from pipeline/feedback.ts, like the two lists above
// mirror extraction.ts: this module reads a log file and nothing else, and a kind a future
// version adds should be visible as ungraded on an old reader rather than change this file's
// dependencies. The five are defined in agents/feedback.md and pinned in
// pipeline/feedback.ts `VERIFY_KINDS`; test/verify-kinds.test.ts holds the two lists equal.
//
// That visibility is only complete where a line names NO kind this reader knows: then the page
// is in `untagged_pages` and its problems are in `verify_untagged_problems`. A newer writer that
// mixes a sixth kind with one of these five logs `untagged: 0` — it recognized its own tag — so
// the old reader sees one known kind, no untagged count, and the sixth-kind problem is silently
// absent from the split rather than visibly ungraded. It needs a sixth kind shipped AND an old
// reader folding a newer log (a retained bench log, a diagnostics read mid-deploy), which is why
// it is written down here rather than coded around: inside one deploy the two lists cannot
// diverge, and the test above is what keeps that true.
const VERIFY_KINDS = ["content_missing", "content_wrong", "structure_wrong", "a11y_only", "alt_quality"] as const;
// The longest a model call can legitimately still be open, used to tell a run that is
// working from one whose process is gone (see `abandoned`). Derived, not picked: each
// adapter abandons a stream at an absolute 15-minute ceiling (providers/bedrock.ts,
// providers/openrouter.ts `MAX_TOTAL_MS`) and OpenRouter retries at most three times, so
// ~45 minutes is the worst case a caller can produce. An hour is that, rounded up — far
// enough past it that this never cuts off a call still running, and short enough that a
// killed run stops claiming to be stuck within one.
const MAX_PLAUSIBLE_CALL_MS = 60 * 60_000;
const ms = (a?: string, b?: string): number =>
a && b ? Math.max(0, new Date(b).getTime() - new Date(a).getTime()) : 0;
// How much of `rechecks.failures` this payload carries. Everything else in this file is a
// count, and these entries are model prose about a page, so they are the one part of it that
// grows with what the document needed rather than with the shape of the run — which is the
// growth `errors` did not have when the same entry was 7 characters of `"unknown"` (#296).
//
// `MAX_VERDICTS` is 20, which is generous against the SAMPLED population and not against the
// binding one — and it is worth being exact about that, because the two grow differently.
// `recheck_sample_size` defaults to 1, so sampled failures accrue at most one per run and a
// session would need twenty rounds of them to fill this. The binding recheck is not sampled:
// extraction.ts runs it on every page that PASSED its check and had a link or alt rewritten,
// so a link-heavy document can refuse more than twenty rewrites inside one round on default
// config. This cap therefore engages on a real run, and what it engages on is exactly the run
// worth capping — twenty refusals with a count of the rest says "the rewrite path is losing
// content systematically" as well as fifty verbatim would, and log.jsonl holds all fifty.
// `verdicts_omitted` says how many were left out, so a capped list is never a short one read
// as whole, on the same terms as `calls_reported` beside `tokens`.
//
// `MAX_VERDICT_CHARS` is 600 because a verdict about one page is one or two sentences — the
// one that prompted #296 is 250 characters — while the number of problems in the array is the
// model's to choose, so the JOIN is what has no bound. Wide enough to hold a real two-problem
// verdict whole, and the cut is marked (so a cut message is 601 characters, the mark being
// extra). Deliberately not markup.ts's 40 (`MAX_EXAMPLE_CHARS`): that pair bounds five
// instances at 40 characters each to recognise a CLASS of defect, and this one has to carry a
// specific page's diagnosis intact, which is the whole reason the prose is here.
const MAX_VERDICTS = 20;
const MAX_VERDICT_CHARS = 600;
// A verifier's verdict as one line, for `verification.rechecks.failures`. The problems in
// FULL and not the first of them: they are one or two sentences of the Feedback Agent's own
// prose about a specific page, no order is claimed among them (extraction.ts logs the list
// as the agent gave it), and the one that would be dropped is as likely as any to be the
// reason the page is wrong. Counted first when there is more than one, so a reader can see
// at a glance whether a correction left one problem behind or five.
//
// Defensive about the shape for the reason every read in this file is: this runs over a log
// line that may have been written by an older Iris or hand-edited, and a `problems` that is
// not an array of strings must produce a sentence rather than `undefined` or a crash.
//
// ONE fallback string for every one of those shapes — a missing `problems`, one holding
// something that is not a string, and one whose strings are all blank — and deliberately not
// three: this file cannot tell them apart in a way a reader could act on, and today's emitter
// cannot produce any of them at all (`ok: false` implies a non-empty `problems`, see the field
// comment). Naming which shape it found would be a distinction no run can make, which is the
// mistake the `"unknown"` this replaces was: a string that read as an answer about the page
// when it was really an answer about the reader. This one says where it looked.
const verdictMessage = (e: LogEvent): string => {
const problems = Array.isArray(e.problems)
? e.problems.filter((p): p is string => typeof p === "string" && p.trim() !== "")
: [];
if (!problems.length) return "no problems on the line";
const joined = problems.length === 1 ? problems[0] : `${problems.length} problems: ${problems.join(" | ")}`;
return joined.length <= MAX_VERDICT_CHARS ? joined : `${joined.slice(0, MAX_VERDICT_CHARS)}…`;
};
export function summarizeRun(
logText: string,
ctx: { sessionId: string; status: string; phase: string; now: number },
): Diagnostics {
const events = parse(logText);
const running = ctx.status === "running" || ctx.status === "queued";
const nowIso = new Date(ctx.now).toISOString();
const startedAt = events[0]?.ts ?? null;
const lastEventAt = events.length ? events[events.length - 1].ts ?? null : null;
// The terminal line of the CURRENT run, not the first one in the file.
//
// A session's log is one append-only file across every round it has (store/runlog.ts),
// so a session that has taken feedback holds several `run_start` … `run_complete`
// pairs. Reading the first terminal event therefore answered "did the FIRST run
// finish?" — which is always yes by the time a feedback round exists, since a round is
// only accepted on a session that already reached `ready_for_review`. Two things rested
// on that answer and got the wrong one: how long the session has been working, which
// stopped counting at the first round's completion however many rounds followed, and
// whether a call is still open, below.
const runStart = events.map((e) => e.type).lastIndexOf("run_start");
const currentRun = runStart === -1 ? events : events.slice(runStart);
const terminal = currentRun.find((e) => e.type === "run_complete" || e.type === "run_failed");
// Counted rather than read off the slice above, because a session's rounds are not
// always laid end to end: a client may POST /feedback during a round's post-delivery
// window (pipeline/orchestrator.ts), and with `max_concurrent_runs` above 1 the second
// round's `run_start` is then appended before the first round's `run_complete`. The
// slice would hold that trailing line and read as finished. A count cannot be fooled by
// the interleaving: as many terminal lines as starts means every round is done.
const roundsStarted = events.filter((e) => e.type === "run_start").length;
const roundsEnded = events.filter((e) => e.type === "run_complete" || e.type === "run_failed").length;
const unfinished = roundsStarted > roundsEnded;
// In-flight detection. Extraction runs several pages concurrently, so more
// than one call can be open at once and start/end events interleave. Match
// them by identity (agent+step+model+capability) rather than position: each end
// event closes the OLDEST matching open start, which is the same pairing a
// FIFO queue would produce. `in_flight` reports the longest-waiting open call
// — the best single answer to "what is this run stuck on?" — and
// `in_flight_count` shows how many are outstanding.
// Over the current run only, for the same reason the terminal lookup is: an earlier
// round's call that never closed — a process killed mid-flight — is not what THIS run
// is stuck on, and reporting it as such is the phantom hang again by another route.
const openCalls: LogEvent[] = [];
// `step` is part of the identity, not decoration. Without it, extraction's three feedback
// jobs — `verify`, `recheck_binding`, `recheck_sampled` — are all the same agent, model and
// capability, and they run across pages concurrently, so page 1's recheck ending would close
// page 3's still-open verify and `in_flight` would name the recheck as what the run is stuck
// on. Adding it strictly narrows the match and cannot make the `i === -1` fallback newly
// reachable: a start and its end spread the same `meta`, so within one run both carry `step`
// or neither does.
const callKey = (e: LogEvent): string =>
`${e.agent ?? "?"}|${e.step ?? "?"}|${e.model ?? "?"}|${e.capability ?? "?"}`;
for (const e of currentRun) {
if (e.type === "model_call_start") {
openCalls.push(e);
} else if (e.type === "model_call") {
const i = openCalls.findIndex((o) => callKey(o) === callKey(e));
// Fall back to dropping the oldest open call if no identity match: an end
// event always closes something, and leaving it open would report a
// phantom hang.
openCalls.splice(i === -1 ? 0 : i, 1);
}
}
const oldestOpenAt = openCalls.map((c) => c.ts ?? "").sort()[0] ?? null;
// Whether this run is still working, which is not the same as whether the session is
// still `running`. A feedback round marks the session `ready_for_review` as soon as the
// document is delivered and then trains the page agent from it
// (pipeline/orchestrator.ts), holding its `max_concurrent_runs` slot throughout. Both
// questions this drives — how long the run has been going, and whether a call is still
// open — answered "it is over" in exactly that window, which is where a hung provider
// call delays every upload behind it and nothing else reports it.
//
// A dead process is the hard case, because nothing it left behind says it died. For a
// run interrupted while the session still read `running` or `queued`, the next boot
// rewrites the status (store/db.ts `failStaleSessions`) and the first clause closes.
// That sweep does NOT touch `ready_for_review` rows — the document is delivered and the
// status is correct — so a process killed inside the post-delivery window leaves a row
// no one will ever correct, and "no terminal line" alone would report its abandoned
// call as hanging forever, with `waiting_ms` and `elapsed_ms` climbing off the clock.
//
// So the claim is bounded by what a call can actually do: each adapter abandons a
// stream at an absolute 15-minute ceiling and OpenRouter retries at most three times,
// which puts the longest a call can legitimately stay open at ~45 minutes. Past an hour
// an open call is not a slow call, it is a process that is gone — and this reports the
// run as over, which is what it is.
const abandoned = oldestOpenAt !== null && ms(oldestOpenAt, nowIso) > MAX_PLAUSIBLE_CALL_MS;
const active = running || (ctx.status === "ready_for_review" && unfinished && !abandoned);
// The clock runs to NOW only where something is plausibly still happening. For a
// `running` session that is the whole of it — a run between two calls is still a run.
// In the post-delivery window it also has to be RECENT, because that window is the one
// place a run can end without saying so: a process killed there leaves a round that
// never terminated and a status no sweep rewrites, so measuring it to `now` has
// `elapsed_ms` counting up for days, `concurrency_factor` decaying toward zero and the
// last phase's duration growing without end — an idle, delivered session reading as one
// that has been working since it was killed.
//
// Recency is measured from the last event rather than from an open CALL, because the
// longest step in this window may not be a model call at all: filing the agent-update
// issue is a GitHub request (github/issue.ts) with no timeout of its own, and a stalled
// one holds the run's `max_concurrent_runs` slot while `openCalls` is empty. Keying on
// an open call would freeze the clock on exactly that run, which is the one still
// occupying the machine.
//
// What it costs: a live run stalled for longer than the ceiling in a step that logs
// nothing is measured to its last event, so its `elapsed_ms` stops climbing. That is
// the right way round — past an hour of silence, "the process is gone" is the better
// guess, and it is the only one that terminates.
const pending = running || (active && ms(lastEventAt ?? undefined, nowIso) <= MAX_PLAUSIBLE_CALL_MS);
const endRef = pending ? nowIso : terminal?.ts ?? lastEventAt ?? nowIso;
// Longest-waiting first (oldest start timestamp).
openCalls.sort((a, b) => (a.ts ?? "").localeCompare(b.ts ?? ""));
const oldest = openCalls[0];
// "Is this run stuck on something?" is a question about the RUN, and a run is not over
// when the session says `ready_for_review`: the document is delivered there, but a
// feedback round then trains the page agent from it (pipeline/orchestrator.ts), holding
// its `max_concurrent_runs` slot until that finishes. Gating this on the session status
// alone therefore blinded the one field that answers the question, in exactly the window
// where a hung provider call delays every upload behind it and nothing else reports it.
//
// Gated on `active` above: the run, not the session status. Reading the file's FIRST
// terminal event rather than this run's would make that gate dead code, since a
// feedback round only ever starts on a session whose earlier run already wrote one.
const inFlight =
active && oldest
? {
agent: oldest.agent ?? "?",
step: oldest.step ?? "?",
model: oldest.model ?? "?",
provider: oldest.provider ?? "?",
capability: oldest.capability ?? "?",
since: oldest.ts ?? nowIso,
waiting_ms: ms(oldest.ts, nowIso),
}
: null;
const inFlightCount = active ? openCalls.length : 0;
// Completed model calls (the `model_call` end events carry duration_ms).
const calls = events.filter((e) => e.type === "model_call");
const durations = calls.map((c) => c.duration_ms ?? 0);
const failed = calls.filter((c) => c.ok === false).length;
const total = durations.reduce((a, b) => a + b, 0);
// Token totals, and how many calls contributed any. Counted over the same `calls`
// as the timings, which includes the failed ones: a truncated call paid for a full
// ceiling of output and a stalled one paid for its prompt, so excluding them would
// under-report the bill on exactly the documents that cost the most.
const tokens = { input: 0, output: 0, cache_read: 0, cache_write: 0, calls_reported: 0 };
const byAgent: Diagnostics["by_agent"] = {};
const byStep: Diagnostics["by_step"] = {};
// One fold, run twice over the same calls under two keys, so `by_agent` and `by_step` cannot
// disagree about a call: every total in either is the same arithmetic over the same events.
// Summing `by_step` and summing `by_agent` gives the same seven numbers over the same set of
// models, which is worth being
// true by construction — a report that quoted a step's share against a differently-collected
// whole would be wrong in a way no reader could see.
const fold = (into: Record<string, CallTotals>, key: string, c: LogEvent): void => {
const cur =
into[key] ??
{
count: 0,
total_ms: 0,
max_ms: 0,
input_tokens: 0,
output_tokens: 0,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
models: [],
};
cur.count += 1;
cur.total_ms += c.duration_ms ?? 0;
cur.max_ms = Math.max(cur.max_ms, c.duration_ms ?? 0);
cur.input_tokens += c.input_tokens ?? 0;
cur.output_tokens += c.output_tokens ?? 0;
cur.cache_read_input_tokens += c.cache_read_input_tokens ?? 0;
cur.cache_creation_input_tokens += c.cache_creation_input_tokens ?? 0;
// Every `model_call` the router writes carries `model`, on the failure branch as well as
// the success one, so a bucket with calls in it and nothing here is a log old enough to
// predate the field rather than a call that went to nobody. Absent rather than `"?"`:
// this list is read to answer "which model", and a placeholder in it would answer.
if (typeof c.model === "string" && c.model !== "" && !cur.models.includes(c.model)) {
cur.models.push(c.model);
}
into[key] = cur;
};
for (const c of calls) {
fold(byAgent, c.agent ?? "?", c);
// `?` for a call whose line carries no step, which today means a log written before
// `step` existed: the router requires one and the type is closed, so a live run cannot
// produce it. Named rather than dropped — a bucket that silently omitted those calls
// would make `by_step` sum to less than `tokens` on an old log and say nothing about why.
fold(byStep, c.step ?? "?", c);
const reported =
c.input_tokens != null ||
c.output_tokens != null ||
c.cache_read_input_tokens != null ||
c.cache_creation_input_tokens != null;
if (reported) tokens.calls_reported += 1;
tokens.input += c.input_tokens ?? 0;
tokens.output += c.output_tokens ?? 0;
tokens.cache_read += c.cache_read_input_tokens ?? 0;
tokens.cache_write += c.cache_creation_input_tokens ?? 0;
}
// Sorted once at the end rather than kept ordered, since first-seen order is an accident of
// which page finished first and two dumps of the same run should diff to nothing.
for (const t of [...Object.values(byAgent), ...Object.values(byStep)]) t.models.sort();
const slowest = [...calls]
.sort((a, b) => (b.duration_ms ?? 0) - (a.duration_ms ?? 0))
.slice(0, 5)
.map((c) => ({
agent: c.agent ?? "?",
// The step too, because the five slowest calls are where a reader goes to ask what a
// long run was waiting on, and the agent name does not answer it: a slow `copy_editor`
// call is a review round or a table join, and those have different remedies.
step: c.step ?? "?",
model: c.model ?? "?",
capability: c.capability ?? "?",
duration_ms: c.duration_ms ?? 0,
ok: c.ok !== false,
}));
// Phase durations from explicit `phase` events (diff to next, last to end).
const phaseEvents = events.filter((e) => e.type === "phase" && e.phase);
const phaseDurations: Record<string, number> = {};
for (let i = 0; i < phaseEvents.length; i++) {
const cur = phaseEvents[i];
const next = phaseEvents[i + 1];
phaseDurations[cur.phase as string] = ms(cur.ts, next ? next.ts : endRef);
}
// The two post-delivery steps report their own failures rather than raising, because
// neither may revoke a document the user already has (pipeline/orchestrator.ts). That
// makes them invisible to the `ok === false` rule, which only sees model calls: the
// throw those catches exist for is an fs read, not a provider error, so a run whose
// training or contribution died would read as clean here and be findable only in the
// raw ndjson.
//
// Failures, and only failures. `ok === false` was written for `model_call`, where it means
// the provider refused — but `page_correction_recheck` carries an `ok` of its own meaning
// "the verifier named no problem", and a second verdict that names one is a measurement
// coming back negative, not a run that went wrong: the sampled kind runs AFTER the
// correction is kept and changes nothing about what ships, and the binding kind's refusal
// is the loop protecting a page that had already passed (`page_links_correction_rejected`).
// Both were landing here, and 31 of 31 on disk across 22 rounds were the sampled kind — so
// on a four-document round, two documents that were clean read as having errors and the
// only thing distinguishing a working measurement from a truncated call was that the
// measurement's `message` said `"unknown"`, this event carrying its diagnosis under
// `problems` (issue #296). Excluded here and reported where its counts already are, as
// `verification.rechecks.failures`, because a non-empty `errors` is the first thing read
// off a run and it has to mean the run is in doubt.
const errors = events
.filter(
(e) =>
e.type === "run_failed" ||
e.type === "feedback_training_failed" ||
e.type === "contribution_failed" ||
(e.ok === false && e.type !== "page_correction_recheck"),
)
// Every event that reaches here today carries `error`: the three named above are built
// from a caught throw, and a failed `model_call` is logged by providers/index.ts with the
// provider's message. So `"unknown"` is what an old log or a future `ok: false` event
// would read as, and not — as of #296 — a standing entry on every run that sampled.
.map((e) => ({ ts: e.ts ?? null, type: e.type ?? "error", message: e.error ?? "unknown" }));
// Which pages the document has no content for — a set, and a set that changes over
// the life of one session's log, because a feedback round can re-extract a page that
// failed earlier and fill the hole. So this is a fold over the events in order rather
// than a filter: `page_extraction_failed` adds, `page_recovered` removes, and what the
// log says LAST about a page is what is true of the document.
//
// `kept: "prior"` is excluded, because that event reports the opposite outcome under
// the same name: a re-extraction that threw left the page's earlier content in place,
// so the document is whole and naming the page here would send a client looking for a
// hole that isn't there (pipeline/extraction.ts reExtractPages). Which is also why a
// recovered page stays recovered: after the hole is filled, the page HAS content, so
// every later failure on it is one of these.
// The verify/correct tally. A fold over the events rather than four filters, so a
// `page_corrected` line with a `result` this predates counts as a correction and lands
// in none of the buckets — which is the honest reading of an old log, and better than
// silently attributing it to one.
const verification: Diagnostics["verification"] = {
pages_verified: 0,
pages_unjudged: 0,
pages_skipped_blank: 0,
pages_verify_error: 0,
verify_failed: 0,
verify_kinds: {
content_missing: 0,
content_wrong: 0,
structure_wrong: 0,
a11y_only: 0,
alt_quality: 0,
untagged_pages: 0,
},
verify_untagged_problems: 0,
verify_inconsistent: {
pages: 0,
content_missing: 0,
content_wrong: 0,
structure_wrong: 0,
a11y_only: 0,
alt_quality: 0,
content_or_structure: 0,
undecided_pages: 0,
},
corrections: 0,
results: { kept: 0, rejected: 0, identical: 0, empty: 0, failed: 0 },
triggers: { verify: 0, links: 0, alt: 0, ids: 0, words: 0, both: 0 },
declined: { pages: 0, problems: 0, problems_offered: 0, code_checked: 0, words: 0, unattributed: 0 },
effects: { alt_only: 0, text: 0, attrs: 0, structure: 0, text_grew: 0, text_shrank: 0 },
rechecks: {
sampled: 0,
sampled_ok: 0,
sampled_unjudged: 0,
sampled_problems_before: 0,
sampled_problems_after: 0,
binding: 0,
binding_ok: 0,
binding_unjudged: 0,
binding_error: 0,
failures: [],
verdicts_omitted: 0,
},
};
const tables: Diagnostics["tables"] = {
joined_in_code: 0,
joined_by_editor: 0,
joined_in_code_with_halves: 0,
code_declined: 0,
code_declined_with_halves: 0,
header_compared: 0,
header_differs: 0,
body_unreadable: 0,
failed: 0,
capped_pending: 0,
};
const editorCeiling: Diagnostics["editor_ceiling"] = {
truncated: 0,
salvaged: 0,
salvaged_closed: 0,
retreated: 0,
declined: 0,
decline_reasons: {
no_edits_list: 0,
no_complete_edit: 0,
unknown_block: 0,
unreadable_edit: 0,
out_of_order: 0,
all_refused: 0,
loss_before_cut: 0,
unrecognized: 0,
},
};
for (const e of events) {
if (e.type === "page_verify_ok") {
verification.pages_verified += 1;
// Strictly `true`, not truthy: this reader trusts nothing on a log line, and a page
// whose flag arrived as a string would otherwise be subtracted from the rejection rate
// on the strength of a typo. A line without the field is a judged page, which is what
// every log written before it says (issue #211).
if (e.unjudged === true) verification.pages_unjudged += 1;
// Strictly the string the emitter writes, and counted inside `unjudged` rather than beside it:
// a line claiming a skip while claiming a verdict was reached is a line this reader does not
// have to reconcile, because the only emitter sets both together (pipeline/extraction.ts). A
// future `skipped` for some other reason lands in `pages_unjudged` and not here, which is the
// right default — this field is named for the one thing it prices.
if (e.unjudged === true && e.skipped === "blank") verification.pages_skipped_blank += 1;
// And the second value that sentence describes, on the same terms (issue #364). Two `if`s rather
// than one `switch` on `skipped`, so that a third value still lands in `pages_unjudged` and in
// neither of these — the default the comment above commits to.
if (e.unjudged === true && e.skipped === "error") verification.pages_verify_error += 1;
} else if (e.type === "page_verify_error") {
// The gate that could not be applied. Only the BINDING step is counted here, and the step
// is matched strictly rather than treated as "not the first check": the `verify` step is
// already counted, one page at a time, off `page_verify_ok`'s `skipped` above, so reading
// both events for it would double it. A `step` this reader does not know adds to neither,
// which is the same default `skipped` has — a new call site gets a line in the log and no
// silent contribution to a rate that was defined without it.
//
// Nothing else on this event feeds a number. It deliberately does NOT reach `errors`: that
// list is read as "the run is in doubt", and a page whose check could not be obtained ships
// its content exactly as extracted, so putting it there would report a delivered document
// as a failed one — which is the whole misattribution issue #364 is about, relocated.
if (e.step === "recheck_binding") verification.rechecks.binding_error += 1;
} else if (e.type === "page_verify_failed") {
verification.pages_verified += 1;
verification.verify_failed += 1;
// One page, so each kind it named counts once however many problems carried that kind.
// Matched against the closed list rather than trusted, for the reason `result` is: a
// `kinds: ["constructor"]` line would otherwise be added to a function.
const named: unknown = e.kinds;
const kinds = Array.isArray(named) ? VERIFY_KINDS.filter((k) => named.includes(k)) : [];
for (const kind of kinds) verification.verify_kinds[kind] += 1;
// A page counts as `untagged_pages` when the line named no kind this reader knows — an
// old log, an agent file whose contract predates the kinds, a model that answered in
// plain strings — and ALSO when it named some and left others untagged, because then the
// kind buckets are missing part of that page's story. Unlike the recheck sums below, an
// absent field is not left alone here: a page in `verify_failed` and in no bucket is
// exactly what this count exists to make visible, and silence would read as a split
// that covered the whole run.
const untagged = typeof e.untagged === "number" && e.untagged > 0 ? e.untagged : 0;
if (kinds.length === 0 || untagged > 0) {
verification.verify_kinds.untagged_pages += 1;
}
// And the problem count beside it, because the page count alone cannot tell a run where
// one problem per page arrived untagged from one where every problem did — both report
// the same `untagged_pages`, and only the second means the split is uninformative. A log
// that predates the kinds carries no count, and on it every problem the line lists is
// untagged by definition, so the line's own `problems` supplies the number; a page with
// neither field readable counts as one, because it is in `verify_failed` and reporting
// zero would read as fully tagged.
if (untagged > 0) verification.verify_untagged_problems += untagged;
else if (kinds.length === 0) {
verification.verify_untagged_problems += Array.isArray(e.problems) ? Math.max(e.problems.length, 1) : 1;
}
} else if (e.type === "page_verify_inconsistent") {
// NOT added to `pages_verified`: the page that wrote this line also wrote a
// `page_verify_ok` line, which is where it is counted. This is a second reading of the
// same verdict, so folding it as a page would count that page twice and make the
// rejection rate's denominator larger than the run's page count.
verification.verify_inconsistent.pages += 1;
// Same closed list and the same reasons as the failure fold above: a kind is counted
// only if this code knows it, and a page counts once per kind however many problems
// carried it.
const named: unknown = e.kinds;
const kinds = Array.isArray(named) ? VERIFY_KINDS.filter((k) => named.includes(k)) : [];
for (const kind of kinds) verification.verify_inconsistent[kind] += 1;
// The pricing field: one per PAGE naming at least one of the three, not one per kind, so
// it can be read against `pages` as a share and against `corrections` as a bill.
const decided = kinds.some(
(k) => k === "content_missing" || k === "content_wrong" || k === "structure_wrong",
);
if (decided) verification.verify_inconsistent.content_or_structure += 1;
// And the unknown above it, which is why this is not the `||` the failure fold uses: a
// page already naming one of those three is DECIDED, whatever else it left untagged, so
// counting it here too would double it in a sum whose two halves are meant to bracket the
// bill. What is undecided is a page carrying a problem with no kind this code knows —
// including a page whose only tags are `a11y_only` or `alt_quality`, since the untagged
// one beside them could be anything — and a line with no readable count at all, which is
// every verdict written in plain prose and the whole of the corpus this was measured on.
const untagged = typeof e.untagged === "number" && e.untagged > 0 ? e.untagged : 0;
if (!decided && (untagged > 0 || kinds.length === 0)) {
verification.verify_inconsistent.undecided_pages += 1;
}
} else if (e.type === "page_corrected") {
verification.corrections += 1;
// The denominator for `declined.problems`, taken from this line rather than from the decline
// event: every correction writes a `page_corrected`, including the ones that answered with
// nothing, so summing here counts the problems the pass was given and not only the problems on
// the pages that disagreed. Reading the rate off the declining pages' own bills would divide by
// a subset chosen by the numerator.
if (typeof e.problems === "number" && e.problems > 0) verification.declined.problems_offered += e.problems;
// Matched against a fixed list rather than tested with `in`, which answers true
// for anything on Object.prototype: a log line reading `result: "constructor"`
// would otherwise be added to a function and turn a count into NaN. The same trap
// util/html.ts uses a null prototype for.
const result = CORRECTION_RESULTS.find((r) => r === e.result);
if (result) verification.results[result] += 1;
const trigger = CORRECTION_TRIGGERS.find((t) => t === e.trigger);
if (trigger) verification.triggers[trigger] += 1;
if (e.text_changed === true) verification.effects.text += 1;
// The direction, gated on `text_changed` rather than on the two sizes alone: a
// correction that swaps one word for a longer one changes the prose and its length, and
// a correction that reorders a sentence changes the prose and not its length, and only
// the flag knows which happened. Both numbers must be present — an old log carries
// neither, and `undefined > undefined` is false, so such a line lands in `text` and in
// neither direction, which is the same reading an unknown `result` gets.
if (
e.text_changed === true &&
typeof e.text_chars_before === "number" &&
typeof e.text_chars_after === "number"
) {
if (e.text_chars_after > e.text_chars_before) verification.effects.text_grew += 1;
else if (e.text_chars_after < e.text_chars_before) verification.effects.text_shrank += 1;
}
if (e.attrs_changed === true) verification.effects.attrs += 1;
if (e.structure_changed === true) verification.effects.structure += 1;
if (
e.alt_changed === true &&
e.text_changed !== true &&
e.attrs_changed !== true &&
e.structure_changed !== true
) {
verification.effects.alt_only += 1;
}
} else if (e.type === "page_correction_declined") {
// One line per page whose correction refused something, so `pages` counts the line and the
// three others count its entries (#373 directive 4).
//
// `pages` is incremented on the line's EXISTENCE and not on a readable `declined` array: the
// event is written only where the reply declined something, so a line whose array cannot be
// read is still a page that disagreed, and skipping it would report the run as compliant
// because the disagreement arrived malformed. The entries are then counted only where they are
// objects, which is the same split `page_verify_failed`'s kinds make.
verification.declined.pages += 1;
const entries = Array.isArray(e.declined) ? e.declined : [];
for (const entry of entries) {
if (typeof entry !== "object" || entry === null) continue;
const source = (entry as { source?: unknown }).source;
verification.declined.problems += 1;
// `verify` is the one source the licence was written for. `links`, `alt` and `ids` were
// checked in code, so a decline naming one of them is wrong by construction — matched against
// the same closed list the triggers use, minus `both`, which is a property of a correction and
// never of a single problem. A `source` this code does not know lands in neither bucket rather
// than in `code_checked`: manufacturing the evidence that the licence is being misused is
// worse than a total that is visibly short.
//
// `words` is checked in code and counted separately, because on that one a decline is a
// legitimate answer rather than a misuse (#334 part B). What Iris verified there is that the
// page carries both spellings; what it cannot know is which the printing shows, and
// `splitWordProblem` says so, `SPELLINGS_CHECKED_IN_CODE` marks the entry as settled in that
// one part only, and `correctPage` spends a sentence telling the model that here — and only
// here — the image showing otherwise is a reason not to act. Three texts, one channel; a
// count of refusals is worth nothing if the request the refusal answers forbids it. Folding
// those into
// `code_checked` would put compliance in the field that exists to count the licence being
// abused — the same mistake `CHECKED_IN_CODE` was added to stop, one field along. It is its
// own count and not silence for the reason the unknown-`source` case is silent in a
// DIFFERENT way: a gap between `problems` and the buckets means "a source this build does not
// know", and a legitimate refusal is not that.
if (source === "links" || source === "alt" || source === "ids") verification.declined.code_checked += 1;
else if (source === "words") verification.declined.words += 1;
else if (source === null || source === undefined) verification.declined.unattributed += 1;
}
} else if (e.type === "page_correction_recheck") {
// Split on the flag the event already carries. A line whose `binding` is neither
// boolean lands in neither bucket, for the same reason an unknown `result` does:
// guessing which population a verdict belongs to is worse than a total that is
// visibly short of the lines in the log.
// Strictly `true`, as on `page_verify_ok`: a recheck subtracted from the pass rate on
// the strength of a string would be the trap the closed lists here exist for.
const unjudged = e.unjudged === true;
// The verdict's own words, kept whichever population the line claims — including a line
// whose `binding` is neither boolean, which the split below counts in neither: what that
// line failed to say is which rate it belongs in, not what is wrong with the page. This
// is the only place in this file the prose survives, and it used to be in `errors` under
// the word `"unknown"` (issue #296, and see the comment on the field).
//
// `ok === false` and strictly so, for the reason every flag here is read strictly. It is
// also the whole condition: `problems` non-empty is implied by it (extraction.ts
// `failedCheck`), so there is no second test for an empty message to write.
if (e.ok === false) {
// Counted whether or not it is carried, so the cap is disclosed rather than being a
// list that quietly stops growing. First N and not last N: the earliest failures are
// the first pass's, and a session's later rounds re-verify a handful of pages the user
// asked about — so a cap that kept the tail would drop the document's own account of
// itself in favour of a follow-up's.
if (verification.rechecks.failures.length < MAX_VERDICTS) {
verification.rechecks.failures.push({
ts: e.ts ?? null,
page: typeof e.page === "number" ? e.page : null,
binding: typeof e.binding === "boolean" ? e.binding : null,
message: verdictMessage(e),
});
} else {
verification.rechecks.verdicts_omitted += 1;
}
}
if (e.binding === true) {
verification.rechecks.binding += 1;
if (e.ok === true) verification.rechecks.binding_ok += 1;
if (unjudged) verification.rechecks.binding_unjudged += 1;
} else if (e.binding === false) {
verification.rechecks.sampled += 1;
if (e.ok === true) verification.rechecks.sampled_ok += 1;
if (unjudged) verification.rechecks.sampled_unjudged += 1;
// Only when the line carries both, so a log from before these existed leaves the two
// sums alone rather than adding a zero to each. A missing `problems_before` counted as
// 0 would read as a page corrected for no reason, which is the opposite of what
// happened, and it would make the pair say the corrections had nothing to fix.
//
// And only when something judged it, for the mirror-image reason: an unjudged sample
// (the first verdict was real and failed, the second reply would not parse) carries a
// true before-count and an `problems_after` of 0 that means "nothing was named", not
// "nothing was left" — a page nobody looked at, summed in as a correction that fixed
// everything it was handed.
if (
!unjudged &&
typeof e.problems_before === "number" &&
typeof e.problems_after === "number"
) {
verification.rechecks.sampled_problems_before += e.problems_before;
verification.rechecks.sampled_problems_after += e.problems_after;
}
}
} else if (e.type === "table_joined") {
// Matched against the two values the emitter writes, not `=== "code"` with an else: an
// unrecognized `by` — an old log from before #278, or a third path added later — has to land in
// neither bucket, because the whole question this field answers is which of the two spent
// output tokens. Guessing it would report a paid join as free, and the free share is the number
// #326 says a later round must be able to re-measure.
if (e.by === "code") tables.joined_in_code += 1;
else if (e.by === "editor") tables.joined_by_editor += 1;
// Inside the `by === "code"` test and not beside it: the emitter writes the halves on a free join
// only, so a paid line carrying this word would be from a build that changed that rule, and
// counting it would put a pair whose bytes are on ANOTHER line into the tally of lines that carry
// their own. The `"logged"`/`"too_large"` reading is the declines' below, for the same reason.
if (e.by === "code" && e.halves === "logged") tables.joined_in_code_with_halves += 1;
} else if (e.type === "table_join_code_declined") {
tables.code_declined += 1;
// Off the state field rather than off the bytes. A replay needs BOTH halves, so two presence
// tests would be two things that can disagree, and one word that the emitter always writes
// partitions every decline this build logged: `logged` here, `too_large` in the difference from
// `code_declined`, with the lines written before the field in there too.
if (e.halves === "logged") tables.code_declined_with_halves += 1;
// The comparison is only counted where the log line says one was possible: `headers_identical`
// is absent when a half held no `<table>` to read, and the counts are absent with it. Read
// strictly as a boolean for the reason every flag here is, and gated on both halves having
// header CELLS, because a continued page that reprinted no header compares an empty signature
// against a real one and writes `false` — a fact about the printing, not two readings
// disagreeing. Counting it would put the commonest legitimate case into the instability number.
//
// On `cells` and not on `rows`, and that is the whole of the difference between this and the
// first draft: an empty `<tr>` inside a `<thead>` is a header row by every test here, so it
// reported one row, no cells and an empty signature — unequal to a real header, and a
// `rows`-based gate let it through into both counts below.
const compared =
typeof e.headers_identical === "boolean" &&
typeof e.header_cells_first === "number" &&
typeof e.header_cells_second === "number" &&
e.header_cells_first > 0 &&
e.header_cells_second > 0;
if (compared) {
tables.header_compared += 1;
if (e.headers_identical === false) tables.header_differs += 1;
}
} else if (e.type === "table_join_failed") {
// `stage` is what separates the run-level line from the per-pair ones, and it is matched against
// the value the emitter writes rather than tested for absence: a line naming a stage this build
// does not know is about something, and calling it a split table would be a guess.
if (e.stage === undefined) tables.failed += 1;
else if (e.stage === "body") tables.body_unreadable += 1;
} else if (e.type === "table_joins_capped") {
// Summed, not counted: the field is how many pairs were left, and one run leaving four is four
// tables a reader meets in halves.
tables.capped_pending += typeof e.pending === "number" ? e.pending : 0;
} else if (e.type === "editor_truncated") {
editorCeiling.truncated += 1;
} else if (e.type === "editor_salvaged") {
editorCeiling.salvaged += 1;
// Strictly `true`, for the reason every flag in this reader is: the emitter omits the field
// rather than writing `false`, so a line carrying some other value is not a claim this reader
// has to interpret.
if (e.closed === true) editorCeiling.salvaged_closed += 1;
// On the presence of `lost_at`, not on its truth: the emitter writes it only when a retreat
// happened, and block 0 is a legitimate value — `if (e.lost_at)` would drop the retreat that
// gave up the whole document, which is the worst one.
if (e.lost_at !== undefined && e.lost_at !== null) editorCeiling.retreated += 1;
} else if (e.type === "editor_salvage_declined") {
editorCeiling.declined += 1;
// The reasons are matched against the emitter's closed list and anything else is counted, not
// dropped — the one place in this reader where an unrecognized value gets a bucket of its own.
// These have to sum to `declined` for the split to be checkable, so a value from a build this
// one has not heard of must be visible in the total rather than absent from it.
//
// `hasOwnProperty` and deliberately NOT `in`, which is the operator this guard has to avoid:
// `in` walks the prototype chain, so `reason: "toString"` would pass it and
// `decline_reasons["toString"] += 1` would add a new own property of NaN to the response —
// breaking the published key set and the "these sum to `declined`" invariant in the same move.
// Called off `Object.prototype` rather than as a method, because the object being tested is one
// a log line names and a `decline_reasons` build that ever gained a key called `hasOwnProperty`
// would take its own guard away.
//
// No special case for a `reason` of literally `"unrecognized"`: it is an own key of the object
// below, so it takes the first branch and increments the same counter the second one would. A
// guard against it would be a branch that cannot change a count.
const reason = e.reason;
if (typeof reason === "string" && Object.prototype.hasOwnProperty.call(editorCeiling.decline_reasons, reason)) {
editorCeiling.decline_reasons[reason as keyof Diagnostics["editor_ceiling"]["decline_reasons"]] += 1;
} else editorCeiling.decline_reasons.unrecognized += 1;
}
}
const failedSet = new Set<number>();
// Blank pages fold the same way and for the same reason: feedback can name a page the
// agent reported empty ("you missed the table on page 4"), and a re-extraction that
// finds content there means the page is not blank after all. So `reextract_start`
// withdraws the earlier answer for the pages it is about to redo, and the `page_blank`
// lines that follow it — written by extractPage, before the round's completion line —
// give the new one. A page that comes back blank again is added straight back.
//
// `staleBlank` covers the one path that produces no new answer: a re-extraction that
// THROWS keeps the page's prior fragment (pipeline/extraction.ts reExtractPages), which
// for a blank page is the empty one, so the page is still blank and the withdrawal has
// to be undone. Without it that page would appear in neither set while having no
// content, which is the reading this whole field exists to prevent.
//
// That re-add is made on every throw, including a round where the reply was unreadable —
// so the last thing the log knows about the page is that the model gave up, and this still
// says blank. Deliberate, and the least wrong of the cheap answers: the field describes the
// DOCUMENT, whose fragment for that page is the empty one an accepted declaration produced,
// and moving the page to `pages_failed` instead would send a client looking for a
// `@page-failed` marker that is not in the body. The round's own account is in the log
// (`page_no_output`, `page_extraction_failed` with `kept: "prior"`).
const blankSet = new Set<number>();
let staleBlank = new Set<number>();
// The no-envelope pages fold on the same three events and for the same reason — see
// `pages_bare_html`. One difference worth naming: a re-extraction that DELIVERS is what takes a
// page out of this set, and it does so by writing no `page_bare_html` line, so the withdrawal at
// `reextract_start` is what does the work here rather than a positive answer of its own. There is
// no `page_enveloped` event and there should not be one; the ordinary case is not worth a line
// per page.
const bareSet = new Set<number>();
let staleBare = new Set<number>();
for (const e of events) {
if (e.type === "page_extraction_failed" && typeof e.page === "number" && e.kept !== "prior") {
failedSet.add(e.page);
} else if (e.type === "page_recovered" && Array.isArray(e.pages)) {
for (const p of e.pages) if (typeof p === "number") failedSet.delete(p);
}
if (e.type === "page_blank" && typeof e.page === "number") {
blankSet.add(e.page);
} else if (e.type === "reextract_start" && Array.isArray(e.pages)) {
staleBlank = new Set(e.pages.filter((p): p is number => typeof p === "number" && blankSet.has(p)));
for (const p of staleBlank) blankSet.delete(p);
} else if (
e.type === "page_extraction_failed" &&
e.kept === "prior" &&
typeof e.page === "number" &&
staleBlank.has(e.page)
) {
blankSet.add(e.page);
}
if (e.type === "page_bare_html" && typeof e.page === "number") {
bareSet.add(e.page);
} else if (e.type === "reextract_start" && Array.isArray(e.pages)) {
staleBare = new Set(e.pages.filter((p): p is number => typeof p === "number" && bareSet.has(p)));
for (const p of staleBare) bareSet.delete(p);
} else if (
e.type === "page_extraction_failed" &&
e.kept === "prior" &&
typeof e.page === "number" &&
staleBare.has(e.page)
) {
bareSet.add(e.page);
}
}
const pagesFailed = [...failedSet].sort((a, b) => a - b);
const pagesBlank = [...blankSet].sort((a, b) => a - b);
const pagesBareHtml = [...bareSet].sort((a, b) => a - b);
// The Copy Editor's fidelity observations, summed over every round it ran — one line per round
// that had any, so a document reviewed in three rounds can contribute three lines about the same
// page. `observed` counts observations and `pages` the distinct pages they name, which is what
// separates one page reported three times from three pages reported once; the same page in two
// rounds is one page here and two observations, and that is the honest reading of it (the round
// that produced each is in the log).
const observed: Diagnostics["fidelity_observed"] = {
observed: 0,
pages: [],
unattached_pages: [],
kinds: { content_missing: 0, content_wrong: 0, structure_wrong: 0, a11y_only: 0, alt_quality: 0, untagged: 0 },
unattached: 0,
unplaced: 0,
};
const observedPages = new Set<number>();
const unattachedPages = new Set<number>();
for (const e of events) {
if (e.type !== "editor_fidelity_observed" || !Array.isArray(e.observations)) continue;
observed.unattached += typeof e.unattached === "number" ? e.unattached : 0;
observed.unplaced += typeof e.unplaced === "number" ? e.unplaced : 0;
// Which pages the editor actually had. Read per line rather than unioned across the run,
// because the attachment is per round: page 4 attached in round 1 and not in round 2 makes an
// observation filed in round 2 a guess, and unioning would launder it into a checkable one.
// A line with no `attached` at all leaves every page unnamed here rather than naming them
// all — there is nothing to tell against, and the count on that line is what says how many
// were guesses.
const attached = Array.isArray(e.attached) ? e.attached : null;
for (const entry of e.observations) {
if (entry === null || typeof entry !== "object") continue;
const rec = entry as Record<string, unknown>;
observed.observed += 1;
if (typeof rec.page === "number") {
observedPages.add(rec.page);
if (attached !== null && !attached.includes(rec.page)) unattachedPages.add(rec.page);
}
// The closed list again, for the reason `verify_kinds` uses it: a `kind` naming a
// function on Object.prototype would otherwise be incremented rather than counted as
// the unrecognized label it is.
const kind = VERIFY_KINDS.find((k) => k === rec.kind);
if (kind) observed.kinds[kind] += 1;
else observed.kinds.untagged += 1;
}
}
observed.pages = [...observedPages].sort((a, b) => a - b);
observed.unattached_pages = [...unattachedPages].sort((a, b) => a - b);
const elapsed = ms(startedAt ?? undefined, endRef);
return {
session_id: ctx.sessionId,
status: ctx.status,
phase: ctx.phase,
started_at: startedAt,
last_event_at: lastEventAt,
elapsed_ms: elapsed,
in_flight: inFlight,
in_flight_count: inFlightCount,
phase_durations_ms: phaseDurations,
model_calls: {
count: calls.length,
failed,
total_ms: total,
avg_ms: calls.length ? Math.round(total / calls.length) : 0,
max_ms: durations.length ? Math.max(...durations) : 0,
concurrency_factor: elapsed > 0 ? Math.round((total / elapsed) * 100) / 100 : 0,
},
tokens,
by_agent: byAgent,
by_step: byStep,
slowest_calls: slowest,
errors,
verification,
tables,
editor_ceiling: editorCeiling,
pages_failed: pagesFailed,
pages_blank: pagesBlank,
pages_bare_html: pagesBareHtml,
fidelity_observed: observed,
};
}