1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540import { test } from "node:test";
import assert from "node:assert/strict";
import { fileURLToPath } from "node:url";
import {
readFileSync,
readdirSync,
writeFileSync,
mkdtempSync,
mkdirSync,
existsSync,
} from "node:fs";
import { dirname, join } from "node:path";
import { tmpdir } from "node:os";
import { perAgentKeyWarning } from "../src/config.ts";
// `providers.per_agent` is the whole model-selection surface of this deployment, and it is
// the one config key whose failure mode is silence. `resolveAgentModel` looks an override up
// by agent name and, finding none, falls back through the provider's `per_capability` to its
// `default_model` — so a key naming an agent nothing dispatches is not a startup error, does
// not appear in `by_agent` (which reports the agents that RAN), and costs what the run would
// have cost anyway. `by_agent.<agent>.models` closes half of that: a finished run now names
// the model each agent used, so a swap that did not happen is visible after the fact. The
// ignored KEY still appears nowhere but this warning, which is also the only account of it
// that arrives before the run is paid for.
//
// It had already happened twice, in the two files an operator reads first, and both were
// relics of the per-content-type fan-out that was withdrawn: `config.example.yaml`
// offered a commented `table:` line described as the way to put a stronger model on the table
// join, and the now-retired specification's own block showed `image_analysis: bedrock`. They
// are stale in different ways, which is why this file pins the set rather than those two
// names — `image_analysis` was
// the triage agent and went with `src/pipeline/triage.ts`; `table` was never dispatched by
// anything, so no removal could have caught it.
//
// So the check is on the call sites, not on a list. src/config.ts has to carry the static
// names as literals (it cannot read the pipeline it configures), and this is what holds that
// literal to the code: derive the same set from every `router.complete` call in src/ and fail
// if the two disagree in either direction.
const ROOT = fileURLToPath(new URL("..", import.meta.url));
const SRC = join(ROOT, "src");
const AGENTS_DIR = join(ROOT, "agents");
// Every `.ts` under src/, so a new pipeline file with a new agent is in scope without being
// added anywhere.
function sources(dir: string): string[] {
const out: string[] = [];
for (const e of readdirSync(dir, { withFileTypes: true })) {
const p = join(dir, e.name);
if (e.isDirectory()) out.push(...sources(p));
else if (e.name.endsWith(".ts")) out.push(p);
}
return out;
}
// The first argument of each `ctx.router.complete(...)` / `this.router.complete(...)` call,
// which is the agent name the override is looked up by. Two shapes reach the router and only
// one of them is a name this can read: a literal or a SCREAMING_CASE const (`PAGE_AGENT`),
// which is resolved to its own literal below; or `agent.name`, the specialist and per-agent
// paths, whose name comes from a file in agents/ at run time and cannot be known here. The
// dynamic ones are counted separately and asserted to exist, because their existence is the
// reason the startup check warns instead of refusing.
function dispatchedAgents(): { literal: Set<string>; dynamic: number } {
const literal = new Set<string>();
let dynamic = 0;
for (const file of sources(SRC)) {
const text = readFileSync(file, "utf8");
// `const NAME = "value";` in this file, for resolving a constant first argument.
const consts = new Map<string, string>();
for (const m of text.matchAll(/const ([A-Z][A-Z0-9_]*) = "([^"]+)";/g)) consts.set(m[1]!, m[2]!);
for (const m of text.matchAll(/router\.complete\(\s*([^,]+),/g)) {
const arg = m[1]!.trim();
const quoted = arg.match(/^"([^"]+)"$/);
if (quoted) literal.add(quoted[1]!);
else if (consts.has(arg)) literal.add(consts.get(arg)!);
else dynamic++;
}
}
return { literal, dynamic };
}
// Read the literal src/config.ts declares, rather than exporting it: the point is that the
// hand-written list agrees with the code, and importing a value the source disagreed with
// would test nothing. Parsed from the source for the same reason test/config-example.test.ts
// parses the YAML it is about.
function declaredAgents(): string[] {
const text = readFileSync(join(SRC, "config.ts"), "utf8");
const m = text.match(/const DISPATCHED_AGENTS = \[([^\]]+)\] as const;/);
assert.ok(m, "DISPATCHED_AGENTS moved or was renamed in src/config.ts");
return [...m[1]!.matchAll(/"([^"]+)"/g)].map((x) => x[1]!);
}
test("the agent names src/config.ts can route are the ones src/ actually dispatches", () => {
const { literal, dynamic } = dispatchedAgents();
const declared = declaredAgents();
assert.deepEqual(
[...declared].sort(),
[...literal].sort(),
"DISPATCHED_AGENTS in src/config.ts disagrees with the router.complete call sites in src/ — " +
"add the new agent there (and to config.example.yaml's list) or drop the one that went",
);
// The reason `perAgentKeyWarning` warns rather than refusing. If this ever reaches 0, the
// set of valid keys is closed and an unknown one could be a startup error instead.
assert.ok(
dynamic > 0,
"no router.complete call takes a run-time agent name any more, so the valid per_agent keys " +
"are now a closed set — an unknown key could be refused at startup rather than warned about",
);
});
// The half the derivation above cannot see: a specialist's name is the file stem of a `.md` in
// `agents_dir` (src/agents/loader.ts `loadAgent`), so the warning has to read that directory
// or it would call today's shipped specialist an unknown agent.
test("a specialist shipped in agents/ is a routable per_agent key", () => {
const shipped = readdirSync(AGENTS_DIR)
.filter((f) => f.endsWith(".md"))
.map((f) => f.replace(/\.md$/, ""));
assert.ok(shipped.length > 0, "agents/ has no .md files; this test's premise is gone");
const specialists = shipped.filter((n) => !declaredAgents().includes(n));
assert.ok(
specialists.length > 0,
"agents/ holds no file beyond the statically dispatched agents, so this test no longer " +
"exercises the library half of the lookup",
);
assert.equal(perAgentKeyWarning({ [specialists[0]!]: "bedrock" }, AGENTS_DIR), undefined);
});
test("per_agent keys that name no agent are warned about, and routable ones are not", () => {
// Silence on the whole dispatched set, which is what makes a warning readable when it comes.
const all = Object.fromEntries(declaredAgents().map((a) => [a, "bedrock"]));
assert.equal(perAgentKeyWarning(all, AGENTS_DIR), undefined);
assert.equal(perAgentKeyWarning({}, AGENTS_DIR), undefined);
assert.equal(perAgentKeyWarning(undefined, AGENTS_DIR), undefined);
// The two names the shipped examples used, and the shape of a typo.
const w = perAgentKeyWarning({ table: "openrouter", image_analysis: "bedrock" }, AGENTS_DIR);
assert.ok(w, "an unroutable per_agent key produced no warning");
assert.match(w, /"table"/);
assert.match(w, /"image_analysis"/);
// The consequence, not just the name: an operator who reads only the first clause has to
// learn that the entry is ignored, since that is the part no log will tell them.
assert.match(w, /ignored/);
// And the way out. The two routable sets are named SEPARATELY — the dispatched agents, then
// the directory and what is in it — because that is the distinction the sentence exists to
// draw, and one merged list has to be sorted, which drops a specialist in the middle of the
// built-ins. An earlier draft said "the last of those beyond <built-ins> are specialist
// files" over a sorted list, where `chartDataAgent` came out second.
for (const agent of declaredAgents()) assert.ok(w.includes(agent), `warning omits ${agent}`);
assert.match(w, /any agent file in .*agents/);
const shipped = readdirSync(AGENTS_DIR)
.filter((f) => f.endsWith(".md"))
.map((f) => f.replace(/\.md$/, ""))
.filter((n) => !declaredAgents().includes(n));
for (const s of shipped) assert.ok(w.includes(s), `warning omits the specialist ${s}`);
// `table` earns its own sentence, because that key was shipped in this repo's own example
// and the join really is a copy_editor call — but only where it answers the key in hand.
assert.match(w, /no table agent/);
// A routable key alongside a broken one is not named as broken, and a typo does not get a
// sentence about tables.
const mixed = perAgentKeyWarning({ page: "bedrock", tabel: "bedrock" }, AGENTS_DIR);
assert.ok(mixed?.includes('"tabel"'));
assert.ok(!/"page"/.test(mixed!), "a routable key was reported as unroutable");
assert.ok(!/table agent/.test(mixed!), "a typo was answered with a sentence about tables");
});
test("an unreadable agents_dir warns about nothing rather than about everything", () => {
// The directory is the likelier thing to be wrong, and a deployment whose library failed to
// check out would otherwise get every specialist entry reported as a bad key on every boot.
const missing = join(mkdtempSync(join(tmpdir(), "iris-agents-")), "nope");
assert.equal(perAgentKeyWarning({ chartDataAgent: "bedrock" }, missing), undefined);
// An empty but readable directory still warns, because there is nothing to have missed.
const empty = mkdtempSync(join(tmpdir(), "iris-agents-empty-"));
assert.match(perAgentKeyWarning({ chartDataAgent: "bedrock" }, empty)!, /"chartDataAgent"/);
// And a specialist added to a library later is routable without a code change, which is the
// property that makes this a warning.
const later = mkdtempSync(join(tmpdir(), "iris-agents-later-"));
mkdirSync(later, { recursive: true });
writeFileSync(join(later, "formFieldAgent.md"), "# Form Field Agent\n");
assert.equal(perAgentKeyWarning({ formFieldAgent: "bedrock" }, later), undefined);
});
// The examples are the reason this file exists, so they are asserted rather than trusted:
// every agent key any shipped example names must be one of the dispatched agents. Both files,
// because both were wrong, and including the COMMENTED block in config.example.yaml — a line
// an operator is told to uncomment is a line that has to work.
test("every per_agent key any example names is an agent Iris dispatches", () => {
const known = new Set([
...declaredAgents(),
...readdirSync(AGENTS_DIR)
.filter((f) => f.endsWith(".md"))
.map((f) => f.replace(/\.md$/, "")),
]);
// A YAML comment marker is stripped so a commented example is read as the operator would
// uncomment it — but only for a block whose own `per_agent:` was commented, and only for
// lines indented deeper than that key. An earlier version scanned everything after the
// FIRST `per_agent:` and stopped at the first line that was neither an entry, a comment nor
// blank, which in config.example.yaml is `openrouter:` some forty lines later: every
// paragraph of prose in between was a candidate, and one indented comment reading
// `# max_tokens: 32000` anywhere in that span would have failed this suite claiming the
// file offers a per_agent key called `max_tokens`. Both files carry more than one block, so
// each is scanned in turn rather than picking one.
const blocks = /^(\s*)(#\s?)?per_agent:/;
for (const file of ["config.example.yaml", "docs/models.md"]) {
const lines = readFileSync(join(ROOT, file), "utf8").split("\n");
let found = 0;
let seenBlock = false;
for (const [i, head] of lines.entries()) {
const b = head.match(blocks);
if (!b) continue;
seenBlock = true;
const indent = b[1]!.length;
const commented = Boolean(b[2]);
for (const raw of lines.slice(i + 1)) {
// A COMMENTED block is the contiguous run of comment lines, because that run is what
// an operator uncomments as a unit — so prose inside it is skipped rather than ending
// the block, and only a blank line or live YAML closes it. config.example.yaml's block
// ends with three prose lines about `copy_editor`, and an entry written after that
// paragraph is still an entry the operator would uncomment; stopping at the prose would
// leave it unchecked while this test passed.
//
// A LIVE block runs to the first line at or above the key's own indent, since that is
// the line that has left the mapping. YAML allows both blank lines and comments
// between two entries, so neither ends it — and neither can BE an entry, so both are
// skipped rather than read: a `#` line in a live block is a comment about the block
// and cannot be an override however it is indented.
const isComment = /^\s*#/.test(raw);
const blank = raw.trim() === "";
if (commented) {
if (!isComment) break;
} else if (blank || isComment) continue;
const line = commented ? raw.replace(/^(\s*)#\s?/, "$1") : raw;
const m = line.match(/^(\s+)([A-Za-z_][A-Za-z0-9_]*)\s*:/);
if (m === null || m[1]!.length <= indent) {
if (commented) continue;
break;
}
found++;
assert.ok(
known.has(m[2]!),
`${file} offers per_agent key "${m[2]}", which no router.complete call dispatches — ` +
`it would be silently ignored. Routable: ${[...known].sort().join(", ")}`,
);
}
}
assert.ok(seenBlock, `${file} no longer contains a per_agent block; drop this or repoint it`);
// Self-arming: config.example.yaml's live block is `per_agent: {}` with no entries, so a
// count above 0 can only come from the COMMENTED example — a stop rule that gave up before
// reaching it would fail here rather than pass silently.
assert.ok(found > 0, `${file}'s per_agent blocks have no entries under them to check`);
}
});
// docs/models.md is a recommendation PER AGENT, so its table is a claim about the same set the
// test above derives — and the sprint report it was written from got that set wrong, listing the
// specialist row in place of `builder` and arriving at five by counting one agent twice. A
// missing row is the failure that matters: an agent nobody has a recommendation for reads as an
// agent with no cost, which is exactly how `builder` and the specialist came to be reported as
// two independent zeroes (they are one gate — see the file).
test("docs/models.md's recommendation table is exactly the agents a deployment can route", () => {
const doc = readFileSync(join(ROOT, "docs/models.md"), "utf8");
// The RECOMMENDATION table specifically, not the file. Every dispatched agent also appears in
// §1's table of call sites, so a check against the whole document passes while the summary a
// reader actually acts on is missing a row — which is the failure being guarded, and the one
// an earlier version of this test did not catch when the `builder` row was taken out of §0.
const section = doc.split(/^## /m).find((s) => s.startsWith("0."));
assert.ok(section, "docs/models.md has no `## 0.` summary section any more");
// Both directions, because both are the same mistake in the reader's hands. A MISSING row
// reads as an agent with no cost; an EXTRA one names a `per_agent` key that would be silently
// ignored, which is the failure the rest of this file exists for — and a recommendation is
// the most likely place someone copies such a key from. Rows whose agent cell is backticked
// are the claim; §0's `specialists` row is deliberately not, since it is a class of agents
// named at run time and not a key anyone can write.
const rows = [...section.matchAll(/^\| `([^`]+)`/gm)].map((m) => m[1]!);
assert.deepEqual(
[...rows].sort(),
[...declaredAgents()].sort(),
"docs/models.md §0's recommendation table and DISPATCHED_AGENTS disagree — a missing row " +
"leaves a reader unable to tell whether that agent was measured, left alone deliberately " +
"or forgotten, and an extra one recommends a per_agent key Iris would silently ignore",
);
});
// The same table's OTHER column is a partition, and the way it goes wrong is one row being
// updated from a new round while its neighbours keep the old round's numbers. That has happened
// twice already in this sprint's reporting: #311 published four shares that summed to 94.6%, and
// §0 carried a set from one round while §6 predicted where a later round would put them. A share
// that does not belong to the same denominator as the one beside it is unusable, and the sum is
// the only free check for it — nothing in the document can tell a reader that 42.0% and 25.0%
// came from different rounds, but they cannot both be true at once.
//
// NOT a check that the figures are current: it passes on any self-consistent set, so it does not
// substitute for the round names §6 attaches to each. It fails on the realistic edit — one row
// moved, the rest left standing.
test("docs/models.md §0's share column is a partition of one round, not a mix of several", () => {
const doc = readFileSync(join(ROOT, "docs/models.md"), "utf8");
const section = doc.split(/^## /m).find((s) => s.startsWith("0."));
assert.ok(section, "docs/models.md has no `## 0.` summary section any more");
// Second cell of each row, which is the share. Read as a number only when it looks like a
// percentage, so a row that stops quoting one (or the header separator) is skipped rather than
// read as zero — a silent 0 would make a broken table sum closer to 100, not further from it.
const shares = [...section.matchAll(/^\|[^|]+\|\s*\*{0,2}([\d.]+)%\*{0,2}\s*\|/gm)].map((m) =>
Number(m[1]!),
);
assert.ok(shares.length >= 4, `§0 has ${shares.length} share cells; expected one per agent`);
const sum = shares.reduce((a, b) => a + b, 0);
// A tenth of a point per row, since each is published rounded to one decimal.
assert.ok(
Math.abs(sum - 100) <= shares.length * 0.1,
`docs/models.md §0's shares sum to ${sum.toFixed(1)}%, not 100% — ${shares.join(" + ")}. ` +
`Either a row was updated from a newer round while its neighbours kept the old one, or an ` +
`agent is missing from the denominator. Both read to an operator as "this is where the ` +
`money is" and neither is.`,
);
});
// §5's per-agent table is the OTHER partition in this document, and it needs the same guard §0 has.
//
// What it does NOT catch, said plainly because it was proposed as the fix for exactly this: the
// version reviewed on PR #327 left `builder` out of §5 altogether, and no arithmetic check could
// have found that. The four rows summed to the stated total to the cent and the shares summed to
// 100.0%, because the total was ITSELF the four-agent subtotal the harness prints — an agent absent
// from the rows *and* the denominator leaves a table that is internally perfect and mis-labelled.
// The only fix for that shape is naming the denominator in the prose, which §5 now does. A test that
// passes on the defect it was written for is worse than no test, so this one claims a different job.
//
// The job it does have is the realistic later edit: a fifth row added while the total stands, a total
// updated while the rows stand, or one column re-run and the other left stale. It also pins the
// arithmetic the section's own headline is read off — the reviewed draft summarised this table as
// "only about a third of the saving comes from the agent that was swapped" when its two money
// columns give 63.8%, and a table that adds up cannot be summarised backwards without one of the two
// being wrong on its face.
//
// Both money columns, because they fail differently: the SWAPPED column is the one an agent goes
// missing from (a new agent appears in a later round and nobody adds a row), and the UNSWAPPED column
// is the one that goes stale (a re-run moves the baseline and only the interesting half is updated).
test("docs/models.md §5's per-agent rows sum to the total row they are published under", () => {
const doc = readFileSync(join(ROOT, "docs/models.md"), "utf8");
const section = doc.split(/^## /m).find((s) => s.startsWith("5."));
assert.ok(section, "docs/models.md has no `## 5.` section any more");
// Cells with the bold markers stripped, since emphasis lands on whichever figures moved.
const rows = section
.split("\n")
.filter((l) => l.startsWith("|"))
.map((l) =>
l
.replace(/\*\*/g, "")
.split("|")
.slice(1, -1)
.map((c) => c.trim()),
);
const money = (cell: string | undefined) => {
const m = cell?.match(/\$([\d,]+\.\d+)/);
return m ? Number(m[1]!.replace(/,/g, "")) : undefined;
};
// Anchored on the total row and walked BACK to its own header separator, rather than picking rows
// that look like agents: §5 carries a second table of Iris's verifier counters whose first cell is
// also a backticked snake_case name (`content_missing`, `editor_truncated`), and a name-shaped
// filter swept those in. Structural membership also means "a row with no dollar figure" is a real
// failure rather than a row this test declined to recognise.
const iTotal = rows.findIndex((r) => /^total\b/.test(r[0] ?? ""));
assert.ok(iTotal > 0, "docs/models.md §5's per-agent table has no `total` row to check against");
const total = rows[iTotal]!;
const agents: string[][] = [];
for (let i = iTotal - 1; i >= 0 && !/^-+$/.test(rows[i]![0] ?? ""); i--) agents.unshift(rows[i]!);
assert.ok(agents.length >= 4, `§5 lists ${agents.length} agent rows; expected one per agent`);
for (const [col, label] of [
[1, "unswapped"],
[2, "swapped"],
] as const) {
const parts = agents.map((r) => money(r[col]));
assert.ok(
parts.every((p) => p !== undefined),
`§5's ${label} column has a row with no dollar figure in it: ` +
`${agents.map((r) => `${r[0]}=${r[col]}`).join(", ")}`,
);
const sum = parts.reduce((a, b) => a! + b!, 0)!;
const stated = money(total[col]);
assert.ok(stated !== undefined, `§5's total row has no ${label} figure`);
// Half a cent per row: the rows are published to four decimals, the total to four.
assert.ok(
Math.abs(sum - stated) <= agents.length * 0.005,
`docs/models.md §5's ${label} agent rows sum to $${sum.toFixed(4)}, but the total row says ` +
`$${stated.toFixed(4)}. An agent that ran and is not in this table reads as an agent that ` +
`cost nothing, and the share column beside it then partitions a subtotal while the prose ` +
`calls it the round.`,
);
}
// And the share column, which is only meaningful over the total the rows above actually sum to.
const shares = agents.map((r) => Number(r[r.length - 1]!.match(/([\d.]+)%/)?.[1]));
assert.ok(
shares.every((s) => Number.isFinite(s)),
`§5's last column is not a share on every agent row: ${agents.map((r) => r[r.length - 1]).join(" | ")}`,
);
const shareSum = shares.reduce((a, b) => a + b, 0);
assert.ok(
Math.abs(shareSum - 100) <= shares.length * 0.1,
`docs/models.md §5's post-swap shares sum to ${shareSum.toFixed(1)}%, not 100% — ` +
`${shares.join(" + ")}.`,
);
// And the figures the section's HEADLINE is actually read off, which is the gap the two sums above
// leave open: each agent's share of the SAVING is a difference of the two money columns, so an edit
// that moves the table and leaves this sentence standing reproduces the reviewed defect exactly —
// both columns would still sum, the shares would still be 100.0%, and everything above would pass
// while the prose said "a third" of a saving that is two thirds. Recomputed from the same rows
// rather than pinned as literals, so it is the relationship that is asserted and not the numbers.
const totalDelta = money(total[1])! - money(total[2])!;
const attributed = [
...section.matchAll(/`([a-z_]+)`\s+(?:is|gives)\s+\$([\d.]+)(?:\s+back)?\s+\((−?[\d.]+)%\)/g),
];
assert.equal(
attributed.length,
agents.length,
`§5 attributes the saving to ${attributed.length} agents but its table has ${agents.length} ` +
`rows — every row's contribution to the saving has to be stated, including a negative one, or ` +
`the sentence adds to less than the total it claims to break down`,
);
for (const [, name, dollars, percent] of attributed) {
const row = agents.find((r) => r[0] === `\`${name}\``);
assert.ok(row, `§5 attributes part of the saving to \`${name}\`, which has no row in its table`);
const delta = money(row[1])! - money(row[2])!;
// Written unsigned with "back" where an agent got dearer, so compare magnitudes here and let the
// percentage carry the sign.
assert.ok(
Math.abs(Math.abs(delta) - Number(dollars)) <= 0.0001,
`§5 says \`${name}\` accounts for $${dollars} of the saving, but its own row is ` +
`${row[1]} → ${row[2]}, a difference of $${Math.abs(delta).toFixed(4)}`,
);
const stated = Number(percent.replace("−", "-"));
const actual = (delta / totalDelta) * 100;
assert.ok(
Math.abs(actual - stated) <= 0.1,
`§5 says \`${name}\` is ${percent}% of the saving; its own columns give ` +
`${actual.toFixed(1)}% ($${delta.toFixed(4)} of $${totalDelta.toFixed(4)}). This is the ` +
`sentence a reader takes the keep-or-revert decision from.`,
);
}
});
// §0's table is the summary, and the sections restate it. That restatement is the document's most
// frequent defect: five of the false statements PR #327 removed were a claim corrected in one place
// and left standing in another, and #327 itself merged with §8 still saying "the revert is the one
// config line" an hour after §5 had been corrected to two. Then #329 flipped `copy_editor` from a
// keep to a recommended swap, which had to be rewritten in four places — the intro, §0, §4 and §8 —
// and nothing in the repo could have told anyone if one had been missed.
//
// So this pins the join, in both of the ways the two statements can disagree: the SHARE (§0's column
// is re-quoted in the section's own opener, and a re-run that moves one leaves the other stale) and
// the DISPOSITION (keep / declined / recommended / applied, which is the thing a reader acts on).
//
// Deliberately not a check that either is correct — §0's share is already pinned as a partition by
// the test above, and no test can say whether "keep" is the right call. This says only that the
// document gives one answer rather than two. That is the whole of the defect it is written for: at no
// point was either copy of a drifted claim unverifiable, and at every point both were present.
//
// Openers only, not every mention. §6 re-quotes all four shares in its drift note and also quotes
// #311's two INCORRECT ones (43.2%, 20.8%) on purpose, so a document-wide sweep would fail on the
// paragraph whose job is to publish a wrong figure — the same trap that made an earlier draft of the
// §5 test above read a table of verifier counters. The cost of scoping this way is real and worth
// naming: §6's drift-note shares go unchecked, and so does any share quoted mid-paragraph.
//
// The agents checked are LISTED rather than discovered, which is the whole of what makes the
// disposition half load-bearing (PR #332 review, note 1). The first version of this test skipped any
// pair it could not classify — `if (want === null || got === null) continue` — so rewording §0's
// cell to "swap pending a decision" disabled the check silently and let §4 go on saying whatever it
// liked. An unclassifiable cell is now a FAILURE for these four, because "§0 stopped stating a
// disposition in words this test knows" is itself the thing worth being told about: either the
// vocabulary below needs a phrase adding, or the summary stopped answering the question a reader
// came for.
//
// Four, not five: `page`, `reader`, `copy_editor` and `feedback` are the agents a model decision has
// been taken or recommended on. `builder` is exempt because it genuinely has no disposition in this
// vocabulary — its §0 cell is a zero with a date on it, not a keep or a swap — and forcing it into
// one would be inventing a decision nobody took. Add an agent here when a decision is taken on it.
const DECIDED = ["page", "reader", "copy_editor", "feedback"] as const;
const DISPOSITIONS = [
// The order of this list decides NOTHING — do not add a phrase here on the assumption that putting
// it earlier wins. It used to: the phrases overlap ("swap recommended, not yet applied" holds
// `applied`), first match won, and reordering reclassified documents. That was the defect, because
// both halves of the check below read this same list in the same order, so an overlapping clause
// made them agree on the wrong word instead of disagreeing. A clause matching two entries now
// FAILS. The text each entry is matched against is still cut at the first comma or dash, so a later
// clause cannot reclassify a paragraph.
//
// `open` was added an hour after this test was written, and by the route the comment above
// predicted: the seat that ran the verifier round retracted its headline, `feedback`'s keep became
// undecided, and both §0 and §4 stopped stating a disposition in the four words this list knew. The
// test failed rather than skipping the pair, which is the whole point of listing DECIDED — a
// decision can be *un*-taken, and "no answer yet" is a disposition a reader has to be told.
["open", /\bopen\b|\bundecided\b/i],
["declined", /declin/i],
["recommended", /recommend/i],
["keep", /\bkeep\b/i],
["applied", /\bswapped\b|\bapplied\b|\blive\b/i],
] as const;
// EVERY match, not the first one. Resolving a clause that carries two dispositions by list order is
// the same off-switch as skipping one that carries none: both halves of the check below read the same
// vocabulary in the same order, so they would shadow identically and *agree on the wrong word* rather
// than disagree. `feedback` reached "swap recommended … still open" in one revision and stopped one
// comma short of this. An ambiguous clause fails and says which two it matched.
function dispositions(text: string): string[] {
const clause = text.split(/,| — |\s—\s/)[0]!;
return DISPOSITIONS.filter(([, re]) => re.test(clause)).map(([d]) => d);
}
test("docs/models.md's sections agree with §0 about each agent's share and disposition", () => {
const doc = readFileSync(join(ROOT, "docs/models.md"), "utf8");
const summary = doc.split(/^## /m).find((s) => s.startsWith("0."));
assert.ok(summary, "docs/models.md has no `## 0.` summary section any more");
// agent -> [share, status cell], from the rows the test above already treats as the claim.
const stated = new Map<string, [number, string]>();
for (const m of summary.matchAll(/^\| `([^`]+)` \|\s*\*{0,2}([\d.]+)%\*{0,2}\s*\|([^|]*)\|/gm)) {
stated.set(m[1]!, [Number(m[2]!), m[3]!]);
}
assert.ok(stated.size >= 4, `§0 yielded ${stated.size} agent rows; expected one per dispatched agent`);
// A section's opener: a bolded lead-in naming a backticked agent and re-quoting its share. Any
// further parenthetical between the share and the dash is tolerated — `(0% in this round)` carries
// a caveat rather than a second figure, and §4's `builder` opener names the specialists too.
const openers = new Map<string, [string, string]>();
for (const m of doc.matchAll(/^\*\*`([a-z_]+)` \((\d[\d.]*)%[^—]*—\s*([^*]+)\*\*/gm)) {
openers.set(m[1]!, [m[2]!, m[3]!]);
}
for (const name of DECIDED) {
const row = stated.get(name);
assert.ok(row, `\`${name}\` has a model decision on it but no row in §0's table`);
const [share0, status] = row;
const opener = openers.get(name);
assert.ok(
opener,
`no section of docs/models.md opens with \`**\`${name}\` (share%) — disposition**\`, so its ` +
`share and its disposition are stated only in §0 and nothing checks the section a reader ` +
`is sent to. \`page\` and \`reader\` were both in this position until PR #332.`,
);
const [share, text] = opener;
assert.equal(
Number(share),
share0,
`\`${name}\` is ${share0}% in §0's table and ${share}% in its own section's opener. One of ` +
`the two was updated from a newer round and the other was not; §0's is the partition that ` +
`is checked to sum to 100%, so the opener is the likelier stale copy.`,
);
// Both sides must classify, exactly once. Neither a null nor a tie here is a pass — see the two
// notes above the vocabulary.
const wants = dispositions(status!);
const gots = dispositions(text);
const want = wants[0] ?? null;
const got = gots[0] ?? null;
const vocabulary = DISPOSITIONS.map(([d]) => d).join(", ");
for (const [side, matches, quoted] of [
["§0's table cell", wants, status!],
["its own section's opener", gots, text],
] as const) {
assert.ok(
matches.length < 2,
`\`${name}\`'s ${side} reads as "${matches.join('" and "')}" at once: ` +
`"${quoted.trim().slice(0, 80)}". Which one wins is decided by the order of DISPOSITIONS, ` +
`and both halves of this check read that same order — so they would agree on the wrong ` +
`word instead of disagreeing, and this test would pass on a document that states two ` +
`dispositions for one agent. Say one thing before the first comma.`,
);
}
assert.ok(
want,
`§0's \`${name}\` cell no longer states a disposition this test can read (${vocabulary}): ` +
`"${status!.trim().slice(0, 80)}". §0 is the table a reader takes the decision from, so ` +
`either say which of those it is, or add the new phrasing to DISPOSITIONS deliberately.`,
);
assert.ok(
got,
`\`${name}\`'s own section opens without a disposition this test can read (${vocabulary}): ` +
`"${text.trim().slice(0, 80)}". Skipping this pair is how the check disabled itself.`,
);
assert.equal(
got,
want,
`§0 says \`${name}\` is "${want}" and its own section says "${got}". This is the sentence a ` +
`reader acts on, and it is the defect that recurred most in this document: a decision ` +
`changed in the summary and left standing in the section, or the reverse.`,
);
}
});
// The last defect in this document that shipped past every gate was a rendering one: fixing a figure
// above, I wrapped a clause in `**` inside a sentence that was already bold, and the outer run was
// left open. It renders as two literal asterisks in front of the sentence stating the price of a
// decision. tsc was clean, all 1328 tests passed, and nothing else in the file reads that paragraph —
// the only thing that caught it was a person reading the diff (PR #332 review round 2).
//
// So this checks the one property a markdown document can lose without anything else noticing: every
// `**` run closes inside the paragraph that opened it. Per paragraph rather than per file, because a
// file-wide count comes back even as soon as a second paragraph breaks the other way, and per
// paragraph is also where the render actually goes wrong.
//
// The unit is a rendered block, NOT a blank-line-delimited chunk. Counting parity over a whole chunk
// lets two odd bullets in one list cancel — an unclosed run in one bullet and another in the next add
// to an even total and the check comes back clean, which is the same off-switch as a guard that skips
// what it cannot classify. So a list item and a table row each start a fresh count; their wrapped
// continuation lines belong to the item they continue, because a `**` legitimately spans those.
//
// Two limits, stated because both will eventually fire on innocent text. This is a parity count, not
// a parser: (1) a deliberate literal `**` in prose fails it, and the fix then is to fence or escape
// that text, not to delete the test — fenced blocks are skipped for the same reason, since they quote
// markup rather than use it. (2) The split below tests what a line *looks* like, so a prose line that
// happens to wrap onto `- and that is the point` or `3. Undecided, because…` starts a fresh count
// mid-sentence, and a `**` run spanning that wrap then reports as two odd blocks. That is a false
// positive and the fix is to reflow the paragraph, not to widen the guard. It is latent in **all
// three** documents this runs over, and the audit is per-document because the answer could differ: in
// docs/models.md the minus signs are U+2212, which the ASCII `[-*+]` class does not match, and no line
// wraps onto an ordered marker — §4's numbered list is where a future edit would hit it first. In
// docs/cost.md the same holds (1 × U+2212, no ASCII minus as a numeric sign) and the five sampling
// bounds are genuine ordered markers. docs/sprint-246.md was audited the same way when the report was
// split out of docs/cost.md (15 × U+2212, no ASCII minus as a sign, and §5's `1./2./3.` genuine): both
// lines were checked on it rather than assumed from its parent, which is what a fourth document added
// to the loop below owes as well. Count occurrences with `grep -o … | wc -l`, not `grep -c`, which
// counts matching LINES — this comment first recorded 15 as "12" for exactly that reason, and 12 is
// the number of lines those 15 signs sit on.
function unclosedBoldRuns(doc: string): string[] {
const unclosed: string[] = [];
let fenced = false;
// [line number, text] per line, so the message can name the run that is left open rather than the
// top of the block — round 3 asked for that and it is the line an editor has to go and look at.
let block: [number, string][] = [];
// A `**` inside an inline code span is not emphasis: `src/**` and `.github/workflows/**` are glob
// patterns, and GFM renders what is between backticks literally by definition. Stripped before
// counting, or every path pattern written in prose reads as a run that never closes. Four lines of
// docs/ci.md are what found this, on the commit that first brought that text under this check —
// it had lived in README.md, which this loop did not cover.
//
// Stripping is per line, like everything else here, and a code span WRAPPED across a line break
// therefore strips the wrong range: the half-span on each line pairs with the next backtick it
// finds, which can swallow a real `**` in between. docs/sprint-246.md had one (`git worktree
// list`, split across two lines), and the run it swallowed was the one this check exists to catch,
// so adding the strip turned a masked defect into a false positive on the same line. It is
// reflowed rather than parsed for, which is what the note above says to do with a false positive
// of this shape — but unlike that one, this failure mode also HIDES defects, so if a wrapped span
// is ever legitimately needed the strip has to become a real scan, not an exemption.
const stripCode = (text: string): string => text.replace(/`+[^`]*`+/g, "");
const finish = () => {
const runs = block.flatMap(([n, text]) => (stripCode(text).match(/\*\*/g) ?? []).map(() => n));
if (runs.length % 2 !== 0) {
const [openerLine, opener] = block[0]!;
unclosed.push(
`line ${runs.at(-1)} (block opens at line ${openerLine}: ${opener.trim().slice(0, 70)})`,
);
}
block = [];
};
for (const [i, line] of doc.split("\n").entries()) {
if (line.startsWith("```")) {
finish();
fenced = !fenced;
continue;
}
if (fenced) continue;
if (line.trim() === "") {
finish();
continue;
}
// A new list item or table row is its own render unit; anything else continues the current one.
if (/^\s*(?:[-*+]|\d+\.)\s/.test(line) || line.trimStart().startsWith("|")) finish();
block.push([i + 1, line]);
}
finish();
return unclosed;
}
// Every measurement document, not just the one the defect happened in. docs/cost.md and
// docs/sprint-246.md are the same kind of writing — bold lead-ins on nearly every paragraph, tables
// whose emphasis lands on whichever figure moved — so they fail the same way, and both were written
// after this check existed. sprint-246.md is most of the prose docs/cost.md used to carry, so leaving
// it out of the loop would have quietly dropped ~250 already-covered lines out of coverage on the
// commit that moved them.
//
// `docs/design-notes.md`, `docs/ci.md`, `docs/verifier-calibration.md` and `docs/github-auth.md` are
// in the loop for the same reason sprint-246.md is: they are ~1,950 lines lifted out of README.md,
// written in exactly this style, and a move is the commit where a `**` run gets cut in half.
//
// One list, read off the directory, for every markdown check in this file. Three things went wrong
// with the hand-written version and each is fixed by the same line:
// - The bold-run and table-swallow checks kept a COPY of it each, so a document added to one and
// not the other was covered by half the guard while reading as covered by both.
// - The link check three functions down enumerated `docs/` with readdirSync while these two named
// files, so the two halves of this file disagreed about what "the docs" meant — and a new
// `docs/*.md` was auto-covered by one and silently uncovered by the others.
// - `docs/API.md` (1,897 lines, bold lead-ins and tables throughout) and CONTRIBUTING.md were
// outside BOTH checks for no stated reason. They pass, so nothing was holding them out.
// A document is now covered by being a document, which is the only rule that cannot go stale.
const PROSE_DOCS = [
"README.md",
"CONTRIBUTING.md",
...readdirSync(join(ROOT, "docs"))
.filter((f) => f.endsWith(".md"))
.map((f) => `docs/${f}`),
];
test("the measurement docs' bold runs close in the block that opens them", () => {
for (const file of PROSE_DOCS) {
const unclosed = unclosedBoldRuns(readFileSync(join(ROOT, file), "utf8"));
assert.deepEqual(
unclosed,
[],
`${file} has ${unclosed.length} block(s) with an odd number of \`**\` runs, so a bold ` +
`run opens and never closes and the asterisks render literally. Each entry names the line ` +
`carrying the run that is left open — a paragraph, one list item or one table row:\n ` +
`${unclosed.join("\n ")}`,
);
}
});
// A GFM table ends at a blank line or at the start of another block-level structure. A plain paragraph
// line is NEITHER, so a paragraph placed directly under a table is swallowed as table rows — one row
// per wrapped line, each cell announced under the table's own column headers, with any `**` run split
// across two rows and rendering as literal asterisks.
//
// This is here because it happened, and because every other gate was green while it was broken. Round 1
// of PR #396 had me mutate docs/cost.md to prove a new assertion could fail; reverting the mutation ate
// the blank line between the price-sheet table and the sentence under it, and tsc, 1511 tests and e2e all
// passed with that document's summary paragraph rendering as five prose rows in a six-column table. The
// `git diff --stat` said so — four insertions against five deletions for a five-line-to-five-line
// paragraph rewrite — and reading the stat rather than the diff is what missed it. A reverted mutation
// needs diffing against the pre-mutation blob, not eyeballing.
//
// Headings, fences, lists, blockquotes and HTML blocks are all block-level starts and end a table
// legitimately, so only a plain paragraph line is a defect. The unit is the source line because that is
// what the renderer consumes; nothing here parses the table.
test("no prose paragraph is swallowed into the table above it", () => {
for (const file of PROSE_DOCS) {
const lines = readFileSync(join(ROOT, file), "utf8").split("\n");
let fenced = false;
const swallowed: string[] = [];
for (const [i, line] of lines.entries()) {
if (line.startsWith("```")) {
fenced = !fenced;
continue;
}
if (fenced) continue;
const previous = lines[i - 1];
if (!previous?.trimStart().startsWith("|")) continue;
// A block-level start ends the table; anything else is consumed as another row.
if (line.trim() === "" || /^\s*(?:\||#{1,6}\s|>|```|<|[-*+]\s|\d+\.\s)/.test(line)) continue;
swallowed.push(`line ${i + 1}: ${line.trim().slice(0, 70)}`);
}
assert.deepEqual(
swallowed,
[],
`${file} has ${swallowed.length} line(s) of prose directly under a table row with no blank line ` +
`between, so GitHub renders them as table rows rather than as a paragraph — each one a cell ` +
`under the table's column headers, with any bold run split across rows and showing literal ` +
`asterisks. Insert a blank line before each:\n ${swallowed.join("\n ")}`,
);
}
});
// docs/sprint-246.md's §3 is the sprint's three-arm page-model comparison, whose rows are an
// ARITHMETIC decomposition rather than a partition: each arm's total is its `verify + correct`
// column plus its `page only` column, and the percentage beside them is the first over the total.
// So the realistic edit — a newer round moves one arm's total and its components are left standing,
// or the reverse — is catchable for free, and it is the edit that has already gone wrong twice in
// this sprint's reporting (#311's four shares summing to 94.6%, and §0 of docs/models.md carrying
// one round's shares while §6 predicted another's).
//
// It lived on docs/cost.md until #395 split that document into a price sheet and this report, which
// is why the section number is unchanged: the table moved file, not position. The per-STEP table that
// replaced it in docs/cost.md is a different shape and has its own check below — the two are not
// interchangeable and neither test covers the other's document.
//
// Deliberately NOT a check that the figures are current or that the round named beside them is the
// one they came from: it passes on any self-consistent set. What it does say is that the table and
// the prose under it cannot drift apart, since the per-page figures the prose quotes are re-derived
// here from the table's own cells rather than pinned as literals.
test("docs/sprint-246.md's cost table decomposes to its own totals, and its prose quotes those totals", () => {
const doc = readFileSync(join(ROOT, "docs/sprint-246.md"), "utf8");
const section = doc.split(/^## /m).find((s) => s.startsWith("3."));
assert.ok(section, "docs/sprint-246.md has no `## 3.` cost section any more");
const money = (cell: string | undefined) => {
const m = cell?.replace(/\*\*/g, "").match(/\$([\d,]+\.\d+)/);
return m ? Number(m[1]!.replace(/,/g, "")) : undefined;
};
// Rows naming a backticked model, which is what an arm is. The header and the separator carry no
// backtick, and no other table in the section does either.
const arms = section
.split("\n")
.filter((l) => /^\| `/.test(l))
.map((l) =>
l
.split("|")
.slice(1, -1)
.map((c) => c.trim()),
);
assert.ok(arms.length >= 3, `§3's table has ${arms.length} arm rows; expected one per model`);
for (const row of arms) {
const [name, total, checked, share, , , pageOnly] = row;
const [t, c, p] = [money(total), money(checked), money(pageOnly)];
assert.ok(
t !== undefined && c !== undefined && p !== undefined,
`§3's ${name} row is missing one of its three dollar figures: ${row.join(" | ")}`,
);
// A cent, since the three are published to four decimals and the round's own totals carry the
// same rounding.
assert.ok(
Math.abs(c! + p! - t!) <= 0.01,
`docs/sprint-246.md §3 says ${name} cost ${total}, but its own components are ${checked} of ` +
`checking and correcting plus ${pageOnly} of page calls — $${(c! + p!).toFixed(4)}. One ` +
`column was updated from a newer round and the others were left standing, and the ` +
`verify+correct SHARE beside them is read off exactly this decomposition.`,
);
const stated = Number(share!.replace(/\*\*/g, "").match(/([\d.]+)%/)?.[1]);
assert.ok(Number.isFinite(stated), `§3's ${name} row has no verify+correct share: ${share}`);
assert.ok(
Math.abs((c! / t!) * 100 - stated) <= 0.5,
`docs/sprint-246.md §3 says checking and correcting is ${share} of ${name}'s bill; its own cells ` +
`give ${((c! / t!) * 100).toFixed(1)}% (${checked} of ${total}). That share is the number ` +
`§5 opens on — "85% of the extraction step is checking and correcting" — so it decides ` +
`which lever the document sends a reader to.`,
);
// And the per-page figure the prose quotes, over the denominator the prose names. The table is
// per 100 pages SUBMITTED and every cross-arm figure in the document is on that denominator, so
// a total that moves has to move the prose with it or the two disagree by a factor of a hundred.
// Scoped to §3, not the whole document: a match anywhere else would let a figure quoted in some
// other section stand in for the sentence this is here to hold, and the failure message would
// then be false about where it looked. Prose only — the table row it came from is excluded, or
// every row would satisfy this against itself.
const perPage = `$${(t! / 100).toFixed(4)}`;
const prose = section
.split("\n")
.filter((l) => !/^\| /.test(l))
.join("\n");
assert.ok(
prose.includes(perPage),
`docs/sprint-246.md §3's ${name} row is ${total} per 100 pages submitted, so ${perPage} a page, ` +
`and no prose in §3 quotes ${perPage}. Either the table moved and the ` +
`per-page sentence under it did not, or a figure is being quoted on the other ` +
`denominator — pages that produced a file, which §3 says is not comparable across arms.`,
);
}
});
// docs/cost.md is now a price sheet whose entire value is that a reader can take a figure off it
// without excavating: one headline, one per-STEP table, and three prose figures read off that table.
// That is a different shape from the three-arm comparison above — the rows are steps of one round
// rather than arms of three, so they decompose to the headline total instead of to each other — and
// the realistic edit is the one this sprint has already made twice: a newer round moves a row and the
// headline, the block subtotals or a share is left standing.
//
// What makes it worth a check rather than a proofread is that the document deliberately states four
// sets of numbers that are NOT independent of the table: the cents-a-page headline is the total over
// 100, each share is a row over the total, the three block subtotals are groups of rows added, and the
// "checking costs two and a half times producing" claim is three of those shares added. Nothing in a
// markdown file tells a reader which of those went stale.
//
// All four are derived off the rows here, the blocks included. Review round 1 of PR #396 caught that
// this comment claimed the block subtotals were covered when nothing read them — the row filter is
// `/^\| /`, so the prose sentence carrying them was excluded and `$6.4071` → `$6.5071` passed. A
// comment naming what a check catches is itself a claim, and the same round found the sentence the
// row-sum assertion anchors on was about the BLOCKS rather than the rows, so the strictest assertion in
// the test was right by coincidence: the two decompositions are numerically equal today and regrouping
// the blocks would have moved the expected value of a row-sum check. The document now states each
// decomposition in its own sentence and this reads the one it means.
//
// Rounding is asserted rather than tolerated away. The per-step cells are published to four decimals,
// so they sum to $0.0001 less than the round's ledger total and the shares sum to 99.9%; the document
// says both of those out loud, and this reads the sum it states rather than accepting any total within
// a cent — a drifted row would otherwise hide inside the allowance.
test("docs/cost.md's price sheet decomposes to the headline it opens with", () => {
const doc = readFileSync(join(ROOT, "docs/cost.md"), "utf8");
const money = (cell: string | undefined) => {
const m = cell?.replace(/\*\*/g, "").match(/\$([\d,]+\.\d+)/);
return m ? Number(m[1]!.replace(/,/g, "")) : undefined;
};
// Step rows are the table lines whose second cell is a dollar figure; the header and separator
// carry none. The `failed` row is one of them on purpose — it is billed spend and belongs in the
// decomposition, which is the error #311 made by excluding it from four numerators.
const rows = doc
.split("\n")
.filter((l) => /^\| /.test(l))
.map((l) =>
l
.split("|")
.slice(1, -1)
.map((c) => c.trim()),
)
.filter((cells) => money(cells[1]) !== undefined);
assert.ok(rows.length >= 8, `docs/cost.md's table has ${rows.length} step rows; expected one per step`);
const total = money(doc.match(/total\s+\*\*(\$[\d,]+\.\d+)\*\*/)?.[1]);
assert.ok(
total !== undefined,
"docs/cost.md's opening paragraph no longer states the round's total as `total **$N**`, so " +
"nothing below can be checked against it — every share and the cents-a-page headline are that " +
"total's denominator.",
);
const summed = rows.reduce((a, cells) => a + money(cells[1])!, 0);
// The sentence about the STEPS, not the one about the three blocks. Those two sums are equal today,
// so anchoring on the wrong one passes and stops meaning anything the moment the blocks are regrouped.
const stated = money(doc.match(/steps sum to (\$[\d,]+\.\d+)/)?.[1]);
assert.ok(
stated !== undefined,
"docs/cost.md no longer states what its step rows sum to (`… steps sum to $N`). That sentence is " +
"the one place the table's own arithmetic is written down for a reader, and it has to be the " +
"sentence about the steps — the block subtotals are a different decomposition of the same rows.",
);
assert.equal(
Number(summed.toFixed(4)),
stated,
`docs/cost.md's step rows sum to $${summed.toFixed(4)}, and the prose says they sum to ` +
`$${stated!.toFixed(4)}. A row moved and the sentence under the table did not.`,
);
assert.ok(
Math.abs(summed - total!) <= 0.01,
`docs/cost.md's step rows sum to $${summed.toFixed(4)} but the headline total is ` +
`$${total!.toFixed(4)} — a cent apart at most is per-cell rounding, more than that is a ` +
`missing or double-counted step.`,
);
const statedShares: number[] = [];
for (const cells of rows) {
const [name, cost, share] = cells;
const stated = Number(share!.replace(/\*\*/g, "").match(/([\d.]+)%/)?.[1]);
assert.ok(Number.isFinite(stated), `docs/cost.md's ${name} row has no share: ${share}`);
statedShares.push(stated);
const actual = (money(cost)! / total!) * 100;
assert.ok(
Math.abs(actual - stated) <= 0.05,
`docs/cost.md says ${name} is ${share} of the bill; ${cost} over the headline ` +
`$${total!.toFixed(4)} is ${actual.toFixed(2)}%. The share column is read off the total the ` +
`document opens with, so one of the two is from a different round.`,
);
}
// The share column's own total, which the document states because it is 99.9% rather than 100% and
// says so instead of rounding one cell up to hide it. Every share above is pinned to its own row, so
// reaching this needs a restated round — and then the column can land on 99.8% with the prose still
// claiming 99.9%, which is the sentence telling a reader the column is a rounded decomposition and
// not a partition. Round 2 of PR #396 asked for it.
const columnSum = statedShares.reduce((a, b) => a + b, 0);
const statedColumnSum = Number(doc.match(/share column\s+sums to ([\d.]+)%/)?.[1]);
assert.ok(
Number.isFinite(statedColumnSum),
"docs/cost.md no longer states what its share column sums to (`the share column sums to N%`), " +
"which is the sentence that tells a reader the column is a decomposition rounded rather than a " +
"partition.",
);
assert.ok(
Math.abs(columnSum - statedColumnSum) <= 0.05,
`docs/cost.md says the share column sums to ${statedColumnSum}%; its own ${statedShares.length} ` +
`share cells come to ${columnSum.toFixed(1)}%. Either a share moved or a step was added, and ` +
`the sentence that explains why the column is not 100% now explains the wrong gap.`,
);
// The three block subtotals, which are the sentence a reader quotes when they want one number for
// "where does the money go" and are the only figures in the document that are a GROUP of rows added.
// The grouping is the document's own, restated here: a step that appears in no block, or in two,
// fails rather than being silently dropped from a subtotal — which is exactly how #311 published four
// shares summing to 94.6%, by leaving each agent's failed spend out of a numerator that kept it in
// the denominator.
const BLOCKS: [string, string[]][] = [
["producing and checking pages", ["extract", "verify", "correct", "recheck_sampled", "table_join"]],
["reviewing and editing the finished document", ["read", "edit"]],
["wasted", ["failed"]],
];
const stepName = (cell: string) => cell.replace(/[`*]/g, "").trim();
const assigned = BLOCKS.flatMap(([, steps]) => steps);
const tableSteps = rows.map((cells) => stepName(cells[0]!));
assert.deepEqual(
[...tableSteps].sort(),
[...assigned].sort(),
`docs/cost.md's table and the three block subtotals under it name different steps. The table has ` +
`${tableSteps.join(", ")}; the blocks account for ${assigned.join(", ")}. A step in no block is ` +
`spend the "where the money goes" sentence silently omits, and a step in two is spend it ` +
`double-counts.`,
);
const blockFigures = [
...(doc.match(/Where the money goes:[\s\S]*?\*\*/)?.[0] ?? "").matchAll(
/([\d.]+)%\s*\((\$[\d,]+\.\d+)\)/g,
),
];
assert.equal(
blockFigures.length,
BLOCKS.length,
`docs/cost.md's "where the money goes" sentence states ${blockFigures.length} percent-and-dollar ` +
`pairs; the ${BLOCKS.length} blocks below it each need one, or a subtotal is going unchecked.`,
);
for (const [i, [label, steps]] of BLOCKS.entries()) {
const [, statedPct, statedDollars] = blockFigures[i]!;
const actual = steps.reduce((a, step) => {
const row = rows.find((cells) => stepName(cells[0]!) === step);
assert.ok(row, `docs/cost.md's table has no \`${step}\` row, so the "${label}" block is unpriced`);
return a + money(row![1])!;
}, 0);
assert.ok(
Math.abs(actual - money(statedDollars)!) <= 0.0001,
`docs/cost.md says "${label}" is ${statedDollars}; its own rows (${steps.join(" + ")}) come to ` +
`$${actual.toFixed(4)}. This is the sentence a reader quotes for where the money goes, and it ` +
`is prose rather than a table cell, so nothing else in the repo would notice it going stale.`,
);
assert.ok(
Math.abs((actual / total!) * 100 - Number(statedPct)) <= 0.05,
`docs/cost.md says "${label}" is ${statedPct}% of the bill; $${actual.toFixed(4)} over the ` +
`headline $${total!.toFixed(4)} is ${((actual / total!) * 100).toFixed(2)}%.`,
);
}
// The headline a reader quotes, in the unit it is written in.
const cents = Number(doc.match(/\*\*([\d.]+)¢ a page\.\*\*/)?.[1]);
assert.ok(Number.isFinite(cents), "docs/cost.md no longer opens with `**N¢ a page.**`");
assert.equal(
cents,
Number(((total! / 100) * 100).toFixed(1)),
`docs/cost.md opens on ${cents}¢ a page, but its own total $${total!.toFixed(4)} over the 100 ` +
`pages it names is ${((total! / 100) * 100).toFixed(1)}¢. The headline is the figure everything ` +
`else in the repo quotes.`,
);
// And the one comparison the document draws between its own rows, which is three shares added.
const claimed = Number(
doc.match(/`verify`\s*\+\s*`correct`\s*\+\s*`recheck_sampled`\s*is\s*\*\*([\d.]+)%\*\*/)?.[1],
);
assert.ok(
Number.isFinite(claimed),
"docs/cost.md no longer states the `verify` + `correct` + `recheck_sampled` share, which is the " +
"claim its 'checking costs more than producing' line rests on.",
);
const checking = ["verify", "correct", "recheck_sampled"].map((step) => {
const row = rows.find((c) => c[0] === `\`${step}\``);
assert.ok(row, `docs/cost.md's table has no \`${step}\` row, so its ${claimed}% cannot be checked`);
return money(row![1])!;
});
const checkingShare = (checking.reduce((a, b) => a + b, 0) / total!) * 100;
assert.ok(
Math.abs(checkingShare - claimed) <= 0.1,
`docs/cost.md says checking a page is ${claimed}% of the bill; its own \`verify\`, \`correct\` ` +
`and \`recheck_sampled\` rows come to ${checkingShare.toFixed(2)}%. That figure is the ` +
`document's headline finding about where the money goes.`,
);
});
// GitHub's own heading slug, near enough for the links this repo writes: lowercase, drop anything
// that is not a word character, hyphen or space, then spaces to hyphens. Inline code and a link
// inside a heading contribute their text, not their markup.
//
// One hyphen per space, and NOT one per run of spaces: a dropped character between two spaces
// leaves both of them, so `### \`a\` / \`b\`` is `a--b` on GitHub. Collapsing instead would have
// been invisible here — a docs link written to the collapsed slug resolves against a guard that
// collapses too, and dies in the browser. Thirteen headings in these docs are of that shape.
function headingAnchors(markdown: string): Set<string> {
const anchors = new Set<string>();
let fenced = false;
for (const line of markdown.split("\n")) {
if (line.startsWith("```")) fenced = !fenced;
if (fenced) continue;
const heading = /^#{1,6}\s+(.*?)\s*$/.exec(line);
if (!heading) continue;
const text = heading[1].replace(/`/g, "").replace(/\[(.*?)\]\(.*?\)/g, "$1");
anchors.add(
text
.toLowerCase()
.replace(/[^\w\- ]+/g, "")
.trim()
.replace(/ /g, "-"),
);
}
return anchors;
}
// The slug rule itself, pinned on a heading rather than on the docs. The link check below does
// fail on a collapsing rule today — seven of the links in docs/API.md point at headings of this
// shape — but it fails by calling a correct link broken, which reads as the link needing repair.
// It also only fails while some document happens to link to such a heading.
test("a heading slug keeps both spaces around a dropped character", () => {
const anchors = headingAnchors("### `page_verify_ok` / `page_verify_failed`\n## 9. Close (a / b)\n");
assert.ok(
anchors.has("page_verify_ok--page_verify_failed"),
`got ${[...anchors].join(", ")} — GitHub replaces each space, so the dropped "/" leaves "--"`,
);
assert.ok(anchors.has("9-close-a--b"), [...anchors].join(", "));
});
// The other way that helper lies, and the reason it is a Set: two headings that slug the same
// collapse into one entry, so a link to either resolves against the check below while GitHub numbers
// the second `#thing-1` and every link written to `#thing` lands on the first. There are none today
// across the 13 prose docs, which is what makes this cheap to keep.
test("no two headings in one doc slug the same", () => {
const clashes: string[] = [];
for (const file of PROSE_DOCS) {
const counts = new Map<string, number>();
let fenced = false;
for (const line of readFileSync(join(ROOT, file), "utf8").split("\n")) {
if (line.startsWith("```")) fenced = !fenced;
if (fenced || !/^#{1,6}\s/.test(line)) continue;
for (const a of headingAnchors(line)) counts.set(a, (counts.get(a) ?? 0) + 1);
}
for (const [a, n] of counts) if (n > 1) clashes.push(`${file}: ${n} headings make #${a}`);
}
assert.deepEqual(
clashes,
[],
`heading anchor(s) are ambiguous, so a link to them cannot say which section it means:\n ` +
clashes.join("\n "),
);
});
// A cross-reference that does not resolve is the failure mode of moving prose between files, and it
// is silent: GitHub renders a dead relative link as a link, and a dead `#anchor` scrolls nowhere.
// The commit that split ~1,950 lines out of README.md into docs/design-notes.md, docs/ci.md,
// docs/verifier-calibration.md and docs/github-auth.md rewrote 11 pointers and created 18 anchors,
// none of which any other gate reads.
//
// Anchors are checked as well as paths because the paths were the easy half. Two of the pointers
// this replaced were *positional* — "see the end of design notes", "the section above" — which
// cannot break and cannot be checked either; they were turned into anchors precisely so that a
// later move fails here instead of quietly pointing at the wrong paragraph.
test("every relative link in the docs resolves, file and anchor", () => {
const files = PROSE_DOCS;
const anchorCache = new Map<string, Set<string>>();
const anchorsFor = (path: string) => {
if (!anchorCache.has(path)) anchorCache.set(path, headingAnchors(readFileSync(path, "utf8")));
return anchorCache.get(path)!;
};
const broken: string[] = [];
let checked = 0;
for (const file of files) {
const text = readFileSync(join(ROOT, file), "utf8");
for (const [, target] of text.matchAll(/\[[^\]]*\]\(([^)\s]+)\)/g)) {
if (/^(?:https?:|mailto:|#!)/.test(target)) continue;
const [path, anchor] = target.split("#");
const abs = path === "" ? join(ROOT, file) : join(ROOT, dirname(file), path);
checked++;
if (!existsSync(abs)) {
broken.push(`${file} -> ${target} (no such file)`);
continue;
}
if (anchor && abs.endsWith(".md") && !anchorsFor(abs).has(anchor)) {
broken.push(`${file} -> ${target} (file exists, no heading makes that anchor)`);
}
}
}
// Without this the check passes on a repo whose links it failed to match at all.
assert.ok(checked > 40, `only ${checked} relative links found across ${files.length} files`);
assert.deepEqual(
broken,
[],
`${broken.length} relative link(s) in the docs do not resolve. A moved section takes its ` +
`anchor with it, so repoint the link rather than deleting it:\n ${broken.join("\n ")}`,
);
});
// docs/API.md's run log was one table of 65 rows, and the largest cell in it ran to 9,176
// characters — a reference nobody could scan and nobody could read. It is now an index of one row per
// event and a section per event, which introduces a way to be wrong that the single table did not
// have: the two halves can disagree. A new event indexed and not written up is a row that scrolls
// nowhere, and one written up and not indexed cannot be found from the top at all. The link check
// above catches the first (a dead anchor) and is blind to the second.
test("every run-log event in docs/API.md is both indexed and written up, once each", () => {
const lines = readFileSync(join(ROOT, "docs/API.md"), "utf8").split("\n");
const from = lines.indexOf("## Run log");
assert.notEqual(from, -1, "docs/API.md has no `## Run log` heading any more");
const after = lines.findIndex((l, i) => i > from && l.startsWith("## "));
assert.ok(after > from, "`## Run log` is the last section in the file, which it should not be");
const body = lines.slice(from, after);
// The index is everything before the first section, so a table inside a section body is not read
// as a malformed index row — which is how this would have failed, naming the wrong file.
const firstSection = body.findIndex((l) => l.startsWith("### "));
assert.ok(firstSection > 0, "the run log has no `### ` sections, so the restructure was undone");
const indexed = body.slice(0, firstSection)
.filter((l) => l.startsWith("|") && l !== "| --- | --- |" && !l.startsWith("| `type`"))
.map((l) => {
const m = /^\| \[(.+?)\]\(#([^)]+)\) \| .+ \|$/.exec(l);
assert.ok(m, `an index row is not \`| [name](#anchor) | summary |\`:\n ${l}`);
return { name: m[1]!, anchor: m[2]! };
});
assert.ok(indexed.length > 60, `only ${indexed.length} events indexed in the run log`);
const headings = body.filter((l) => l.startsWith("### ")).map((l) => l.slice(4));
// "Once each" is not implied by the comparison below: two same-order lists are deepEqual with a
// repeat in both. It is also the case where the anchors go wrong and nothing else notices —
// GitHub disambiguates a second `### `page_blank`` to `#page_blank-1`, while `headingAnchors`
// slugs each heading alone and returns `page_blank` for both, so the second row scrolls to the
// first section and the link check above is satisfied because `#page_blank` does exist.
assert.equal(
new Set(headings).size,
headings.length,
"two run-log sections have the same heading; GitHub numbers the second anchor and links break",
);
assert.deepEqual(
indexed.map((e) => e.name),
headings,
"the run log's index and its sections name different events, or name them in a different order",
);
assert.deepEqual(
indexed.map((e) => e.anchor),
headings.map((h) => [...headingAnchors(`### ${h}`)][0]),
"an index row's anchor is not the slug of the section it names",
);
// A heading with nothing under it: the shape a half-finished move leaves.
const empty: string[] = [];
for (const [i, line] of body.entries()) {
if (!line.startsWith("### ")) continue;
let j = i + 1;
while (j < body.length && !body[j]!.startsWith("### ") && body[j]!.trim() === "") j++;
if (j >= body.length || body[j]!.startsWith("### ")) empty.push(line.slice(4));
}
assert.deepEqual(empty, [], `run-log section(s) with no text under the heading: ${empty.join(", ")}`);
});
// The run log now claims to document EVERY event src/ emits, which is the claim this test exists for:
// the sentence that first made it was wrong by 40 events, and a reader who greps a `run_start` line
// would have concluded the log could not carry it. It was replaced by a paragraph that counted the
// gap instead, and the gap has since been closed section by section. So the strong claim is back, and
// this time nothing about it is maintained by hand — the numbers are read back out of that paragraph,
// and the coverage itself is asserted rather than counted. A new event fails here until it is written
// up.
test("docs/API.md's run log documents every event src/ emits, and says so in numbers that are current", () => {
// THREE emit shapes reach the log, and each is invisible to a grep for the others:
// 1. `ctx.log.event("name", …)` everywhere in the pipeline — 108 names.
// 2. `this.onEvent?.("model_call_start", meta)` in src/providers — `model_call_start` and
// `model_call`, the two the run log documents best.
// 3. a RunLog method that skips event() and names its own type. `agentCall` does, on every one of
// 13 call sites, so `agent_call` is one of the commonest lines in the log while its name
// appears nowhere either grep above can reach.
//
// The first two are searched across src/, because those calls are made from everywhere. The third
// is searched in src/store/runlog.ts ALONE, and every literal `type` in that file counts however
// its object literal is ordered — `{ phase, type: "x" }` is the same event as `{ type: "x", phase }`
// and a pattern keyed on `this.write({ type:` would see only the second. Scoping it to that file is
// a claim about the rest of src/, so the claim is checked below rather than assumed.
const RUNLOG = join(SRC, "store/runlog.ts");
const emitted = new Set<string>();
// Where each name is emitted FROM, not just that it is. The run log's coverage paragraph makes its
// claim about the undocumented events by file rather than by name — three of them are not named for
// the family they belong to — so the check needs the file, and the same loop already has it.
const where = new Map<string, Set<string>>();
const emit = (name: string, file: string) => {
emitted.add(name);
if (!where.has(name)) where.set(name, new Set());
where.get(name)!.add(file.slice(ROOT.length));
};
// Shapes 1 and 2 differ only in the name, so the two searches below are nearly the same pattern —
// and both spell `onEvent(` without the optional chain as well as with it. Nothing writes it that
// way today, but it is one character from a spelling that IS here.
//
// THE TWO ARE DELIBERATELY NOT THE SAME WIDTH, because they fail in opposite directions.
//
// EMIT_LITERAL stays wide: it is anchored on a string literal, and a method signature —
// `onEvent(type: string, data: …): void` in an interface, where today's `TelemetryFn` is a type
// alias — has none, so nothing that is not a call can reach it. That matters because an event this
// search misses is an event the coverage claim below is never made over, and it goes missing with no
// message at all: a receiverless `onEvent?.("new_event")` would leave `undocumented` empty and the
// suite green while the run log says it documents everything. Silent staleness is the exact failure
// the `agent_call` history left behind, so this half concedes nothing it does not have to.
//
// EMIT_CALL requires a receiver. With no literal to anchor on it cannot tell a call from a
// signature, and a match on a signature accuses a type-only change of emitting under a name the
// test cannot read. What that narrowing gives up is a receiverless COMPUTED bridge, and losing that
// costs a warning rather than a claim — the same one-directional standing as APPENDS above.
const EMIT_NAMED = String.raw`(?:\.event|onEvent(?:\?\.)?)\(\s*`;
// The name class is `[^"]`, not `[a-z0-9_]`: a `log.event("foo-bar")` cannot have a run-log section,
// because every heading there is asserted to be snake_case — so it must fail as undocumented and be
// renamed, rather than slip past the search and out of the claim.
const EMIT_LITERAL = new RegExp(EMIT_NAMED + String.raw`"([^"]+)"`, "g");
const EMIT_CALL = new RegExp(String.raw`\.(?:event|onEvent(?:\?\.)?)\(\s*(?!")([^\s,)][^,)]*)`, "g");
for (const file of sources(SRC)) {
const text = readFileSync(file, "utf8");
for (const m of text.matchAll(EMIT_LITERAL)) emit(m[1]!, file);
}
const runlog = readFileSync(RUNLOG, "utf8");
// `[^"]` for the same reason as EMIT_LITERAL: an off-convention name must fail loudly as
// undocumented rather than fall out of the search and out of the claim.
for (const m of runlog.matchAll(/\btype:\s*"([^"]+)"/g)) emit(m[1]!, RUNLOG);
// What makes one file enough: nothing else in src/ reaches for an append-shaped fs call. If a
// second file ever writes the log, shape 3 stops being findable where this looks and the count
// goes quietly stale — the failure `agent_call` already demonstrated once.
//
// This cannot be an invariant, and the comment does not claim to be one: an `open`/`write` pair, a
// spawned process or a stream handed in from elsewhere would all slip past. It is the four spellings
// a writer would actually reach for, and it fails loudly on a plain mention — one message asking
// someone to check, against a silently stale count, is the right direction to be wrong in.
const APPENDS = /appendFileSync\(|appendFile\(|createWriteStream\(|flags?:\s*"a/;
const appenders = sources(SRC)
.filter((f) => APPENDS.test(readFileSync(f, "utf8")))
.map((f) => f.slice(ROOT.length));
assert.deepEqual(
appenders,
["src/store/runlog.ts"],
`something outside src/store/runlog.ts opens a file for appending: ${appenders.join(", ")}. If ` +
"any of them writes the run log, reading shape 3 from runlog.ts alone no longer sees every " +
"event — widen that search with the writers rather than adjusting the count.",
);
// All three shapes above read a LITERAL name, so a call whose first argument is an expression is
// invisible to every one of them — and the run log now claims to document every event, which a name
// the grep cannot see would make quietly false. Two such call sites exist and both are the router's
// telemetry bridge, which forwards names that originate in a literal `this.onEvent?.("…")` in
// src/providers, so shape 2 still sees all of them.
//
// Same standing as APPENDS above, and the same reason: not an invariant (a name built from a literal
// in a lookup table, or a bridge added inside a class, would each slip past), but the shape a caller
// would actually write, failing loudly on a third bridge rather than dropping its events from a
// coverage claim that says it has none.
//
// Each entry is file + the argument's own text and carries NO line number, deliberately: an import
// added to orchestrator.ts moves the bridge a line, and this would then fail an unrelated PR with a
// message accusing it of emitting under an unreadable name. Dropping to a set of paths would go too
// far the other way — a THIRD bridge in a file that already has one would vanish into it, and the
// duplicate entry is what catches that.
const bridges = sources(SRC)
.flatMap((file) => {
const text = readFileSync(file, "utf8");
return [...text.matchAll(EMIT_CALL)].map((m) => `${file.slice(ROOT.length)} ${m[1]!.trim()}`);
})
.sort();
assert.deepEqual(
bridges,
["src/pipeline/orchestrator.ts type", "src/tools/calibrate.ts type"],
`an event is emitted under a name this test cannot read: ${bridges.join(", ")}. The run log claims to ` +
"document every event src/ emits, and a computed name is documented or not without this test " +
"being able to tell — give the new call site a literal name, or teach the searches above where " +
"its names come from before the claim goes stale.",
);
assert.ok(emitted.size > 90, `only ${emitted.size} event names found in src/ — the grep missed`);
// Losing shape 3 would SHRINK a count rather than fail anything, which is how `agent_call` went
// undocumented for as long as it did: an event with no run-log section just stops being counted, while
// a broken shape-2 pattern fails `ghosts` loudly because `model_call` has one. `agent_call` has a
// section now, so a broken shape 3 would reach `ghosts` as well — this assertion stays because it
// says WHICH of the two failures it is, and because the next event added this way will again have
// no section to be missed from.
assert.ok(
emitted.has("agent_call"),
"`agent_call` is not in the emitted set. Either the shape-3 search above stopped matching " +
`${RUNLOG}, or that literal \`type\` was renamed — check which, because the first means ` +
"fixing the search and not the count.",
);
const api = readFileSync(join(ROOT, "docs/API.md"), "utf8");
const lines = api.split("\n");
const from = lines.indexOf("## Run log");
// Named, because the alternative is loud but wrong: a renamed heading leaves `from` at -1, the
// slice below empty, and every event in src/ reported as undocumented.
assert.notEqual(from, -1, "docs/API.md has no `## Run log` heading any more");
const after = lines.findIndex((l, i) => i > from && l.startsWith("## "));
const headings = lines.slice(from, after).filter((l) => l.startsWith("### ")).map((l) => l.slice(4));
// Every run-log heading is event names and nothing else, which is what makes reading its code spans as
// event names safe. A heading that also named a field — `### `quality_report` (`score`)` — would
// have to fail HERE, as a heading this test cannot parse, and not two lines down as an event src/
// no longer emits. If the run log ever needs such a heading, widen this shape and the extraction
// together.
for (const h of headings) {
assert.match(
h,
/^`[a-z0-9_]+`( \/ `[a-z0-9_]+`)*$/,
`a run-log heading is not event names only: \`### ${h}\`. Every code span in a run-log heading is read ` +
"below as an event name, so anything else in one is reported as a deleted event.",
);
}
const documented = new Set(
headings.flatMap((h) => [...h.matchAll(/`([a-z0-9_]+)`/g)].map((m) => m[1]!)),
);
// A section for an event nothing emits any more is dead documentation, and reads as current.
const ghosts = [...documented].filter((n) => !emitted.has(n)).sort();
assert.deepEqual(ghosts, [], `the run log documents event(s) src/ no longer emits: ${ghosts.join(", ")}`);
// The whole claim, and the one assertion a new event fails. Everything below it is about the
// paragraph that states it; this is the property. Kept as a list rather than a count so the failure
// names what to write up.
const undocumented = [...emitted].filter((n) => !documented.has(n)).sort();
assert.deepEqual(
undocumented,
[],
`the run log says it documents every event src/ emits, and these have no section: ${undocumented.join(", ")}. ` +
"Write them up, or replace that claim with one that is true — a count of the gap is what this " +
"paragraph used to carry, and it went stale every time the gap moved.",
);
// Read the numbers out of THAT paragraph, not out of the file: a check against the whole document
// would pass on a paragraph that had lost the sentence entirely, since both numbers appear
// elsewhere in these 4,000 lines.
//
// Scoped from that opening to the index table rather than to the next blank line: the coverage
// prose is two paragraphs, and the second is where the promise about what is checked is made.
const opens = "**The index is the whole log.**";
const paraStart = lines.findIndex((l) => l.startsWith(opens));
assert.ok(paraStart > 0, `the run log no longer opens its coverage paragraph with ${opens}`);
const paraEnd = lines.findIndex((l, i) => i > paraStart && l.startsWith("| `type`"));
assert.ok(paraEnd > paraStart, "the run log's coverage prose is no longer followed by the index table");
const para = lines.slice(paraStart, paraEnd).join(" ").replace(/\s+/g, " ");
const stated =
/`src\/` emits \*\*(\d+)\*\* event types and every one of them has a section below — \*\*(\d+)\*\* sections/.exec(
para,
);
assert.ok(
stated,
"the run log's coverage paragraph was reworded, so its numbers are no longer checked. Keep the shape " +
"`emits **N** event types and every one of them has a section below — **N** sections`, or move " +
"the check with the words.",
);
assert.deepEqual(
stated.slice(1, 3).map(Number),
[emitted.size, headings.length],
`the run log says ${stated.slice(1, 3).join("/")} (emitted/sections) and src/ says ` +
`${[emitted.size, headings.length].join("/")}`,
);
// A few sections make a claim about src/ rather than about the log — which file a line comes from,
// and what can reach that file — and those are checked where they are written. Bodies keyed by
// every event name in the heading, since a section can cover a pair.
const sections: { names: string[]; body: string[] }[] = [];
for (const line of lines.slice(from, after)) {
if (line.startsWith("### ")) {
sections.push({ names: [...line.slice(4).matchAll(/`([a-z0-9_]+)`/g)].map((m) => m[1]!), body: [] });
} else if (sections.length) {
sections[sections.length - 1]!.body.push(line);
}
}
const bodyOf = (name: string): string => {
const found = sections.find((s) => s.names.includes(name));
assert.ok(found, `the run log has no section for \`${name}\`, so the claim below cannot be checked`);
return found!.body.join(" ").replace(/\s+/g, " ");
};
// Three events are named for neither their family nor the file they come from, and each section
// says which file that is. Read one-directionally, name -> file: these files emit others too.
const ATTRIBUTED: [string, string][] = [
["contribution_failed", "src/pipeline/orchestrator.ts"],
["feedback_training_failed", "src/pipeline/orchestrator.ts"],
["calibrate_call_failed", "src/pipeline/calibration.ts"],
];
for (const [name, file] of ATTRIBUTED) {
assert.ok(
bodyOf(name).includes(`\`${file}\``),
`the run log's \`${name}\` section no longer names ${file} as where the line comes from`,
);
assert.deepEqual([...(where.get(name) ?? [])], [file], `the run log attributes \`${name}\` to ${file}`);
}
// "Calibration is a tool — `src/tools/calibrate.ts` — and not a phase of a run", which is what makes
// the section's "an ordinary run never writes this line" true. A claim about the pipeline rather
// than about the log, and the one a later change would silently falsify.
//
// Both import forms, because a DYNAMIC import is the likeliest way a run reaches this: the module
// is only wanted on the calibration path, and `await import(...)` inside the branch that wants it
// is what someone adding it would reach for. A static-only pattern would let exactly that through.
const IMPORTS_CALIBRATION = /(?:from|import\()\s*"[^"]*\/calibration\.ts"/;
const importers = sources(SRC)
.filter((f) => IMPORTS_CALIBRATION.test(readFileSync(f, "utf8")))
.map((f) => f.slice(ROOT.length));
assert.deepEqual(
importers,
["src/tools/calibrate.ts"],
`the run log says an ordinary run never writes \`calibrate_call_failed\` because calibration is a tool. ` +
`These import it: ${importers.join(", ")}. If a run reaches it now, that sentence is wrong.`,
);
assert.ok(
bodyOf("calibrate_call_failed").includes("`src/tools/calibrate.ts`"),
"the run log's `calibrate_call_failed` section no longer names the tool that reaches calibration",
);
// `agent_update_blocked` is documented as ONE line with two shapes, told apart by a `reason` only
// the eval-gate one carries. Two emit sites, exactly one naming `reason`, is what makes that rule
// usable — a third site, or a `reason` on the first, would make the section wrong about how to read
// a line while every number above stayed right.
const feedbackSrc = readFileSync(join(SRC, "pipeline/feedback.ts"), "utf8");
const blocked = [...feedbackSrc.matchAll(/\.event\("agent_update_blocked",\s*\{([^}]*)\}/g)].map((m) => m[1]!);
assert.equal(
blocked.length,
2,
`the run log documents \`agent_update_blocked\` as two shapes and feedback.ts emits it ${blocked.length} time(s)`,
);
assert.deepEqual(
blocked.map((fields) => /\breason:/.test(fields)),
[false, true],
"the run log says the regression-gate shape of `agent_update_blocked` carries no `reason` and the eval-gate " +
"one carries `reason: \"eval_regression\"`. The two emit sites no longer split that way.",
);
// And the units of `failures`, which is the collision the two sections warn about: a count on
// `regression_gate`, the list behind it on `agent_update_blocked`. One name, two shapes, both lines
// written for the same blocked update.
assert.ok(
/\.event\("regression_gate",[^}]*failures: failures\.length/.test(feedbackSrc),
"the run log says `regression_gate`'s `failures` is a count; feedback.ts no longer logs `failures.length`",
);
assert.ok(
/failures: gate\.failures/.test(blocked[0]!),
"the run log says `agent_update_blocked`'s `failures` is the list of strings, not a count; the " +
"regression-gate emit site no longer passes `gate.failures`",
);
// The run log's `agent_trained` section says a run reaches that branch only if something outside the
// pipeline seeds the session's tmp agents directory: the branch is behind `sessionBuilt`, which
// `loadAgent` sets from a file existing there, and the one line that writes such a file is inside
// the branch. Same shape of claim as APPENDS above and the same caveat — these are the write
// spellings a seeder would reach for, not an invariant.
const seeders = sources(SRC)
.filter((f) => /(writeFileSync|writeFile|copyFileSync|renameSync|cpSync)\([^;]*tmpAgentsDir/.test(readFileSync(f, "utf8")))
.map((f) => f.slice(ROOT.length));
assert.deepEqual(
seeders,
["src/pipeline/feedback.ts"],
`the run log says the only line writing into tmp/<id>/agents is the training branch itself. These write ` +
`there: ${seeders.join(", ")}. If one of them seeds it, \`agent_trained\` is reachable and that ` +
"paragraph is wrong.",
);
});