1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { runExtraction } from "../src/pipeline/extraction.ts";
import { STANDARD } from "../src/pipeline/contribute.ts";
import type { PipelineContext } from "../src/pipeline/context.ts";
import type { Paths } from "../src/store/paths.ts";
// Specialist dispatch turns a free-text name the MODEL wrote ("chartDataAgent",
// "chart", "table.md ") into a library file to run. Every outcome is a page that
// looks plausible either way, so the log line is the only observable difference
// between "routed", "correctly declined" and "silently missed" — and getting the
// wrong one of those three is the failure mode.
//
// test/e2e.sh covers the two happy directions end to end (a resolvable name and a
// near-miss). These cover the branches a real model reaches through sloppier
// output: a standard type, whitespace around it, and an unusable name.
const SUGGEST_MARK = "suggested-by-test";
interface Recorded {
events: { type: string; data: Record<string, unknown> }[];
calls: { agent: string; prompt: string }[];
}
// One page, one library specialist (chartDataAgent.md), plus a `table.md` that the
// real library no longer ships. It is written here on purpose: `table` is a standard
// type, so the decline must hold even when a file of that name IS present and would
// resolve — which is the case a directory-listing-based decline would get wrong.
// No feedback.md, so verifyAgentOutput short-circuits to ok and these tests stay
// about dispatch.
function makeCtx(dir: string, suggestedName: string | null): { ctx: PipelineContext; rec: Recorded } {
const agentsDir = join(dir, "agents");
const fragDir = join(dir, "fragments");
const inputDir = join(dir, "input");
for (const d of [agentsDir, fragDir, inputDir]) mkdirSync(d, { recursive: true });
writeFileSync(join(agentsDir, "page.md"), "# Page Agent\n\n## Required capability\nvision\n");
writeFileSync(join(agentsDir, "chartDataAgent.md"), "# Chart Agent\n\n## Required capability\nvision\n");
writeFileSync(join(agentsDir, "table.md"), "# Table Agent\n\n## Required capability\nvision\n");
writeFileSync(join(inputDir, "page-001.png"), "not-a-real-png");
const rec: Recorded = { events: [], calls: [] };
const ctx = {
sessionId: "ses_test",
images: [{ name: "page-001.png", order: 1, path: join(inputDir, "page-001.png") }],
extractionConcurrency: 1,
recheckSampleSize: 1,
maxReviewIterations: 1,
paths: {
agentsDir,
tmpAgentsDir: () => join(dir, "tmp-agents"),
agentMemory: (agent: string) => join(dir, `mem-${agent.replace(/\.md$/, "")}.json`),
sessionFragments: () => fragDir,
} as unknown as Paths,
router: {
complete: async (agent: string, _cap: string, messages: { role: string; content: string }[]) => {
const sys = messages.find((m) => m.role === "system")?.content ?? "";
const prompt = messages.map((m) => m.content).join("\n");
rec.calls.push({ agent, prompt });
// A specialist asked for just its own content type.
if (prompt.includes("Extract ONLY the content your contract covers")) {
return { text: JSON.stringify({ no_content: false, html: `<p>${SUGGEST_MARK} fragment</p>` }) };
}
// The page agent, called a second time to splice a fragment in.
if (sys.includes("You merge a higher-fidelity HTML fragment")) {
return { text: JSON.stringify({ html: `<p>page</p><p>${SUGGEST_MARK} merged</p>` }) };
}
// The general page pass, optionally naming a specialist it wants.
return {
text: JSON.stringify({
html: "<p>page</p>",
log: "",
...(suggestedName === null ? {} : { suggested_agent: { name: suggestedName, reason: "test" } }),
}),
};
},
},
log: {
event: (type: string, data: Record<string, unknown> = {}) => rec.events.push({ type, data }),
agentCall: () => {},
},
} as unknown as PipelineContext;
return { ctx, rec };
}
async function withTemp<T>(fn: (dir: string) => Promise<T>): Promise<T> {
const dir = mkdtempSync(join(tmpdir(), "iris-dispatch-"));
try {
return await fn(dir);
} finally {
rmSync(dir, { recursive: true, force: true });
}
}
const ev = (rec: Recorded, type: string) => rec.events.filter((e) => e.type === type);
const ranSpecialist = (rec: Recorded) =>
rec.calls.some((c) => c.prompt.includes("Extract ONLY the content your contract covers"));
test("a standard content type is declined, not dispatched", async () => {
await withTemp(async (dir) => {
const { ctx, rec } = makeCtx(dir, "table");
const { fragments, suggestions } = await runExtraction(ctx);
// `table.md` IS in the library, so this would resolve — the decline is a
// deliberate policy, not a lookup failure: the general page pass already
// rendered the table, and splicing a second rendering of it over the first is
// exactly the duplication the page prompt forbids.
assert.deepEqual(ev(rec, "specialist_declined").map((e) => e.data.agent), ["table"]);
assert.equal(ev(rec, "specialist_dispatched").length, 0);
assert.equal(ranSpecialist(rec), false, "a standard agent was run anyway");
assert.equal(fragments[0].innerHtml, "<p>page</p>", "the page output was rewritten");
// A declined suggestion IS still collected (`dispatched` is false), and
// runContribution's own STANDARD filter is what stops it becoming a new-agent
// issue. Asserted as-is rather than "corrected" here: extraction reports what
// the model asked for, one filter owns the standard-type policy, and moving it
// would put the same decision in two places.
assert.deepEqual(suggestions, [{ name: "table", reason: "test", image: "page-001.png" }]);
});
});
test("whitespace and a .md extension around a standard type are still declined", async () => {
await withTemp(async (dir) => {
// The regression this pins: normalizing as `.replace(/\.md$/, "").trim()`
// leaves "table.md " as "table.md", which STANDARD does not contain but
// loadAgent resolves to agents/table.md — so the decline is skipped and a
// standard specialist runs and merges. Two extra model calls and a duplicated
// table, from a trailing space. " table " declines under either order, which is
// what makes this shape the one worth asserting.
const { ctx, rec } = makeCtx(dir, " table.md \n");
const { fragments } = await runExtraction(ctx);
assert.deepEqual(ev(rec, "specialist_declined").map((e) => e.data.agent), ["table"]);
assert.equal(ranSpecialist(rec), false, "a padded standard name got past the decline");
assert.equal(fragments[0].innerHtml, "<p>page</p>");
});
});
test("a case variant of a standard type is still declined", async () => {
await withTemp(async (dir) => {
// `STANDARD` spells one of its nine entries `formField`, so these names are prose
// descriptions of content types rather than filenames and a model writing "Table" is
// ordinary output, not sloppy output. An exact-match decline sends it on to the file
// lookup, where this fixture's `table.md` resolves on a case-insensitive volume and
// the standard specialist runs — splicing a second rendering of the table over the
// one the page pass already produced.
//
// The decline event is asserted rather than just the absence of a dispatch, because
// the two failure modes are platform-dependent (dispatch on macOS, an unresolved
// miss on Linux) while the correct outcome is the same everywhere.
const { ctx, rec } = makeCtx(dir, "Table");
const { fragments } = await runExtraction(ctx);
assert.deepEqual(ev(rec, "specialist_declined").map((e) => e.data.agent), ["Table"]);
assert.equal(ranSpecialist(rec), false, "a case variant of a standard name was dispatched");
assert.equal(ev(rec, "specialist_unresolved").length, 0, "the variant was treated as a near-miss, not policy");
assert.equal(fragments[0].innerHtml, "<p>page</p>", "the page output was rewritten");
});
});
test("an unusable name is logged as unresolved rather than silently dropped", async () => {
await withTemp(async (dir) => {
// A model that emits `"name": " "` (or ".md") has still SUGGESTED something;
// it just gave nothing to route on. Without this line the page is
// indistinguishable from one that never suggested anything at all.
const { ctx, rec } = makeCtx(dir, " .md ");
await runExtraction(ctx);
const events = ev(rec, "specialist_unresolved");
assert.equal(events.length, 1);
assert.equal(events[0].data.reason, "empty name");
assert.equal(events[0].data.agent, " .md ", "the raw string it could not use is reported");
assert.equal(ranSpecialist(rec), false);
});
});
test("a resolvable non-standard name dispatches and merges", async () => {
await withTemp(async (dir) => {
const { ctx, rec } = makeCtx(dir, "chartDataAgent");
const { fragments, suggestions } = await runExtraction(ctx);
assert.deepEqual(ev(rec, "specialist_dispatched").map((e) => e.data), [
{ agent: "chartDataAgent.md", image: "page-001.png", merged: true },
]);
assert.match(fragments[0].innerHtml, new RegExp(`${SUGGEST_MARK} merged`), "the merge result is not in the page");
assert.match(fragments[0].log, /merged chartDataAgent/);
// A dispatched suggestion is already covered by the library, so it must not be
// re-filed as a new agent to build.
assert.deepEqual(suggestions, []);
});
});
test("a miss reports what was dispatchable and what would have been declined, separately", async () => {
await withTemp(async (dir) => {
// "tables" is the commonest near-miss shape: a plural of a standard type. It is
// not in STANDARD (so it is never declined) and resolves to no file, so it
// arrives here — and a log line that omitted `table` would hide the one name
// that explains the miss.
//
// The two lists stay apart because they answer different questions.
// `candidates` means "what you could have asked for", which is true only of real
// files; a standard type is declined by policy before the file is even looked up.
// Merging them would make the first claim false for most of the list.
const { ctx, rec } = makeCtx(dir, "tables");
const { suggestions } = await runExtraction(ctx);
const miss = ev(rec, "specialist_unresolved")[0];
assert.equal(miss?.data.agent, "tables");
assert.deepEqual(miss?.data.candidates, ["chartDataAgent", "table"]);
assert.deepEqual(miss?.data.declined_types, [...STANDARD].sort());
assert.equal(ranSpecialist(rec), false);
// Unresolved IS reported for contribution: this is the path that proposes a new
// agent for a type the library genuinely lacks.
assert.deepEqual(suggestions, [{ name: "tables", reason: "test", image: "page-001.png" }]);
});
});
test("the near-miss explanation survives with no standard agent files on disk", async () => {
await withTemp(async (dir) => {
// The real library ships no standard agent files — the nine were deleted as
// unreachable. So `declined_types` must come from STANDARD rather
// than from a directory listing: reading the directory, `table` appears nowhere
// and the commonest miss ("tables") becomes unexplainable in exactly the
// deployment everyone runs.
//
// Same scenario as above with `table.md` removed, which is the point of running
// both: there, the file's presence proves the DECLINE is not file-driven; here,
// its absence proves the EXPLANATION is not either. `candidates` correctly loses
// `table` — it is no longer a file — while `declined_types` keeps it.
const { ctx, rec } = makeCtx(dir, "tables");
rmSync(join(dir, "agents", "table.md"));
await runExtraction(ctx);
const data = ev(rec, "specialist_unresolved")[0]?.data as Record<string, string[]>;
assert.deepEqual(data.candidates, ["chartDataAgent"]);
assert.deepEqual(data.declined_types, [...STANDARD].sort());
assert.ok(data.declined_types.includes("table"), "the name that explains the miss is absent");
});
});
test("no suggestion means no dispatch events at all", async () => {
await withTemp(async (dir) => {
const { ctx, rec } = makeCtx(dir, null);
await runExtraction(ctx);
for (const type of ["specialist_dispatched", "specialist_declined", "specialist_unresolved"]) {
assert.deepEqual(ev(rec, type), [], `${type} logged without a suggestion`);
}
});
});