1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391// Mock GitHub + mock OpenRouter for the end-to-end curl test (test/e2e.sh).
// These stand in for the only two external dependencies so the whole API can be
// exercised offline. Not used in production.
import { createServer } from "node:http";
const GH_PORT = Number(process.env.MOCK_GH_PORT ?? 9301);
const OR_PORT = Number(process.env.MOCK_OR_PORT ?? 9302);
function readBody(req) {
return new Promise((resolve) => {
let b = "";
req.on("data", (c) => (b += c));
req.on("end", () => resolve(b));
});
}
function json(res, status, obj) {
res.writeHead(status, { "content-type": "application/json" });
res.end(JSON.stringify(obj));
}
// ---- Mock GitHub (covers both api.github.com and github.com OAuth paths) ----
const forks = new Set(); // repos that have been forked to the test user
let prNumber = 140;
// Body of the most recent POST /login/device/code, readable via
// GET /__last_device_scope so e2e.sh can assert that the service requested NO scope.
//
// `null` until the route is actually hit, NOT `{}`: with `{}` the reported
// `present:false` would be indistinguishable from "the device flow was never started",
// so the no-scope assertion could pass without the service having sent anything. The
// probe reports `recorded` separately for exactly that reason.
let lastDeviceBody = null;
const gh = createServer(async (req, res) => {
const url = new URL(req.url, `http://localhost:${GH_PORT}`);
const p = url.pathname;
const m = req.method;
// What the last device-flow start asked GitHub for. Recorded rather than
// asserted here so e2e.sh can check what the SERVICE sends — the request
// body is otherwise invisible from outside, and a reintroduced scope is a silent
// problem: the flow succeeds either way.
//
// `recorded` is what makes the assertion non-vacuous: "no scope was sent" and "no
// request was sent" are otherwise the same answer, so a break that stopped the flow
// reaching here would read as a pass.
if (m === "GET" && p === "/__last_device_scope")
return json(res, 200, {
recorded: lastDeviceBody !== null,
present: lastDeviceBody !== null && "scope" in lastDeviceBody,
scope: lastDeviceBody?.scope ?? null,
});
// OAuth / device flow
if (m === "POST" && p === "/login/device/code") {
try {
lastDeviceBody = JSON.parse((await readBody(req)) || "{}");
} catch {
lastDeviceBody = {};
}
return json(res, 200, {
device_code: "DEVICECODE123",
user_code: "WXYZ-1234",
verification_uri: "https://github.com/login/device",
expires_in: 900,
interval: 1,
});
}
if (m === "POST" && p === "/login/oauth/access_token")
// No `scope` and no `refresh_token`/`expires_in`: the shape a GitHub App with
// user-token expiry disabled actually returns.
return json(res, 200, { access_token: "gho_testtoken", token_type: "bearer" });
// Authenticated user (api base): identifies the caller AND getAuthenticated()
if (m === "GET" && p === "/user") return json(res, 200, { id: 4242, login: "iris-tester" });
// repos.get
let mm;
if (m === "GET" && (mm = p.match(/^\/repos\/([^/]+)\/([^/]+)$/))) {
const [, owner, repo] = mm;
if (owner === "iris-tester") {
if (forks.has(repo)) return json(res, 200, { fork: true, default_branch: "main", html_url: `https://github.com/iris-tester/${repo}` });
return json(res, 404, { message: "Not Found" });
}
return json(res, 200, { fork: false, default_branch: "main", html_url: `https://github.com/${owner}/${repo}` });
}
// repos.createFork
if (m === "POST" && (mm = p.match(/^\/repos\/([^/]+)\/([^/]+)\/forks$/))) {
forks.add(mm[2]);
return json(res, 202, { fork: true, default_branch: "main", html_url: `https://github.com/iris-tester/${mm[2]}` });
}
// git.getRef GET /repos/:o/:r/git/ref/heads/:branch
if (m === "GET" && p.match(/^\/repos\/[^/]+\/[^/]+\/git\/ref\//))
return json(res, 200, { ref: "refs/heads/main", object: { sha: "baseSHA0000000000000000000000000000000000" } });
// git.createRef
if (m === "POST" && p.match(/^\/repos\/[^/]+\/[^/]+\/git\/refs$/))
return json(res, 201, { ref: "refs/heads/new", object: { sha: "newSHA00000000000000000000000000000000000" } });
// repos.getContent -> 404 so createOrUpdate treats it as a new file
if (m === "GET" && p.match(/^\/repos\/[^/]+\/[^/]+\/contents\//)) return json(res, 404, { message: "Not Found" });
// repos.createOrUpdateFileContents
if (m === "PUT" && p.match(/^\/repos\/[^/]+\/[^/]+\/contents\//)) {
await readBody(req);
return json(res, 201, { content: { path: p }, commit: { sha: "commitSHA" } });
}
// pulls.create
if (m === "POST" && (mm = p.match(/^\/repos\/([^/]+)\/([^/]+)\/pulls$/))) {
await readBody(req);
prNumber += 1;
return json(res, 201, { number: prNumber, html_url: `https://github.com/${mm[1]}/${mm[2]}/pull/${prNumber}` });
}
json(res, 404, { message: `mock-github: unhandled ${m} ${p}` });
});
// ---- Mock OpenRouter (OpenAI-compatible chat completions) ----
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
// The adapter streams, so the mock has to as well. Deliberately not one event
// carrying the whole answer: the content is split across several deltas so the
// adapter's newline framing and accumulation are actually exercised, and the reply
// opens with the keepalive COMMENT that OpenRouter really sends — which the adapter
// must parse without treating it as output.
function sse(res, content, finishReason) {
res.writeHead(200, {
"content-type": "text/event-stream",
"cache-control": "no-cache",
connection: "keep-alive",
});
res.write(": OPENROUTER PROCESSING\n\n");
const points = Array.from(content); // split on code points, never mid-character
const per = Math.max(1, Math.ceil(points.length / 3));
for (let i = 0; i < points.length; i += per) {
const piece = points.slice(i, i + per).join("");
res.write(`data: ${JSON.stringify({ choices: [{ delta: { content: piece } }] })}\n\n`);
}
res.write(`data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: finishReason }] })}\n\n`);
res.write("data: [DONE]\n\n");
res.end();
}
// When set, every completion returns a TRUNCATED response (see below). Toggled at
// runtime via POST /__truncate so one mock process can serve both a normal run and
// a truncation run — the mock boots once for the whole e2e.
let truncateNext = false;
// When set to a page number, ONLY that page's extraction returns a truncated response,
// so a run can end with a document that is missing one page while the others are whole —
// the case per-page containment exists for (issue #135). Set via POST /__fail-page with
// `{"page":2}`, cleared with `{"page":null}`; while it is set a re-extraction of that
// page fails too, and clearing it lets a later feedback round recover the page.
let failPage = null;
// When set, the page agent's response carries a `suggested_agent` with this name,
// so the e2e can drive specialist dispatch. Set to a real library agent to make
// dispatch succeed, or to a name no file matches to make it MISS — the case the
// service used to handle silently. Toggled via POST /__suggest.
let suggestAgent = null;
// When set, page 1's output drops the #240 structural defects below and every page is written
// cleanly. The e2e uses it to prove the run log stays QUIET on a clean document — the
// `delivered_markup` line is gated on having something to say, and with every session's page 1
// dirty there would be nothing behind that gate. Toggled via POST /__clean-markup.
let cleanMarkup = false;
// When set, page 2's output ends with a `[page not fully transcribed]` marker, which is what
// agents/page.md tells the page agent to emit when a page holds more than it can return — the
// last thing in the fragment, with the "log" field saying what was left. It is the one marker
// no pass in the review loop may resolve, so a document carrying it cannot finish the loop
// clean at any budget, and `iris:unfinished-page` (#264) counts it as the measured floor under
// `unresolved_rate`. Toggled via POST /__unfinished-page, so it lands on one session's document
// and not on every document in the run. Behind a toggle rather than always on for the reason
// `__clean-markup` exists: most of this suite's assertions are about documents that ship
// clean.
let unfinishedPage = false;
// When set, the Feedback Agent's TASK: classify call fails with a 500 — the first
// model call of the post-delivery training step. Lets the e2e prove that training
// which dies takes nothing with it: the document has already been delivered and the
// session is already `ready_for_review` by the time this runs. Toggled via
// POST /__fail-training.
let failTraining = false;
const or = createServer(async (req, res) => {
if (req.method === "POST" && new URL(req.url, "http://x").pathname === "/__truncate") {
truncateNext = !truncateNext;
return json(res, 200, { truncate: truncateNext });
}
if (req.method === "POST" && new URL(req.url, "http://x").pathname === "/__fail-page") {
const raw = await readBody(req);
const p = JSON.parse(raw || "{}").page;
failPage = typeof p === "number" ? p : null;
return json(res, 200, { fail_page: failPage });
}
if (req.method === "POST" && new URL(req.url, "http://x").pathname === "/__unfinished-page") {
const raw = await readBody(req);
unfinishedPage = JSON.parse(raw || "{}").on === true;
return json(res, 200, { unfinished_page: unfinishedPage });
}
if (req.method === "POST" && new URL(req.url, "http://x").pathname === "/__suggest") {
const raw = await readBody(req);
suggestAgent = JSON.parse(raw || "{}").name ?? null;
return json(res, 200, { suggest: suggestAgent });
}
if (req.method === "POST" && new URL(req.url, "http://x").pathname === "/__clean-markup") {
const raw = await readBody(req);
cleanMarkup = JSON.parse(raw || "{}").clean === true;
return json(res, 200, { clean_markup: cleanMarkup });
}
if (req.method === "POST" && new URL(req.url, "http://x").pathname === "/__fail-training") {
const raw = await readBody(req);
failTraining = JSON.parse(raw || "{}").fail === true;
return json(res, 200, { fail_training: failTraining });
}
const body = await readBody(req);
let sys = "";
let user = "";
let imageParts = 0;
let wantsStream = false;
try {
wantsStream = JSON.parse(body).stream === true;
const msgs = JSON.parse(body).messages;
sys = msgs.find((x) => x.role === "system")?.content ?? "";
const u = msgs.find((x) => x.role === "user")?.content;
// Vision requests send content as an array of parts: one text part plus one
// image_url part per attached image.
user = typeof u === "string" ? u : (u ?? []).map((p) => p.text ?? "").join(" ");
if (Array.isArray(u)) imageParts = u.filter((p) => p.type === "image_url").length;
} catch {}
let content = "{}";
// Set when THIS call is the armed page's extraction (see failPage), so the truncation
// below applies to one page of the document rather than to every call in the run.
let truncateThisPage = false;
// Feedback Agent, TASK: scope — route feedback to extraction or the review loop.
// Keyed off the feedback text so the e2e can drive either path: a message naming
// a page and a misread is source-level, anything else is document-level.
if (failTraining && user.includes("TASK: classify")) {
// A provider error on the training step. Not a truncation and not a bad body: a
// plain 500, the way an overloaded upstream answers.
return json(res, 500, { error: { message: "e2e: training call refused" } });
}
if (user.includes("TASK: scope")) {
const m = user.match(/misread on page (\d+)/i);
content = m
? JSON.stringify({ target: "extraction", pages: [Number(m[1])], reason: "source misread" })
: JSON.stringify({ target: "document", pages: [], reason: "document-level wording" });
} else if (sys.includes("convert an ENTIRE document page")) {
// Echo the page number back so the assembled document proves page ORDER was
// preserved. Pages are extracted in parallel, so respond SLOWEST-FIRST:
// page 1 is delayed the longest, meaning completion order is the reverse of
// document order. If ordering were driven by completion, the output would
// come out backwards and the e2e ordering assertion would catch it.
const m = user.match(/page (\d+) of (\d+)/);
const page = m ? Number(m[1]) : 1;
const total = m ? Number(m[2]) : 1;
await sleep(Math.max(0, (total - page + 1) * 120));
truncateThisPage = page === failPage;
// A re-extraction prompt carries the page's previous output. Mark the result
// so the e2e can prove that ONLY the targeted page was re-extracted.
const revised = user.includes("## Your previous output for this page");
content = JSON.stringify({
html:
`<h1>Quarterly Report</h1>\n<p>Revenue grew this quarter.</p>\n` +
`<p>Page marker ${page}.${revised ? " Revised." : ""}</p>\n` +
// A reference to a section no page transcribes — the #234 defect, on every
// page, so the delivered document has three references to one dead id. Nothing
// repairs it: anchors.ts repoints a fragment only when some page claims the id,
// and there is no axe rule for a same-document link that lands nowhere. It is
// here to prove the orchestrator MEASURES it, and the repetition is what proves
// the two units apart — three references, one id.
`<p><a href="#appendix-a">See Appendix A</a></p>` +
// Two images per page, and both of them CORRECT, which is what makes them worth
// having here: the generic-alt rule (#290) claims to fire on nothing this pipeline
// writes, and a rule that only ever reports when it fires cannot be seen to have
// run. So the e2e reads `extraction_complete.alts_checked` as the denominator of
// that zero — one real description per page, plus a decorative `alt=""` that must
// NOT be in the denominator, since an empty alt is a decision rather than a
// missing one. The per-page correction path is covered by unit tests instead: a
// placeholder here would buy a page call on every run and put a correction in the
// log that every other assertion in this file would have to know about.
`\n<img src="chart.png" alt="Bar chart of revenue by region, rising each quarter">\n` +
`<img src="rule.png" alt="">` +
// The #240 defects, on page 1 only so the counts are exact. Both are invisible to
// the lint gate by construction, which is the whole point of measuring them on the
// delivered bytes: the parser closes the `<div>` at end of document before axe sees
// a tree, and a table with a header block and no rows is perfectly well formed —
// there is no rule for a table that announces nine columns and holds nothing.
//
// The div is unclosed rather than the table: an unclosed `<table>` foster-parents
// everything after it out of the table, which would reorder the delivered text and
// break the page-order assertions this same document exists to make.
(page === 1 && !cleanMarkup
? `\n<div>\n<table><caption>Table 1. Revenue by region</caption>\n` +
`<thead><tr><th scope="col">Region</th><th scope="col">Revenue</th></tr></thead></table>\n` +
// And one #255 defect, in the same conditional and on page 1 for the same reason: a
// reference to an id no page defines. Invisible to the gate in a different way from
// the two above — nothing is malformed and nothing was repaired, but axe reports a
// dead `aria-describedby` as `incomplete` (`aria-valid-attr-value` is `reviewOnFail`)
// and never as a violation, so the run reaches ready_for_review with a clean lint and
// a paragraph promising a description that does not exist.
`<p aria-describedby="revenue-note">Revenue is described in the note.</p>\n`
: ``) +
// Last in the fragment, which is where agents/page.md puts it: "make [page not fully
// transcribed] the last thing you emit". Page 2 only, so the delivered document holds
// exactly one and the e2e can count it.
(unfinishedPage && page === 2 ? `\n<p>[page not fully transcribed]</p>` : ``),
log: unfinishedPage && page === 2 ? "Stopped after the second table; the rest of the page is not in the fragment." : "",
// Only when the e2e has armed it, and only on page 1, so the run yields
// exactly one dispatch attempt to assert on.
...(suggestAgent && page === 1
? { suggested_agent: { name: suggestAgent, reason: "e2e-driven dispatch" } }
: {}),
});
} else if (user.includes("Extract ONLY the content your contract covers")) {
// A dispatched library specialist. Returns a marked fragment so the e2e can
// tell a dispatch that ran from one that was skipped.
content = JSON.stringify({ no_content: false, html: `<p>Specialist fragment.</p>` });
} else if (sys.includes("You merge a higher-fidelity HTML fragment")) {
// The merge step, which folds the specialist fragment back into the page.
content = JSON.stringify({
html:
`<h1>Quarterly Report</h1>\n<p>Revenue grew this quarter.</p>\n` +
`<p>Page marker 1.</p>\n<p>Specialist fragment.</p>`,
});
} else if (sys.includes("Reader Agent")) {
// Normally clean. When the run carries feedback asking for a copy-edit pass,
// report ONE issue attributed to page 2 — that drives the editor and lets the
// e2e prove only page 2's image was attached. Issues are reported on every
// round (the mock document never changes), so the loop runs to its cap.
content = user.includes("headings need a copy-edit pass")
? JSON.stringify({
issues: [
{
issue: "The revenue table on page 2 has no column headers.",
pages: [2],
severity: "high",
suggested_action: "add <th scope=\"col\"> to the table",
},
],
})
: JSON.stringify({ issues: [] });
} else if (sys.includes("Copy Editor Agent")) {
// Echo how many page images were attached so the e2e can assert the payload
// was scoped to the attributed page rather than the whole document.
const attached = imageParts;
const rewritten =
`<h1>Quarterly Report</h1>\n<p>Revenue grew this quarter.</p>\n` +
`<p>Page marker 1.</p>\n<p>Page marker 2.</p>\n<p>Page marker 3.</p>\n` +
// A placeholder where a description belongs, written by the ONE component that can put one
// into a delivered document after the page agents are finished with it (#290). Every page's
// own images are described properly above, so this is the only reason the run's
// `delivered_alt` line can exist — which is what makes that line a measurement of the
// delivered bytes rather than a second copy of `extraction_complete.alts_generic`. Deliberate
// like page 1's unclosed `<div>`: axe passes it (`image-alt` asks only whether the attribute
// is present), so nothing else in the run can see it.
`<img src="chart.png" alt="image">\n` +
`<p>Editor saw ${attached} image(s).</p>`;
// Answered in the shape the request actually asks for (issue #250). An ordinary round shows
// the body as numbered blocks and wants back only the ones that changed, so the reply is an
// `edits` array — and this mock is the only place the e2e drives that code at all, which is
// the point: the guarantees only the e2e checks (axe clean on the served document, no internal
// comments in the delivered HTML) were being checked against a reply shape the pipeline no
// longer asks for. A request with no markers in it is the per-section fallback or the table
// join, and neither has blocks to name, so those still answer with a body.
const blocks = [...user.matchAll(/<!--\s*@block\s+(\d+)\s*-->/g)].map((m) => Number(m[1]));
content = blocks.length
? JSON.stringify({
// The whole document rewritten, expressed as a patch: everything into the first block
// and the rest emptied. That keeps this scenario what it has always been — a round whose
// correction replaces the body, so an in-fragment marker is gone unless something
// downstream re-states it — while exercising `applyEditorPatch` on the way.
edits: blocks.map((n) => ({ block: n, html: n === blocks[0] ? rewritten : "" })),
})
: JSON.stringify({ html: rewritten });
}
// Truncation: a 200 carrying PARTIAL content plus finish_reason "length" —
// exactly what a model returns when it stops at the output ceiling. The payload
// is deliberately plausible-looking JSON cut mid-tag, which is what makes this
// dangerous: without the provider-level guard it would be assembled into the
// deliverable as if it were genuine content.
if (truncateNext || truncateThisPage) {
const cut = '{"html":"<table><tr><td>cut off mid';
return wantsStream
? sse(res, cut, "length")
: json(res, 200, { choices: [{ message: { content: cut }, finish_reason: "length" }] });
}
if (wantsStream) return sse(res, content, "stop");
json(res, 200, { choices: [{ message: { content } }] });
});
gh.listen(GH_PORT, () => console.log(`mock-github on ${GH_PORT}`));
or.listen(OR_PORT, () => console.log(`mock-openrouter on ${OR_PORT}`));