📦 EqualifyEverything / equalify-iris

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444import type { Capability } from "../config.ts";

export interface Message {
  role: "system" | "user" | "assistant";
  content: string;
  // The leading part of `content` that is byte-identical from call to call, declared so
  // an adapter can put a cache breakpoint after it (providers/promptCache.ts). Only
  // meaningful on a user message, and only worth setting where a caller genuinely sends
  // the same head repeatedly — the verify task re-states the whole contract of the agent
  // it is judging on every page of a document, which is the case this exists for.
  //
  // It is a PREFIX OF `content`, not a replacement for part of it: `content` stays the
  // complete message, so anything reading a Message sees exactly the text it saw before
  // this field existed, and an adapter that ignores the field sends exactly the same
  // request. An adapter that honours it splits `content` at the prefix and sends two
  // text blocks whose concatenation is the same string. A value that is not actually a
  // prefix of `content` is ignored rather than trusted, since acting on it would send a
  // prompt the caller did not write.
  cachedPrefix?: string;
}

export interface Image {
  // Raw bytes of the source image plus its media type (e.g. image/png).
  data: Buffer;
  media_type: string;
}

// Which job in the pipeline bought this call.
//
// The run log already says which AGENT answered, and that is a different question. An agent
// file is a contract, and one contract serves several jobs: `feedback` judges a freshly
// extracted page, re-judges a corrected one, routes user feedback and classifies a lesson,
// while `copy_editor` runs the review round AND merges tables split across a page break. So
// `by_agent` in diagnostics cannot price a step. Extraction's per-page fidelity check books
// to `feedback`, which made the extraction step read as 41% of a document when its jobs
// together are 57.2% (#280); the same shape hid a whole new step's cost inside the review
// loop's name when table joins shipped (#243, sprint #246 iteration 13, which nearly
// published the review loop as having got more expensive).
//
// Split finely rather than coarsely, and on purpose: buckets ADD. A reader who wants
// "extraction" sums the seven extraction steps below, and can. A reader given one
// `extraction` bucket cannot get the split back out, which is the defect this exists to fix —
// so where two call sites do different work for different reasons, they get different names
// even when the same agent answers both.
//
// A closed union, and required on every call (providers/index.ts): an optional step would
// give a new call site a silent `?` bucket, and an unattributed bucket is exactly the thing
// that hides the answer. Adding a call site means naming its job here.
export type PipelineStep =
  // Extraction (src/pipeline/extraction.ts, src/pipeline/feedback.ts)
  | "extract" // first render of a page image
  | "verify" // the per-page fidelity check on that render
  | "correct" // re-render of a page its check rejected
  // Two re-checks of a corrected page, kept apart because they buy different things and
  // diagnostics already reports them apart (`verification.rechecks.binding` / `.sampled`).
  // The binding one runs on a page that PASSED and was re-rendered only to recover a link,
  // and its verdict decides whether that rewrite ships; the sampled one is bought once per
  // batch by default (correction.ts `recheckSampler`) and decides nothing. Folding them would put a
  // measurement that is deliberately capped at ~1% of a document in the same bucket as one
  // that runs whenever a link needs recovering — which is the whole failure this type exists
  // to stop.
  | "recheck_binding"
  | "recheck_sampled"
  | "specialist" // a library specialist agent's pass over one page
  | "specialist_merge" // splicing that specialist's fragment into the page body
  // Review (src/pipeline/review.ts, src/pipeline/tables.ts)
  | "read" // the Reader's pass over the assembled document
  | "edit" // the Copy Editor's whole-document round
  | "edit_section" // its per-section retry after a round did not fit
  | "table_join" // merging a table split across a page break
  // User feedback (src/pipeline/feedback.ts)
  | "feedback_scope" // deciding which pages a user's feedback is about
  | "feedback_learn" // classifying a correction into a reusable lesson
  // Maintenance paths, which a delivered document does not pay for
  | "agent_update" // drafting a proposed change to an agent file
  | "agent_regression" // re-running an agent on a fixture to gate that change
  | "agent_calibrate" // the calibration harness judging a seeded defect
  | "contribute"; // drafting a brand-new specialist agent

// What a call actually consumed. Named after Anthropic's fields because that is the
// vocabulary the models themselves report in; other adapters normalize onto it.
//
// Two things to know before adding these up. First, `input_tokens` EXCLUDES cached
// tokens — the whole prompt is input + cache_read + cache_creation, so summing only
// `input_tokens` on a cache-hitting deployment understates the prompt. Second, the
// four numbers bill at four different rates, so they are deliberately kept apart
// rather than folded into one total.
//
// Every field is optional because reporting is not guaranteed: an upstream may omit
// usage entirely, and a call that fails partway through knows its prompt size but
// never learns its output size. Absent means "not reported", not zero — see
// `tokens.calls_reported` in diagnostics, which exists so a partial sum cannot be
// mistaken for a complete one.
export interface Usage {
  input_tokens?: number;
  output_tokens?: number;
  cache_read_input_tokens?: number;
  cache_creation_input_tokens?: number;
}

// A fact an adapter learned while serving one call that only the CALLER can record. Same
// shape of problem as `onUsage`, and unreportable for the same reason a return value cannot
// carry it: it is learned mid-call, it is worth having whether the call then succeeds or
// throws, and the layer that knows the session and writes the run log is the router
// (providers/index.ts) — an adapter has no logger and should not grow one.
//
// A discriminated union with one member today. A union rather than a free-form string so the
// router's handling is a `switch` a new kind has to be added to, instead of an unknown fact
// silently becoming a log field nobody declared.
export type ProviderNote = {
  // This call ran below the output ceiling its deployment configured, because Bedrock refused
  // that ceiling for this model and stated a lower one (providers/bedrock.ts, issues #249
  // and #254).
  //
  // This is the run's only record that `providers.<provider>.max_tokens` is wrong for the
  // model it is pointed at. The retry is what makes the pages arrive, which also means the
  // config error has no consequence anyone downstream can see: without this the run log shows
  // one call where two requests were made, a duration covering both, and nothing about the
  // ceiling — a deployment can run for a month at a number nobody chose, with a dense page
  // truncating occasionally, and the log will say only that a page truncated.
  kind: "output_ceiling_clamped";
  model: string;
  // The ceiling that was asked for and the one that was granted. Both, because either alone is
  // unactionable: the pair is what says which way to move `max_tokens` and how far.
  asked: number;
  stated: number;
  // Whether THIS call is the one that learned it, by having a request refused and re-sent. The
  // condition and its cost are different facts and both are worth having: the condition holds
  // for every call to a clamped model, while the cost — a rejected round-trip inside one
  // `complete`, and a `duration_ms` covering two requests — is paid by the first call in a
  // process and by none after it.
  refused: boolean;
};

export interface CompletionRequest {
  capability: Capability;
  messages: Message[];
  images?: Image[];
  schema?: Record<string, unknown>; // JSON Schema for structured_output
  model: string; // resolved by the router from deployment config
  // Called by the adapter whenever the running token totals change, with the full
  // snapshot known so far (the latest call wins). A return value cannot serve here:
  // the prompt's size is known at the start of a stream and the output's only at the
  // end, so a call that stalls or truncates — exactly the expensive kind — would
  // report nothing at all if usage only rode the successful return path.
  onUsage?: (usage: Usage) => void;
  // A ceiling for THIS call, below the deployment's. Only ever lowers it: an adapter takes the
  // smallest of the deployment's `max_tokens`, whatever ceiling the model has stated for itself,
  // and this — so a caller can bound one call's output without knowing which of the other two is
  // currently in force, and cannot raise a limit by asking.
  //
  // It exists because the deployment's ceiling is the wrong instrument for a call whose answer has
  // a known size. A page correction re-emits a page it has already seen, so its output has a tight
  // prior — the first pass's own output — and one that ran away to the full 32,000-token ceiling
  // cost $0.51 and shipped the uncorrected page anyway (issue #285): a bill 5.13x the first pass
  // for text that was discarded. Raising `max_tokens`, which is what the truncation error advises,
  // would only buy a larger discarded reply. See `correctionCeiling` in pipeline/extraction.ts for
  // the one caller and the corpus the multiple was chosen on.
  maxOutputTokens?: number;
  // Called for a fact worth recording that is neither usage nor an error — see ProviderNote.
  // Called once per occurrence, and deliberately NOT deduplicated the way the adapter's own
  // stderr warning is: Bedrock says the paragraph about a wrong ceiling once per process
  // (`warnedCeilings`), because five paragraphs about one config problem read as five problems,
  // but every call that runs at the clamped ceiling still owes the run log its own line.
  // Copying that dedup here would put the fact on one `model_call` per process and leave every
  // document after the first reading clean.
  onNote?: (note: ProviderNote) => void;
}

export interface CompletionResult {
  text: string;
  model: string;
  provider: string;
  // Absent when the upstream reported nothing.
  usage?: Usage;
}

// The provider interface (see docs/models.md). An agent declares a capability; the
// deployment decides which provider serves it.
export interface ModelProvider {
  name: string;
  capabilities: Capability[];
  // Which wire format the calls go out on, for a provider that has more than one.
  // Only Bedrock does (`providers.bedrock.api`, an Anthropic-native body or Bedrock's
  // own Converse API), and it is here rather than private to that adapter because the
  // router puts it on the `model_call` log event: the point of that switch is comparing
  // two dialects against each other, and a comparison whose run log does not say which
  // side produced a number is not one. Undefined for a provider with a single API.
  dialect?: string;
  complete(request: CompletionRequest): Promise<CompletionResult>;
}

// What the standing "raise it" advice is worth when the ceiling was reached with no reply at all,
// which is a different failure wearing the same stop reason. A response cut mid-document is an
// answer that did not fit, and a larger ceiling is exactly the remedy. Zero characters is not a
// long answer: the ceiling was spent before the answer began — on reasoning a model streams as its
// own channel, which the adapters do not read as text (providers/bedrock.ts `readConverse` builds
// the reply out of `delta.text` alone) — so raising the number is a bet that the thinking ends
// inside the new ceiling, and a lost bet is billed for the whole of the new one. Measured on a
// 100-page benchmark round, where one page's extraction spent 32,000 output tokens and returned 0
// characters (issue #293); the advice as it stood sent an operator to buy a larger burn.
//
// Appended to the standing sentence rather than replacing it, for the same reason `note` below is:
// `isTruncatedResponseError` matches the fixed part of that sentence and two callers act on the
// match, so this one must keep reading as a truncation.
//
// The instruction comes FIRST and the explanation second, because this message is quoted somewhere
// that cuts it: a lost page ships a `@page-failed` comment carrying the first 300 characters of it
// (pipeline/extraction.ts `failedPage`), which a maintainer greps before they reach the run log. In
// the other order that comment kept "Raise providers.<provider>.max_tokens." in full and dropped the
// clause taking it back — the one thing this sentence exists to deliver.
const EMPTY_REPLY =
  "No text was returned at all, so raising that ceiling is not the remedy: look at the model's " +
  "reasoning behaviour and at the size of what it was asked to produce. The ceiling was spent " +
  "before the reply began, and a larger one buys more of whatever consumed it.";

// A model stopped because it hit the output-token ceiling, not because it was
// finished. This is a 200 response carrying partial content, so nothing below the
// provider layer can tell it from a complete answer — a page of HTML truncated
// mid-tag still parses well enough to be assembled into the deliverable, where it
// reads as content the source never had. Both adapters raise this rather than
// return the fragment, so the failure is visible in diagnostics and to the caller.
export class TruncatedResponseError extends Error {
  readonly provider: string;
  readonly model: string;
  readonly maxTokens: number;
  readonly chars: number;
  // What the model DID emit before the ceiling cut it. Carried rather than dropped, because the
  // caller cannot ask again — the next round would put the same question to the same model — so
  // this is the only evidence that will ever exist about why the answer did not fit, and a round
  // that hits it has already been paid for in full (issue #277).
  //
  // One caller now reads it rather than only logging an excerpt of it. The Copy Editor answers with
  // a list of independent block edits, so a reply cut inside the list still carries every edit the
  // model finished writing, and `salvageRound` (pipeline/review.ts) applies those and asks again
  // only for the part the reply never reached (issue #295). That is a property of one contract and
  // not of this field: raising rather than returning the fragment is still right, because whether a
  // fragment means anything is the caller's question and the answer for a page of HTML is no.
  //
  // The length is derived from it rather than passed alongside it, so `chars` — which the message
  // quotes and which `sectionRound` sizes the next request from — cannot disagree with the text it
  // describes.
  readonly text: string;

  // `note` is for the cases where "raise it" is the wrong instruction, of which there are now
  // two: a ceiling the MODEL enforces, below the one the deployment asked for, which cannot be
  // raised at all (issue #249), and a ceiling the CALLER asked for because it knows how large the
  // answer should be, where the number to look at is the caller's (`maxOutputTokens` above, issue
  // #285). Both are chosen in `truncationRemedy`, providers/bedrock.ts.
  //
  // Appended rather than replacing the sentence, because `isTruncatedResponseError` matches the
  // fixed part of it and the review loop acts on that. So a noted message says "raise it" and
  // then takes it back, deliberately: the alternative is a wrong instruction that reads as the
  // only one, and the note is what an operator needs either way.
  constructor(provider: string, model: string, maxTokens: number, text: string, note?: string) {
    super(
      `${provider}: response hit the ${maxTokens}-token output ceiling and was truncated ` +
        `(${text.length} chars returned). Raise providers.${provider}.max_tokens.` +
        (text === "" ? ` ${EMPTY_REPLY}` : "") +
        (note ? ` ${note}` : ""),
    );
    this.name = "TruncatedResponseError";
    this.provider = provider;
    this.model = model;
    this.maxTokens = maxTokens;
    this.chars = text.length;
    this.text = text;
  }
}

// How a log line quotes the `text` above: a few hundred characters at each end, whitespace folded
// so the line stays one line. Shared by both paths that report a truncation of their own — the Copy
// Editor's round (pipeline/review.ts, issue #277) and a page's correction (pipeline/extraction.ts,
// issue #293) — because two copies of the budget rule drift, and a reader comparing an editor
// truncation with a page one needs the same width on both to compare them at all.
//
// It is the user's own document coming back, so every caller keeps it in the run log on the
// deployment and **never** puts it on `GET /v1/quality` — the same confinement, for the same reason,
// as `prose_joined`'s `word_split_examples`. The width is what that question needs and no more.
// Not exported: `replyExcerpt` below is the only thing that may spend this budget, and a caller
// holding the number would be a caller that could quote a reply on its own terms.
const REPLY_EXCERPT = 240;

// One budget, spent either as two ends or as the whole fragment: at or under the pair's width the
// two excerpts would overlap or abut, so the fragment is quoted entire under `reply_head` with no
// `reply_tail` at all — rather than reported as a head whose middle and end are missing while
// `chars` says there was more. Either way the log carries at most `2 x REPLY_EXCERPT` characters.
//
// Both fields are absent on a reply that returned nothing. `reply_head: ""` would read as a model
// that answered with an empty string, which is a thing a model can do and is not this; `chars: 0`
// and `EMPTY_REPLY`'s sentence are what say a ceiling was spent with the answer never started.
export function replyExcerpt(text: string): { reply_head?: string; reply_tail?: string } {
  if (text === "") return {};
  const fold = (s: string) => s.replace(/\s+/g, " ").trim();
  const split = text.length > 2 * REPLY_EXCERPT;
  return {
    reply_head: fold(text.slice(0, split ? REPLY_EXCERPT : 2 * REPLY_EXCERPT)),
    ...(split ? { reply_tail: fold(text.slice(-REPLY_EXCERPT)) } : {}),
  };
}

// The same fact as a predicate, and the one the review loop acts on: a round whose
// response hit the ceiling is a round that produced nothing, which costs the round rather
// than the document (pipeline/review.ts, issue #143).
//
// `instanceof` is the check, because unlike `isRequestTooLargeError` below this error is
// Iris's own — raised in two places, both in this repo, with a message written here. The
// message fallback is for an error that reached the caller having lost its prototype: a
// boundary that re-wraps what it caught, or a second copy of this module in one process.
// It matches the fixed part of that one sentence rather than a phrasing some upstream
// chose, so it cannot be tripped by a provider rewording anything.
export function isTruncatedResponseError(e: unknown): boolean {
  if (e instanceof TruncatedResponseError) return true;
  const message = e instanceof Error ? e.message : String(e);
  return message.includes("output ceiling and was truncated");
}

// A call the upstream REFUSED for size, before processing any of it: too much input
// for the model's context window, or too many bytes for the endpoint to accept at all.
// Both are the same fact to a caller — this request is too big — and the same remedy.
//
// Worth telling apart from every other failure because it is the one a caller can
// answer: the payload is Iris's own doing (page images, a whole document body), so
// dropping part of it and asking again is a real recovery, and the refusal is usually
// cheap — no prompt was read, so nothing was billed and it comes back in under a second.
//
// "Usually" because one member of this set is not a refusal at all: Bedrock's Converse
// API can report `model_context_window_exceeded` as a STOP REASON, after a full
// generation that was billed in both directions (providers/bedrock.ts). It is matched
// here on purpose — the remedy is identical, and it is the only one Iris has — but it
// means the `editor_images_refused` event this routes to can name a call that cost a
// round of output rather than nothing. The event carries the error message, which is
// what tells the two apart.
//
// Matched on the message, because neither adapter is given anything better. Bedrock
// raises a ValidationException whose message is "Input is too long for requested
// model." with no code distinguishing it from any other validation failure, and
// OpenRouter forwards a 400 body from whichever upstream served the request, worded
// differently again ("maximum context length is N tokens"). Both reach a caller as a
// plain Error carrying that text.
//
// The byte-size phrasings are not redundant with the token ones, and the count-based
// bound upstream does not cover them: MAX_EDITOR_IMAGES is derived from what a
// rasterized page costs in TOKENS, while the request ceiling these APIs enforce is in
// bytes. Screenshots rather than rendered PDF pages are the shape that gets there —
// at the per-image ceiling GET /v1/limits publishes, a dozen of them is tens of
// megabytes once base64-encoded, which is refused for size without ever being weighed
// in tokens.
//
// Deliberately a small set of phrasings rather than anything cleverer. The caller
// (pipeline/review.ts) reacts by retrying with a smaller payload, so a false positive
// costs one extra call that fails the same way, and a false negative is exactly the
// behaviour that existed before this function.
export function isRequestTooLargeError(e: unknown): boolean {
  const message = (e instanceof Error ? e.message : String(e)).toLowerCase();
  return (
    // Over the context window.
    message.includes("input is too long") ||
    message.includes("prompt is too long") ||
    message.includes("context length") ||
    message.includes("context window") ||
    // The Anthropic body's own wording for the same thing, which is what an `invoke`
    // deployment gets: "input length and `max_tokens` exceed context limit: 199000 + 32000 >
    // 200000, decrease input length or `max_tokens` and try again". Matched here for two
    // reasons. It IS a prompt-size refusal, so the image-drop recovery in pipeline/review.ts
    // is the right answer to it and had no way to reach it before. And because this predicate
    // is checked first, matching it also keeps `refusedForOutputCeiling`
    // (providers/bedrock.ts) from reading it as a refusal over the model's output ceiling: it
    // names `max_tokens` and a limit, which is otherwise exactly that shape, and the
    // resulting diagnosis would be "the model is not being asked to do work it cannot do"
    // about a prompt that does not fit.
    message.includes("context limit") ||
    message.includes("too many tokens") ||
    // Over the transport's or endpoint's size limit.
    message.includes("payload size") ||
    message.includes("too large") ||
    // A 413 with no reason phrase ("failed with status code 413"). The status word is
    // required rather than matching a bare 413, because a TruncatedResponseError's
    // message carries a character count and a model's max_tokens — either of which can
    // be that number, and retrying a truncation with fewer images fixes nothing.
    (/\b413\b/.test(message) && /status|http|code/.test(message))
  );
}

// A streamed call was abandoned. Three ways, because they are three different
// diagnoses and an operator reading one of these needs to know which: nothing ever
// arrived ("first_output"), output started and then stopped ("idle"), or output kept
// coming without the message ever finishing ("total").
//
// This type exists because the alternative is unreadable. Aborting a call makes the
// underlying client throw something opaque — the AWS SDK a bare
// `Error("Request aborted")`, fetch() a DOMException — which the orchestrator stores
// verbatim as the session's error and the UI shows to the user, naming neither the
// cause, the phase, nor anything to do about it. A slow document rewrite and a
// genuinely dead connection produced the identical string.
export type StallKind = "first_output" | "idle" | "total";

export class StalledStreamError extends Error {
  readonly provider: string;
  readonly model: string;
  readonly kind: StallKind;
  readonly limitMs: number;
  readonly chars: number;

  constructor(args: {
    provider: string;
    model: string;
    kind: StallKind;
    limitMs: number;
    chars: number;
  }) {
    const seconds = Math.round(args.limitMs / 1000);
    const streamed = args.chars
      ? `${args.chars} chars had streamed`
      : "nothing had streamed";
    let message: string;
    if (args.kind === "first_output") {
      message =
        `${args.provider}: no output arrived within ${seconds}s on ${args.model}, so the call ` +
        `was abandoned before it produced anything. The request was accepted and then went ` +
        `quiet — a queue that never cleared, or a model that never started.`;
    } else if (args.kind === "idle") {
      message =
        `${args.provider}: the model stopped sending output for ${seconds}s ` +
        `(${streamed}) on ${args.model}, so the call was abandoned. The connection ` +
        `stalled rather than the work being too slow — a healthy stream is never ` +
        `silent this long.`;
    } else {
      message =
        `${args.provider}: the call was still producing output after ${seconds}s ` +
        `(${streamed}) on ${args.model} without ever finishing its message, and hit the ` +
        `absolute ceiling. Nothing stalled — the work itself did not converge, which usually ` +
        `means the document is too large to correct in one call.`;
    }
    super(message);
    this.name = "StalledStreamError";
    this.provider = args.provider;
    this.model = args.model;
    this.kind = args.kind;
    this.limitMs = args.limitMs;
    this.chars = args.chars;
  }
}