1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303import { MAX_LINKS_PER_PAGE, type PdfLink } from "../util/pdf.ts";
import { decodeEntities } from "../util/html.ts";
// Carrying a PDF's links into the delivered HTML.
//
// A link in a PDF is an annotation — a rectangle plus a URI — laid over the page,
// not something drawn on it. Rasterizing the page for the vision model therefore
// throws every link target away: "read the full report" arrives as three words that
// happen to be blue, and the model has nothing to make an <a href> out of. The
// result was a document where the text survived and every link in it was gone, which
// for a screen-reader user is worse than it sounds: the destination is not merely
// hidden, there is no indication a destination existed.
//
// So the targets are extracted from the file directly (util/pdf.ts) and handed to the
// page agent alongside the image, as ground truth it cannot see. Attaching them is
// left to the model rather than done by string substitution on its output, because
// deciding WHICH text an annotation's rectangle covers, in the structure the model
// chose to emit, is the same judgement the extraction itself is: the anchor text is
// approximate (poppler's text layer, split at line breaks), and a search-and-wrap
// over generated HTML would as happily wrap a heading, a table cell, or half an
// attribute value.
//
// What is checked deterministically is whether the links arrived — `missingLinks`
// below — because that has an exact answer, and a link the model dropped is fed back
// to it as a fidelity problem alongside the source image (see extraction.ts).
// A link's URL, reduced to a form two spellings of the same URL agree on. Entities
// are decoded because the model writes `&` into an href where poppler reports
// `&`, and the scheme is lowercased because it is case-insensitive. Nothing else is
// touched: a path, query, or fragment IS case-sensitive, and %-encoding is not
// normalized either, because `%2F` and `/` are genuinely different path segments.
//
// The trailing slash is dropped so `https://example.org/` and `https://example.org`
// are one link. That is a real equivalence for an empty path, and the alternative is
// reporting a link as missing over a character a browser resolves identically.
function normalizeHref(href: string): string {
return decodeEntities(href)
.trim()
.replace(/^[A-Za-z][A-Za-z0-9+.-]*:/, (scheme) => scheme.toLowerCase())
.replace(/\/+$/, "");
}
// An attribute's value in source HTML: double-quoted, single-quoted, or bare. One
// definition because `unresolvedRefs` scans two attribute names with it and a difference
// between the two would be a difference in what counts as a reference versus a target.
const VALUE = `\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'>]+))`;
// What may sit immediately before those attribute names. Deliberately NOT anchors.ts's
// `ATTR_SEP`, and the difference is the whole reason this is written out: that constant is
// measured for what can precede an attribute in a TAG, and this scan does not read tags —
// it reads every byte of the delivered document, values and text included.
//
// So this is ATTR_SEP without `/`. That character is in ATTR_SEP because `<br/id="x">`
// parses an `id`; it is out here because a `/` is far commoner inside a URL, where
// `<a href="https://x.example/id=intro">` registers a phantom `intro` and silences a
// genuinely dead `#intro` — the direction this function's comment below calls the worse one.
// The mirror cost is that an id written `<br/id="x">` goes unseen here, so a live `#x` is
// reported as dangling; that over-reports, which is the direction to fail in, and it needs a
// page agent to write an attribute with no space in front of it.
//
// `<` is not in either class, and for the same reason `/` is out of this one: the first
// attribute of a tag is preceded by whitespace anyway (`<p id="x">`), so `<` catches nothing
// a separator does not — while `<id="foo">` is a tag whose NAME the tokenizer reads as
// `id="foo"`, which would be one more phantom id. An earlier version of this class had it,
// with a rationale that did not survive being written down.
const SEP = `[\\s"']`;
// Does this href name a scheme? Both comparisons below are about URLs that came from
// (or claim to have come from) the source file's annotations, and only an absolute
// href can be one of those.
function isAbsolute(href: string): boolean {
return /^[a-z][a-z0-9+.-]*:/i.test(href);
}
// Every href in a fragment of HTML, normalized. A scan rather than a parse because
// this runs on model output mid-pipeline, where the fragment may not be well-formed
// yet — the same reason anchors.ts keeps a scan alongside its parser. Unquoted
// attribute values are included since a model writes them from time to time.
function hrefsIn(html: string): Set<string> {
const found = new Set<string>();
for (const m of html.matchAll(/\bhref\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+))/gi)) {
found.add(normalizeHref(m[1] ?? m[2] ?? m[3] ?? ""));
}
return found;
}
// The page's links, as the section of the page-agent prompt that carries them, plus
// what was dropped to bound it. Empty section when the page has no links, so an
// image upload (or a PDF with none) sends exactly the prompt it sent before.
export function pageLinkContext(links: PdfLink[] = []): {
section: string;
shown: PdfLink[];
dropped: number;
} {
if (links.length === 0) return { section: "", shown: [], dropped: 0 };
const shown = links.slice(0, MAX_LINKS_PER_PAGE);
const dropped = links.length - shown.length;
const list = shown
.map((l, i) => `${i + 1}. ${l.text ? `"${l.text}"` : "(no text found under the link)"} -> ${l.href}`)
.join("\n");
const section =
`\n\n## Links on this page (from the source file's own link annotations)\n` +
`The image cannot show where a link points — a PDF link is an annotation over the page, ` +
`not part of the picture — so the source file's link targets are listed here. The URLs are ` +
`exact. The anchor text is APPROXIMATE: it comes from a separate text extraction, so it may ` +
`be split across lines, clipped short, or differ in spacing from what you see.\n\n` +
`${list}\n\n` +
(dropped > 0 ? `(…and ${dropped} more link${dropped === 1 ? "" : "s"} on this page.)\n\n` : "") +
`Wrap the matching visible text in <a href="…"> using each URL EXACTLY as written above — ` +
`never shorten, re-encode, or guess one. Link the text the page itself presents as the link ` +
`(a phrase, not the whole paragraph or heading around it), and keep the link INSIDE the ` +
`structure you would have chosen anyway: a link never changes what an element is, never ` +
`duplicates content, and never wraps a whole table, list, or section. Do not invent links ` +
`for anything not listed; the one exception is a URL printed visibly in the text, which may ` +
`link to itself. If you genuinely cannot tell which text a link belongs to, attach it to the ` +
`text that plainly matches its purpose and say so in the "log" field rather than dropping it.\n`;
return { section, shown, dropped };
}
// The links whose URL is nowhere in the produced HTML. Deduplicated by URL: one <a>
// is enough to say the target survived, and a page that repeats a URL under two
// phrases would otherwise report a miss for a link that is present.
export function missingLinks(links: PdfLink[] = [], html: string): PdfLink[] {
if (links.length === 0) return [];
const present = hrefsIn(html);
const seen = new Set<string>();
const missing: PdfLink[] = [];
for (const l of links.slice(0, MAX_LINKS_PER_PAGE)) {
const key = normalizeHref(l.href);
if (present.has(key) || seen.has(key)) continue;
seen.add(key);
missing.push(l);
}
return missing;
}
// A dropped link, phrased for the self-correction pass — which sees this text and the
// source image, so it is told where to look, not just what is wrong.
export function missingLinkProblem(link: PdfLink): string {
const where = link.text ? `the text "${link.text}"` : "text on this page";
return (
`The source file has a link on ${where} pointing to ${link.href}, and your output does not ` +
`link to it. Wrap that text in <a href="${link.href}"> — exactly that URL — without changing ` +
`anything else about the page.`
);
}
// Links a rewrite of the document lost: absolute URLs the earlier body linked to and
// the later one does not.
//
// The review loop is where a link is most quietly at risk. The Copy Editor rewrites
// the WHOLE body from the HTML plus the source images — and an href is the one piece
// of content that is not in either the text it is rewriting or the image it is
// checking against, so a link it drops cannot be noticed by the Reader (which never
// sees the source) or recovered from the page (which never showed the URL). It is
// told to preserve them; this reports it when it did not.
//
// Only absolute URLs, deliberately. In-document references churn legitimately —
// anchors.ts renames colliding ids as pages are joined, and the editor renumbers
// footnotes when it fixes their structure — so including them would report ordinary
// work as loss and bury the case that matters.
export function droppedHrefs(before: string, after: string): string[] {
const kept = hrefsIn(after);
return [...hrefsIn(before)].filter((h) => isAbsolute(h) && !kept.has(h)).sort();
}
// Every in-document reference in the delivered document, and whether it lands (#234).
//
// This is the question no component asked. `missingLinks` asks whether the source file's
// URLs arrived, `droppedHrefs` asks whether a rewrite lost one, `unexpectedHrefs` asks
// whether one was invented — all three about absolute URLs, and all three deliberately
// silent about `href="#…"`, which anchors.ts owns. anchors.ts, in turn, resolves a
// reference only when at least one page CLAIMS the id: `resolve` returns any other token
// unchanged, so a reference nobody can have meant is neither repointed nor reported
// (`ambiguous: []` on a document whose whole table of contents was dead). Each component
// is right about its own question; the end state — does every reference in the document
// that ships land on an id in that document — belonged to nobody.
//
// Two shapes, counted apart because they have different causes and different remedies:
//
// `empty` `href="#"`. A link the agent wrote knowing it had no target, which is
// what a model produces when it is asked for the shape of a table of
// contents and the destinations are not on this page. Not recoverable by
// renaming anything: there is no target to find.
// `dangling` a fragment naming an id the document does not contain — a target that
// moved, was never emitted (a footnote marker whose note the page did not
// transcribe), or is in a part of the document this run did not have.
//
// Neither is an axe violation, so a document full of them lints clean and reads as
// finished. `#` and `#top` are the two fragments a browser resolves without an element
// (HTML's "top of the document"), so `#top` is not dangling and `#` is reported as its own
// number rather than as a missing id.
//
// Comments are stripped before anything is counted. The delivered document carries the
// @unresolved list and the other markers, which are model-written prose about the
// document and can quote markup — an `<a>` inside a comment is not a link, and counting
// one inflates both the numerator and the denominator.
//
// Every count here is per REFERENCE, so `refs` is the denominator of `empty + dangling`
// and "12 of 226 go nowhere" reads as a maintainer expects. `ids` is the odd one out and
// deliberately so: it is the distinct set, because it goes into a log line capped at 20
// and a table of contents pointing forty times at one missing section would otherwise
// spend the whole cap on a single fact.
//
// Both attribute scans require whitespace, `<`, or a closing quote in front of the name,
// which matters more than it looks. `\bid=` matches inside ANOTHER attribute's value — a
// source PDF whose annotation carries `?id=intro` would register `intro` as an id the
// document contains, silencing a genuinely dead `#intro`, and `data-id="s1"` masks `#s1`
// the same way (`-` is a non-word character, so `\b` is satisfied). That failure mode is
// the worst one available here: it hides exactly the defect this function exists to
// surface.
//
// What it gives up, so the next person to touch the regex has the whole argument: a quote
// is in the class because a browser reads `class="a"id="x"` as two attributes and a page
// agent's output is delivered unserialized when no id collides (anchors.ts), so without it
// a real id would go unseen. The cost is that a quote can also be an OPENING one, so a
// value whose text begins or contains `id=` registers a phantom id: `alt="id=intro"` on a
// scanned form field, or `title='he said "id=y"'`. Those fail in the worse direction —
// they hide a dead reference rather than invent one — and the first is the likelier of the
// two, since a page can transcribe `id=` literally.
//
// The class is `SEP` above, which is close to but deliberately not anchors.ts's measured
// `ATTR_SEP` — see there for the two-character difference and why each class excludes what
// the other includes. Sharing that constant outright was tried and it made this scan worse:
// its `/` fires inside a URL path, which is a phantom id, which is the failure mode this
// paragraph is about.
//
// Left as a scan anyway. Every remaining case needs a delivered document that both
// contains such a value AND has a dead reference to the id that value names, nothing here
// is thresholded, so the cost is a maintainer's glance rather than a run. The exact fix,
// if this ever carries a threshold, is a parse: unlike the rest of this file, this function
// alone runs on the FINAL document, which is well-formed enough for jsdom — `[id]` and
// `a[href]` off a tree would settle every one of these, at the price of parsing a large
// document a second time. The lint step's tree is not that tree: axe is given the document
// WITHOUT the `@` markers and these bytes have them, so the parse has to be its own.
export function unresolvedRefs(html: string): {
refs: number;
empty: number;
dangling: number;
ids: string[];
} {
const text = html.replace(/<!--[\s\S]*?-->/g, " ");
const ids = new Set<string>();
for (const m of text.matchAll(new RegExp(`(?<=${SEP})id${VALUE}`, "gi"))) {
ids.add(normalizeFragment(m[1] ?? m[2] ?? m[3] ?? ""));
}
let refs = 0;
let empty = 0;
let dangling = 0;
const failed = new Set<string>();
for (const m of text.matchAll(new RegExp(`(?<=${SEP})href${VALUE}`, "gi"))) {
const href = decodeEntities(m[1] ?? m[2] ?? m[3] ?? "").trim();
if (!href.startsWith("#")) continue;
refs++;
const token = normalizeFragment(href.slice(1));
if (token === "") empty++;
// `top` is matched ASCII case-insensitively by a browser, unlike an id, which is
// why only this comparison is folded and `ids.has` is left exact.
else if (token.toLowerCase() !== "top" && !ids.has(token)) {
dangling++;
failed.add(token);
}
}
return { refs, empty, dangling, ids: [...failed].sort() };
}
// An id or a fragment, reduced to the form the other spelling of it agrees on. Entities
// because a model writes `&` into an href where the id it means says `&`; percent
// decoding because a fragment is percent-encoded in a URL and an id is not, so
// `href="#f%C3%BC"` and `id="fü"` are the same reference. A malformed escape is left as
// written rather than thrown on: reporting it as dangling on account of a `%` is worse
// than comparing the bytes.
function normalizeFragment(value: string): string {
const decoded = decodeEntities(value).trim();
try {
return decodeURIComponent(decoded);
} catch {
return decoded;
}
}
// Absolute URLs the output links to that no annotation on this page accounts for.
//
// Reported for visibility, NOT corrected, and only for a page that had annotations —
// where there is a ground truth to be outside of. Two innocent things land here: a
// URL printed visibly in the text and linked to itself (which the prompt above
// permits), and a link poppler could not attribute to any text, e.g. one over an
// image. The third is a model that invented a URL, which is the reason this is
// logged at all: it is the failure mode this feature introduces, and without the
// list there is no symptom — a fabricated href looks exactly like a real one in the
// delivered document.
//
// Only hrefs with a scheme are considered. In-document references (`#fn-1`) are the
// page agent's own anchors and anchors.ts owns them; a relative href (`page2.html`)
// is not a URL this page's annotations could have supplied either way, and counting
// one as unaccounted-for dilutes the single signal that is supposed to read as
// "possibly fabricated".
export function unexpectedHrefs(links: PdfLink[] = [], html: string): string[] {
if (links.length === 0) return [];
const expected = new Set(links.map((l) => normalizeHref(l.href)));
return [...hrefsIn(html)].filter((h) => isAbsolute(h) && !expected.has(h)).sort();
}