1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223import { test } from "node:test";
import assert from "node:assert/strict";
import { runExtraction } from "../src/pipeline/extraction.ts";
import { stripSoftHyphens } from "../src/util/html.ts";
import { ev, makeCtx, ORDINARY, withTemp as inTemp } from "./extraction-seams.ts";
// A word the printing broke across a column, carried into the markup as U+00AD (issue #334).
// `agents/page.md` forbids it in as many words, with a worked example, and three models from three
// different labs do it anyway — 63 occurrences on 9 pages of one 100-page arm, reaching 23 of 62
// delivered documents, on pages Iris's own fidelity check passed.
//
// What makes it worth code rather than more prose is that it is decidable with no image, no word
// list and no second model: there is no output where a soft hyphen is the right answer. It is also
// the quieter of the two ways a model can get this wrong. The visible version (`Govern-ment`) is at
// least on screen; this one renders as nothing, so the page reads as clean while find-in-page
// silently fails — and the words it lands on are table row labels and column headings, which are the
// words a reader searches for.
//
// EVERY SOFT HYPHEN IN THIS FILE IS WRITTEN `\u00ad`, never as the character. A test whose
// expectation is an invisible character is a test the next reader cannot check, and a stray copy of
// one pasted into a fixture would make the strip look like it had found something real.
const SHY = "\u00ad";
test("every spelling a model can write a soft hyphen in, and nothing else", () => {
// The raw character and the three entity forms, because a strip that reads only the codepoint
// leaves `­` in the markup — where it renders the same way and defeats find-in-page the same
// way. Leading zeros and case are both legal.
const cases = [`a${SHY}b`, "a­b", "a­b", "a­b", "a­b", "a­b", "a­b", "a­b"];
for (const one of cases) {
assert.deepEqual(stripSoftHyphens(one), { html: "ab", removed: 1 }, one);
}
});
test("what a soft-hyphen strip must not touch", () => {
// A hyphen the word itself owns, and the one the page prompt says to keep at a page's edge.
assert.deepEqual(stripSoftHyphens("non-tax"), { html: "non-tax", removed: 0 });
// The literal text `­`, which is what a page ABOUT html entities prints. The `&` is consumed
// by `amp`, so there is no `&` left immediately before `shy;` for the pattern to match — asserted
// rather than reasoned about, because getting it wrong deletes visible content from a real page.
assert.deepEqual(stripSoftHyphens("<code>&shy;</code>"), { html: "<code>&shy;</code>", removed: 0 });
// `­` with no semicolon is legal HTML5 and deliberately NOT matched: requiring the semicolon is
// what keeps the line above safe from a `&shy` one character shorter.
assert.deepEqual(stripSoftHyphens("a­b"), { html: "a­b", removed: 0 });
// Not a soft hyphen: the hyphen-minus, the non-breaking hyphen, and the zero-width space a model
// reaching for an invisible break might use instead. Out of scope on purpose — this issue is about
// one character, and each of those has its own argument.
assert.deepEqual(stripSoftHyphens("a-b\u2011c\u200bd"), { html: "a-b\u2011c\u200bd", removed: 0 });
// An href, which is the attribute where a false positive would cost something rather than just be
// wrong: `normalizeHref` compares a page's links against the ones poppler reported, so a character
// taken out of a URL reads as a link the page dropped and buys a correction pass for a page that had
// nothing wrong with it. A query string's escaped ampersand is the shape that comes closest, and it
// is safe for the same reason `&shy;` is.
const href = `<a href="?y=2026&q=1">report</a>`;
assert.deepEqual(stripSoftHyphens(href), { html: href, removed: 0 });
});
test("a soft hyphen inside an attribute goes, href included", () => {
// Deliberate, and it costs nothing: these are the entity spellings, so a browser resolving that
// href would put U+00AD into the URL it requests. No URL carries one, which makes it a
// transcription error either way, and leaving one in an `alt` would defeat the same search the
// visible text is being repaired for.
assert.deepEqual(stripSoftHyphens(`<img alt="Insur­ance" src="a­.png">`), {
html: `<img alt="Insurance" src="a.png">`,
removed: 2,
});
});
test("the count is every occurrence, not every page that had one", () => {
const { html, removed } = stripSoftHyphens(`Commu${SHY}nications and Govern­ment and Insur­ance`);
assert.equal(html, "Communications and Government and Insurance");
assert.equal(removed, 3);
});
// --- The seams -------------------------------------------------------------------------------
//
// Four calls in the extraction phase turn a model's reply into markup Iris keeps, and the strip is on
// all four rather than once on the way out of the phase. The reason is that a page's own output is an
// INPUT further along: the first render is what the correction pass is shown as "your previous
// output" and what the specialist merge is shown as the current page. Strip only at the exit and the
// model is handed its own soft hyphen back, together with an instruction to carry over everything the
// problem list does not name exactly as it stands.
// The harness lives in `extraction-seams.ts`: #374's repairs test the same four seams.
test("the first render's soft hyphens never reach the fragment, in text or in an attribute", async () => {
await inTemp("iris-shy-", async (dir) => {
// The shape the census found: table column headings set in narrow columns, one of them broken
// inside an `alt`. The attribute matters as much as the text — a soft hyphen there defeats the
// same search, and it is announced by whatever the screen reader makes of it.
const render =
`<h2>Commu${SHY}nications</h2>` +
`<p>Govern­ment and agri­culture and manu­facturing ${"content ".repeat(20)}</p>` +
`<p><img src="chart.png" alt="Insur${SHY}ance by state"></p>`;
const { ctx, rec } = makeCtx(dir, { render });
const { fragments } = await runExtraction(ctx);
assert.equal(
fragments[0].innerHtml,
`<h2>Communications</h2>` +
`<p>Government and agriculture and manufacturing ${"content ".repeat(20)}</p>` +
`<p><img src="chart.png" alt="Insurance by state"></p>`,
);
assert.deepEqual(ev(rec, "page_soft_hyphens").map((e) => e.data), [
{ image: "page-001.png", page: 1, where: "extract", removed: 5 },
]);
});
});
test("a page with none of them says nothing about it", async () => {
await inTemp("iris-shy-", async (dir) => {
const { ctx, rec } = makeCtx(dir, { render: ORDINARY });
const { fragments } = await runExtraction(ctx);
assert.equal(fragments[0].innerHtml, ORDINARY);
// So a run with no line of this kind is a run where no reply carried one, rather than a run
// where the count happened to be written as zero.
assert.deepEqual(ev(rec, "page_soft_hyphens"), []);
});
});
test("the correction pass is stripped too, and the strip runs before its reply is compared", async () => {
await inTemp("iris-shy-", async (dir) => {
// The correction reply is the page it was given, plus soft hyphens: a pass that repaired nothing
// and re-typed two words on its way past. Adopted as a string, it would be `moved` — the
// rejected page shipping with an invisible defect added, and #328's marker not fired, because
// something DID change. Stripped first, it is what it actually is: identical.
const { ctx, rec } = makeCtx(dir, {
render: ORDINARY,
problems: ["The table on this page lost its six aggregate rows."],
correction: ORDINARY.replace("Page 1", `Page${SHY} 1`).replace("content ", "con­tent "),
});
const { fragments, uncorrectedPages } = await runExtraction(ctx);
assert.equal(fragments[0].innerHtml, ORDINARY, "the correction's soft hyphens are not in the page");
assert.deepEqual(ev(rec, "page_soft_hyphens").map((e) => e.data), [
{ image: "page-001.png", page: 1, where: "correct", removed: 2 },
]);
assert.equal(ev(rec, "page_corrected")[0].data.result, "identical");
assert.deepEqual(uncorrectedPages, [1], "a pass that only added soft hyphens repaired nothing");
});
});
test("both specialist seams are stripped", async () => {
await inTemp("iris-shy-", async (dir) => {
// The specialist fragment is stripped where it is read, before it goes into the merge prompt, so
// the merge agent is never shown one to copy — and the merge reply is stripped as well, because a
// merge agent re-typing a word it is joining is the same transcription step that produces these.
const { ctx, rec } = makeCtx(dir, {
render: ORDINARY,
specialist: {
fragment: `<table><caption>Compos­ite index</caption><tr><td>1</td></tr></table>`,
merged: `${ORDINARY}<table><caption>Compos${SHY}ite index</caption><tr><td>col­lections</td></tr></table>`,
},
});
const { fragments } = await runExtraction(ctx);
assert.equal(
fragments[0].innerHtml,
`${ORDINARY}<table><caption>Composite index</caption><tr><td>collections</td></tr></table>`,
);
assert.deepEqual(ev(rec, "page_soft_hyphens").map((e) => e.data), [
{ image: "page-001.png", page: 1, where: "specialist", removed: 1 },
{ image: "page-001.png", page: 1, where: "specialist_merge", removed: 2 },
]);
});
});
test("a fragment whose only text is soft hyphens is a page with nothing on it", async () => {
await inTemp("iris-shy-", async (dir) => {
// The reason the strip is ahead of the emptiness check rather than after it. U+00AD is not
// whitespace to `visibleText`, so `<p>\u00ad</p>` reads as a page with content on it: without
// the strip in front, this run reports a page delivered and the document carries an empty
// paragraph. Stripped first, it is the failure it is, and the run says the page is lost.
const { ctx, rec } = makeCtx(dir, { render: `<p>${SHY}${SHY}</p>` }, true);
const { fragments, failedPages } = await runExtraction(ctx);
assert.deepEqual(failedPages, [1]);
assert.match(fragments[0].innerHtml, /@page-failed 1:/, "and the document says the page is missing");
// ONE, on two draws. A reply of nothing but soft hyphens claims nothing about the page, so it is
// redrawn once (#365 directive 5) and this fixture answers the same way twice — which is the
// invariant that matters here: `page_no_output` counts pages given up on, not draws discarded, so
// every count taken off this line still means what it meant before the redraw existed.
assert.equal(ev(rec, "page_no_output").length, 1);
assert.deepEqual(ev(rec, "page_redrawn").map((e) => e.data.shape), ["empty_html"]);
// Counted before it was discarded, so a page lost this way is still attributable — once per draw,
// because the strip runs on each reply and the second one carried the same two hyphens. `redrawn`
// is what keeps that pair readable: `where` attributes a count to the call it was billed under, a
// redraw makes two `extract` calls for one page, and a per-occurrence census (#334's is offline and
// by hand) would otherwise read four hyphens off one page's markup. Absent on the first draw, so a
// run with no redraw has exactly the lines it had before.
assert.deepEqual(
ev(rec, "page_soft_hyphens").map((e) => [e.data.removed, e.data.redrawn]),
[
[2, undefined],
[2, true],
],
);
});
});
test("...unless the reply SAID the page is empty, in which case it is a blank page and not a lost one", async () => {
await inTemp("iris-shy-", async (dir) => {
// The same reply as the test above plus a declaration, and the distinction the whole no-content
// branch exists to keep: a failed page is work to redo, a blank page is nothing to do. The
// emptiness gate reads the stripped markup and `blankDeclaration` re-derives the same reading
// from the reply, so the two have to be handed the same fragment — given the raw one it answers
// "this carries content", refuses the declaration, and a page the model correctly reported empty
// comes out as a page that FAILED.
const { ctx, rec } = makeCtx(
dir,
{ render: `<p>${SHY}${SHY}</p>`, log: "This page is blank.", blank: true },
true,
);
const { failedPages } = await runExtraction(ctx);
assert.deepEqual(failedPages, [], "declared blank, so nothing was lost");
assert.equal(ev(rec, "page_no_output").length, 0);
assert.equal(ev(rec, "page_blank").length, 1);
// And the markup on that line is the RAW reply: the field says which shape the declaration
// arrived in, which is a fact about what the model sent and not about what Iris did next.
assert.equal(ev(rec, "page_blank")[0].data.dropped, `<p>${SHY}${SHY}</p>`);
});
});