1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407import { test } from "node:test";
import assert from "node:assert/strict";
import { runExtraction } from "../src/pipeline/extraction.ts";
import { stripStyleAttributes, tightenDigitGroups } from "../src/util/html.ts";
import { ev, makeCtx, ORDINARY, withTemp } from "./extraction-seams.ts";
// U+00A0, written as an escape for the reason `soft-hyphens.test.ts` writes U+00AD as one.
const NBSP = "\u00a0";
// #374 item 7: two repairs to markup a page agent wrote, on the same terms as #334's soft-hyphen
// strip — decidable on the reply alone, with no image, no second model and no page-specific
// knowledge, and with no output where the thing being removed is the right answer.
//
// The measurements behind them, on one 91-page arm of #374's census:
//
// * 549 thousands separators split by the printer's alignment space (`4, 271`) on 6 pages from
// `claude-sonnet-4-6`, 166 from `kimi-k2.5`, none from `gpt-5.6-luna`. 546 of the 549 sit in a
// table cell whose whole content is a figure. `p028` and `p029` are the SAME table transcribed
// twice, 174 groups spaced on one page and 39 tight on the other, which is what says this is a
// per-cell coin flip rather than a considered house style.
// * 52 `style` attributes, 46 of them `padding-left` and 40 of those on one page's row headings
// (`p089`), plus an empty `<span>` given a coloured background as a legend swatch.
//
// What each repair CANNOT do is the part these tests pin hardest. Closing the digit gap must not
// touch prose, because the same pattern loose in a sentence turns `In 1954, 105 cases` into
// `In 1954,105 cases`; and stripping a style attribute does not rebuild what the declaration was
// carrying, which is why the properties are reported rather than silently dropped.
// --- The digit-group repair ------------------------------------------------------------------
test("a thousands separator split by the printer's space closes up, inside a numeric cell", () => {
for (const [before, after, n] of [
// The shape the census found, in a `<td>` and in a `<th>`, since a stub can hold a figure too.
[`<td>4, 271</td>`, `<td>4,271</td>`, 1],
[`<th>4, 271</th>`, `<th>4,271</th>`, 1],
// Every group in the cell, counted per group: a figure can be split twice.
[`<td>1, 234, 567</td>`, `<td>1,234,567</td>`, 2],
// What a column of money and percentages prints around its figures. The bracketing characters
// are why the cell test allows them: without them the cell is not recognised as a figure and its
// gap survives.
[`<td>$4, 271</td>`, `<td>$4,271</td>`, 1],
[`<td>(1, 234)</td>`, `<td>(1,234)</td>`, 1],
[`<td>-1, 234</td>`, `<td>-1,234</td>`, 1],
[`<td>85, 000%</td>`, `<td>85,000%</td>`, 1],
// A decimal point in the same cell is not in the way.
[`<td>4, 271.50</td>`, `<td>4,271.50</td>`, 1],
// Attributes are kept exactly, quoted `>` included — the cell is rewritten, not re-emitted from
// parts. An `align` attribute is presentational and forbidden elsewhere; it is not this repair's
// to remove, and a repair that quietly took a second thing with it would be unattributable.
[`<td align="right" data-x="a>b">4, 271</td>`, `<td align="right" data-x="a>b">4,271</td>`, 1],
// The non-breaking spellings of the space, for the reason `stripSoftHyphens` covers its entity
// forms: they render identically and defeat find-in-page identically. The raw character is
// written as `NBSP` and never as itself: an expectation whose only difference from the plain-space
// case above is invisible is one the next reader cannot check.
[`<td>4,${NBSP}271</td>`, `<td>4,271</td>`, 1],
[`<td>4, 271</td>`, `<td>4,271</td>`, 1],
[`<td>4, 271</td>`, `<td>4,271</td>`, 1],
[`<td>4, 271</td>`, `<td>4,271</td>`, 1],
// More than one space, and a newline, which is what a wrapped reply looks like.
[`<td>4, 271</td>`, `<td>4,271</td>`, 1],
[`<td>4,\n271</td>`, `<td>4,271</td>`, 1],
// Two cells in one row are two separate scopes.
[`<tr><td>4, 271</td><td>9, 100</td></tr>`, `<tr><td>4,271</td><td>9,100</td></tr>`, 2],
] as [string, string, number][]) {
assert.deepEqual(tightenDigitGroups(before), { html: after, tightened: n }, before);
}
});
test("what the digit-group repair must not touch", () => {
for (const one of [
// THE case this is scoped for. A sentence's comma is followed by a space and three digits as
// often as a figure's is, and the global version of this pattern rewrites the sentence: `In
// 1954, 105 cases were filed` becomes `In 1954,105 cases were filed`, which is a number the page
// does not print and a date that has lost its year. The `(?!\d)` guard does not save it — `105`
// is three digits — so the guard that matters is the enclosing cell holding nothing but a figure.
`<p>In 1954, 105 cases were filed</p>`,
`In 1954, 105 cases were filed`,
// The same sentence inside a cell: still prose, still untouched, because the cell's content is
// not a figure.
`<td>In 1954, 105 cases were filed</td>`,
`<td>Table 5, 100 counties reporting</td>`,
// A list of years in a numeric cell — four digits after the comma, so no group matches. This is
// what the `(?!\d)` lookahead is for, and it is the case a `\d{3}` pattern without it corrupts.
`<td>1954, 1955</td>`,
`<td>1954, 1955, 1956</td>`,
// A list of short numbers: three digits are required, and these are one.
`<td>1, 2, 3</td>`,
`<td>1, 22</td>`,
// A cell carrying a tag is out of scope entirely, which is the stated limit: on the census that
// is 3 of 549 groups, all of them cells with a footnote marker beside the figure. They stay as
// written rather than being handled by a second, looser pattern nothing has measured.
`<td>4, 271<sup>1</sup></td>`,
`<td><span>4, 271</span></td>`,
// Not inside a cell at all.
`<caption>4, 271</caption>`,
`<li>4, 271</li>`,
// A `<td>` closed by `</th>` is not a cell this recognises: the closing tag is matched against
// the name that opened it, so a page with broken markup is left for the lint to report rather
// than half-repaired here.
`<td>4, 271</th>`,
// Nothing to do: already tight, or no separator at all.
`<td>4,271</td>`,
`<td>4271</td>`,
`<td>4.271</td>`,
`<td></td>`,
`<td> </td>`,
]) {
assert.deepEqual(tightenDigitGroups(one), { html: one, tightened: 0 }, one);
}
});
// --- The style strip ------------------------------------------------------------------------
test("a style attribute comes out, and its properties are reported", () => {
// `p089`'s shape: forty row headings whose rank is in the indentation. The strip loses nothing a
// reader was getting — a style attribute is not announced — but `padding-left` in the log is what
// says this page's row groups were in ink, and that rebuilding them is a re-ask against the image.
assert.deepEqual(stripStyleAttributes(`<th style="padding-left:2em" scope="row">Ohio</th>`), {
html: `<th scope="row">Ohio</th>`,
stripped: 1,
spans: 0,
cellsEmptied: 0,
props: ["padding-left"],
});
// Several declarations in one attribute, and a value with a colon in it, which must not be read as
// a second property. `props` is a sorted set and not a reading order: the empty-span pass runs
// before the attribute walk, so first-seen order would look like document order without being it.
assert.deepEqual(
stripStyleAttributes(`<td style="padding-left:2em;background:url(http://x/y.png)">1</td>`),
{ html: `<td>1</td>`, stripped: 1, spans: 0, cellsEmptied: 0, props: ["background", "padding-left"] },
);
// Every quoting a model writes one in, including unquoted (legal for a value with no space) and a
// shouted attribute name.
for (const one of [
`<td style="color:red">1</td>`,
`<td style='color:red'>1</td>`,
`<td style=color:red>1</td>`,
`<td STYLE="color:red">1</td>`,
`<td style = "color:red">1</td>`,
]) {
assert.deepEqual(stripStyleAttributes(one), { html: `<td>1</td>`, stripped: 1, spans: 0, cellsEmptied: 0, props: ["color"] }, one);
}
// Two on one page: counted twice, properties deduped, so the list says what the page was doing
// rather than how many times it did it.
assert.deepEqual(
stripStyleAttributes(`<th style="padding-left:2em">A</th><th style="padding-left:4em">B</th>`),
{ html: `<th>A</th><th>B</th>`, stripped: 2, spans: 0, cellsEmptied: 0, props: ["padding-left"] },
);
});
test("a span this strip empties goes with it, and one it did not empty stays", () => {
// The legend swatch: an empty `<span>` with a coloured background, which paints nothing for a
// reader who cannot see it and announces nothing either. Removing the attribute and leaving
// `<span></span>` behind would swap one defect for a different one.
assert.deepEqual(stripStyleAttributes(`<p><span style="background:#ccc"></span> under 5%</p>`), {
html: `<p> under 5%</p>`,
stripped: 1,
spans: 1,
cellsEmptied: 0,
props: ["background"],
});
// Scoped to the residue THIS strip creates, which is the whole of the rule. An empty span the model
// wrote empty is not this repair's business.
assert.deepEqual(stripStyleAttributes(`<p><span></span>x</p>`), {
html: `<p><span></span>x</p>`,
stripped: 0,
spans: 0,
cellsEmptied: 0,
props: [],
});
// A span with another attribute is still a span someone put there on purpose: the style goes, the
// element stays, and whether a bare `class` should go too is a different question from this one.
assert.deepEqual(stripStyleAttributes(`<span class="swatch" style="background:#ccc"></span>`), {
html: `<span class="swatch"></span>`,
stripped: 1,
spans: 0,
cellsEmptied: 0,
props: ["background"],
});
// A span with content is not empty, whatever the strip took off it.
assert.deepEqual(stripStyleAttributes(`<span style="font-weight:bold">Southeast</span>`), {
html: `<span>Southeast</span>`,
stripped: 1,
spans: 0,
cellsEmptied: 0,
props: ["font-weight"],
});
});
test("the element goes and the whitespace stays, because only one of those two errors is visible", () => {
// A styled span holding one space is removed AS AN ELEMENT and its space handed back. The two things
// that space can be are not distinguishable from the markup, so the side with the smaller error wins.
// Here it is layout inside a legend, and putting it back changes nothing a reader sees: HTML collapses
// it against the space already after the swatch.
assert.deepEqual(stripStyleAttributes(`<p><span style="background:#ccc"> </span> under 5%</p>`), {
html: `<p> under 5%</p>`,
stripped: 1,
spans: 1,
cellsEmptied: 0,
props: ["background"],
});
// And here it is the word boundary. Taking it out delivers `Ohio5%` — text the page prints nowhere,
// produced BY the repair, which is the harm the whole printed-marks family exists to stop. This
// assertion is the one a future tidier has to argue with: an earlier draft of this file pinned the
// joining behaviour on the reasoning that the space is layout, which is true of the case above and
// false of this one.
assert.deepEqual(stripStyleAttributes(`<p>Ohio<span style="background:#ccc"> </span>5%</p>`).html, `<p>Ohio 5%</p>`);
assert.deepEqual(stripStyleAttributes(`<p>a<span style="color:red">\n</span>b</p>`).html, `<p>a\nb</p>`);
});
test("a styled span whose only content is another one goes too, and both are counted", () => {
// One pass cannot see this: `replace` is past the outer span's start by the time the inner is removed,
// so the outer would reach only the attribute walk and survive as the bare `<span></span>` this rule
// exists to prevent — with `spans` short by one on a mark that is just as gone.
assert.deepEqual(stripStyleAttributes(`<p><span style="a:b"><span style="c:d"></span></span>x</p>`), {
html: `<p>x</p>`,
stripped: 2,
spans: 2,
cellsEmptied: 0,
props: ["a", "c"],
});
// And in a cell, which is where the count matters: the mark is gone either way, so the number that
// names it must not depend on how deeply the model wrapped it.
assert.equal(stripStyleAttributes(`<td><span style="a:b"><span style="c:d"></span></span></td>`).cellsEmptied, 1);
// The repeat does not reach a span the model wrote empty, at any depth: it has no style attribute, so
// no pass matches it and the element it sits in still has content.
assert.deepEqual(stripStyleAttributes(`<p><span style="a:b"><span></span></span>x</p>`), {
html: `<p><span><span></span></span>x</p>`,
stripped: 1,
spans: 0,
cellsEmptied: 0,
props: ["a"],
});
// Whitespace still comes back, from however many levels: what is removed is elements.
assert.equal(
stripStyleAttributes(`<p>Ohio<span style="a:b"> <span style="c:d"> </span> </span>5%</p>`).html,
`<p>Ohio 5%</p>`,
);
});
test("a cell the strip leaves empty is counted, because that is the encoding a reader cannot undo", () => {
// A legend swatch written as a cell rather than as a `<p>`. `agents/page.md` now says an empty cell
// claims the paper printed nothing there, so the one place this strip's residue is not neutral gets
// its own number. The delivered markup is unchanged by the counting — the cell held no text before
// the strip either, which is why this is a log line and not a refusal.
const cell = stripStyleAttributes(`<td><span style="background:#ccc"></span></td><td>under 5%</td>`);
assert.equal(cell.html, `<td></td><td>under 5%</td>`);
assert.equal(cell.cellsEmptied, 1);
assert.equal(cell.spans, 1);
// Whitespace inside the cell does not make it a cell with something in it, and the count says so —
// the space is handed back by the rule above, and the cell still asserts a blank.
assert.equal(stripStyleAttributes(`<th><span style="background:#ccc"> </span></th>`).cellsEmptied, 1);
// A cell that was ALREADY empty is not this strip's doing and is not counted.
assert.equal(stripStyleAttributes(`<td></td><td style="color:red">1</td>`).cellsEmptied, 0);
// Nor is a cell that still holds text, or one whose span survives with another attribute.
assert.equal(stripStyleAttributes(`<td><span style="background:#ccc">5%</span></td>`).cellsEmptied, 0);
assert.equal(stripStyleAttributes(`<td><span class="sw" style="background:#ccc"></span></td>`).cellsEmptied, 0);
});
test("inside an element whose content is text, a tag is not a tag", () => {
// The same case as the printed `style="…"` above, with the `<` left bare. A page transcribing a
// report on markup can put a tag's source in a `<textarea>`, and the parser reads every character of
// it as text — so a strip that rewrites it there is rewriting what the page says. `anchors.ts` skips
// the same family for its own reasons; the two lists differ on purpose, and this pins the difference.
for (const one of [
`<p>x <textarea readonly><td style="color:red"></textarea></p>`,
`<p>x <script>var a = "<td style='color:red'>";</script></p>`,
`<title><span style="color:red"></span></title>`,
// No close tag: the element runs to the end, which is the parser's rule and the safe direction here.
`<p>x <xmp><td style="color:red">`,
]) {
assert.deepEqual(stripStyleAttributes(one), { html: one, stripped: 0, spans: 0, cellsEmptied: 0, props: [] }, one);
}
// The skip is the CONTENT, not the element: a style attribute on the raw-text element's own tag is a
// real attribute and goes. `<template>` and `<option>` interiors are parsed as markup, so they are
// not in the set at all — skipping them would leave the strip a hole rather than close a false one.
assert.equal(stripStyleAttributes(`<textarea style="color:red">x</textarea>`).html, `<textarea>x</textarea>`);
assert.equal(stripStyleAttributes(`<template><td style="color:red">1</td></template>`).stripped, 1);
assert.equal(stripStyleAttributes(`<select><option style="color:red">a</option></select>`).stripped, 1);
});
test("what the style strip must not touch", () => {
for (const one of [
// The text `style="…"` where the page PRINTS it — a report on markup, a code sample. Rewriting
// that is the same fault as repairing a misspelling: it is what the page says.
`<p>write <code>style="color:red"</code> in the tag</p>`,
`<p><td style="color:red"></p>`,
// The same text inside an attribute VALUE, which is why the attribute region is walked pair by
// pair instead of searched: a pattern that scans the whole tag for a style attribute finds this
// one and cuts a hole in the alt.
`<img alt='the style="color:red" attribute' src="a.png">`,
// A `<style>` element is a different thing from a style attribute and is not this function's:
// `flatten.ts`'s SILENT set is what keeps its content out of the text.
`<style>td { color: red }</style>`,
// An attribute whose NAME merely ends in style.
`<td data-style="x">1</td>`,
// Nothing to strip.
`<td class="num">1</td>`,
`plain text`,
]) {
assert.deepEqual(stripStyleAttributes(one), { html: one, stripped: 0, spans: 0, cellsEmptied: 0, props: [] }, one);
}
});
test("a tag the style strip rewrites keeps everything else it carried", () => {
// Rebuilt from its parsed attributes, so this asserts what survives that: the other attributes,
// their quoting, a quoted `>` inside a value, and a self-closing slash `ATTR` cannot match.
assert.deepEqual(
stripStyleAttributes(`<img src="a.png" alt="a > b" style="border:0" loading=lazy />`).html,
`<img src="a.png" alt="a > b" loading=lazy />`,
);
// A boolean attribute has no value at all and must not acquire one.
assert.deepEqual(stripStyleAttributes(`<input disabled style="color:red">`).html, `<input disabled>`);
// The whitespace between attributes is normalized on a tag this touched — stated here rather than
// in a comment alone, because it is the visible cost of parsing pairs instead of cutting a
// substring out, and a tag with no style attribute is returned byte-for-byte by the guard above.
assert.deepEqual(stripStyleAttributes(`<td class="a" style="color:red">1</td>`).html, `<td class="a">1</td>`);
});
// --- The seams ------------------------------------------------------------------------------
//
// Both repairs run at all four calls that turn a reply into markup Iris keeps, for the reason the
// soft-hyphen strip does: a page's own output is an INPUT further along. `soft-hyphens.test.ts` makes
// that argument at length and covers each seam one at a time; these tests cover the two new repairs
// on the seams and the fields their events carry.
test("the first render's styling and split figures never reach the fragment", async () => {
await withTemp("iris-marks-", async (dir) => {
const render =
`<table><caption>Table 8.</caption>` +
`<tr><th style="padding-left:2em" scope="row">Ohio</th><td>4, 271</td></tr>` +
`<tr><th scope="row">Iowa</th><td>1, 234, 567</td></tr>` +
`</table><p><span style="background:#ccc"></span> under 5% ${"content ".repeat(20)}</p>`;
const { ctx, rec } = makeCtx(dir, { render });
const { fragments } = await runExtraction(ctx);
assert.equal(
fragments[0].innerHtml,
`<table><caption>Table 8.</caption>` +
`<tr><th scope="row">Ohio</th><td>4,271</td></tr>` +
`<tr><th scope="row">Iowa</th><td>1,234,567</td></tr>` +
`</table><p> under 5% ${"content ".repeat(20)}</p>`,
);
assert.deepEqual(ev(rec, "page_style_attributes").map((e) => e.data), [
{
image: "page-001.png",
page: 1,
where: "extract",
stripped: 2,
spans: 1,
cells_emptied: 0,
props: ["background", "padding-left"],
},
]);
assert.deepEqual(ev(rec, "page_digit_groups").map((e) => e.data), [
{ image: "page-001.png", page: 1, where: "extract", tightened: 3 },
]);
});
});
test("a page with neither says nothing about either", async () => {
await withTemp("iris-marks-", async (dir) => {
const { ctx, rec } = makeCtx(dir, { render: ORDINARY });
const { fragments } = await runExtraction(ctx);
assert.equal(fragments[0].innerHtml, ORDINARY);
// So a run with no line of either kind is a run in which no reply carried one, rather than a run
// where the count happened to be written as zero.
assert.deepEqual(ev(rec, "page_style_attributes"), []);
assert.deepEqual(ev(rec, "page_digit_groups"), []);
});
});
test("the correction pass and both specialist seams are repaired too", async () => {
await withTemp("iris-marks-", async (dir) => {
const { ctx, rec } = makeCtx(dir, {
render: ORDINARY,
problems: ["The table on this page lost its six aggregate rows."],
// A correction pass that re-typed the page it was given and split a figure on the way past.
correction: `${ORDINARY}<table><tr><td style="text-align:right">4, 271</td></tr></table>`,
specialist: {
fragment: `<table><tr><td>9, 100</td></tr></table>`,
merged: `${ORDINARY}<table><tr><td style="color:red">9, 100</td></tr></table>`,
},
});
const { fragments } = await runExtraction(ctx);
// The page that ships is the correction's, which replaced the merged one — so the fragment is
// where the LAST seam is observable and the events are where the other three are. Both repairs
// reached it: the figure the correction split is closed and its style attribute is gone.
assert.equal(
fragments[0].innerHtml,
`${ORDINARY}<table><tr><td>4,271</td></tr></table>`,
"the correction's own reply is repaired before it is adopted",
);
assert.deepEqual(
ev(rec, "page_digit_groups").map((e) => [e.data.where, e.data.tightened]),
[["specialist", 1], ["specialist_merge", 1], ["correct", 1]],
"the digit repair fires at each seam that carried one, correction last",
);
assert.deepEqual(
ev(rec, "page_style_attributes").map((e) => [e.data.where, e.data.props]),
[["specialist_merge", ["color"]], ["correct", ["text-align"]]],
"and the style strip too, on the two replies that carried an attribute",
);
});
});