1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463import { cutPoints } from "./sections.ts";
// A sentence the source printed across a page turn arrives here in two pieces, and neither page
// could have mended it: the page-break marker is the FIRST thing a page emits, so everything
// standing before it came off a sheet that call was never shown (issue #248, and `agents/page.md`,
// which tells a page to transcribe its own edge exactly and leave the join to a pass holding both
// halves). Measured on the last bench round's artifacts — 4 chunks × 25 pages, 90 markers — 22
// markers stood where a sentence carried on, 13 of those with the sentence's tail in the element
// immediately before the marker, and 2 of the 13 split a word:
//
// "…tourist courts. Simi-" || "larly, the more populous States do not tax…"
//
// A reader hears "Simi", then a page-break announcement, then "larly".
//
// This is a text question with no judgement in it, so no model call is spent on it — unlike the
// table join (tables.ts), which has to decide whether two headers describe one table.
//
// WHICH WAY THE TEXT MOVES is the decision this file embodies, and it is a decision about what a
// page anchor means. `<hr>` cannot sit inside a `<p>`, so the halves cannot be joined without text
// crossing the marker, and there are only two directions. The previous page's tail moves FORWARD,
// after the marker: the sentence is then whole and `#page-74` lands on the marker immediately
// before it, so a reader following that anchor hears a few words of page 73 before page 74's own
// text. The alternative — pulling page 74's head back before the marker — makes `#page-74` land
// AFTER the sentence it should open on, which costs that reader the beginning of it. A few words of
// provenance drift is the cheaper mistake.
//
// "A few words" is held to rather than hoped for: `MAX_MOVED_CHARS` bounds what may cross a marker,
// because the one shape that would move a whole page's text is a paragraph with no sentence boundary
// in it, and the argument above does not cover that.
//
// Nothing is created and nothing is dropped: every character of both halves is delivered, in order,
// and the only edit is which element holds them. That includes the hyphen a word-split leaves
// behind — see `joinAt`.
// What the join did, for the run log. Counted rather than argued because the before-numbers are on
// file (90 markers, 22 mid-sentence, 13 tail-adjacent, 2 word-splits): a round on a build carrying
// this should show `candidates` still around 22 with `joined` around 13, and the 9 the marker does
// not interrupt declined as `interrupted`.
export interface ProseJoinReport {
// Page-break markers standing where a page begins, which is where `agents/page.md` puts them.
markers: number;
// Boundaries where the first thing on the next page is a `<p>` beginning with a lowercase letter
// — the issue's own test for "a sentence carries on here", whatever stands before the marker.
//
// A lowercase letter is a signal only a cased script has, so this never fires on Hangul, Chinese,
// Japanese, Arabic or Hebrew, and their sentences ship split as they do today. That is a join
// missed rather than a join got wrong, and it is left that way deliberately: the 22 above were
// measured on an English corpus, and what the signal should be for a caseless script is a
// question no run has asked yet. `candidates: 0` on a document with many `markers` is what that
// looks like in the log.
candidates: number;
joined: number;
// Of the joins, how many had no marker between the halves at all: a page that prints no number
// emits none, so a document of unnumbered scans has page turns with nothing marking them. Those
// halves are already adjacent, so the join there is only a merge of two paragraphs into the one
// sentence they hold, and no anchor moves.
unmarked: number;
// Joins where the break fell inside a word, so the tail ended with a hyphen.
wordSplits: number;
declined: {
// Something other than a `<p>` stands immediately before the marker — a footnote list, in all
// 9 of the measured cases. The marker is then not what interrupts the sentence, and moving
// text across the notes would reorder the page. A page that failed extraction lands here too:
// its fragment is the `@page-failed` comment, which is exactly such a node.
interrupted: number;
// The `<p>` before the marker ends a sentence, so the lowercase start after it is something
// else: a caption, a continued list item, a line of verse.
notContinuing: number;
// A page between the halves returned nothing at all, so the middle of the sentence may be what
// is missing and these two edges do not meet. Narrower than "a page is missing": a page that
// FAILED extraction ships a `@page-failed` comment (extraction.ts), which is a node standing
// between the halves and declines as `interrupted` above. What reaches here is the page that
// came back empty with no marker at all (#194) — dropped from the body, and then visible only
// as a hole in the numbering.
pageGap: number;
// The tail cannot be cut out without unbalancing markup: the sentence begins inside an inline
// element that opened earlier in the paragraph, or inside a footnote reference.
noCut: number;
// The whole paragraph is the tail, and it carries an attribute that would go with it. `id` is
// the one that matters — dropping it breaks whatever refers to it, and `namespaceAnchors` has
// just been repointing references at exactly these — but the guard is on any attribute, since
// the words would arrive under the NEXT paragraph's attributes and this pass has no business
// deciding which of them the words can do without.
attrsKept: number;
// The two paragraphs disagree about `lang`. Moving words between them would deliver them in a
// language nothing said they were in, and `bodyLang` reads these same attributes to decide what
// the document declares (assembly.ts, #163). Agreement is the ordinary case in a document that
// declares one at all: `agents/page.md` puts it on every top-level element of such a page.
langMismatch: number;
// One of the two pages is being delivered exactly as its agent wrote it, because the parser and
// its bytes disagree about the page's structure (`skipped_pages`, anchors.ts). A pass that reads
// that structure to find the paragraph at its edge is reading the half of the disagreement the
// browser will not honour, so it keeps its hands off the page entirely.
asWritten: number;
// More text would cross the marker than moving it forward can be justified for. The argument for
// that direction is that it costs a reader following `#page-74` a few words of page 73 — and
// where the paragraph has no sentence boundary in it at all the WHOLE of it moves, so a page of
// unpunctuated prose would deliver its entire text after the next page's anchor, which is not a
// few words and not what the direction was chosen for. The bound is on the characters moved
// rather than on that shape alone, since a single sentence long enough to reach it costs the
// same reader the same thing.
tooFar: number;
};
// The joined words, for the two the corpus had and any others: a word split across a page turn is
// the shape worth eyeballing, and the count alone cannot be checked against the document. Text
// out of the user's own document, so bounded the way `emptyTableCaptions` bounds its examples,
// and it never reaches `GET /v1/quality`.
wordSplitExamples: string[];
}
const MAX_EXAMPLES = 5;
const MAX_EXAMPLE_CHARS = 40;
// How much text may cross a marker. Generous against what it has to allow — the longest tail in the
// reference corpus is a fraction of this, and a sentence needing more than 500 characters of it
// after the last full stop is not a sentence in the prose this runs on — and it is the whole of the
// guard against the shape the direction was NOT chosen for: a paragraph with no sentence boundary in
// it moves entire, so without a bound a page of unpunctuated prose ships its text after the next
// page's anchor. Counted when it bites (`tooFar`), so a corpus that wants a different number says so.
const MAX_MOVED_CHARS = 500;
function emptyProseJoin(): ProseJoinReport {
return {
markers: 0,
candidates: 0,
joined: 0,
unmarked: 0,
wordSplits: 0,
declined: {
interrupted: 0,
notContinuing: 0,
pageGap: 0,
noCut: 0,
attrsKept: 0,
langMismatch: 0,
asWritten: 0,
tooFar: 0,
},
wordSplitExamples: [],
};
}
// Attribute lists are read quote-aware rather than as `[^>]*`, because a `>` inside an attribute
// value is legal and a greedy read of it would take the rest of the document for a tag.
const ATTRS = `(?:"[^"]*"|'[^']*'|[^>"'])*`;
const PARAGRAPH = new RegExp(`^(\\s*<p${ATTRS}>)([\\s\\S]*)(</p>\\s*)$`, "i");
const HR = new RegExp(`^\\s*<hr${ATTRS}>\\s*$`, "i");
const PAGE_BREAK_ROLE = /\brole\s*=\s*["']?\s*doc-pagebreak\b/i;
// Attributes on a start tag, read the way `bodyLang` reads them — attribute by attribute rather
// than searched for, because a search for ` lang=` finds one inside another attribute's value.
const ATTR = /([^\s=/>]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]*)))?/g;
function attrs(tag: string): Map<string, string> {
const out = new Map<string, string>();
const inner = tag.replace(/^\s*<[a-zA-Z][^\s/>]*/, "").replace(/\/?>\s*$/, "");
ATTR.lastIndex = 0;
for (let m = ATTR.exec(inner); m; m = ATTR.exec(inner)) {
// The first spelling wins, which is what a parser does with a repeated attribute.
if (!out.has(m[1]!.toLowerCase())) out.set(m[1]!.toLowerCase(), m[2] ?? m[3] ?? m[4] ?? "");
}
return out;
}
// The two a paragraph may carry and still move whole: they say how its words are read, and the
// paragraph it joins carries the same ones or the join is declined for disagreeing about them.
const TRAVELS = new Set(["lang", "dir"]);
// The top-level nodes of one page's fragment, as slices that concatenate back to exactly the string
// they came from — so a page nothing is done to comes back byte-identical, and the whitespace
// between nodes belongs to whichever slice held it.
function topLevelNodes(html: string): string[] {
const out: string[] = [];
let start = 0;
for (const end of cutPoints(html)) {
if (end <= start) continue;
out.push(html.slice(start, end));
start = end;
}
if (start < html.length) out.push(html.slice(start));
return out;
}
// One character of a paragraph's text, with where it came from and what was open around it. The two
// questions the join has to answer are both about position — is this character inside an inline
// element, is it inside a footnote reference — and neither survives `replace(/<[^>]*>/g, "")`.
interface Char {
ch: string;
at: number; // index in the paragraph's inner HTML
depth: number; // elements open at this character, within the paragraph
inSup: boolean;
}
// Void elements cannot be open around anything, so they must not raise the depth. `<br>` inside a
// paragraph is the common one and a paragraph whose last line ends in one would otherwise look
// permanently nested.
const VOID = new Set([
"area", "base", "br", "col", "embed", "hr", "img", "input",
"link", "meta", "param", "source", "track", "wbr",
]);
function scan(inner: string): Char[] {
const out: Char[] = [];
const stack: string[] = [];
let i = 0;
while (i < inner.length) {
if (inner.startsWith("<!--", i)) {
const end = inner.indexOf("-->", i);
i = end < 0 ? inner.length : end + 3;
continue;
}
const tag = /^<(\/?)([a-zA-Z][^\s/>]*)((?:"[^"]*"|'[^']*'|[^>"'])*)(\/?)>/.exec(inner.slice(i));
if (tag) {
const name = tag[2]!.toLowerCase();
if (tag[1]) {
const at = stack.lastIndexOf(name);
if (at >= 0) stack.length = at;
} else if (!tag[4] && !VOID.has(name)) {
stack.push(name);
}
i += tag[0].length;
continue;
}
out.push({
ch: inner[i]!,
at: i,
depth: stack.length,
// A footnote reference is `<sup><a href="#fn-N" id="fnref-N">N</a></sup>` (agents/page.md), so
// its digit is the last character of a paragraph that ends a sentence and cites a note. Read
// as text, "…tourist courts.<sup>1</sup>" ends in a digit and looks like a sentence still
// running — which would move a whole finished sentence onto the next page.
inSup: stack.includes("sup"),
});
i += 1;
}
return out;
}
// The paragraph's own words, with footnote references left out for the reasons above.
function words(chars: Char[]): Char[] {
return chars.filter((c) => !c.inSup);
}
// Ends mid-sentence: the last thing the paragraph says is a letter, a digit, a comma or a hyphen.
// Deliberately a positive test rather than "not a full stop" — a paragraph ending in a colon
// introduces what follows rather than continuing into it, and one ending in a semicolon or a dash
// is its own clause, so neither is joined to the next page.
const CONTINUES = /[\p{L}\p{N},\-‐‑]$/u;
// The word-split case: the tail's last character is a hyphen the printer put there to fill a line.
const HYPHEN = /[-‐‑]$/;
// Where one sentence ends and the next begins, allowing for the punctuation that closes a quotation
// or a parenthesis first.
const SENTENCE_END = /[.?!]["'’”)\]]*\s+/g;
// The head of the paragraph and the tail that carries on into the next page, or null where the tail
// cannot be taken out without breaking markup.
//
// The cut lands immediately after the whitespace that separates the two sentences, and only where
// nothing is open at that point: everything after it is then balanced on its own, including any
// inline element the continuing sentence opens with. A sentence that begins INSIDE an element that
// opened earlier in the paragraph has no such cut, and is declined rather than cut anyway.
function splitTail(inner: string): { head: string; tail: string } | null {
const chars = words(scan(inner));
const text = chars.map((c) => c.ch).join("");
let cut: number | null = null;
let sawBoundary = false;
SENTENCE_END.lastIndex = 0;
for (let m = SENTENCE_END.exec(text); m; m = SENTENCE_END.exec(text)) {
sawBoundary = true;
// The last whitespace character of the separator, in the paragraph's own indexing. The cut goes
// after it, so the tail keeps any inline element the next sentence opens with.
const last = chars[m.index + m[0].length - 1]!;
if (last.depth === 0) cut = last.at + 1;
SENTENCE_END.lastIndex = m.index + m[0].length;
}
// A boundary was found but every one of them sits inside an element that opened earlier, so there
// is nowhere to cut that leaves both halves balanced.
if (cut === null && sawBoundary) return null;
// No boundary at all: the whole paragraph continues the sentence the previous page began, so the
// whole of it moves.
if (cut === null) return { head: "", tail: inner };
// The whitespace that separated the two sentences goes to the seam rather than being left
// dangling at the end of the head — `joinAt` puts one space back where the sentence continues.
// The only characters this loses are ones no reader receives: whitespace at the end of a block
// element is collapsed away before it reaches anybody.
return { head: inner.slice(0, cut).replace(/\s+$/, ""), tail: inner.slice(cut) };
}
// Whether this page's fragment opens with the page-break marker, and where its first real content
// is. `agents/page.md` requires the marker to be the first thing a page emits, which is what makes
// the element after it the head of that page's text.
function opening(nodes: string[]): { marker: number | null; content: number | null } {
const real = nodes.map((n, i) => ({ n, i })).filter(({ n }) => n.trim().length > 0);
const first = real[0];
if (!first) return { marker: null, content: null };
const hr = HR.exec(first.n);
if (hr && PAGE_BREAK_ROLE.test(hr[0])) {
return { marker: first.i, content: real[1]?.i ?? null };
}
return { marker: null, content: first.i };
}
// One page's HTML with its order, which is what the page-gap rule is decided on: a page that came
// back empty contributes no fragment to the body, so the only trace of it at this seam is a hole in
// the numbering. Not a page that FAILED extraction — that one ships a `@page-failed` comment, so the
// numbering stays contiguous and the comment declines the boundary as an interruption instead.
export interface PageHtml {
order: number;
html: string;
// Set where this page is being shipped byte for byte as its agent wrote it (`skipped_pages`,
// anchors.ts). Carried on the page rather than passed as a separate set of numbers, so a caller
// cannot hand over the pages and forget which of them are untouchable.
asWritten?: boolean;
}
export function joinPageBreakProse(pages: PageHtml[]): { pages: string[]; report: ProseJoinReport } {
const report = emptyProseJoin();
const nodes = pages.map((p) => topLevelNodes(p.html));
for (let i = 0; i + 1 < pages.length; i += 1) {
const before = nodes[i]!;
const after = nodes[i + 1]!;
const { marker, content } = opening(after);
if (marker !== null) report.markers += 1;
if (content === null) continue;
// Does the next page begin in the middle of a sentence? The measured test, and the one that
// decides whether this boundary is counted at all.
const head = PARAGRAPH.exec(after[content]!);
if (!head) continue;
const headText = words(scan(head[2]!)).map((c) => c.ch).join("").trimStart();
if (!/^\p{Ll}/u.test(headText)) continue;
report.candidates += 1;
// And is the sentence's tail the thing immediately before it?
const lastIdx = before.map((n, k) => ({ n, k })).filter(({ n }) => n.trim().length > 0).at(-1)?.k;
const tailNode = lastIdx === undefined ? null : PARAGRAPH.exec(before[lastIdx]!);
if (lastIdx === undefined || !tailNode) {
report.declined.interrupted += 1;
continue;
}
const tailChars = words(scan(tailNode[2]!));
const tailText = tailChars.map((c) => c.ch).join("").trimEnd();
if (!CONTINUES.test(tailText)) {
report.declined.notContinuing += 1;
continue;
}
// A page is missing between these two, so the middle of the sentence may be what is missing.
// Both edges are then transcribed correctly and joining them would invent a sentence neither
// page printed.
if (pages[i + 1]!.order !== pages[i]!.order + 1) {
report.declined.pageGap += 1;
continue;
}
// Either page is being delivered exactly as written, so nothing may edit its bytes.
if (pages[i]!.asWritten || pages[i + 1]!.asWritten) {
report.declined.asWritten += 1;
continue;
}
// The words would arrive under the other paragraph's `lang`/`dir`, so the two must agree about
// them. Absent on both is agreement, and is the ordinary English document.
const tailAttrs = attrs(tailNode[1]!);
const headAttrs = attrs(head[1]!);
if ([...TRAVELS].some((a) => tailAttrs.get(a) !== headAttrs.get(a))) {
report.declined.langMismatch += 1;
continue;
}
const split = splitTail(tailNode[2]!);
if (!split) {
report.declined.noCut += 1;
continue;
}
// Measured on the moved TEXT, not on its markup: what a reader following the next page's anchor
// has to listen through before reaching that page is words, and a tail carrying a footnote
// reference is mostly tag.
if (words(scan(split.tail)).length > MAX_MOVED_CHARS) {
report.declined.tooFar += 1;
continue;
}
// The whole paragraph would move, and it carries something the move cannot take with it. Its
// `lang` and `dir` can go, since the paragraph it joins was just held to the same ones.
if (!split.head.trim() && [...tailAttrs.keys()].some((a) => !TRAVELS.has(a))) {
report.declined.attrsKept += 1;
continue;
}
const joinedWord = HYPHEN.test(tailText);
before[lastIdx] = split.head.trim()
? tailNode[1]! + split.head + tailNode[3]!
: // Nothing of this paragraph stays behind, so the element goes with its text rather than
// shipping as an empty `<p>`. Its whitespace is kept so the pages still read apart.
(before[lastIdx]!.match(/^\s*/)?.[0] ?? "") + (before[lastIdx]!.match(/\s*$/)?.[0] ?? "");
after[content] = head[1]! + joinAt(split.tail, head[2]!, joinedWord) + head[3]!;
report.joined += 1;
if (marker === null) report.unmarked += 1;
if (joinedWord) {
report.wordSplits += 1;
if (report.wordSplitExamples.length < MAX_EXAMPLES) {
report.wordSplitExamples.push(wordSplitExample(tailText, headText));
}
}
}
return { pages: pages.map((_, i) => nodes[i]!.join("")), report };
}
// The two halves in one paragraph. A space between them, except where the printer broke a word, and
// then nothing — so "Simi-" and "larly," are delivered as one word.
//
// THE HYPHEN STAYS. It is the one character where the printing itself is ambiguous: "Simi-" +
// "larly" wants it gone and "public-" + "sector" wants it kept, and nothing at this seam can tell
// which of the two a hyphen at a page's edge is — the page prompt reaches the same wall from the
// other side and answers it the same way ("Where you cannot tell whose hyphen it is, keep it — a
// hyphen too many is a printing some page might have, and two words run into one is a word no page
// printed"). Dropping it would also be the one place this pass deleted a character the source
// printed. What the join fixes is the interruption: a reader hears "Simi-larly" as one word instead
// of hearing "Simi", a page-break announcement, and then "larly". Whether the hyphen can be decided
// after all is a separate question, and `word_splits` in the run log is what makes it answerable.
//
// Both branches close the whitespace at the seam, and the word-split branch HAS to: a fragment's `<p>`
// is not on one line — pretty-printed HTML is what a model emits, and nothing between the extractor
// and here collapses it — so the tail arrives as "Simi-\n ". Left in, the delivered document says
// "Simi- larly", which is WORSE than the split it replaced: a screen reader still reads "Simi",
// pause, "larly", and now there is no page-break announcement to explain the pause. It would also
// make `word_split_examples` report a closure the document does not contain, which is the one datum
// that lets a later pass decide the hyphen with evidence.
//
// The whitespace is found at the TEXT seam rather than at the string's ends, because a tag may stand
// between the last word and `</p>`: `<em>Simi-\n </em>`, or a comment, or the mirror of either at the
// head's start. Trimming the string's ends leaves the whitespace inside and delivers "Simi- larly"
// again, and the `<em>` is what decides the word's own emphasis, so it cannot simply be moved.
// `scan` gives the index of every text character, so what is dropped is exactly the whitespace after
// the tail's last word and before the head's first — no markup, and nothing a reader receives.
//
// One shape is left as it arrives, recorded rather than guarded: `<p>… Simi-<br></p>` delivers
// "Simi-<br>larly", a line break inside the joined word. Closing it would mean deleting the `<br>`
// the page emitted, and no input produces the shape — `agents/page.md` steers prose away from `<br>`
// entirely, so reaching it needs a hyphenated word broken at a page's edge inside verse or an
// address. A branch nothing can reach is one no test can honestly cover.
function joinAt(tail: string, head: string, joinedWord: boolean): string {
const closed = closeSeam(tail, "end");
const rest = closeSeam(head, "start");
return joinedWord ? closed + rest : `${closed} ${rest}`;
}
// The fragment with the whitespace at one end's text seam removed: every text character after the
// last word (`"end"`) or before the first (`"start"`) goes, and every tag stays where it was.
function closeSeam(html: string, which: "start" | "end"): string {
const chars = scan(html);
const wordChars = chars.filter((c) => !/\s/.test(c.ch));
const edge = which === "end" ? wordChars.at(-1) : wordChars[0];
if (!edge) return html;
const drop = new Set(chars.filter((c) => (which === "end" ? c.at > edge.at : c.at < edge.at)).map((c) => c.at));
if (drop.size === 0) return html;
// Indexed by code unit, which is how `scan` counted: splitting by code point would put an emoji or
// a rare CJK character one index out of step with the offsets being dropped.
let out = "";
for (let i = 0; i < html.length; i += 1) if (!drop.has(i)) out += html[i];
return out;
}
// The word the two halves make, for the log: the last word of the tail and the first of the head,
// which is what a reader would have heard split in two.
function wordSplitExample(tailText: string, headText: string): string {
const tail = /\S+$/.exec(tailText)?.[0] ?? "";
const head = /^\S+/.exec(headText)?.[0] ?? "";
return (tail + head).slice(0, MAX_EXAMPLE_CHARS);
}