1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140// Shared plumbing. Deliberately dependency-free — Node 24 has everything this
// needs, and a harness that measures another service should not be the thing
// that breaks in a lockfile update.
import { createHash } from "node:crypto";
import { execFile } from "node:child_process";
import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
export const sha256 = (buf) => createHash("sha256").update(buf).digest("hex");
export const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
// Progress goes to stderr so stdout stays a clean data stream (a report can be
// piped into jq while the run is still narrating itself).
export function log(...parts) {
process.stderr.write(`[${new Date().toISOString()}] ${parts.join(" ")}\n`);
}
export function ensureDir(path) {
mkdirSync(path, { recursive: true });
return path;
}
// A malformed line is skipped rather than fatal: these files are append-only logs
// that a killed run can leave half-written, and losing the last line is better
// than refusing to resume.
export function readJsonl(path) {
if (!existsSync(path)) return [];
const out = [];
for (const line of readFileSync(path, "utf8").split("\n")) {
if (!line.trim()) continue;
try {
out.push(JSON.parse(line));
} catch {
// skip
}
}
return out;
}
export function appendJsonl(path, obj) {
appendFileSync(path, `${JSON.stringify(obj)}\n`);
}
// Iris's errors are structured — `{code, message, details}` — and the good ones say
// exactly what was wrong and what to do about it. Interpolated into a string they arrive
// as "[object Object]", which hides the one sentence that would have explained the
// failure, and makes every distinct structured error group into a single useless class.
// The structured form is what gets stored; this is for reading and grouping.
export function errorText(err) {
if (err == null) return null;
if (typeof err === "string") return err;
if (typeof err.message === "string") return err.code ? `${err.code}: ${err.message}` : err.message;
return JSON.stringify(err);
}
// prepared.jsonl is append-only, so a URL re-attempted with `--retry` has more than one
// attempt in it. Only the most recent one counts: otherwise a URL that failed the fetch
// once and succeeded on retry is tallied as both a failure and a success, and appears
// twice in the corpus. Keyed on `prepared_at` because every record one attempt writes —
// the URL-level outcome and each of its chunk children — shares that single timestamp.
export function latestAttempts(records) {
const newest = new Map();
for (const r of records) {
const at = r.prepared_at ?? "";
if (at > (newest.get(r.url) ?? "")) newest.set(r.url, at);
}
return records.filter((r) => (r.prepared_at ?? "") === newest.get(r.url));
}
// Never rejects: a non-zero exit is data, not an exception. Callers branch on
// `code` because "this PDF is broken" is an expected outcome at corpus scale.
export function exec(cmd, args, opts = {}) {
return new Promise((resolve) => {
execFile(cmd, args, { maxBuffer: 64 * 1024 * 1024, ...opts }, (err, stdout, stderr) => {
resolve({ code: err?.code ?? (err ? 1 : 0), stdout: stdout ?? "", stderr: stderr ?? String(err ?? "") });
});
});
}
export async function hasCommand(cmd) {
const { code } = await exec("sh", ["-c", `command -v ${cmd}`]);
return code === 0;
}
// Bounded-concurrency map that preserves input order in its output. Workers pull
// from a shared cursor rather than being pre-partitioned, so one slow item does
// not idle a lane — which matters here, where item cost varies by 100x.
export async function pool(items, limit, worker) {
const out = new Array(items.length);
let next = 0;
const lanes = Math.max(1, Math.min(limit, items.length));
await Promise.all(
Array.from({ length: lanes }, async () => {
for (;;) {
const i = next++;
if (i >= items.length) return;
out[i] = await worker(items[i], i);
}
}),
);
return out;
}
// --flag value / --flag=value / --bool. Unknown flags are returned rather than
// rejected so each script can decide what it accepts.
export function args(argv = process.argv.slice(2)) {
const out = { _: [] };
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
if (!a.startsWith("--")) {
out._.push(a);
continue;
}
const eq = a.indexOf("=");
if (eq !== -1) {
out[a.slice(2, eq)] = a.slice(eq + 1);
} else if (argv[i + 1] != null && !argv[i + 1].startsWith("--")) {
out[a.slice(2)] = argv[++i];
} else {
out[a.slice(2)] = true;
}
}
return out;
}
export const num = (v, fallback) => {
const n = Number(v);
return Number.isFinite(n) && n > 0 ? n : fallback;
};
// Percentile over an unsorted array of numbers. Used for the latency summary,
// where the mean is the least interesting number: the tail is what decides how
// long a 2000-document campaign actually takes.
export function pct(values, p) {
if (!values.length) return null;
const s = [...values].sort((a, b) => a - b);
return s[Math.min(s.length - 1, Math.floor((p / 100) * s.length))];
}