1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170// `runAxe` degrades rather than fails: when axe cannot run it returns no verdict —
// `ok: false` and no `violations` at all, with `error` set — and the document ships
// unchecked. That trade is deliberate — a linter that cannot load must not cost a user
// their document — and it only stays honest if the failure can be chased down afterwards.
// (It used to report `ok: true, violations: []`, i.e. a pass, which is #164.)
//
// It could not be. The first report of this happening on a real document (#144) carried
// one sentence: "Octal escape sequences are not allowed in strict mode". A JavaScript
// SyntaxError, which is not the case the code documented as reachable (a stack overflow
// on a page too deeply nested), which names no document, and which does not say whether
// axe's own source failed to evaluate or axe choked walking the output. Those two answers
// point at a version bump and at a page of HTML respectively, and nobody could tell which
// from the log. So the report now carries which step threw, the error's class, and the top
// of its stack; this test holds all three to the log line an operator actually reads.
import { test } from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { createRequire } from "node:module";
import { runAxe, trimStackPaths } from "../src/pipeline/lint.ts";
import { runAssembly, wrapDocument } from "../src/pipeline/assembly.ts";
import type { Fragment } from "../src/pipeline/fragment.ts";
import type { PipelineContext } from "../src/pipeline/context.ts";
function frag(order: number, innerHtml: string): Fragment {
return { image: `page-00${order}.png`, order, agent: "page.md", region: "page", innerHtml, edges: [], log: "" };
}
function recorder(): { ctx: PipelineContext; events: { type: string; data: Record<string, unknown> }[] } {
const events: { type: string; data: Record<string, unknown> }[] = [];
const ctx = {
log: { event: (type: string, data: Record<string, unknown> = {}) => events.push({ type, data }) },
} as unknown as PipelineContext;
return { ctx, events };
}
// Deep enough that walking the tree overflows the stack. Every threshold here moves with
// the stack the caller already spent, so if axe manages to run the degradation never
// happened and there is nothing to report — each test below returns instead of asserting
// against a gate that worked.
const DEEP = `${"<div>".repeat(6000)}<p>Buried</p>`;
test("a lint that could not run says which step threw, and what threw", async () => {
const lint = await runAxe(wrapDocument(DEEP));
if (lint.error === undefined) return;
// The distinction the message could not make. `run` is axe walking THIS document, which
// sends an operator to the page of HTML; `inject` would be axe's own source failing to
// evaluate, which sends them to a version. It is read off which call threw rather than
// out of the text, which is why lint.ts keeps the two in separate try blocks.
assert.equal(lint.errorWhere, "run", "the step is guessed from the message rather than recorded");
// The degradation is still a degradation — the session continues — but it no longer
// reports a pass: `ok: true, violations: []` was the pair a flawless document produces
// (#164). What the caller gets instead is no verdict, and no violation list to map over.
assert.equal(lint.ok, false, "a lint that threw is still reported as one the document passed");
assert.equal(lint.violations, undefined, "a check that did not happen still reports a violation count");
assert.ok(lint.errorName, "the error class is missing, which is the first thing to look at");
assert.equal(lint.errorName, "RangeError", "a stack overflow is what this document provokes");
assert.ok(lint.errorStack?.includes(lint.errorName), "the stack does not even name its own error");
// Bounded, because this ends up in every run's log: enough frames to name the throwing
// library and its caller, not a page of jsdom internals in a session record.
const lines = lint.errorStack!.split("\n").length;
assert.ok(lines <= 7, `the whole stack was logged (${lines} lines)`);
// And no absolute paths, because this log is served to the session's owner
// (`GET /v1/sessions/{id}/logs`) and where the app is installed is not theirs to read.
// The part that answers the question — which library threw — is what has to survive.
assert.doesNotMatch(lint.errorStack!, /\s\/[A-Za-z]|\(\/|file:\/\//, `an absolute path was logged: ${lint.errorStack}`);
assert.ok(!lint.errorStack!.includes(process.cwd()), "the frames name the install directory");
assert.match(lint.errorStack!, /node_modules\/(jsdom|axe-core)\//, "the trim took the library name with it");
});
test("the three fields reach the log line an operator reads", async () => {
// The fields only matter where they are read, and that is `runAssembly`'s `assembly`
// event — the same line that records `lint_ok: true` for a gate that never ran.
const { ctx, events } = recorder();
const { lint } = await runAssembly(ctx, [frag(1, DEEP), frag(2, `<p>B</p>`)]);
const logged = events.find((e) => e.type === "assembly")!;
if (lint.error === undefined) {
assert.ok(!("lint_error_where" in logged.data), "a gate that ran reported a failure step anyway");
return;
}
assert.equal(logged.data.lint_error_where, lint.errorWhere);
assert.equal(logged.data.lint_error_name, lint.errorName);
assert.equal(logged.data.lint_error_stack, lint.errorStack);
// Paired with the reading that makes them necessary. This run used to be on the record as
// clean — `lint_ok: true, violations: 0` — which is why the fields above had to carry the
// whole disclosure on their own; now the verdict itself says there is none.
assert.equal(logged.data.lint_ok, false);
assert.ok(!("violations" in logged.data), "a lint that did not run still logged a violation count");
});
test("an ordinary run carries none of them", async () => {
// A key on every run is noise, and noise is how the one line that mattered gets skipped.
const { ctx, events } = recorder();
await runAssembly(ctx, [frag(1, `<h1>Report</h1><p>Clean</p>`)]);
const clean = events.find((e) => e.type === "assembly")!;
for (const key of ["lint_error", "lint_error_where", "lint_error_name", "lint_error_stack"]) {
assert.ok(!(key in clean.data), `an ordinary run logged ${key}: ${JSON.stringify(clean.data)}`);
}
});
test("a document that parses and lints reports no failure fields at all", async () => {
const lint = await runAxe(wrapDocument(`<h1>Report</h1><p>Body</p>`));
assert.equal(lint.error, undefined);
assert.equal(lint.errorWhere, undefined);
assert.equal(lint.errorName, undefined);
assert.equal(lint.errorStack, undefined);
});
test("every frame shape loses the install path and keeps what the frame is logged for", () => {
// The test above exercises one shape, because one shape is all this environment can
// provoke: the deep-nesting failure is six jsdom frames. The others are reachable through
// `runAxe` only on an "inject"-step throw, or a jsdom whose stack shape changed — so they
// are pinned directly rather than trimmed on faith.
const cwd = process.cwd();
for (const [shape, frame, expected] of [
// A dependency: which library threw is the question the stack is logged to answer.
["a dependency frame", ` at Y (${cwd}/node_modules/jsdom/lib/jsdom/x.js:12:3)`, " at Y (node_modules/jsdom/lib/jsdom/x.js:12:3)"],
["a dependency installed elsewhere", " at Y (/opt/app/node_modules/jsdom/lib/x.js:12:3)", " at Y (node_modules/jsdom/lib/x.js:12:3)"],
// A store-based installer nests one node_modules inside another. Cut at the last of
// them, so the frame names the library once rather than twice or, worse, glues the two
// segments together — this repo installs flat, so what would reach it is a deployment
// that installs with pnpm.
[
"a nested dependency layout",
" at Y (/opt/x/node_modules/.pnpm/jsdom@25.0.1/node_modules/jsdom/lib/a.js:1:1)",
" at Y (node_modules/jsdom/lib/a.js:1:1)",
],
// The app's own frames: the path within the repo is the useful half and discloses nothing.
["an app frame", ` at runAxe (${cwd}/src/pipeline/lint.ts:52:10)`, " at runAxe (src/pipeline/lint.ts:52:10)"],
// ESM stacks carry a URL. A scheme left standing on its own reads as a path that
// escaped the trim, which is the note this closes.
["an ESM url", ` at runAxe (file://${cwd}/src/pipeline/lint.ts:52:10)`, " at runAxe (src/pipeline/lint.ts:52:10)"],
// Outside both trees there is no relative form to keep, so the file name is all of it.
["a frame from neither tree", " at z (/opt/other/lib/x.js:3:1)", " at z (x.js:3:1)"],
["a runtime frame, already relative", " at node:internal/modules/run_main:123:5", " at node:internal/modules/run_main:123:5"],
] as [string, string, string][]) {
assert.equal(trimStackPaths(frame), expected, `the trim mishandles ${shape}`);
assert.ok(!trimStackPaths(frame).includes(cwd), `${shape} still names the install directory`);
}
// And the whole stack at once, since the rules run in sequence over one string.
const trimmed = trimStackPaths(
`RangeError: Maximum call stack size exceeded\n at Y (${cwd}/node_modules/jsdom/lib/x.js:1:1)\n at runAxe (file://${cwd}/src/pipeline/lint.ts:52:10)`,
);
assert.doesNotMatch(trimmed, /file:\/\/|\s\/|\(\//, `an absolute path or bare scheme survived: ${trimmed}`);
assert.match(trimmed, /RangeError: Maximum call stack size exceeded/, "the message was mangled");
});
test("axe-core and jsdom are pinned to exact versions, and to the ones installed", () => {
// This function is a gate. What it reports decides whether a document ships with a
// violation; its rule set is tuned against axe internals (which of the three
// duplicate-id rules claims which element, `duplicate-id-aria` arriving in
// `incomplete`, which shapes `heading-order` reports), and the same rule ids are what
// `GET /v1/quality` publishes deployment-wide. On a caret range that behaviour can
// change on any redeploy with no commit to point at — including a change that makes a
// failure like #144 appear or disappear — and an operator checking out this sha could
// not be sure they had the same linter. equalify-iris-bench ports this configuration so
// its accuracy numbers mean what Iris's mean, which only holds if both can name a version.
const pkg = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")) as {
dependencies: Record<string, string>;
};
const require = createRequire(import.meta.url);
for (const name of ["axe-core", "jsdom"]) {
const spec = pkg.dependencies[name];
assert.ok(spec, `${name} is not a direct dependency`);
assert.match(spec, /^\d+\.\d+\.\d+$/, `${name} is on a range (${spec}); a gate cannot float`);
const installed = (require(`${name}/package.json`) as { version: string }).version;
assert.equal(installed, spec, `${name} in node_modules is not what package.json pins`);
}
});