1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
recordExample,
eligibleExamples,
examplesForPrompt,
lessonSlug,
CORROBORATION_THRESHOLD,
} from "../src/pipeline/memory.ts";
import type { Paths } from "../src/store/paths.ts";
// memory.ts only ever calls paths.agentMemory(agentFile), so a duck-typed stub is
// enough to exercise it against a temp directory.
function fakePaths(dir: string): Paths {
return { agentMemory: (agent: string) => join(dir, `${agent.replace(/\.md$/, "")}.json`) } as unknown as Paths;
}
function withTemp(fn: (paths: Paths) => void): void {
const dir = mkdtempSync(join(tmpdir(), "iris-mem-"));
try {
fn(fakePaths(dir));
} finally {
rmSync(dir, { recursive: true, force: true });
}
}
const LESSON = "Mark decorative images with empty alt text";
test("a generalizable lesson needs corroboration before it is eligible to inject", () => {
withTemp((paths) => {
recordExample(paths, {
agent: "page.md",
kind: "generalizable",
instruction: LESSON,
before: "<img src='x'>",
after: "<img src='x' alt=''>",
feedback: "this image is decorative",
session: "ses_1",
});
assert.equal(eligibleExamples(paths, "page.md").length, 0, "one session: not yet corroborated");
assert.equal(examplesForPrompt(paths, "page.md"), "", "nothing eligible -> empty injection");
// Same lesson, a different session -> corroborated.
const entry = recordExample(paths, {
agent: "page.md",
kind: "generalizable",
instruction: LESSON,
before: "<img src='y'>",
after: "<img src='y' alt=''>",
feedback: "another decorative image",
session: "ses_2",
});
assert.equal(entry.count, CORROBORATION_THRESHOLD, "two distinct sessions counted");
const eligible = eligibleExamples(paths, "page.md");
assert.equal(eligible.length, 1, "corroborated -> eligible");
assert.match(examplesForPrompt(paths, "page.md"), /decorative|alt/i);
});
});
test("re-recording within the same session does not inflate the corroboration count", () => {
withTemp((paths) => {
recordExample(paths, { agent: "page.md", kind: "generalizable", instruction: LESSON, before: "", after: "", feedback: "x", session: "ses_1" });
const entry = recordExample(paths, { agent: "page.md", kind: "generalizable", instruction: LESSON, before: "", after: "", feedback: "x again", session: "ses_1" });
assert.equal(entry.count, 1, "same session should count once");
assert.equal(eligibleExamples(paths, "page.md").length, 0);
});
});
test("an a11y_policy lesson is eligible immediately, without corroboration", () => {
withTemp((paths) => {
recordExample(paths, {
agent: "page.md",
kind: "a11y_policy",
instruction: "Headings must not skip levels",
before: "",
after: "",
feedback: "h1 jumped to h3",
session: "ses_1",
});
assert.equal(eligibleExamples(paths, "page.md", ["a11y_policy"]).length, 1);
assert.match(examplesForPrompt(paths, "page.md", ["a11y_policy"]), /Headings must not skip levels/);
});
});
// ---------------------------------------------------------------------------
// lessonSlug: the dedupe key of the issue a lesson gets proposed on
// ---------------------------------------------------------------------------
//
// The slug's job is to make the title of an `Agent update proposal:` issue depend on
// WHICH lesson it proposes. Without it that title had exactly one possible value on the
// feedback path (`agentFile` is hardcoded to page.md), so the first open issue deduped
// away every later lesson from every user for as long as it stayed open.
//
// Two properties matter and they pull against each other: the slug must agree with the
// bank's own notion of lesson identity — or a repeat report opens a second issue — and it
// must differ between different lessons, or one issue swallows unrelated ones.
test("the slug agrees with the bank on which lessons are the same lesson", () => {
withTemp((paths) => {
// `recordExample` dedupes on the instruction normalized (case, punctuation and
// whitespace folded), so two reports it corroborates into ONE entry have to slug
// identically — otherwise they land on two issues while the bank counts them as one.
const a = "Preserve all hyperlinks from the source document";
const b = " preserve ALL hyperlinks, from the source document! ";
for (const [i, instruction] of [a, b].entries()) {
const entry = recordExample(paths, {
agent: "page.md",
kind: "generalizable",
instruction,
before: "",
after: "",
feedback: "links were dropped",
session: `ses_${i}`,
});
assert.equal(entry.count, i + 1, "the bank did not treat these as the same lesson");
}
assert.equal(lessonSlug(a), lessonSlug(b));
});
});
test("different lessons get different slugs", () => {
// The whole point: two lessons for the same agent must not compute one title. Both of
// these are real page.md lessons from user feedback on the UIC deployment.
assert.notEqual(
lessonSlug("Preserve all hyperlinks from the source document"),
lessonSlug("Transcribe visible text faithfully, including typos in the source"),
);
});
test("a slug is short, whole-worded, and does not end mid-thought", () => {
const slug = lessonSlug(
"Preserve all hyperlinks from the source document, including inline links within body text, " +
"so that anchor elements and their href values are carried over faithfully",
);
assert.ok(slug.length <= 60, `slug too long for a title: ${slug.length} chars`);
assert.equal(slug, "preserve all hyperlinks from the source document");
// Truncation must not leave a dangling connective ("…from the source document
// including"), which reads as a bug in the title of a public issue.
assert.doesNotMatch(slug, /\b(and|or|the|a|an|of|to|in|for|from|with|including)$/);
});
test("an unsluggable lesson yields nothing rather than a fragment", () => {
// The empty case has a caller behind it: `createAgentUpdateIssue` falls back to the
// bare title when the slug is empty, rather than titling an issue with a fragment.
assert.equal(lessonSlug(""), "");
assert.equal(lessonSlug(" ...!!! "), "");
assert.equal(lessonSlug("Use <th> for header cells"), "use th for header cells");
// One word longer than the cap is kept whole: a single word still discriminates
// between lessons, an empty slug does not.
const long = "a".repeat(80);
assert.equal(lessonSlug(long), long);
});
// The bank is shared: it is keyed by AGENT FILE, not by session, and
// `defaults.max_concurrent_runs` allows more than one run at a time — so a page being
// extracted in one run reads this file while another run records a lesson to it. A
// plain write is not atomic, and `loadExamples` answers a partial read with `[]`: that
// page would be extracted with no lessons at all, accessibility-policy ones included,
// and nothing would say so.
test("a lesson is written atomically, so a concurrent reader never sees half of it", () => {
const dir = mkdtempSync(join(tmpdir(), "iris-memory-atomic-"));
try {
const p = fakePaths(dir);
for (let i = 0; i < 8; i++) {
recordExample(p, {
agent: "page.md",
kind: "a11y_policy",
instruction: `Lesson number ${i} with enough words to make the file worth tearing.`,
before: "<h2>Sub</h2>",
after: "<h3>Sub</h3>",
feedback: "the subheadings are all h2",
session: `ses_${i}`,
});
}
const file = join(dir, "page.json");
// Whatever a reader opens must parse: the write goes to a temporary and is renamed
// into place, so the path is never a half-written file.
const raw = readFileSync(file, "utf8");
assert.doesNotThrow(() => JSON.parse(raw), "the bank on disk is not parseable JSON");
assert.equal(JSON.parse(raw).length, 8);
// And the temporary does not survive to be mistaken for the bank.
assert.deepEqual(
readdirSync(dir).filter((f) => f.includes(".tmp")),
[],
"a temporary file was left behind",
);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});