1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308#!/usr/bin/env node
// Apply a review round's verdicts to the corpus, mechanically.
//
// The seeking agent may not promote its own items and the reviewer may not edit
// the corpus. Something has to move an item from candidate to accepted, and it
// must not be a judgment call: this does it, from the review records only.
//
// accept -> accepted
// revise -> needs-revision
// reject -> rejected
//
// Usage:
// node tools/apply-verdicts.mjs --round 3
// node tools/apply-verdicts.mjs --round 3 --dry-run
// node tools/apply-verdicts.mjs --selftest
//
// Exit codes: 0 applied, 1 nothing to apply, 2 refused because the round's
// records are unusable, 3 bad usage or self-test failure.
import { readFileSync, writeFileSync, existsSync, mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { dirname, join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
const HERE = dirname(fileURLToPath(import.meta.url))
const PROJECT = resolve(HERE, '..')
const VERDICT_STATUS = {
accept: 'accepted',
revise: 'needs-revision',
reject: 'rejected',
}
// An accepted item must still satisfy the rules the schema puts on acceptance.
// The reviewer can be wrong; these are not negotiable, so a bad promotion is
// refused here rather than surfacing as a schema error later.
function blocksAcceptance(item) {
if (item.leaky === true) return 'the item is marked leaky'
const passes = Array.isArray(item.gold_alt_passes) ? item.gold_alt_passes : []
const adjudicated = typeof item.adjudication === 'string' &&
item.adjudication.trim() !== ''
if (adjudicated) return null
if (passes.length < 2) {
return 'the item has a single gold standard pass and no adjudication'
}
const alts = new Set(passes
.filter((p) => p !== null && typeof p === 'object')
.map((p) => p.alt)
.filter((a) => typeof a === 'string'))
if (alts.size > 1) {
return 'its gold standard passes disagree and there is no adjudication'
}
return null
}
function readJsonl(path) {
const rows = []
const errors = []
let text
try {
text = readFileSync(path, 'utf8')
} catch (e) {
return { rows, errors: [`${path}: cannot be read, ${e.code ?? e.message}`] }
}
text.split('\n').forEach((raw, i) => {
const line = raw.trim()
if (line === '') return
try {
rows.push(JSON.parse(line))
} catch (e) {
errors.push(`${path}:${i + 1}: invalid JSON, ${e.message}`)
}
})
return { rows, errors }
}
function apply(corpusPath, reviewPath, round) {
const notes = []
const corpus = readJsonl(corpusPath)
const review = readJsonl(reviewPath)
const errors = [...corpus.errors, ...review.errors]
if (errors.length > 0) return { errors, notes, changed: 0, lines: null }
const byId = new Map()
for (const item of corpus.rows) {
if (typeof item.id === 'string') byId.set(item.id, item)
}
const verdicts = new Map()
for (const rec of review.rows) {
if (rec.round !== round) {
errors.push(`${reviewPath}: record for ${rec.item_id} says round ` +
`${rec.round}, expected ${round}`)
continue
}
if (!(rec.verdict in VERDICT_STATUS)) {
errors.push(`${reviewPath}: unknown verdict "${rec.verdict}" for ` +
`${rec.item_id}`)
continue
}
if (!byId.has(rec.item_id)) {
errors.push(`${reviewPath}: no corpus item ${rec.item_id}`)
continue
}
// A later record for the same item in the same round wins, so a reviewer
// that revisits an item does not leave the outcome ambiguous.
verdicts.set(rec.item_id, rec.verdict)
}
if (errors.length > 0) return { errors, notes, changed: 0, lines: null }
let changed = 0
for (const [id, verdict] of verdicts) {
const item = byId.get(id)
const want = VERDICT_STATUS[verdict]
if (want === 'accepted') {
const blocker = blocksAcceptance(item)
if (blocker) {
errors.push(`${id}: cannot be accepted because ${blocker}`)
continue
}
}
if (item.status !== want) {
notes.push(`${id}: ${item.status} -> ${want}`)
item.status = want
changed++
}
}
if (errors.length > 0) return { errors, notes, changed: 0, lines: null }
const lines = corpus.rows.map((r) => JSON.stringify(r)).join('\n') + '\n'
return { errors, notes, changed, lines }
}
function selftest() {
let failures = 0
const dir = mkdtempSync(join(tmpdir(), 'alt-apply-'))
const corpusPath = join(dir, 'corpus.jsonl')
const reviewPath = join(dir, 'review.jsonl')
const item = (id, over) => ({
id, status: 'candidate', leaky: false, adjudication: null,
gold_alt_passes: [{ author: 'a', alt: 'x', rationale: 'r' },
{ author: 'b', alt: 'x', rationale: 'r' }],
...over,
})
const write = (path, rows) => writeFileSync(path,
rows.map((r) => JSON.stringify(r)).join('\n') + '\n')
const check = (name, cond, detail) => {
if (cond) process.stdout.write(`PASS ${name}\n`)
else { process.stdout.write(`FAIL ${name}: ${detail}\n`); failures++ }
}
// Verdicts map to statuses.
write(corpusPath, [item('fi-0001'), item('fi-0002'), item('fi-0003')])
write(reviewPath, [
{ item_id: 'fi-0001', round: 1, verdict: 'accept' },
{ item_id: 'fi-0002', round: 1, verdict: 'revise' },
{ item_id: 'fi-0003', round: 1, verdict: 'reject' },
])
let r = apply(corpusPath, reviewPath, 1)
const statuses = r.lines === null ? [] : r.lines.trim().split('\n')
.map((l) => JSON.parse(l).status)
check('verdicts map to statuses',
r.errors.length === 0 &&
statuses.join(',') === 'accepted,needs-revision,rejected',
`${r.errors.join('; ')} statuses ${statuses.join(',')}`)
// A leaky item is never promoted, however the reviewer voted.
write(corpusPath, [item('fi-0001', { leaky: true })])
write(reviewPath, [{ item_id: 'fi-0001', round: 1, verdict: 'accept' }])
r = apply(corpusPath, reviewPath, 1)
check('refuses to accept a leaky item',
r.errors.length === 1 && r.errors[0].includes('leaky'),
r.errors.join('; ') || 'no error raised')
// A single-pass item is never promoted without an adjudication.
write(corpusPath, [item('fi-0001',
{ gold_alt_passes: [{ author: 'a', alt: 'x', rationale: 'r' }] })])
write(reviewPath, [{ item_id: 'fi-0001', round: 1, verdict: 'accept' }])
r = apply(corpusPath, reviewPath, 1)
check('refuses to accept a single-pass item',
r.errors.length === 1 && r.errors[0].includes('single gold standard pass'),
r.errors.join('; ') || 'no error raised')
// Disagreeing passes are a legal candidate state, and never an accepted one.
// The disagreement is what a blind second pass exists to surface, so it has to
// survive in the corpus until a seeking agent adjudicates it.
write(corpusPath, [item('fi-0001', {
gold_alt_passes: [{ author: 'pass-a', alt: 'Search', rationale: 'r' },
{ author: 'pass-b', alt: '', rationale: 'r' }],
})])
write(reviewPath, [{ item_id: 'fi-0001', round: 1, verdict: 'accept' }])
r = apply(corpusPath, reviewPath, 1)
check('refuses to accept an unadjudicated disagreement',
r.errors.length === 1 && r.errors[0].includes('disagree'),
r.errors.join('; ') || 'no error raised')
// With the disagreement adjudicated, the same item promotes.
write(corpusPath, [item('fi-0001', {
adjudication: 'Adjacent text repeats the label, so the empty pass wins.',
gold_alt_passes: [{ author: 'pass-a', alt: 'Search', rationale: 'r' },
{ author: 'pass-b', alt: '', rationale: 'r' }],
})])
write(reviewPath, [{ item_id: 'fi-0001', round: 1, verdict: 'accept' }])
r = apply(corpusPath, reviewPath, 1)
check('accepts an adjudicated disagreement',
r.errors.length === 0 && r.changed === 1,
r.errors.join('; ') || `changed ${r.changed}`)
// Round mismatch is refused rather than applied to the wrong round.
write(corpusPath, [item('fi-0001')])
write(reviewPath, [{ item_id: 'fi-0001', round: 2, verdict: 'accept' }])
r = apply(corpusPath, reviewPath, 1)
check('refuses a round mismatch',
r.errors.length === 1 && r.errors[0].includes('expected 1'),
r.errors.join('; ') || 'no error raised')
// An unknown item id is refused.
write(corpusPath, [item('fi-0001')])
write(reviewPath, [{ item_id: 'fi-9999', round: 1, verdict: 'accept' }])
r = apply(corpusPath, reviewPath, 1)
check('refuses an unknown item id',
r.errors.length === 1 && r.errors[0].includes('fi-9999'),
r.errors.join('; ') || 'no error raised')
// Nothing is written when any record is refused: all or nothing.
check('refusal writes nothing', r.lines === null && r.changed === 0,
`changed ${r.changed}`)
// Re-applying the same round is a no-op.
write(corpusPath, [item('fi-0001', { status: 'accepted' })])
write(reviewPath, [{ item_id: 'fi-0001', round: 1, verdict: 'accept' }])
r = apply(corpusPath, reviewPath, 1)
check('re-applying is idempotent', r.errors.length === 0 && r.changed === 0,
`changed ${r.changed}, errors ${r.errors.join('; ')}`)
rmSync(dir, { recursive: true, force: true })
process.stdout.write(failures === 0
? '\napply-verdicts self-test passed\n'
: `\napply-verdicts self-test failed, ${failures} case(s)\n`)
return failures === 0 ? 0 : 3
}
function main(argv) {
let round = null
let dryRun = false
let corpusPath = join(PROJECT, 'corpus', 'functional-images.jsonl')
let roundsDir = join(PROJECT, 'rounds')
for (let i = 0; i < argv.length; i++) {
const arg = argv[i]
if (arg === '--selftest') return selftest()
else if (arg === '--dry-run') dryRun = true
else if (arg === '--round') round = Number(argv[++i])
else if (arg === '--corpus') corpusPath = resolve(argv[++i] ?? '')
else if (arg === '--rounds') roundsDir = resolve(argv[++i] ?? '')
else {
process.stderr.write(`apply-verdicts.mjs: unknown argument "${arg}"\n` +
'usage: apply-verdicts.mjs --round N [--dry-run] [--corpus FILE] ' +
'[--rounds DIR]\n')
return 3
}
}
if (!Number.isInteger(round) || round < 1) {
process.stderr.write('apply-verdicts.mjs: --round needs a whole number of ' +
'1 or more\n')
return 3
}
const reviewPath = join(roundsDir,
`round-${String(round).padStart(2, '0')}-review.jsonl`)
if (!existsSync(corpusPath)) {
process.stdout.write(`no corpus at ${corpusPath}, nothing to apply\n`)
return 1
}
if (!existsSync(reviewPath)) {
process.stdout.write(`no review records at ${reviewPath}, nothing to apply\n`)
return 1
}
const { errors, notes, changed, lines } = apply(corpusPath, reviewPath, round)
if (errors.length > 0) {
process.stdout.write(`refused to apply round ${round}:\n`)
for (const e of errors) process.stdout.write(` ${e}\n`)
process.stdout.write('\nNo corpus changes were written. Fix the review ' +
'records, then apply again.\n')
return 2
}
for (const n of notes) process.stdout.write(` ${n}\n`)
if (changed === 0) {
process.stdout.write(`round ${round}: no status changes needed\n`)
return 0
}
if (dryRun) {
process.stdout.write(`round ${round}: ${changed} status changes, not ` +
'written, --dry-run\n')
return 0
}
writeFileSync(corpusPath, lines)
process.stdout.write(`round ${round}: ${changed} status changes written\n`)
return 0
}
process.exit(main(process.argv.slice(2)))