|
|
@@ -4,7 +4,7 @@ import { fileURLToPath } from 'node:url'
|
|
|
import { dirname, join } from 'node:path'
|
|
|
import { describe, expect, it } from 'vitest'
|
|
|
import { type HarvestedLog, type InputScript, runScenario } from './snapshot-harness.ts'
|
|
|
-import { type NormalizeContext, normalizeSessionLog, normalizeStdout } from './snapshot-normalize.ts'
|
|
|
+import { type NormalizeContext, normalizeSessionLog, normalizeStdout, scrubRequestHeaders } from './snapshot-normalize.ts'
|
|
|
|
|
|
/**
|
|
|
* ACP snapshot tests (REPLAY by default, keyless). Each scenario under
|
|
|
@@ -16,6 +16,16 @@ import { type NormalizeContext, normalizeSessionLog, normalizeStdout } from './s
|
|
|
* golden: the fixture doubles as the replay source (recorded scenarios) and the
|
|
|
* expected produced log (both sides normalized before comparing).
|
|
|
*
|
|
|
+ * Request-header content (the composed system prompt + tool schemas riding on
|
|
|
+ * `request/header` events) is pinned by exactly ONE scenario — the one with
|
|
|
+ * `pinsHeader` — and scrubbed to `{{system}}`/`{{tools}}` tokens in every
|
|
|
+ * other fixture and compare, so a prompt or tool-schema edit churns one
|
|
|
+ * committed line instead of every fixture. A per-run uniformity guard keeps
|
|
|
+ * the single pin sound: every live header must equal the pinned one, and no
|
|
|
+ * header-delta may appear outside the pinning scenario (see the
|
|
|
+ * pinned-header RFC,
|
|
|
+ * docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md).
|
|
|
+ *
|
|
|
* `pnpm run test:snapshot:record` (DSH_SNAPSHOT=record + -u) re-records the
|
|
|
* `session.jsonl` fixtures against the real API and refreshes the stdout golden
|
|
|
* in one pass.
|
|
|
@@ -55,12 +65,31 @@ interface Scenario {
|
|
|
* harvested child logs back to those files. Defaults to 0.
|
|
|
*/
|
|
|
childSessions?: number
|
|
|
+ /**
|
|
|
+ * Whether THIS scenario's fixtures keep the full request-header content (the
|
|
|
+ * composed system prompt and tool schema list on `request/header` /
|
|
|
+ * `request/header-delta` events) and compare it verbatim. Exactly one
|
|
|
+ * scenario pins it; every other scenario stores and compares that content as
|
|
|
+ * `{{system}}`/`{{tools}}` tokens ({@link scrubRequestHeaders}), so a system
|
|
|
+ * prompt or tool-schema change shows up as ONE committed-fixture diff, not
|
|
|
+ * one per scenario. One pin suffices because header composition is
|
|
|
+ * suite-uniform (parent, spawn child, and fork child all compose the same
|
|
|
+ * prompt-modulo-cwd and the same tools) — and that premise is ASSERTED, not
|
|
|
+ * assumed: every non-pinning run's live headers must equal the pinned
|
|
|
+ * fixture's (normalized), so a session-dependent header (say, a restricted
|
|
|
+ * subagent toolset) fails loud until it gets its own pinning scenario.
|
|
|
+ * Defaults to false.
|
|
|
+ */
|
|
|
+ pinsHeader?: boolean
|
|
|
}
|
|
|
|
|
|
const SCENARIOS: Scenario[] = [
|
|
|
{ name: 'handshake', hasModelTurn: false, recorded: false },
|
|
|
{ name: 'reject-extra-dirs', hasModelTurn: false, recorded: false },
|
|
|
- { name: 'text-turn', hasModelTurn: true, recorded: true },
|
|
|
+ // text-turn is the pinned-header scenario: the minimal single text turn,
|
|
|
+ // whose fixture is the ONE place the full system prompt + tool schemas are
|
|
|
+ // committed and compared verbatim.
|
|
|
+ { name: 'text-turn', hasModelTurn: true, recorded: true, pinsHeader: true },
|
|
|
{ name: 'tool-call-turn', hasModelTurn: true, recorded: true },
|
|
|
{ name: 'fs-terminal-card', hasModelTurn: true, recorded: true },
|
|
|
{ name: 'todo-plan', hasModelTurn: true, recorded: true },
|
|
|
@@ -119,6 +148,10 @@ const SCENARIOS: Scenario[] = [
|
|
|
{ name: 'hook-codex-stop-continue', hasModelTurn: true, recorded: true },
|
|
|
]
|
|
|
|
|
|
+/** The single header-pinning scenario. Guarded here (and by a meta-test) so the pin cannot silently vanish. */
|
|
|
+const pinningScenario = SCENARIOS.find(s => s.pinsHeader === true)
|
|
|
+if (pinningScenario === undefined) throw new Error('acp.snapshot: no scenario pins the request-header content')
|
|
|
+
|
|
|
/** The sibling child-fixture paths for a scenario (`session.1.jsonl` …). */
|
|
|
function childFixturePaths(dir: string, childSessions: number): string[] {
|
|
|
return Array.from({ length: childSessions }, (_, i) => join(dir, `session.${i + 1}.jsonl`))
|
|
|
@@ -146,6 +179,30 @@ function fixtureContext(fixture: string): NormalizeContext {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+/**
|
|
|
+ * The `data.header` payload of every `request/header` event in a session
|
|
|
+ * JSONL, in log order, with the log's volatile values scrubbed first
|
|
|
+ * ({@link normalizeSessionLog}) so headers harvested from different runs —
|
|
|
+ * each embedding its own temp cwd in the composed prompt — compare on equal
|
|
|
+ * footing.
|
|
|
+ */
|
|
|
+function normalizedHeaders(rawLog: string, ctx: NormalizeContext): unknown[] {
|
|
|
+ return normalizeSessionLog(rawLog, ctx)
|
|
|
+ .split('\n')
|
|
|
+ .filter(line => line.trim().length > 0)
|
|
|
+ .map(line => JSON.parse(line) as { type?: unknown; data?: { header?: unknown } })
|
|
|
+ .filter(record => record.type === 'request/header')
|
|
|
+ .map(record => record.data?.header)
|
|
|
+}
|
|
|
+
|
|
|
+/** Count the `request/header-delta` events in a session JSONL. */
|
|
|
+function headerDeltaCount(rawLog: string): number {
|
|
|
+ return rawLog.split('\n')
|
|
|
+ .filter(line => line.trim().length > 0)
|
|
|
+ .filter(line => (JSON.parse(line) as { type?: unknown }).type === 'request/header-delta')
|
|
|
+ .length
|
|
|
+}
|
|
|
+
|
|
|
for (const scenario of SCENARIOS) {
|
|
|
describe(`snapshot: ${scenario.name}`, () => {
|
|
|
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the
|
|
|
@@ -181,14 +238,19 @@ for (const scenario of SCENARIOS) {
|
|
|
// RECORD mode (recorded model scenarios only): persist the freshly-harvested
|
|
|
// logs back to their fixtures — the primary to session.jsonl, each child to
|
|
|
// session.<n>.jsonl in harvest order. `--update` refreshes the Vitest
|
|
|
- // goldens but NOT these fixtures, so write them here.
|
|
|
+ // goldens but NOT these fixtures, so write them here. A non-pinning
|
|
|
+ // scenario's fixtures are written header-scrubbed, so a re-record can
|
|
|
+ // never smuggle the full prompt/schema content back into every fixture.
|
|
|
+ const scrub = scenario.pinsHeader === true
|
|
|
+ ? (log: string): string => log
|
|
|
+ : scrubRequestHeaders
|
|
|
if (RECORDING && scenario.recorded && scenario.hasModelTurn) {
|
|
|
expect(result.sessionLogs.length, 'record produced no session log to harvest').toBeGreaterThan(0)
|
|
|
expect(result.sessionLogs.length, `expected ${childSessions + 1} session logs (parent + children)`)
|
|
|
.toBe(childSessions + 1)
|
|
|
- await writeFile(join(dir, 'session.jsonl'), (result.sessionLogs[0] as HarvestedLog).content)
|
|
|
+ await writeFile(join(dir, 'session.jsonl'), scrub((result.sessionLogs[0] as HarvestedLog).content))
|
|
|
for (let i = 1; i < result.sessionLogs.length; i++) {
|
|
|
- await writeFile(join(dir, `session.${i}.jsonl`), (result.sessionLogs[i] as HarvestedLog).content)
|
|
|
+ await writeFile(join(dir, `session.${i}.jsonl`), scrub((result.sessionLogs[i] as HarvestedLog).content))
|
|
|
}
|
|
|
}
|
|
|
|
|
|
@@ -203,15 +265,49 @@ for (const scenario of SCENARIOS) {
|
|
|
// 1:1. Each side passes through normalizeSessionLog, scrubbed against ITS
|
|
|
// OWN volatile values — the live run's via `ctx`, the committed fixture's
|
|
|
// via its own header (a committed file cannot share the live run's ids).
|
|
|
+ // Unless this scenario pins the header, both sides ALSO pass through
|
|
|
+ // scrubRequestHeaders: the live log carries the real prompt/schemas, the
|
|
|
+ // fixture carries the `{{system}}`/`{{tools}}` tokens, and the scrub is
|
|
|
+ // idempotent — so the compare checks the header's presence, position,
|
|
|
+ // reason, and config, but not its bulk content (pinned once, in the
|
|
|
+ // `pinsHeader` scenario).
|
|
|
expect(result.sessionLogs.length, 'this scenario must persist a session log').toBe(childSessions + 1)
|
|
|
const fixtureFiles = ['session.jsonl', ...Array.from({ length: childSessions }, (_, i) => `session.${i + 1}.jsonl`)]
|
|
|
for (let i = 0; i < fixtureFiles.length; i++) {
|
|
|
- const harvested = (result.sessionLogs[i] as HarvestedLog).content
|
|
|
- const fixture = await readFile(join(dir, fixtureFiles[i] as string), 'utf8')
|
|
|
+ const harvested = scrub((result.sessionLogs[i] as HarvestedLog).content)
|
|
|
+ const fixture = scrub(await readFile(join(dir, fixtureFiles[i] as string), 'utf8'))
|
|
|
expect(normalizeSessionLog(harvested, ctx), `${fixtureFiles[i]} mismatch`)
|
|
|
.toEqual(normalizeSessionLog(fixture, fixtureContext(fixture)))
|
|
|
}
|
|
|
}
|
|
|
+
|
|
|
+ // Header-uniformity guard: the single pin is sound only while every
|
|
|
+ // session in the suite composes the SAME header and keeps it for the
|
|
|
+ // whole run. Assert both halves live. (1) Every request/header the run
|
|
|
+ // produced (parent, spawn child, fork child, initial or resume) must
|
|
|
+ // equal the pinned fixture's header after each side is normalized
|
|
|
+ // against its own volatile values. (2) No request/header-delta may
|
|
|
+ // appear at all — a mid-run header change diverges from the pin by
|
|
|
+ // construction, and its content would be invisible under the scrub. If
|
|
|
+ // either fails, either the header changed (update the pin: re-record or
|
|
|
+ // hand-edit the pinning scenario's fixture) or composition became
|
|
|
+ // session-dependent by design (give the divergent shape its own
|
|
|
+ // pinning scenario).
|
|
|
+ if (scenario.pinsHeader !== true) {
|
|
|
+ const pinnedFixture = await readFile(join(SNAPSHOTS_DIR, pinningScenario.name, 'session.jsonl'), 'utf8')
|
|
|
+ const pinned = normalizedHeaders(pinnedFixture, fixtureContext(pinnedFixture))
|
|
|
+ expect(pinned.length, `the pinning fixture (${pinningScenario.name}) must carry exactly one request/header`)
|
|
|
+ .toBe(1)
|
|
|
+ for (const log of result.sessionLogs) {
|
|
|
+ expect(headerDeltaCount(log.content), `session ${log.id}: a request/header-delta in a non-pinning scenario`)
|
|
|
+ .toBe(0)
|
|
|
+ const headers = normalizedHeaders(log.content, ctx)
|
|
|
+ for (const [k, header] of headers.entries()) {
|
|
|
+ expect(header, `session ${log.id}: request/header #${k + 1} diverged from the pinned (${pinningScenario.name}) header`)
|
|
|
+ .toEqual(pinned[0])
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
})
|
|
|
})
|
|
|
}
|
|
|
@@ -253,4 +349,37 @@ describe('snapshot fixtures', () => {
|
|
|
}
|
|
|
}
|
|
|
})
|
|
|
+
|
|
|
+ it('exactly one scenario pins the request-header content', () => {
|
|
|
+ // Zero pins would drop the prompt/schema surface from the suite entirely;
|
|
|
+ // two would split it. The single pin is the design (pinned-header RFC).
|
|
|
+ expect(SCENARIOS.filter(s => s.pinsHeader === true).map(s => s.name)).toEqual(['text-turn'])
|
|
|
+ })
|
|
|
+
|
|
|
+ it('committed fixtures carry request-header content ONLY in the pinning scenario', async () => {
|
|
|
+ // The whole point of the pin: a system-prompt or tool-schema change must
|
|
|
+ // churn exactly one committed line. A non-pinning fixture that carries the
|
|
|
+ // full header (a hand-recorded file, or a header line hand-edited out of
|
|
|
+ // its canonical JSON form) silently reopens the suite-wide churn, so fail
|
|
|
+ // loud here: every non-pinning session*.jsonl must be a fixed point of
|
|
|
+ // scrubRequestHeaders (apply the scrub to fix a violation), and the
|
|
|
+ // pinning scenario's fixtures must NOT be (their content IS the pin).
|
|
|
+ for (const scenario of SCENARIOS) {
|
|
|
+ const dir = join(SNAPSHOTS_DIR, scenario.name)
|
|
|
+ const files = [
|
|
|
+ 'session.jsonl',
|
|
|
+ ...Array.from({ length: scenario.childSessions ?? 0 }, (_, i) => `session.${i + 1}.jsonl`),
|
|
|
+ ]
|
|
|
+ for (const file of files) {
|
|
|
+ const fixture = await readFile(join(dir, file), 'utf8')
|
|
|
+ if (scenario.pinsHeader === true) {
|
|
|
+ expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} must PIN the full header content`)
|
|
|
+ .not.toEqual(fixture)
|
|
|
+ } else {
|
|
|
+ expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} carries unscrubbed header content`)
|
|
|
+ .toEqual(fixture)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ })
|
|
|
})
|
Tianyi Cui