|
|
@@ -80,14 +80,6 @@ export interface Scenario {
|
|
|
* false (replay derives from the fixture's `assistant/chunk` events).
|
|
|
*/
|
|
|
overridden?: boolean
|
|
|
- /**
|
|
|
- * How many SUBAGENT child sessions this scenario records beyond the top-level
|
|
|
- * one (0 for a single-session scenario). Each child rides in a sibling fixture
|
|
|
- * `session.<n>.jsonl` (1-based); replay forwards them to `dsh-llm-replay` so
|
|
|
- * each child session replays from its own script, and record mode writes the
|
|
|
- * harvested child logs back to those files. Defaults to 0.
|
|
|
- */
|
|
|
- childSessions?: number
|
|
|
/**
|
|
|
* Whether THIS scenario pins its header class's model-facing request-header
|
|
|
* content. Its actual composed prompt is maintained as a readable
|
|
|
@@ -150,14 +142,40 @@ export interface SnapshotSuiteOptions {
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * The sibling child-fixture paths for a scenario (`session.1.jsonl` …).
|
|
|
+ * Validate and order a scenario directory's session-fixture filenames.
|
|
|
+ *
|
|
|
+ * The primary fixture is always `session.jsonl`; child sessions are discovered
|
|
|
+ * from contiguous `session.1.jsonl` … filenames. The directory is the source of
|
|
|
+ * truth, so scenario tables do not duplicate a child count that can drift from
|
|
|
+ * the files. A session-like JSONL with any other suffix fails loud.
|
|
|
*
|
|
|
- * @param dir The scenario's snapshots directory (`<snapshotsDir>/<name>`).
|
|
|
- * @param childSessions How many subagent child sessions the scenario records.
|
|
|
- * @returns One path per child, 1-based, in fixture order.
|
|
|
+ * @param names File names in one scenario directory.
|
|
|
+ * @returns The primary and child fixture names in replay/harvest order.
|
|
|
*/
|
|
|
-export function childFixturePaths(dir: string, childSessions: number): string[] {
|
|
|
- return Array.from({ length: childSessions }, (_, i) => join(dir, `session.${i + 1}.jsonl`))
|
|
|
+export function sessionFixtureNames(names: readonly string[]): string[] {
|
|
|
+ if (!names.includes('session.jsonl')) throw new Error('missing session.jsonl')
|
|
|
+ const children: { name: string; index: number }[] = []
|
|
|
+ for (const name of names) {
|
|
|
+ if (name === 'session.jsonl') continue
|
|
|
+ if (!name.startsWith('session.') || !name.endsWith('.jsonl')) continue
|
|
|
+ const match = /^session\.([1-9]\d*)\.jsonl$/.exec(name)
|
|
|
+ if (match === null) throw new Error(`invalid child session fixture name: ${name}`)
|
|
|
+ children.push({ name, index: Number(match[1]) })
|
|
|
+ }
|
|
|
+ children.sort((a, b) => a.index - b.index)
|
|
|
+ for (const [offset, child] of children.entries()) {
|
|
|
+ const expected = offset + 1
|
|
|
+ if (child.index !== expected) {
|
|
|
+ throw new Error(`child session fixtures must be contiguous: expected session.${expected}.jsonl, found ${child.name}`)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return ['session.jsonl', ...children.map(child => child.name)]
|
|
|
+}
|
|
|
+
|
|
|
+/** Read one scenario directory's validated session-fixture inventory. */
|
|
|
+async function sessionFixtures(dir: string): Promise<string[]> {
|
|
|
+ const entries = await readdir(dir, { withFileTypes: true })
|
|
|
+ return sessionFixtureNames(entries.filter(entry => entry.isFile()).map(entry => entry.name))
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
@@ -392,7 +410,9 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
|
|
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
|
|
|
const overrideFile = join(dir, 'replay.override.json')
|
|
|
const workspaceDir = join(dir, 'workspace')
|
|
|
- const childSessions = scenario.childSessions ?? 0
|
|
|
+ const fixtureFiles = await sessionFixtures(dir)
|
|
|
+ const childFixtureFiles = fixtureFiles.slice(1)
|
|
|
+ const childSessions = childFixtureFiles.length
|
|
|
const comparesLog = scenario.comparesLog ?? scenario.hasModelTurn
|
|
|
const result = await runScenario(input, {
|
|
|
agent,
|
|
|
@@ -401,7 +421,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
|
|
...existsSync(overrideFile) ? { overrideFile } : {},
|
|
|
// In REPLAY, forward the recorded child fixtures so each subagent session
|
|
|
// replays from its own script. In RECORD they are harvested, not read.
|
|
|
- ...!RECORDING && childSessions > 0 ? { childFiles: childFixturePaths(dir, childSessions) } : {},
|
|
|
+ ...!RECORDING && childSessions > 0 ? { childFiles: childFixtureFiles.map(file => join(dir, file)) } : {},
|
|
|
...existsSync(workspaceDir) ? { workspaceDir } : {},
|
|
|
// A scenario booting an overlay tree passes its own live config; the
|
|
|
// bin's replay swap derives the sibling `*cordis.snapshot.yml` from it.
|
|
|
@@ -432,7 +452,6 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
|
|
const scrub = scenario.pinsHeader === true
|
|
|
? scrubSystemPrompts
|
|
|
: scrubRequestHeaders
|
|
|
- const fixtureFiles = ['session.jsonl', ...Array.from({ length: childSessions }, (_, i) => `session.${i + 1}.jsonl`)]
|
|
|
const existingFixtures = REFRESHING
|
|
|
? await Promise.all(fixtureFiles.map(file => readFile(join(dir, file), 'utf8')))
|
|
|
: []
|
|
|
@@ -543,7 +562,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
|
|
expect(onDisk).toEqual(registered)
|
|
|
})
|
|
|
|
|
|
- it('every registered scenario has its required fixture files', () => {
|
|
|
+ it('every registered scenario has its required fixture files', async () => {
|
|
|
// Every scenario has an input script and an stdout golden. EVERY scenario
|
|
|
// also needs `session.jsonl`: the suite boots `llm-replay` with that path
|
|
|
// as the replay source for ALL scenarios (the factory passes
|
|
|
@@ -556,7 +575,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
|
|
// `overridden` flag: required when set, forbidden when not — the harness
|
|
|
// forwards the file purely on existence, so an unregistered stray sidecar
|
|
|
// would silently replace the derived script.
|
|
|
- for (const { name, overridden, childSessions, pinsHeader } of scenarios) {
|
|
|
+ for (const { name, overridden, pinsHeader } of scenarios) {
|
|
|
const dir = join(snapshotsDir, name)
|
|
|
expect(existsSync(join(dir, 'input.json')), `${name}/input.json`).toBe(true)
|
|
|
expect(existsSync(join(dir, 'stdout.golden.jsonl')), `${name}/stdout.golden.jsonl`).toBe(true)
|
|
|
@@ -565,11 +584,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
|
|
.toBe(overridden === true)
|
|
|
expect(existsSync(join(dir, SYSTEM_PROMPT_SNAPSHOT)), `${name}/${SYSTEM_PROMPT_SNAPSHOT} presence must match \`pinsHeader\``)
|
|
|
.toBe(pinsHeader === true)
|
|
|
- // A nested-agent scenario ships one child fixture per recorded subagent
|
|
|
- // session (`session.1.jsonl` …), the replay source for that child session.
|
|
|
- for (const childFixture of childFixturePaths(dir, childSessions ?? 0)) {
|
|
|
- expect(existsSync(childFixture), childFixture).toBe(true)
|
|
|
- }
|
|
|
+ await expect(sessionFixtures(dir), `${name}: session fixture inventory`).resolves.toBeDefined()
|
|
|
}
|
|
|
})
|
|
|
|
|
|
@@ -615,10 +630,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
|
|
// all header bulk. Fixed-point checks make both storage rules fail loud.
|
|
|
for (const scenario of scenarios) {
|
|
|
const dir = join(snapshotsDir, scenario.name)
|
|
|
- const files = [
|
|
|
- 'session.jsonl',
|
|
|
- ...Array.from({ length: scenario.childSessions ?? 0 }, (_, i) => `session.${i + 1}.jsonl`),
|
|
|
- ]
|
|
|
+ const files = await sessionFixtures(dir)
|
|
|
for (const file of files) {
|
|
|
const fixture = await readFile(join(dir, file), 'utf8')
|
|
|
expect(scrubSystemPrompts(fixture), `${scenario.name}/${file} carries an unscrubbed system prompt`)
|
Tianyi Cui