|
|
@@ -1,9 +1,12 @@
|
|
|
/** Required performance budgets for cold Session preparation, first history, and Agent resume. */
|
|
|
|
|
|
-import { copyFile, mkdir, mkdtemp, rm } from 'node:fs/promises'
|
|
|
+import { copyFile, mkdir, mkdtemp, readFile, readdir, rm } from 'node:fs/promises'
|
|
|
import { tmpdir } from 'node:os'
|
|
|
import { join } from 'node:path'
|
|
|
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
|
|
+import { Context } from '@deepseek-ai/cordis'
|
|
|
+import { Session, SessionId, SESSION_FORMAT_VERSION } from '@deepseek-ai/dsh-session'
|
|
|
+import JsonlSessionPersistence from '@deepseek-ai/dsh-session-persistence-jsonl'
|
|
|
import {
|
|
|
runBuiltBenchmarkWorker,
|
|
|
type BuiltBenchmarkWorkerRun,
|
|
|
@@ -18,6 +21,7 @@ import type {
|
|
|
} from './session-open.worker.ts'
|
|
|
import {
|
|
|
SYNTHETIC_CURRENT_GENERATION,
|
|
|
+ SYNTHETIC_SESSION_ID,
|
|
|
SYNTHETIC_SESSION_DIRECTORY,
|
|
|
SYNTHETIC_CURRENT_FILENAME,
|
|
|
SYNTHETIC_V0_FILENAME,
|
|
|
@@ -152,7 +156,10 @@ function requireReport(
|
|
|
if (run.report !== undefined) return run.report
|
|
|
const stderrLines = run.stderr.trim().split('\n')
|
|
|
const fatal = stderrLines.filter(line => /FATAL ERROR|heap limit|out of memory/i.test(line))
|
|
|
- const detail = (fatal.length > 0 ? fatal : stderrLines.slice(-10)).join('\n')
|
|
|
+ const context = stderrLines.length <= 20
|
|
|
+ ? stderrLines
|
|
|
+ : [...stderrLines.slice(0, 10), '... stderr middle omitted ...', ...stderrLines.slice(-10)]
|
|
|
+ const detail = (fatal.length > 0 ? fatal : context).join('\n')
|
|
|
const limit = heapLimitMb === undefined ? 'normal heap' : `${String(heapLimitMb)} MB old space`
|
|
|
throw new Error(
|
|
|
`${scenario} failed under ${limit}: exit=${String(run.exitCode)}, signal=${String(run.signal)}, `
|
|
|
@@ -293,6 +300,67 @@ describe('standard hosted reopen calibration', () => {
|
|
|
})
|
|
|
})
|
|
|
|
|
|
+describe('Session opening benchmark prerequisites', () => {
|
|
|
+ it('retains the exception headline and bounded stderr tail when a worker fails', () => {
|
|
|
+ const headline = 'SessionFormatUnsupportedError: source chronology cannot be migrated'
|
|
|
+ const stderr = [headline, ...Array.from({ length: 30 }, (_, index) => 'stack frame ' + String(index)), 'Node.js test'].join('\n')
|
|
|
+ const run: WorkerRun = { report: undefined, exitCode: 1, signal: null, timedOut: false, stderr }
|
|
|
+ expect(() => requireReport(run, 'agent-resume')).toThrow(headline)
|
|
|
+ expect(() => requireReport(run, 'agent-resume')).toThrow('Node.js test')
|
|
|
+ expect(() => requireReport(run, 'agent-resume')).not.toThrow('stack frame 15')
|
|
|
+ expect(() => requireReport({ ...run, stderr: 'FATAL ERROR: heap limit' }, 'agent-resume', 128))
|
|
|
+ .toThrow('128 MB old space')
|
|
|
+ })
|
|
|
+
|
|
|
+ it('migrates the generated workload and reopens its successor without changing V0', async () => {
|
|
|
+ const root = await mkdtemp(join(tmpdir(), 'dsh-session-bench-fixture-'))
|
|
|
+ const contexts: Context[] = []
|
|
|
+ const mount = async () => {
|
|
|
+ const ctx = new Context()
|
|
|
+ contexts.push(ctx)
|
|
|
+ await ctx.plugin(JsonlSessionPersistence, { root, compression: 'zstd' })
|
|
|
+ return ctx.sessionPersistence
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ const facts = await writeSyntheticReleasedV0Session(root, { turns: 2, textDeltas: 12 })
|
|
|
+ expect({ events: facts.events, rows: facts.rows, frames: facts.frames }).toEqual({ events: 54, rows: 28, frames: 29 })
|
|
|
+ const original = await readFile(facts.path)
|
|
|
+ const directory = join(root, SYNTHETIC_SESSION_DIRECTORY)
|
|
|
+ const persistence = await mount()
|
|
|
+ const read = await persistence.open(SessionId(SYNTHETIC_SESSION_ID), 'read')
|
|
|
+ const initial = await read.read()
|
|
|
+ const session = Session.fromRestore(read.header.id, initial.events, read.header, read.inheritedEventCount, initial.eventState)
|
|
|
+ expect(read.header.version).toBe(SESSION_FORMAT_VERSION)
|
|
|
+ expect(initial.events.filter(event => event.type === 'system/message')).toHaveLength(1)
|
|
|
+ expect(session.deriveMessages().map(({ id, role, content }) => ({ id, role, content }))).toEqual(
|
|
|
+ [1, 2].flatMap(turn => [
|
|
|
+ { id: 'user-' + String(turn), role: 'user', content: [{ type: 'text', text: 'prompt ' + String(turn) }] },
|
|
|
+ { id: 'assistant-' + String(turn), role: 'assistant', content: [
|
|
|
+ { type: 'reasoning', text: 'r0 r1 r2 ' },
|
|
|
+ { type: 'text', text: Array.from({ length: 12 }, (_, index) => 'w' + String(index) + ' ').join('') },
|
|
|
+ ] },
|
|
|
+ ]),
|
|
|
+ )
|
|
|
+ await read.close()
|
|
|
+ expect(await readdir(directory)).toEqual([SYNTHETIC_V0_FILENAME])
|
|
|
+ const writer = await persistence.open(SessionId(SYNTHETIC_SESSION_ID), 'write')
|
|
|
+ expect((await writer.read()).events).toEqual(initial.events)
|
|
|
+ await writer.close()
|
|
|
+ expect(await readdir(directory)).toContain(SYNTHETIC_CURRENT_FILENAME)
|
|
|
+ const reopened = await (await mount()).open(SessionId(SYNTHETIC_SESSION_ID), 'read')
|
|
|
+ expect((await reopened.read()).events).toEqual(initial.events)
|
|
|
+ await reopened.close()
|
|
|
+ expect(await readFile(facts.path)).toEqual(original)
|
|
|
+ } finally {
|
|
|
+ try {
|
|
|
+ for (const ctx of contexts.reverse()) await ctx.fiber.dispose()
|
|
|
+ } finally {
|
|
|
+ await rm(root, { recursive: true, force: true })
|
|
|
+ }
|
|
|
+ }
|
|
|
+ })
|
|
|
+})
|
|
|
+
|
|
|
describe('opening a large Session for first open and post-upgrade reopen', () => {
|
|
|
const suite = new SessionOpenBenchmarkSuite()
|
|
|
|
Tianyi Cui