|
|
@@ -1,218 +1,74 @@
|
|
|
-/**
|
|
|
- * Performance gate for the cold Client fold of a large Session format v2
|
|
|
- * history window: every registered Chat Definition runs over a synthesized
|
|
|
- * window in which each assistant reply embeds its compact stream. The gate
|
|
|
- * bounds the wall time and requires the fold to scale with the number of
|
|
|
- * compact stream records rather than with the number of streamed deltas.
|
|
|
- */
|
|
|
+/** Required performance budget for the compiled cold Client conversation fold. */
|
|
|
|
|
|
+import { join } from 'node:path'
|
|
|
import { describe, expect, it } from 'vitest'
|
|
|
-import { AssistantStreamAccumulator } from '@deepseek-ai/dsh-llm/assistant-stream'
|
|
|
-import type { StreamChunk } from '@deepseek-ai/dsh-llm'
|
|
|
-import type { SessionEvent } from '@deepseek-ai/dsh-session/types'
|
|
|
-import type { ChatSnapshot } from '@deepseek-ai/dsh-client-ui-chat/client'
|
|
|
-import type { SessionEventLikeEntry } from '@deepseek-ai/dsh-api-session-controller/client'
|
|
|
import {
|
|
|
- ConversationNodeAssembler,
|
|
|
- inspectRequestPrompt,
|
|
|
- type ConversationNodeDefinition,
|
|
|
- type ConversationViewDefinition,
|
|
|
-} from '@deepseek-ai/dsh-client-ui-conversation/client'
|
|
|
-import { assistantDefinition } from '../../packages/client/ui-chat/src/client/conversation-nodes/assistant.ts'
|
|
|
-import { chatViewDefinition } from '../../packages/client/ui-chat/src/client/conversation-nodes/chat-snapshot-builder.ts'
|
|
|
-import { commandDefinition } from '../../packages/client/ui-chat/src/client/conversation-nodes/command.ts'
|
|
|
-import { compactionDefinition } from '../../packages/client/ui-chat/src/client/conversation-nodes/compaction.ts'
|
|
|
-import { unknownFallbackDefinition } from '../../packages/client/ui-chat/src/client/conversation-nodes/fallback.ts'
|
|
|
-import { nextStepInboxDefinition } from '../../packages/client/ui-chat/src/client/conversation-nodes/inbox.ts'
|
|
|
-import { messageDefinition } from '../../packages/client/ui-chat/src/client/conversation-nodes/message.ts'
|
|
|
-import { requestPromptDefinition } from '../../packages/client/ui-chat/src/client/conversation-nodes/request-prompt.ts'
|
|
|
-import { retryDefinition } from '../../packages/client/ui-chat/src/client/conversation-nodes/retry.ts'
|
|
|
-import { toolDefinition } from '../../packages/client/ui-chat/src/client/conversation-nodes/tool.ts'
|
|
|
-import { turnErrorDefinition } from '../../packages/client/ui-chat/src/client/conversation-nodes/turn-error.ts'
|
|
|
-import { turnMaxTokensDefinition } from '../../packages/client/ui-chat/src/client/conversation-nodes/turn-max-tokens.ts'
|
|
|
-import { turnProcessDefinition } from '../../packages/client/ui-chat/src/client/conversation-nodes/turn-process.ts'
|
|
|
-import { turnTailDefinition } from '../../packages/client/ui-chat/src/client/conversation-nodes/turn-tail.ts'
|
|
|
+ runBuiltBenchmarkWorker,
|
|
|
+ type BuiltBenchmarkWorkerRun,
|
|
|
+} from '../support/built-worker.ts'
|
|
|
+import type { ConversationFoldWorkerReport } from './conversation-fold.worker.client.ts'
|
|
|
|
|
|
/** Replies in the folded window; each carries one reasoning block and one text block. */
|
|
|
const TURNS = 200
|
|
|
-
|
|
|
-/** Text deltas per reply in the large workload; the reply also streams `deltas / 4` reasoning deltas. */
|
|
|
+/** Text deltas per reply in the large workload; each reply adds one quarter as many reasoning deltas. */
|
|
|
const LARGE_DELTAS = 2_000
|
|
|
-
|
|
|
/** Text deltas per reply in the small workload used as the scaling reference. */
|
|
|
const SMALL_DELTAS = 100
|
|
|
+/** Fresh object graphs measured in one compiled worker; the fastest sample removes scheduler delay. */
|
|
|
+const ATTEMPTS = 3
|
|
|
+/** A stuck fold worker is reaped before the outer benchmark deadline. */
|
|
|
+const WORKER_TIMEOUT_MS = 60_000
|
|
|
|
|
|
/**
|
|
|
- * Wall-clock budget for folding the large window (200 replies, 500,000 streamed
|
|
|
- * deltas compacted into 800 stream records). The pre-stack fold processed the
|
|
|
- * equivalent packed chunk rows in a few milliseconds; the budget leaves room
|
|
|
- * for the complete Definition set and slower CI hosts while staying below the
|
|
|
- * per-delta replay that needed hundreds of milliseconds for this window.
|
|
|
+ * The large window contains 500,000 streamed deltas compacted into 1,600
|
|
|
+ * stream records. The budget separates the record-proportional fold from the
|
|
|
+ * per-delta replay that needs hundreds of milliseconds for the same window.
|
|
|
*/
|
|
|
const LARGE_FOLD_BUDGET_MS = 150
|
|
|
|
|
|
/**
|
|
|
- * Maximum ratio between folding the large and the small window. Both windows
|
|
|
- * hold the same number of events and compact records, so a fold that scales
|
|
|
- * with records plus the joined text stays a few times the small fold
|
|
|
- * (about 2.5× measured); a fold that replays every delta grows with the 20×
|
|
|
- * delta count (about 11× measured).
|
|
|
+ * Both windows contain equal event and compact-record counts. A fold over
|
|
|
+ * records plus joined text measures about 2.5×; replaying every delta measures
|
|
|
+ * about 11× as the delta count grows 20×.
|
|
|
*/
|
|
|
const MAX_DELTA_SCALING = 5
|
|
|
|
|
|
-/** Attempts per workload; the gate compares minima so scheduler noise only adds. */
|
|
|
-const ATTEMPTS = 3
|
|
|
-
|
|
|
-const TIME_ZERO = 1_700_000_000_000
|
|
|
-
|
|
|
-class BenchEventDefinitions {
|
|
|
- readonly definitions: readonly ConversationNodeDefinition[] = [
|
|
|
- nextStepInboxDefinition,
|
|
|
- messageDefinition,
|
|
|
- requestPromptDefinition(inspectRequestPrompt),
|
|
|
- assistantDefinition,
|
|
|
- turnProcessDefinition,
|
|
|
- toolDefinition,
|
|
|
- commandDefinition,
|
|
|
- compactionDefinition,
|
|
|
- retryDefinition,
|
|
|
- turnErrorDefinition,
|
|
|
- turnMaxTokensDefinition,
|
|
|
- turnTailDefinition,
|
|
|
- ]
|
|
|
-
|
|
|
- entries(): readonly ConversationNodeDefinition[] {
|
|
|
- return this.definitions
|
|
|
- }
|
|
|
-
|
|
|
- fallbackEntry(): ConversationNodeDefinition {
|
|
|
- return unknownFallbackDefinition
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-class BenchViewDefinitions {
|
|
|
- entries(): readonly ConversationViewDefinition[] {
|
|
|
- return [chatViewDefinition]
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-function entry(seq: number, type: string, data: unknown, extra: Record<string, unknown> = {}): SessionEventLikeEntry {
|
|
|
- return {
|
|
|
- type: 'event',
|
|
|
- event: { seq, time: TIME_ZERO + seq, type, data, ...extra } as unknown as SessionEvent,
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-/**
|
|
|
- * Synthesize one v2 history window: `turns` completed replies whose compact
|
|
|
- * streams are accumulated from `deltas` text deltas and `deltas / 4`
|
|
|
- * reasoning deltas each.
|
|
|
- */
|
|
|
-function synthesizeWindow(turns: number, deltas: number): { readonly entries: readonly SessionEventLikeEntry[]; readonly records: number } {
|
|
|
- const entries: SessionEventLikeEntry[] = []
|
|
|
- let seq = 0
|
|
|
- let records = 0
|
|
|
- const push = (type: string, data: unknown, extra: Record<string, unknown> = {}): void => {
|
|
|
- entries.push(entry(seq, type, data, extra))
|
|
|
- seq += 1
|
|
|
- }
|
|
|
- const reasoningDeltas = Math.floor(deltas / 4)
|
|
|
- for (let turn = 1; turn <= turns; turn += 1) {
|
|
|
- push('turn/start', { turn })
|
|
|
- push('user/message', {
|
|
|
- id: `user-${String(turn)}`,
|
|
|
- role: 'user',
|
|
|
- content: [{ type: 'text', text: `prompt ${String(turn)}` }],
|
|
|
- source: { kind: 'user' },
|
|
|
- }, { surfaceOp: 'append' })
|
|
|
- push('step/start', { turn, step: 1 })
|
|
|
- const accumulator = new AssistantStreamAccumulator()
|
|
|
- let time = TIME_ZERO + seq * 1_000
|
|
|
- const stream = (chunk: StreamChunk): void => {
|
|
|
- accumulator.push({ time, chunk })
|
|
|
- time += 1
|
|
|
- }
|
|
|
- stream({ type: 'block-start', index: 0, blockType: 'reasoning' })
|
|
|
- let reasoning = ''
|
|
|
- for (let index = 0; index < reasoningDeltas; index += 1) {
|
|
|
- const delta = `r${String(index)} `
|
|
|
- reasoning += delta
|
|
|
- stream({ type: 'reasoning-delta', index: 0, text: delta })
|
|
|
- }
|
|
|
- stream({ type: 'block-end', index: 0, block: { type: 'reasoning', text: reasoning } })
|
|
|
- stream({ type: 'block-start', index: 1, blockType: 'text' })
|
|
|
- let text = ''
|
|
|
- for (let index = 0; index < deltas; index += 1) {
|
|
|
- const delta = `w${String(index)} `
|
|
|
- text += delta
|
|
|
- stream({ type: 'text-delta', index: 1, text: delta })
|
|
|
- }
|
|
|
- stream({ type: 'block-end', index: 1, block: { type: 'text', text } })
|
|
|
- const usage = { inputTokens: 100, outputTokens: deltas }
|
|
|
- stream({ type: 'usage', usage })
|
|
|
- stream({ type: 'finish', reason: { kind: 'stop' } })
|
|
|
- const snapshot = accumulator.snapshot()
|
|
|
- records += snapshot.length
|
|
|
- push('assistant/message', {
|
|
|
- turn,
|
|
|
- step: 1,
|
|
|
- message: {
|
|
|
- id: `assistant-${String(turn)}`,
|
|
|
- role: 'assistant',
|
|
|
- content: [{ type: 'reasoning', text: reasoning }, { type: 'text', text }],
|
|
|
- source: { kind: 'model', provider: 'bench', model: 'bench' },
|
|
|
- },
|
|
|
- usage,
|
|
|
- stream: snapshot,
|
|
|
- }, { surfaceOp: 'append' })
|
|
|
- push('step/end', { turn, step: 1 })
|
|
|
- push('turn/end', { turn, reason: { kind: 'completed' } })
|
|
|
- }
|
|
|
- return { entries, records }
|
|
|
-}
|
|
|
-
|
|
|
-function foldOnce(entries: readonly SessionEventLikeEntry[]): { readonly ms: number; readonly nodes: number } {
|
|
|
- const started = performance.now()
|
|
|
- const assembler = new ConversationNodeAssembler(new BenchEventDefinitions(), new BenchViewDefinitions())
|
|
|
- assembler.replaceWindow(entries, false)
|
|
|
- assembler.activateTarget('chat')
|
|
|
- const snapshot = assembler.snapshot('chat') as ChatSnapshot | undefined
|
|
|
- return { ms: performance.now() - started, nodes: snapshot?.order.length ?? 0 }
|
|
|
-}
|
|
|
-
|
|
|
-function bestOf(entries: readonly SessionEventLikeEntry[]): { readonly ms: number; readonly nodes: number } {
|
|
|
- let best = foldOnce(entries)
|
|
|
- for (let attempt = 1; attempt < ATTEMPTS; attempt += 1) {
|
|
|
- const next = foldOnce(entries)
|
|
|
- if (next.ms < best.ms) best = next
|
|
|
- }
|
|
|
- return best
|
|
|
+const WORKER = join(
|
|
|
+ import.meta.dirname,
|
|
|
+ '..',
|
|
|
+ '..',
|
|
|
+ '.dsh-build',
|
|
|
+ 'benchmarks',
|
|
|
+ 'conversation-fold',
|
|
|
+ 'conversation-fold.worker.js',
|
|
|
+)
|
|
|
+
|
|
|
+function requireReport(
|
|
|
+ run: BuiltBenchmarkWorkerRun<ConversationFoldWorkerReport>,
|
|
|
+): ConversationFoldWorkerReport {
|
|
|
+ if (run.report !== undefined) return run.report
|
|
|
+ const stderrLines = run.stderr.trim().split('\n')
|
|
|
+ throw new Error(
|
|
|
+ `conversation-fold worker failed: exit=${String(run.exitCode)}, signal=${String(run.signal)}, `
|
|
|
+ + `timedOut=${String(run.timedOut)}\n${stderrLines.slice(-10).join('\n')}`,
|
|
|
+ )
|
|
|
}
|
|
|
|
|
|
describe('cold Chat fold of a large v2 history window', () => {
|
|
|
- it(`folds ${String(TURNS)} replies with ${String(LARGE_DELTAS)} deltas each within ${String(LARGE_FOLD_BUDGET_MS)} ms and scales with compact records`, () => {
|
|
|
- const small = synthesizeWindow(TURNS, SMALL_DELTAS)
|
|
|
- const large = synthesizeWindow(TURNS, LARGE_DELTAS)
|
|
|
- expect(large.entries.length).toBe(small.entries.length)
|
|
|
- expect(large.records).toBe(small.records)
|
|
|
-
|
|
|
- const smallFold = bestOf(small.entries)
|
|
|
- const largeFold = bestOf(large.entries)
|
|
|
- const scaling = largeFold.ms / Math.max(smallFold.ms, 1)
|
|
|
+ it(`folds ${String(TURNS)} replies with ${String(LARGE_DELTAS)} deltas each within ${String(LARGE_FOLD_BUDGET_MS)} ms and scales with compact records`, async () => {
|
|
|
+ const report = requireReport(await runBuiltBenchmarkWorker<ConversationFoldWorkerReport>({
|
|
|
+ worker: WORKER,
|
|
|
+ args: [String(TURNS), String(SMALL_DELTAS), String(LARGE_DELTAS), String(ATTEMPTS)],
|
|
|
+ timeoutMs: WORKER_TIMEOUT_MS,
|
|
|
+ }))
|
|
|
console.log(JSON.stringify({
|
|
|
benchmark: 'conversation-fold/large-window',
|
|
|
- events: large.entries.length,
|
|
|
- compactRecords: large.records,
|
|
|
- streamedDeltas: TURNS * (LARGE_DELTAS + Math.floor(LARGE_DELTAS / 4)),
|
|
|
- chatNodes: largeFold.nodes,
|
|
|
- smallFoldMs: Math.round(smallFold.ms * 10) / 10,
|
|
|
- largeFoldMs: Math.round(largeFold.ms * 10) / 10,
|
|
|
- scaling: Math.round(scaling * 100) / 100,
|
|
|
+ ...report,
|
|
|
budgetMs: LARGE_FOLD_BUDGET_MS,
|
|
|
maxScaling: MAX_DELTA_SCALING,
|
|
|
}))
|
|
|
- expect(largeFold.nodes).toBeGreaterThan(0)
|
|
|
- expect(largeFold.ms).toBeLessThanOrEqual(LARGE_FOLD_BUDGET_MS)
|
|
|
- expect(scaling).toBeLessThanOrEqual(MAX_DELTA_SCALING)
|
|
|
+ expect(report.chatNodes).toBeGreaterThan(0)
|
|
|
+ expect(report.largeFoldMs).toBeLessThanOrEqual(LARGE_FOLD_BUDGET_MS)
|
|
|
+ expect(report.scaling).toBeLessThanOrEqual(MAX_DELTA_SCALING)
|
|
|
})
|
|
|
})
|
imccyu