Просмотр исходного кода

test(perf): freeze released benchmark rows

imccyu 1 месяц назад
Родитель
Сommit
ec954d495e

+ 2 - 2
.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.i18n.yaml

@@ -2,5 +2,5 @@
 # side as of the last confirmed-consistent state. Both languages carry equal authority;
 # after editing either side, bring the other along and re-record with:
 #   pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.md
-2026-09-04-session-open-performance-gate.md: 701d1a1fe275659b36e7999bc847602ac329a2c8
-2026-09-04-session-open-performance-gate.zh.md: 0ff80764b6a78089664cd5853fe0cc1ad65e7ccd
+2026-09-04-session-open-performance-gate.md: 30c4345a7c1d62c8af0d3d66505e8614935e9618
+2026-09-04-session-open-performance-gate.zh.md: 61e7ed1e87ffcd5c083c6bd4ab6516dbd7031837

+ 1 - 1
.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.md

@@ -16,7 +16,7 @@ Linux pull requests run a required `node 24 / benchmarks` job that executes `pnp
 
 Required performance gates live under top-level `benchmarks/`, grouped by measured user path rather than package ownership. Host files use `*.bench.ts`, Client-face files use `*.bench.client.ts`, and scenario-specific workers and fixtures stay beside their benchmark without a benchmark suffix. Package-local `.perf.ts` files remain non-gating diagnostics; `scripts/` owns orchestration rather than benchmark cases.
 
-The Session benchmarks synthesize a released-v0 input from fixed parameters: 200 turns with 500 text deltas and 125 reasoning deltas per turn, for 127,400 logical events. The input uses Zstandard with fixed logical-row grouping and frame partitioning, so every run processes the same events, bytes, and frame distribution. Setup writes the input into a private temporary directory for each sample before timing starts; benchmarks never use recorded Sessions.
+The Session benchmarks synthesize a released-v0 input from fixed parameters: 200 turns with 500 text deltas and 125 reasoning deltas per turn, for 127,400 logical events. The input uses Zstandard with fixed logical-row grouping and frame partitioning, so every run processes the same events, bytes, and frame distribution. The fixture constructs the immutable released-v0 physical rows directly instead of depending on a current-runtime historical encoder; compression and every measured read or migration entry point still use production code. Setup writes the input into a private temporary directory for each sample before timing starts; benchmarks never use recorded Sessions.
 
 Every Session endpoint runs at two user-lifecycle points. `first-open` starts with only the released V0 generation and therefore includes migration and successor publication. Setup produces `post-upgrade-reopen` once through that same production migration outside measurement, then copies both the unchanged V0 predecessor and published V2 successor into each sample root. Reopen samples use a fresh process, so they measure an upgraded user's later disk open without migration or process-local caches.
 

+ 1 - 1
.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.zh.md

@@ -16,7 +16,7 @@ Linux pull request 运行必需的 `node 24 / benchmarks` job,执行 `pnpm run
 
 必需性能 gate 位于顶层 `benchmarks/`,按被测用户路径而非 package 归属组织。Host 文件使用 `*.bench.ts`,Client 面文件使用 `*.bench.client.ts`,场景专属 worker 与 fixture 留在对应 benchmark 旁且不带 benchmark 后缀。包内 `.perf.ts` 文件仍是非门禁诊断;`scripts/` 负责编排而不承载 benchmark case。
 
-Session benchmark 使用固定参数合成 released-v0 输入:200 轮,每轮 500 个 text delta 与 125 个 reasoning delta,共 127,400 个逻辑事件。输入使用 Zstandard,并固定 logical rows 的分组与 frame 拆分,使每次运行处理相同的事件、字节与 frame 分布。输入在计时前写入每个样本独占的临时目录;benchmark 不使用录制的 Session。
+Session benchmark 使用固定参数合成 released-v0 输入:200 轮,每轮 500 个 text delta 与 125 个 reasoning delta,共 127,400 个逻辑事件。输入使用 Zstandard,并固定 logical rows 的分组与 frame 拆分,使每次运行处理相同的事件、字节与 frame 分布。fixture 直接构造不可变的 released-v0 physical rows,不依赖当前 runtime 的历史 encoder;压缩以及所有被测读取和 migration 入口仍使用生产代码。输入在计时前写入每个样本独占的临时目录;benchmark 不使用录制的 Session。
 
 每个 Session endpoint 都针对用户生命周期中的两个时点运行。`first-open` 最初只有 released V0 generation,因此包含 migration 与后继 generation 发布。测试准备阶段在计时外通过同一套生产 migration 生成一次 `post-upgrade-reopen`,再把未改动的 V0 前代和已发布的 V2 后继一起复制到每个样本目录。Reopen 样本使用全新进程,因此测量用户升级完成后的磁盘再次打开,不包含 migration 或进程内 cache。
 

+ 101 - 64
benchmarks/session-open/synthetic-released-v0-session.ts

@@ -2,8 +2,6 @@
 
 import { mkdir, writeFile } from 'node:fs/promises'
 import { join } from 'node:path'
-import { releasedV0SessionFormatCodec } from '@deepseek-ai/dsh-session-format-v0-to-v1'
-import type { SessionFormatEvent } from '@deepseek-ai/dsh-session-format'
 import { compressZstdFrame } from '../../packages/session/session-persistence-jsonl/src/zstd.ts'
 
 /** Fixed workload parameters used by every Session-opening scenario. */
@@ -25,14 +23,7 @@ const TIME_ZERO = 1_700_000_000_000
 /** One body frame per row preserves the historical many-frame workload deterministically. */
 const ROWS_PER_FRAME = 1
 
-interface SyntheticEvent {
-  readonly type: string
-  readonly seq: number
-  readonly time: number
-  readonly data: unknown
-  readonly sourceEventSeqs?: readonly number[]
-  readonly surfaceOp?: 'append'
-}
+type SyntheticPhysicalRow = Readonly<Record<string, unknown>>
 
 /** Complete metadata returned after writing one synthetic source generation. */
 export interface SyntheticV0SessionWrite {
@@ -44,50 +35,52 @@ export interface SyntheticV0SessionWrite {
   readonly frames: number
 }
 
-function synthesizeEvents(shape: SyntheticV0SessionShape): readonly SyntheticEvent[] {
-  const events: SyntheticEvent[] = []
-  let seq = 0
-  let time = TIME_ZERO
-  const push = (type: string, data: unknown, extra: Partial<SyntheticEvent> = {}): number => {
-    events.push({ type, seq, time, data, ...extra })
-    seq += 1
-    time += 1
-    return seq - 1
+/** Owns the immutable released-v0 physical layout used only as benchmark input. */
+class ReleasedV0FixtureBuilder {
+  readonly rows: SyntheticPhysicalRow[] = []
+  private seq = 0
+  private time = TIME_ZERO
+
+  get eventCount(): number {
+    return this.seq
+  }
+
+  appendTurns(shape: SyntheticV0SessionShape): void {
+    const reasoningDeltas = Math.floor(shape.textDeltas / 4)
+    for (let turn = 1; turn <= shape.turns; turn += 1) {
+      this.appendTurn(turn, reasoningDeltas, shape.textDeltas)
+    }
   }
-  const reasoningDeltas = Math.floor(shape.textDeltas / 4)
-  for (let turn = 1; turn <= shape.turns; turn += 1) {
-    push('turn/start', { turn })
-    push('user/message', {
+
+  private appendTurn(turn: number, reasoningDeltaCount: number, textDeltaCount: number): void {
+    this.appendEvent('turn/start', { turn })
+    this.appendEvent('user/message', {
       id: `user-${String(turn)}`,
       role: 'user',
       content: [{ type: 'text', text: `prompt ${String(turn)}` }],
       source: { kind: 'user' },
     }, { surfaceOp: 'append' })
-    push('step/start', { turn, step: 1 })
-    const chunkSeqs: number[] = []
-    const chunk = (value: unknown): void => {
-      chunkSeqs.push(push('assistant/chunk', { turn, step: 1, chunk: value }))
-    }
-    chunk({ type: 'block-start', index: 0, blockType: 'reasoning' })
-    let reasoning = ''
-    for (let index = 0; index < reasoningDeltas; index += 1) {
-      const delta = `r${String(index)} `
-      reasoning += delta
-      chunk({ type: 'reasoning-delta', index: 0, text: delta })
-    }
-    chunk({ type: 'block-end', index: 0, block: { type: 'reasoning', text: reasoning } })
-    chunk({ type: 'block-start', index: 1, blockType: 'text' })
-    let text = ''
-    for (let index = 0; index < shape.textDeltas; index += 1) {
-      const delta = `w${String(index)} `
-      text += delta
-      chunk({ type: 'text-delta', index: 1, text: delta })
-    }
-    chunk({ type: 'block-end', index: 1, block: { type: 'text', text } })
-    const usage = { inputTokens: 100, outputTokens: shape.textDeltas }
-    chunk({ type: 'usage', usage })
-    chunk({ type: 'finish', reason: { kind: 'stop' } })
-    push('assistant/message', {
+    this.appendEvent('step/start', { turn, step: 1 })
+    const firstChunkSeq = this.appendChunk(turn, { type: 'block-start', index: 0, blockType: 'reasoning' })
+    const reasoningDeltas = Array.from(
+      { length: reasoningDeltaCount },
+      (_, index) => `r${String(index)} `,
+    )
+    this.appendDeltas('reasoning', turn, 0, reasoningDeltas)
+    const reasoning = reasoningDeltas.join('')
+    this.appendChunk(turn, { type: 'block-end', index: 0, block: { type: 'reasoning', text: reasoning } })
+    this.appendChunk(turn, { type: 'block-start', index: 1, blockType: 'text' })
+    const textDeltas = Array.from(
+      { length: textDeltaCount },
+      (_, index) => `w${String(index)} `,
+    )
+    this.appendDeltas('text', turn, 1, textDeltas)
+    const text = textDeltas.join('')
+    this.appendChunk(turn, { type: 'block-end', index: 1, block: { type: 'text', text } })
+    const usage = { inputTokens: 100, outputTokens: textDeltaCount }
+    this.appendChunk(turn, { type: 'usage', usage })
+    const lastChunkSeq = this.appendChunk(turn, { type: 'finish', reason: { kind: 'stop' } })
+    this.appendEvent('assistant/message', {
       turn,
       step: 1,
       message: {
@@ -97,11 +90,58 @@ function synthesizeEvents(shape: SyntheticV0SessionShape): readonly SyntheticEve
         source: { kind: 'model', provider: 'bench', model: 'bench' },
       },
       usage,
-    }, { sourceEventSeqs: chunkSeqs, surfaceOp: 'append' })
-    push('step/end', { turn, step: 1 })
-    push('turn/end', { turn, reason: { kind: 'completed' } })
+    }, { sourceEventSeqs: [[firstChunkSeq, lastChunkSeq]], surfaceOp: 'append' })
+    this.appendEvent('step/end', { turn, step: 1 })
+    this.appendEvent('turn/end', { turn, reason: { kind: 'completed' } })
+  }
+
+  private appendChunk(turn: number, chunk: unknown): number {
+    return this.appendEvent('assistant/chunk', { turn, step: 1, chunk })
+  }
+
+  private appendDeltas(
+    kind: 'reasoning' | 'text',
+    turn: number,
+    index: number,
+    deltas: readonly string[],
+  ): void {
+    if (deltas.length < 3) {
+      for (const delta of deltas) {
+        this.appendChunk(turn, {
+          type: `${kind}-delta`,
+          index,
+          text: delta,
+        })
+      }
+      return
+    }
+    this.rows.push({
+      type: `${kind}-chunks`,
+      seq0: this.seq,
+      time0: this.time,
+      data: {
+        turn,
+        step: 1,
+        index,
+        dt: Array.from({ length: deltas.length - 1 }, () => 1),
+        texts: deltas,
+      },
+    })
+    this.seq += deltas.length
+    this.time += deltas.length
+  }
+
+  private appendEvent(
+    type: string,
+    data: unknown,
+    extra: Readonly<Record<string, unknown>> = {},
+  ): number {
+    const seq = this.seq
+    this.rows.push({ type, seq, time: this.time, data, ...extra })
+    this.seq += 1
+    this.time += 1
+    return seq
   }
-  return events
 }
 
 /**
@@ -114,24 +154,21 @@ export async function writeSyntheticReleasedV0Session(
   root: string,
   shape: SyntheticV0SessionShape,
 ): Promise<SyntheticV0SessionWrite> {
-  const events = synthesizeEvents(shape)
+  const fixture = new ReleasedV0FixtureBuilder()
+  fixture.appendTurns(shape)
   const header = {
+    type: 'session',
     version: 0,
     id: SYNTHETIC_SESSION_ID,
     createdAt: TIME_ZERO,
     cwd: SYNTHETIC_SESSION_CWD,
-    isSeeded: false,
     delegationDepth: 0,
   }
-  const encoded = releasedV0SessionFormatCodec.encodeArtifact(
-    { header, inheritedEventCount: 0, events: events as unknown as readonly SessionFormatEvent[] },
-    { packChunks: true },
-  )
-  const headerLine = `${JSON.stringify(encoded.header)}\n`
+  const headerLine = `${JSON.stringify(header)}\n`
   const bodyFrames: Buffer[] = []
   let logicalBytes = Buffer.byteLength(headerLine)
-  for (let index = 0; index < encoded.rows.length; index += ROWS_PER_FRAME) {
-    const rows = encoded.rows.slice(index, index + ROWS_PER_FRAME)
+  for (let index = 0; index < fixture.rows.length; index += ROWS_PER_FRAME) {
+    const rows = fixture.rows.slice(index, index + ROWS_PER_FRAME)
     const text = `${rows.map(row => JSON.stringify(row)).join('\n')}\n`
     logicalBytes += Buffer.byteLength(text)
     bodyFrames.push(await compressZstdFrame(text))
@@ -148,8 +185,8 @@ export async function writeSyntheticReleasedV0Session(
     path,
     compressedBytes: physical.byteLength,
     logicalBytes,
-    events: events.length,
-    rows: encoded.rows.length,
-    frames: encoded.rows.length + 1,
+    events: fixture.eventCount,
+    rows: fixture.rows.length,
+    frames: fixture.rows.length + 1,
   }
 }