complex-history.perf.ts 52 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444
  1. // Opt-in browser benchmark for high-cardinality workspace and history
  2. // rendering. It reports measurements without timing assertions because host
  3. // speed is not a correctness contract; structural assertions keep the number
  4. // of workspaces and history entries from silently shrinking.
  5. import { mkdtemp, rm, writeFile } from 'node:fs/promises'
  6. import { tmpdir } from 'node:os'
  7. import { join } from 'node:path'
  8. import { performance } from 'node:perf_hooks'
  9. import type { Browser, CDPSession, Locator, Page } from 'playwright'
  10. import { chromium } from 'playwright'
  11. import { afterAll, beforeAll, describe, expect, it } from 'vitest'
  12. import type { StreamChunk } from '@deepseek-ai/dsh-llm'
  13. import {
  14. ToolCallId,
  15. createAssistantMessage,
  16. createToolResultMessage,
  17. createUserMessage,
  18. expandAssistantStream,
  19. } from '@deepseek-ai/dsh-llm'
  20. import type { ReplayEntry, ReplayOverrideDoc } from '@deepseek-ai/dsh-llm-replay'
  21. import type { SessionEvent, SessionSeq } from '@deepseek-ai/dsh-session'
  22. import {
  23. SESSION_FORMAT_VERSION,
  24. Session,
  25. SessionId,
  26. } from '@deepseek-ai/dsh-session'
  27. // Carries the session/title event declaration into the fixture builder.
  28. import type {} from '@deepseek-ai/dsh-session-title'
  29. import {
  30. launchWebScaffold,
  31. seedSession,
  32. watchConsole,
  33. webSnapshotMode,
  34. type WebScaffold,
  35. } from './scaffold.ts'
  36. import { connectFreshWorkspace, newEnglishPage } from './support.ts'
  37. const SIDEBAR_SESSION_COUNT = 1_000
  38. const LONG_SESSION_ID = 'perf-long-history'
  39. const LONG_SESSION_TITLE = 'LONG_PERF_SENTINEL 500-turn session'
  40. const LONG_HISTORY_TURNS = 500
  41. const TOOL_TURN_INTERVAL = 10
  42. const TOOLS_PER_TOOL_TURN = 10
  43. const EXPECTED_TOOL_CALLS = LONG_HISTORY_TURNS / TOOL_TURN_INTERVAL * TOOLS_PER_TOOL_TURN
  44. const EXPECTED_TRAJECTORY_ROWS = 2_100
  45. const DEFAULT_HISTORY_TURNS = 24
  46. const PERF_REPLAY_CONTEXT_WINDOW = 10_000_000
  47. const STREAM_PACE_MS = 8
  48. const STREAM_DELTA_COUNT = 120
  49. const COMPARISON_TURNS = 8
  50. const COMPARISON_DELTA_COUNT = 24
  51. const COMPARISON_TOOL_INTERVAL = 3
  52. const SOAK_TURNS = 100
  53. const POST_SOAK_RENDER_TURN = SOAK_TURNS + 1
  54. const SOAK_DELTA_COUNT = 8
  55. const SOAK_TOOL_INTERVAL = 10
  56. const SOAK_CHECKPOINT_INTERVAL = 10
  57. const LONG_CONTINUATION_USER_PREFIX = 'LONG_CONTINUATION_USER'
  58. const LONG_CONTINUATION_FIRST_PREFIX = 'LONG_CONTINUATION_FIRST'
  59. const LONG_CONTINUATION_DONE_PREFIX = 'LONG_CONTINUATION_DONE'
  60. const SOAK_USER_PREFIX = 'SOAK_CONVERSATION_USER'
  61. const SOAK_FIRST_PREFIX = 'SOAK_CONVERSATION_FIRST'
  62. const SOAK_DONE_PREFIX = 'SOAK_CONVERSATION_DONE'
  63. const LIVE_PROMPT_MARKER = 'STREAM_PERF_USER_INPUT'
  64. const STREAM_FIRST_MARKER = 'STREAM_PERF_FIRST'
  65. const STREAM_DONE_MARKER = 'STREAM_PERF_DONE'
  66. const LIVE_PROMPT = [
  67. LIVE_PROMPT_MARKER,
  68. 'Analyze the following mixed-language project context and return a concise diagnostic.',
  69. ...Array.from(
  70. { length: 48 },
  71. (_, index) =>
  72. `Context ${String(index + 1).padStart(2, '0')}: 用户正在检查长会话中的增量渲染性能。`
  73. + ` Preserve item ${String(index)} and compare ${'payload'.repeat(8)}.`,
  74. ),
  75. '```ts',
  76. ...Array.from(
  77. { length: 40 },
  78. (_, index) => `const sample_${String(index)} = ${JSON.stringify(`value-${String(index)}-${'x'.repeat(32)}`)}`,
  79. ),
  80. '```',
  81. ].join('\n')
  82. const STREAM_DELTAS = Array.from({ length: STREAM_DELTA_COUNT }, (_, index) => {
  83. if (index === 0) return `${STREAM_FIRST_MARKER} `
  84. if (index === STREAM_DELTA_COUNT - 1) return `${STREAM_DONE_MARKER}.`
  85. return `chunk-${String(index).padStart(3, '0')} ${'response'.repeat(3)} `
  86. })
  87. interface ChromiumMetrics {
  88. readonly [name: string]: number
  89. }
  90. interface Measurement {
  91. readonly wallMs: number
  92. readonly taskMs: number
  93. readonly scriptMs: number
  94. readonly layoutMs: number
  95. readonly recalcStyleMs: number
  96. readonly devtoolsMs: number
  97. readonly nodesDelta: number
  98. readonly listenersDelta: number
  99. readonly heapDeltaMb: number
  100. readonly totalNodes: number
  101. readonly heapMb: number
  102. }
  103. interface MutationProbeResult {
  104. readonly batches: number
  105. readonly records: number
  106. }
  107. interface UserRenderProbeResult {
  108. readonly trustedClick: boolean
  109. readonly sendToDomMs: number
  110. readonly sendToPaintMs: number
  111. readonly domToPaintMs: number
  112. readonly mutationBatches: number
  113. readonly mutationRecords: number
  114. }
  115. interface RetainedBrowserState {
  116. readonly domElements: number
  117. readonly nodes: number
  118. readonly listeners: number
  119. readonly heapMb: number
  120. }
  121. interface ContinuedTurnReport {
  122. readonly ordinal: number
  123. readonly resultingTurns: number
  124. readonly kind: 'text' | 'tool'
  125. readonly promptChars: number
  126. readonly composerFill: Measurement
  127. readonly stream: MutationProbeResult & Measurement & {
  128. readonly paceMs: number
  129. readonly deltaChunks: number
  130. readonly persistedChunks: number
  131. readonly toolCalls: number
  132. readonly toolResults: number
  133. readonly clickToUserEchoMs: number
  134. readonly clickToFirstChunkMs: number
  135. readonly firstChunkToSettledMs: number
  136. }
  137. }
  138. interface ConversationTurnSpec {
  139. readonly prompt: string
  140. readonly deltas: readonly string[]
  141. readonly userMarker: string
  142. readonly firstMarker: string
  143. readonly doneMarker: string
  144. readonly toolResultMarker?: string
  145. }
  146. interface RetainedCheckpoint {
  147. readonly turns: number
  148. readonly state: RetainedBrowserState
  149. }
  150. interface ConversationReport {
  151. readonly startingTurns: number
  152. readonly turnsAdded: number
  153. readonly toolTurns: number
  154. readonly retainedBefore: RetainedBrowserState
  155. readonly retainedAfter: RetainedBrowserState
  156. readonly retainedDelta: RetainedBrowserState
  157. readonly checkpoints: readonly RetainedCheckpoint[]
  158. readonly turns: readonly ContinuedTurnReport[]
  159. }
  160. interface PerformanceWorld {
  161. readonly scaffold: WebScaffold
  162. readonly page: Page
  163. readonly tripwire: ReturnType<typeof watchConsole>
  164. readonly sessionEvents: SessionEvent[]
  165. readonly setupMs: number
  166. readonly replayDir?: string
  167. }
  168. interface PerformanceWorldOptions {
  169. readonly browser: Browser
  170. readonly replay?: ReplayOverrideDoc
  171. readonly sidebarSessions?: number
  172. readonly seedLongHistory?: boolean
  173. }
  174. function text(value: string): { type: 'text'; text: string }[] {
  175. return [{ type: 'text', text: value }]
  176. }
  177. function appendTitle(session: Session, title: string, messageSeq: SessionSeq): void {
  178. session.append('session/title', {
  179. title,
  180. messageSeqs: [messageSeq],
  181. source: { kind: 'fallback' },
  182. })
  183. }
  184. function appendRequestHeader(session: Session, turn: number, step: number): void {
  185. session.append('request/header', {
  186. header: {
  187. config: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
  188. system: `Synthetic performance system prompt for turn ${String(turn)}, step ${String(step)}.`,
  189. },
  190. reason: turn === 1 && step === 1 ? 'initial' : 'change',
  191. })
  192. }
  193. function appendAssistant(
  194. session: Session,
  195. turn: number,
  196. step: number,
  197. body: string,
  198. ): void {
  199. session.append('assistant/message', {
  200. stream: [],
  201. turn,
  202. step,
  203. message: createAssistantMessage({
  204. content: text(body),
  205. source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
  206. }),
  207. usage: {
  208. inputTokens: 4_000 + turn * 10,
  209. outputTokens: 200 + step * 20,
  210. cacheReadTokens: turn % 2 === 0 ? 2_000 : 0,
  211. },
  212. }, { surfaceOp: 'append' })
  213. }
  214. function appendToolStep(
  215. session: Session,
  216. turn: number,
  217. step: number,
  218. toolCount: number,
  219. ): void {
  220. const calls = Array.from({ length: toolCount }, (_, index) => {
  221. const callId = ToolCallId(`perf-call-${String(turn)}-${String(index)}`)
  222. const args = JSON.stringify({
  223. turn,
  224. index,
  225. payload: 'x'.repeat(120),
  226. })
  227. return { callId, index, args }
  228. })
  229. session.append('assistant/message', {
  230. stream: [],
  231. turn,
  232. step,
  233. message: createAssistantMessage({
  234. content: [
  235. {
  236. type: 'reasoning',
  237. text: `Dispatching ${String(toolCount)} synthetic tools for turn ${String(turn)}.`,
  238. },
  239. ...calls.map(({ callId, args }) => ({
  240. type: 'tool-call' as const,
  241. id: callId,
  242. name: 'synthetic_tool',
  243. arguments: args,
  244. })),
  245. ],
  246. source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
  247. }),
  248. usage: {
  249. inputTokens: 6_000 + turn * 10,
  250. outputTokens: 500,
  251. cacheReadTokens: 3_000,
  252. reasoningTokens: 50,
  253. },
  254. }, { surfaceOp: 'append' })
  255. const callEvents = calls.map(({ callId, args }) =>
  256. session.append('tool/call', {
  257. turn,
  258. step,
  259. callId,
  260. name: 'synthetic_tool',
  261. arguments: args,
  262. }))
  263. for (const [index, call] of calls.entries()) {
  264. const source = callEvents[index]
  265. if (source === undefined) throw new Error(`missing synthetic tool call ${String(index)}`)
  266. session.append('tool/result', {
  267. turn,
  268. step,
  269. message: createToolResultMessage({
  270. callId: call.callId,
  271. content: text(
  272. `synthetic result turn=${String(turn)} index=${String(call.index)} ${'r'.repeat(400)}`,
  273. ),
  274. isError: false,
  275. }),
  276. }, { surfaceOp: 'append', sourceEventSeqs: [source.seq] })
  277. }
  278. }
  279. function fencedCode(turn: number): string {
  280. if (turn % 25 !== 0) return ''
  281. const lines = Array.from(
  282. { length: 80 },
  283. (_, index) => `const value_${String(index)} = ${String(turn + index)}`,
  284. )
  285. return `\n\n\`\`\`ts\n${lines.join('\n')}\n\`\`\``
  286. }
  287. function fixtureLog(session: Session): string {
  288. const header = {
  289. type: 'session',
  290. version: SESSION_FORMAT_VERSION,
  291. id: '{{sessionId}}',
  292. createdAt: Date.now() - 60_000,
  293. cwd: '{{cwd}}',
  294. isSeeded: false,
  295. delegationDepth: 0,
  296. }
  297. return [
  298. JSON.stringify(header),
  299. ...session.snapshotEvents().map(event => JSON.stringify(event)),
  300. '',
  301. ].join('\n')
  302. }
  303. function smallSidebarFixture(): string {
  304. const session = Session.create(SessionId('perf-small-template'))
  305. session.append('turn/start', {
  306. turn: 1,
  307. })
  308. const user = session.append('user/message', createUserMessage({
  309. content: text('Inspect this compact synthetic session.'),
  310. source: { kind: 'user' },
  311. }), { surfaceOp: 'append' })
  312. appendTitle(session, 'Synthetic sidebar session', user.seq)
  313. session.append('step/start', { turn: 1, step: 1 })
  314. appendRequestHeader(session, 1, 1)
  315. appendToolStep(session, 1, 1, 2)
  316. session.append('step/end', { turn: 1, step: 1 })
  317. session.append('step/start', { turn: 1, step: 2 })
  318. appendRequestHeader(session, 1, 2)
  319. appendAssistant(session, 1, 2, 'Synthetic sidebar fixture complete.')
  320. session.append('step/end', { turn: 1, step: 2 })
  321. session.append('turn/end', { turn: 1, reason: { kind: 'completed' } })
  322. return fixtureLog(session)
  323. }
  324. function longHistoryFixture(): string {
  325. const session = Session.create(SessionId(LONG_SESSION_ID))
  326. for (let turn = 1; turn <= LONG_HISTORY_TURNS; turn += 1) {
  327. session.append('turn/start', {
  328. turn,
  329. })
  330. const user = session.append('user/message', createUserMessage({
  331. content: text(
  332. `LONG_PERF_SENTINEL turn ${String(turn)}: analyze payload ${'u'.repeat(200)}`,
  333. ),
  334. source: { kind: 'user' },
  335. }), { surfaceOp: 'append' })
  336. if (turn === 1) appendTitle(session, LONG_SESSION_TITLE, user.seq)
  337. session.append('step/start', { turn, step: 1 })
  338. appendRequestHeader(session, turn, 1)
  339. if (turn % TOOL_TURN_INTERVAL === 0) {
  340. appendToolStep(session, turn, 1, TOOLS_PER_TOOL_TURN)
  341. session.append('step/end', { turn, step: 1 })
  342. session.append('step/start', { turn, step: 2 })
  343. appendRequestHeader(session, turn, 2)
  344. appendAssistant(
  345. session,
  346. turn,
  347. 2,
  348. `All synthetic tools completed for turn ${String(turn)}. ${'z'.repeat(320)}${fencedCode(turn)}`,
  349. )
  350. session.append('step/end', { turn, step: 2 })
  351. } else {
  352. appendAssistant(
  353. session,
  354. turn,
  355. 1,
  356. `Synthetic assistant response for turn ${String(turn)}. ${'a'.repeat(320)}${fencedCode(turn)}`,
  357. )
  358. session.append('step/end', { turn, step: 1 })
  359. }
  360. session.append('turn/end', { turn, reason: { kind: 'completed' } })
  361. }
  362. return fixtureLog(session)
  363. }
  364. function textStream(deltas: readonly string[], inputTokens: number): StreamChunk[] {
  365. const response = deltas.join('')
  366. return [
  367. { type: 'block-start', index: 0, blockType: 'text' },
  368. ...deltas.map(text => ({
  369. type: 'text-delta' as const,
  370. index: 0,
  371. text,
  372. })),
  373. {
  374. type: 'block-end',
  375. index: 0,
  376. block: { type: 'text', text: response },
  377. },
  378. {
  379. type: 'usage',
  380. usage: {
  381. inputTokens,
  382. outputTokens: Math.ceil(response.length / 4),
  383. },
  384. },
  385. { type: 'finish', reason: { kind: 'stop' } },
  386. ]
  387. }
  388. function comparisonPrompt(index: number): string {
  389. if (index === COMPARISON_TURNS) return LIVE_PROMPT
  390. return (`${LONG_CONTINUATION_USER_PREFIX}_${String(index)} `
  391. + `继续分析这个长会话的第 ${String(index)} 个增量问题,并保留当前滚动和输入响应。 `
  392. + 'context '.repeat(80)).trimEnd()
  393. }
  394. function comparisonDeltas(index: number): string[] {
  395. if (index === COMPARISON_TURNS) return STREAM_DELTAS
  396. return Array.from({ length: COMPARISON_DELTA_COUNT }, (_, chunkIndex) => {
  397. if (chunkIndex === 0) return `${LONG_CONTINUATION_FIRST_PREFIX}_${String(index)} `
  398. if (chunkIndex === COMPARISON_DELTA_COUNT - 1) {
  399. return `${LONG_CONTINUATION_DONE_PREFIX}_${String(index)}.`
  400. }
  401. return `turn-${String(index)}-chunk-${String(chunkIndex).padStart(2, '0')} ${'response'.repeat(2)} `
  402. })
  403. }
  404. function comparisonTurn(index: number): ConversationTurnSpec {
  405. const toolResultMarker = index % COMPARISON_TOOL_INTERVAL === 0
  406. ? `LONG_CONTINUATION_TOOL_RESULT_${String(index)}`
  407. : undefined
  408. return {
  409. prompt: comparisonPrompt(index),
  410. deltas: comparisonDeltas(index),
  411. userMarker: index === COMPARISON_TURNS
  412. ? LIVE_PROMPT_MARKER
  413. : `${LONG_CONTINUATION_USER_PREFIX}_${String(index)}`,
  414. firstMarker: index === COMPARISON_TURNS
  415. ? STREAM_FIRST_MARKER
  416. : `${LONG_CONTINUATION_FIRST_PREFIX}_${String(index)}`,
  417. doneMarker: index === COMPARISON_TURNS
  418. ? STREAM_DONE_MARKER
  419. : `${LONG_CONTINUATION_DONE_PREFIX}_${String(index)}`,
  420. ...toolResultMarker === undefined ? {} : { toolResultMarker },
  421. }
  422. }
  423. function soakTurn(index: number): ConversationTurnSpec {
  424. const suffix = String(index).padStart(3, '0')
  425. const toolResultMarker = index % SOAK_TOOL_INTERVAL === 0
  426. ? `SOAK_CONVERSATION_TOOL_RESULT_${suffix}`
  427. : undefined
  428. return {
  429. prompt: (`${SOAK_USER_PREFIX}_${suffix} `
  430. + `持续对话第 ${String(index)} 轮,检查增量渲染与保留状态。 `
  431. + 'context '.repeat(20)).trimEnd(),
  432. deltas: Array.from({ length: SOAK_DELTA_COUNT }, (_, chunkIndex) => {
  433. if (chunkIndex === 0) return `${SOAK_FIRST_PREFIX}_${suffix} `
  434. if (chunkIndex === SOAK_DELTA_COUNT - 1) return `${SOAK_DONE_PREFIX}_${suffix}.`
  435. return `soak-${suffix}-${String(chunkIndex).padStart(2, '0')} response `
  436. }),
  437. userMarker: `${SOAK_USER_PREFIX}_${suffix}`,
  438. firstMarker: `${SOAK_FIRST_PREFIX}_${suffix}`,
  439. doneMarker: `${SOAK_DONE_PREFIX}_${suffix}`,
  440. ...toolResultMarker === undefined ? {} : { toolResultMarker },
  441. }
  442. }
  443. function toolStream(index: number, marker: string): StreamChunk[] {
  444. const callId = ToolCallId(`performance-tool-${marker.toLowerCase()}-${String(index)}`)
  445. const args = JSON.stringify({
  446. command: `printf '${marker}\\n'`,
  447. description: `Emit performance marker ${String(index)}`,
  448. })
  449. return [
  450. { type: 'block-start', index: 0, blockType: 'tool-call' },
  451. {
  452. type: 'tool-call-delta',
  453. index: 0,
  454. id: callId,
  455. name: 'bash',
  456. argumentsDelta: args,
  457. },
  458. {
  459. type: 'block-end',
  460. index: 0,
  461. block: { type: 'tool-call', id: callId, name: 'bash', arguments: args },
  462. },
  463. { type: 'usage', usage: { inputTokens: 256, outputTokens: 32 } },
  464. { type: 'finish', reason: { kind: 'tool-calls' } },
  465. ]
  466. }
  467. function performanceReplayOverride(
  468. turnCount: number,
  469. turnSpec: (index: number) => ConversationTurnSpec,
  470. ): ReplayOverrideDoc {
  471. const continuationEntries = Array.from(
  472. { length: turnCount },
  473. (_, offset): ReplayEntry[] => {
  474. const index = offset + 1
  475. const spec = turnSpec(index)
  476. const finalResponse: ReplayEntry = {
  477. kind: 'chunks',
  478. chunks: textStream(spec.deltas, Math.ceil(spec.prompt.length / 4)),
  479. }
  480. return spec.toolResultMarker === undefined
  481. ? [finalResponse]
  482. : [{ kind: 'chunks', chunks: toolStream(index, spec.toolResultMarker) }, finalResponse]
  483. },
  484. ).flat()
  485. return continuationEntries
  486. }
  487. function rounded(value: number): number {
  488. return Math.round(value * 1_000) / 1_000
  489. }
  490. async function chromiumMetrics(cdp: CDPSession): Promise<ChromiumMetrics> {
  491. const payload = await cdp.send('Performance.getMetrics')
  492. return Object.fromEntries(payload.metrics.map(metric => [metric.name, metric.value]))
  493. }
  494. async function retainedBrowserState(
  495. cdp: CDPSession,
  496. page: Page,
  497. ): Promise<RetainedBrowserState> {
  498. await cdp.send('HeapProfiler.collectGarbage')
  499. const metrics = await chromiumMetrics(cdp)
  500. return {
  501. domElements: await page.evaluate(() => document.querySelectorAll('*').length),
  502. nodes: requiredMetric(metrics, 'Nodes'),
  503. listeners: requiredMetric(metrics, 'JSEventListeners'),
  504. heapMb: rounded(requiredMetric(metrics, 'JSHeapUsedSize') / 1_048_576),
  505. }
  506. }
  507. function requiredMetric(metrics: ChromiumMetrics, name: string): number {
  508. const value = metrics[name]
  509. if (value === undefined) throw new Error(`Chromium performance metric ${name} is unavailable`)
  510. return value
  511. }
  512. function metricDelta(
  513. before: ChromiumMetrics,
  514. after: ChromiumMetrics,
  515. wallMs: number,
  516. ): Measurement {
  517. return {
  518. wallMs: rounded(wallMs),
  519. taskMs: rounded((requiredMetric(after, 'TaskDuration') - requiredMetric(before, 'TaskDuration')) * 1_000),
  520. scriptMs: rounded((requiredMetric(after, 'ScriptDuration') - requiredMetric(before, 'ScriptDuration')) * 1_000),
  521. layoutMs: rounded((requiredMetric(after, 'LayoutDuration') - requiredMetric(before, 'LayoutDuration')) * 1_000),
  522. recalcStyleMs: rounded(
  523. (requiredMetric(after, 'RecalcStyleDuration') - requiredMetric(before, 'RecalcStyleDuration')) * 1_000,
  524. ),
  525. devtoolsMs: rounded(
  526. (requiredMetric(after, 'DevToolsCommandDuration') - requiredMetric(before, 'DevToolsCommandDuration')) * 1_000,
  527. ),
  528. nodesDelta: requiredMetric(after, 'Nodes') - requiredMetric(before, 'Nodes'),
  529. listenersDelta: requiredMetric(after, 'JSEventListeners') - requiredMetric(before, 'JSEventListeners'),
  530. heapDeltaMb: rounded(
  531. (requiredMetric(after, 'JSHeapUsedSize') - requiredMetric(before, 'JSHeapUsedSize')) / 1_048_576,
  532. ),
  533. totalNodes: requiredMetric(after, 'Nodes'),
  534. heapMb: rounded(requiredMetric(after, 'JSHeapUsedSize') / 1_048_576),
  535. }
  536. }
  537. async function measure<T>(
  538. cdp: CDPSession,
  539. action: () => Promise<T>,
  540. ): Promise<{ measurement: Measurement; value: T }> {
  541. const before = await chromiumMetrics(cdp)
  542. const started = performance.now()
  543. const value = await action()
  544. const wallMs = performance.now() - started
  545. const after = await chromiumMetrics(cdp)
  546. return { measurement: metricDelta(before, after, wallMs), value }
  547. }
  548. async function startMutationProbe(page: Page): Promise<void> {
  549. await page.evaluate(() => {
  550. const target = document.querySelector('[class*="centerCol"]')
  551. if (target === null) throw new Error('stream mutation probe target is unavailable')
  552. const probe = {
  553. batches: 0,
  554. records: 0,
  555. observer: undefined as MutationObserver | undefined,
  556. }
  557. const observer = new MutationObserver((records) => {
  558. probe.batches += 1
  559. probe.records += records.length
  560. })
  561. probe.observer = observer
  562. observer.observe(target, {
  563. attributes: true,
  564. attributeFilter: ['data-streaming'],
  565. characterData: true,
  566. childList: true,
  567. subtree: true,
  568. })
  569. Reflect.set(globalThis, '__dshPerfMutationProbe', probe)
  570. })
  571. }
  572. async function stopMutationProbe(page: Page): Promise<MutationProbeResult> {
  573. return page.evaluate(() => {
  574. const probe = Reflect.get(globalThis, '__dshPerfMutationProbe') as
  575. | { batches: number; records: number; observer: MutationObserver }
  576. | undefined
  577. if (probe === undefined) throw new Error('stream mutation probe was not started')
  578. probe.observer.disconnect()
  579. Reflect.deleteProperty(globalThis, '__dshPerfMutationProbe')
  580. return { batches: probe.batches, records: probe.records }
  581. })
  582. }
  583. async function startUserRenderProbe(
  584. page: Page,
  585. marker: string,
  586. ): Promise<void> {
  587. const send = page.getByRole('button', { name: 'Send message', exact: true })
  588. await send.waitFor({ timeout: 15_000 })
  589. await expect.poll(() => send.isEnabled(), { timeout: 15_000 }).toBe(true)
  590. await page.evaluate((expectedMarker) => {
  591. const target = document.querySelector('[data-conversation-scroll]')
  592. if (target === null) throw new Error('user render probe target is unavailable')
  593. const probe: {
  594. sendAt?: number
  595. domAt?: number
  596. paintAt?: number
  597. trustedClick?: boolean
  598. batches: number
  599. records: number
  600. observer?: MutationObserver
  601. pollForMarker?: () => void
  602. } = { batches: 0, records: 0 }
  603. const transcriptContainsMarker = (): boolean => {
  604. // The composer projects the draft into a backdrop and hidden sizing
  605. // mirror; only a matching text node outside that card proves delivery.
  606. const walker = document.createTreeWalker(target, NodeFilter.SHOW_TEXT)
  607. for (let node = walker.nextNode(); node !== null; node = walker.nextNode()) {
  608. if (!node.textContent?.includes(expectedMarker)) continue
  609. const parent = node.parentElement
  610. if (parent !== null && parent.closest('[data-composer-card]') === null) return true
  611. }
  612. return false
  613. }
  614. const markDomVisible = (): void => {
  615. if (
  616. probe.sendAt === undefined
  617. || probe.domAt !== undefined
  618. || !transcriptContainsMarker()
  619. ) return
  620. probe.domAt = globalThis.performance.now()
  621. // The second callback runs only after a render opportunity for the DOM
  622. // insertion observed before the first callback.
  623. requestAnimationFrame(() => {
  624. requestAnimationFrame(() => {
  625. probe.paintAt = globalThis.performance.now()
  626. observer.disconnect()
  627. })
  628. })
  629. }
  630. const pollForMarker = (): void => {
  631. markDomVisible()
  632. if (probe.domAt === undefined) requestAnimationFrame(pollForMarker)
  633. }
  634. const observer = new MutationObserver((records) => {
  635. if (probe.sendAt === undefined) return
  636. probe.batches += 1
  637. probe.records += records.length
  638. markDomVisible()
  639. })
  640. probe.observer = observer
  641. probe.pollForMarker = pollForMarker
  642. observer.observe(target, {
  643. attributes: true,
  644. attributeFilter: ['data-streaming'],
  645. characterData: true,
  646. childList: true,
  647. subtree: true,
  648. })
  649. Reflect.set(globalThis, '__dshPerfUserRenderProbe', probe)
  650. }, marker)
  651. await send.evaluate((button) => {
  652. button.addEventListener('click', (event) => {
  653. const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
  654. | { sendAt?: number; trustedClick?: boolean; pollForMarker?: () => void }
  655. | undefined
  656. if (probe?.pollForMarker === undefined) throw new Error('user render probe was not started')
  657. probe.trustedClick = event.isTrusted
  658. probe.sendAt = globalThis.performance.now()
  659. requestAnimationFrame(probe.pollForMarker)
  660. }, { capture: true, once: true })
  661. })
  662. }
  663. async function triggerUserRenderProbe(page: Page): Promise<void> {
  664. const send = page.getByRole('button', { name: 'Send message', exact: true })
  665. await send.click()
  666. }
  667. async function stopUserRenderProbe(
  668. page: Page,
  669. marker: string,
  670. ): Promise<UserRenderProbeResult> {
  671. try {
  672. await page.waitForFunction(() => {
  673. const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
  674. | { paintAt?: number }
  675. | undefined
  676. return probe?.paintAt !== undefined
  677. }, undefined, { timeout: 15_000 })
  678. } catch (error) {
  679. const diagnostic = await page.evaluate((expectedMarker) => {
  680. const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
  681. | {
  682. sendAt?: number
  683. domAt?: number
  684. paintAt?: number
  685. trustedClick?: boolean
  686. batches: number
  687. records: number
  688. }
  689. | undefined
  690. const target = document.querySelector('[data-conversation-scroll]')
  691. let markerOutsideComposer = false
  692. if (target !== null) {
  693. const walker = document.createTreeWalker(target, NodeFilter.SHOW_TEXT)
  694. for (let node = walker.nextNode(); node !== null; node = walker.nextNode()) {
  695. if (!node.textContent?.includes(expectedMarker)) continue
  696. const parent = node.parentElement
  697. if (parent !== null && parent.closest('[data-composer-card]') === null) {
  698. markerOutsideComposer = true
  699. break
  700. }
  701. }
  702. }
  703. return {
  704. probe,
  705. markerOutsideComposer,
  706. markerInComposer: document.querySelector('[data-composer-card]')
  707. ?.textContent?.includes(expectedMarker) ?? false,
  708. }
  709. }, marker)
  710. throw new Error(`user render probe timed out: ${JSON.stringify(diagnostic)}`, { cause: error })
  711. }
  712. return page.evaluate(() => {
  713. const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
  714. | {
  715. sendAt?: number
  716. domAt?: number
  717. paintAt?: number
  718. trustedClick?: boolean
  719. batches: number
  720. records: number
  721. }
  722. | undefined
  723. Reflect.deleteProperty(globalThis, '__dshPerfUserRenderProbe')
  724. if (
  725. probe?.trustedClick !== true
  726. || probe.sendAt === undefined
  727. || probe.domAt === undefined
  728. || probe.paintAt === undefined
  729. ) {
  730. throw new Error('user render probe did not observe a trusted click, DOM insertion, and paint')
  731. }
  732. return {
  733. trustedClick: probe.trustedClick,
  734. sendToDomMs: probe.domAt - probe.sendAt,
  735. sendToPaintMs: probe.paintAt - probe.sendAt,
  736. domToPaintMs: probe.paintAt - probe.domAt,
  737. mutationBatches: probe.batches,
  738. mutationRecords: probe.records,
  739. }
  740. })
  741. }
  742. async function stableCount(
  743. locator: Locator,
  744. accepts: (count: number) => boolean,
  745. timeoutMs = 60_000,
  746. ): Promise<number> {
  747. const deadline = performance.now() + timeoutMs
  748. let previous = -1
  749. let stableReads = 0
  750. while (performance.now() < deadline) {
  751. const count = await locator.count()
  752. stableReads = accepts(count) && count === previous ? stableReads + 1 : 0
  753. if (stableReads >= 4) return count
  754. previous = count
  755. await new Promise(resolve => setTimeout(resolve, 50))
  756. }
  757. throw new Error(`browser row count did not stabilize; last count ${String(previous)}`)
  758. }
  759. async function conversationTurns(page: Page): Promise<number> {
  760. // Loaded-window turn count: one mounted turn-tail footer per settled turn in
  761. // the window (context keys are `${kind.length}:${kind}${id}`). The stats
  762. // strip cannot serve as this probe: its counts ride the whole-log
  763. // sessionStats projection and stay fixed across paging by design.
  764. return stableCount(page.locator('[data-chat-flow-key^="9:turn-tail"]'), count => count > 0)
  765. }
  766. function retainedDelta(
  767. before: RetainedBrowserState,
  768. after: RetainedBrowserState,
  769. ): RetainedBrowserState {
  770. return {
  771. domElements: after.domElements - before.domElements,
  772. nodes: after.nodes - before.nodes,
  773. listeners: after.listeners - before.listeners,
  774. heapMb: rounded(after.heapMb - before.heapMb),
  775. }
  776. }
  777. async function launchPerformanceWorld(
  778. options: PerformanceWorldOptions,
  779. ): Promise<PerformanceWorld> {
  780. const setupStarted = performance.now()
  781. let replayDir: string | undefined
  782. let scaffold: WebScaffold | undefined
  783. let page: Page | undefined
  784. try {
  785. if (options.replay === undefined) {
  786. scaffold = await launchWebScaffold()
  787. } else {
  788. replayDir = await mkdtemp(join(tmpdir(), 'dsh-web-perf-replay-'))
  789. const replayOverride = join(replayDir, 'replay.override.json')
  790. await writeFile(replayOverride, JSON.stringify(options.replay))
  791. scaffold = await launchWebScaffold({
  792. // The override-only fixture supplies one positional script for this
  793. // world's single live session.
  794. replayFixture: join(replayDir, 'override-only.jsonl'),
  795. replayOverride,
  796. paceMs: STREAM_PACE_MS,
  797. replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
  798. })
  799. }
  800. const sessionEvents: SessionEvent[] = []
  801. scaffold.ctx.on('session/event', (_session, event: SessionEvent) => {
  802. sessionEvents.push(event)
  803. })
  804. if ((options.sidebarSessions ?? 0) > 0) {
  805. const small = smallSidebarFixture()
  806. for (let index = 0; index < (options.sidebarSessions ?? 0); index += 1) {
  807. await seedSession(scaffold, small, `perf-sidebar-${String(index).padStart(4, '0')}`)
  808. }
  809. }
  810. if (options.seedLongHistory === true) {
  811. await seedSession(scaffold, longHistoryFixture(), LONG_SESSION_ID)
  812. }
  813. const setupMs = performance.now() - setupStarted
  814. page = await newEnglishPage(options.browser)
  815. return {
  816. scaffold,
  817. page,
  818. tripwire: watchConsole(page),
  819. sessionEvents,
  820. setupMs,
  821. ...replayDir === undefined ? {} : { replayDir },
  822. }
  823. } catch (error) {
  824. const failures: unknown[] = [error]
  825. if (page !== undefined) {
  826. try {
  827. await page.close()
  828. } catch (cleanupError) {
  829. failures.push(cleanupError)
  830. }
  831. }
  832. if (scaffold !== undefined) {
  833. try {
  834. await scaffold.close()
  835. } catch (cleanupError) {
  836. failures.push(cleanupError)
  837. }
  838. }
  839. if (replayDir !== undefined) {
  840. try {
  841. await rm(replayDir, { recursive: true, force: true })
  842. } catch (cleanupError) {
  843. failures.push(cleanupError)
  844. }
  845. }
  846. if (failures.length === 1) throw error
  847. throw new AggregateError(failures, 'web performance setup and cleanup failed')
  848. }
  849. }
  850. async function closePerformanceWorld(world: PerformanceWorld): Promise<void> {
  851. const failures: unknown[] = []
  852. await world.page.close().catch((error: unknown) => failures.push(error))
  853. await world.scaffold.close().catch((error: unknown) => failures.push(error))
  854. if (world.replayDir !== undefined) {
  855. await rm(world.replayDir, { recursive: true, force: true })
  856. .catch((error: unknown) => failures.push(error))
  857. }
  858. if (failures.length === 1) throw failures[0]
  859. if (failures.length > 1) throw new AggregateError(failures, 'web performance teardown failed')
  860. }
  861. async function openPerformancePage(
  862. world: PerformanceWorld,
  863. expectedSessions: number,
  864. ): Promise<Locator> {
  865. await world.page.goto(world.scaffold.authenticatedUrl, { waitUntil: 'load' })
  866. await world.page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
  867. const group = world.page.getByRole('treeitem').first()
  868. await expect.poll(() => group.textContent(), { timeout: 30_000 })
  869. .toContain(`${String(expectedSessions)} ${expectedSessions === 1 ? 'session' : 'sessions'}`)
  870. return group
  871. }
  872. async function openLongHistory(page: Page): Promise<number> {
  873. await page.getByRole('textbox', { name: 'Search name, keywords...', exact: true })
  874. .fill('LONG_PERF_SENTINEL')
  875. const results = page.getByRole('tree', { name: 'Search results' }).getByRole('treeitem')
  876. await expect.poll(() => results.count(), { timeout: 60_000 }).toBe(1)
  877. await results.first().click()
  878. await page.getByRole('tab', { name: 'Chat', exact: true }).waitFor({ timeout: 30_000 })
  879. return conversationTurns(page)
  880. }
  881. async function continueConversation(
  882. world: PerformanceWorld,
  883. cdp: CDPSession,
  884. options: {
  885. readonly startingTurns: number
  886. readonly turnCount: number
  887. readonly turnSpec: (index: number) => ConversationTurnSpec
  888. readonly expectedSessionId?: SessionId
  889. readonly checkpointInterval?: number
  890. },
  891. ): Promise<ConversationReport> {
  892. const composer = world.page.locator('[data-composer-input][contenteditable="true"]').last()
  893. await composer.waitFor({ timeout: 15_000 })
  894. const retainedBefore = await retainedBrowserState(cdp, world.page)
  895. const checkpoints: RetainedCheckpoint[] = [{ turns: options.startingTurns, state: retainedBefore }]
  896. const turns: ContinuedTurnReport[] = []
  897. let liveSessionId = options.expectedSessionId
  898. for (let index = 1; index <= options.turnCount; index += 1) {
  899. const spec = options.turnSpec(index)
  900. const composerFill = await measure(cdp, async () => {
  901. await composer.fill(spec.prompt)
  902. await expect.poll(() => composer.textContent()).toBe(spec.prompt)
  903. return ((await composer.textContent()) ?? '').length
  904. })
  905. expect(composerFill.value).toBe(spec.prompt.length)
  906. const eventStart = world.sessionEvents.length
  907. await startMutationProbe(world.page)
  908. const streamBefore = await chromiumMetrics(cdp)
  909. const streamStarted = performance.now()
  910. const settled = world.scaffold.whenTurnSettled(60_000)
  911. await world.page.getByRole('button', { name: 'Send message', exact: true }).click()
  912. await world.page.getByText(spec.userMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  913. const clickToUserEchoMs = performance.now() - streamStarted
  914. await world.page.getByText(spec.firstMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  915. const clickToFirstChunkMs = performance.now() - streamStarted
  916. const settledSessionId = await settled
  917. if (liveSessionId === undefined) {
  918. liveSessionId = settledSessionId
  919. } else {
  920. expect(settledSessionId).toBe(liveSessionId)
  921. }
  922. await expect.poll(
  923. () => world.page.locator('[data-streaming="true"]').count(),
  924. { timeout: 15_000 },
  925. ).toBe(0)
  926. await world.page.getByText(spec.doneMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  927. const clickToSettledMs = performance.now() - streamStarted
  928. const streamAfter = await chromiumMetrics(cdp)
  929. const mutations = await stopMutationProbe(world.page)
  930. const turnEvents = world.sessionEvents.slice(eventStart)
  931. const chunks = turnEvents.flatMap(event => (
  932. event.type === 'assistant/message' || event.type === 'assistant/attempt'
  933. ? expandAssistantStream(event.data.stream)
  934. : []
  935. ))
  936. const toolCalls = turnEvents.filter(event => event.type === 'tool/call')
  937. const toolResults = turnEvents.filter(event => event.type === 'tool/result')
  938. const toolTurn = spec.toolResultMarker !== undefined
  939. expect(chunks).toHaveLength(spec.deltas.length + (toolTurn ? 9 : 4))
  940. expect(toolCalls).toHaveLength(toolTurn ? 1 : 0)
  941. expect(toolResults).toHaveLength(toolTurn ? 1 : 0)
  942. if (spec.toolResultMarker !== undefined) {
  943. expect(toolCalls[0]?.data.name).toBe('bash')
  944. const toolResult = toolResults[0]
  945. if (toolResult?.type !== 'tool/result') {
  946. throw new Error(`continued turn ${String(index)} did not log its tool result`)
  947. }
  948. const resultBlock = toolResult.data.message.content.find(block => block.type === 'tool-result')
  949. expect(resultBlock?.isError).toBe(false)
  950. expect(resultBlock?.content
  951. .filter(block => block.type === 'text')
  952. .map(block => block.text)
  953. .join('')).toContain(spec.toolResultMarker)
  954. }
  955. const user = turnEvents.find(
  956. event => event.type === 'user/message' && event.data.source.kind === 'user',
  957. )
  958. if (user?.type !== 'user/message') {
  959. throw new Error(`continued turn ${String(index)} did not log its user message`)
  960. }
  961. expect(user.data.content
  962. .filter(block => block.type === 'text')
  963. .map(block => block.text)
  964. .join('')).toBe(spec.prompt)
  965. const resultingTurns = await conversationTurns(world.page)
  966. expect(resultingTurns).toBe(options.startingTurns + index)
  967. turns.push({
  968. ordinal: index,
  969. resultingTurns,
  970. kind: toolTurn ? 'tool' : 'text',
  971. promptChars: spec.prompt.length,
  972. composerFill: composerFill.measurement,
  973. stream: {
  974. paceMs: STREAM_PACE_MS,
  975. deltaChunks: spec.deltas.length,
  976. persistedChunks: chunks.length,
  977. toolCalls: toolCalls.length,
  978. toolResults: toolResults.length,
  979. clickToUserEchoMs: rounded(clickToUserEchoMs),
  980. clickToFirstChunkMs: rounded(clickToFirstChunkMs),
  981. firstChunkToSettledMs: rounded(clickToSettledMs - clickToFirstChunkMs),
  982. ...mutations,
  983. ...metricDelta(streamBefore, streamAfter, clickToSettledMs),
  984. },
  985. })
  986. if (options.checkpointInterval !== undefined && index % options.checkpointInterval === 0) {
  987. checkpoints.push({
  988. turns: options.startingTurns + index,
  989. state: await retainedBrowserState(cdp, world.page),
  990. })
  991. }
  992. }
  993. const lastCheckpoint = checkpoints.at(-1)
  994. const retainedAfter = lastCheckpoint?.turns === options.startingTurns + options.turnCount
  995. ? lastCheckpoint.state
  996. : await retainedBrowserState(cdp, world.page)
  997. if (lastCheckpoint?.turns !== options.startingTurns + options.turnCount) {
  998. checkpoints.push({ turns: options.startingTurns + options.turnCount, state: retainedAfter })
  999. }
  1000. return {
  1001. startingTurns: options.startingTurns,
  1002. turnsAdded: options.turnCount,
  1003. toolTurns: turns.filter(turn => turn.kind === 'tool').length,
  1004. retainedBefore,
  1005. retainedAfter,
  1006. retainedDelta: retainedDelta(retainedBefore, retainedAfter),
  1007. checkpoints,
  1008. turns,
  1009. }
  1010. }
  1011. async function measurePostSoakUserRender(
  1012. world: PerformanceWorld,
  1013. cdp: CDPSession,
  1014. ): Promise<object> {
  1015. const spec = soakTurn(POST_SOAK_RENDER_TURN)
  1016. if (spec.toolResultMarker !== undefined) {
  1017. throw new Error('post-soak render probe must remain a text-only turn')
  1018. }
  1019. const composer = world.page.locator('[data-composer-input][contenteditable="true"]').last()
  1020. const composerFill = await measure(cdp, async () => {
  1021. await composer.fill(spec.prompt)
  1022. await expect.poll(() => composer.textContent()).toBe(spec.prompt)
  1023. return ((await composer.textContent()) ?? '').length
  1024. })
  1025. expect(composerFill.value).toBe(spec.prompt.length)
  1026. const eventStart = world.sessionEvents.length
  1027. await startUserRenderProbe(world.page, spec.userMarker)
  1028. const browserBefore = await chromiumMetrics(cdp)
  1029. const fullTurnStarted = performance.now()
  1030. const settled = world.scaffold.whenTurnSettled(60_000)
  1031. await triggerUserRenderProbe(world.page)
  1032. const browserTiming = await stopUserRenderProbe(world.page, spec.userMarker)
  1033. const browserAfter = await chromiumMetrics(cdp)
  1034. const browserAfterPaintSample = metricDelta(
  1035. browserBefore,
  1036. browserAfter,
  1037. performance.now() - fullTurnStarted,
  1038. )
  1039. await world.page.getByText(spec.firstMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  1040. const settledSessionId = await settled
  1041. await expect.poll(
  1042. () => world.page.locator('[data-streaming="true"]').count(),
  1043. { timeout: 15_000 },
  1044. ).toBe(0)
  1045. await world.page.getByText(spec.doneMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  1046. const fullTurnMs = performance.now() - fullTurnStarted
  1047. const turnEvents = world.sessionEvents.slice(eventStart)
  1048. const chunks = turnEvents.flatMap(event => (
  1049. event.type === 'assistant/message' || event.type === 'assistant/attempt'
  1050. ? expandAssistantStream(event.data.stream)
  1051. : []
  1052. ))
  1053. const user = turnEvents.find(
  1054. event => event.type === 'user/message' && event.data.source.kind === 'user',
  1055. )
  1056. if (user?.type !== 'user/message') throw new Error('post-soak render probe did not log its user message')
  1057. expect(user.data.content
  1058. .filter(block => block.type === 'text')
  1059. .map(block => block.text)
  1060. .join('')).toBe(spec.prompt)
  1061. expect(chunks).toHaveLength(spec.deltas.length + 4)
  1062. expect(turnEvents.filter(event => event.type === 'tool/call')).toHaveLength(0)
  1063. expect(turnEvents.filter(event => event.type === 'tool/result')).toHaveLength(0)
  1064. expect(world.scaffold.ctx.agents.get(settledSessionId)).toBeDefined()
  1065. expect(await conversationTurns(world.page)).toBe(POST_SOAK_RENDER_TURN)
  1066. return {
  1067. ordinal: POST_SOAK_RENDER_TURN,
  1068. resultingTurns: POST_SOAK_RENDER_TURN,
  1069. promptChars: spec.prompt.length,
  1070. composerFill: composerFill.measurement,
  1071. trustedClick: browserTiming.trustedClick,
  1072. sendToDomMs: rounded(browserTiming.sendToDomMs),
  1073. sendToPaintMs: rounded(browserTiming.sendToPaintMs),
  1074. domToPaintMs: rounded(browserTiming.domToPaintMs),
  1075. mutationBatchesThroughPaint: browserTiming.mutationBatches,
  1076. mutationRecordsThroughPaint: browserTiming.mutationRecords,
  1077. browserAfterPaintSample,
  1078. fullTurnMs: rounded(fullTurnMs),
  1079. persistedChunks: chunks.length,
  1080. }
  1081. }
  1082. function average(values: readonly number[]): number {
  1083. return rounded(values.reduce((sum, value) => sum + value, 0) / values.length)
  1084. }
  1085. function p95(values: readonly number[]): number {
  1086. const sorted = [...values].sort((left, right) => left - right)
  1087. return sorted[Math.max(0, Math.ceil(sorted.length * 0.95) - 1)] ?? 0
  1088. }
  1089. function summarizeTurnWindows(
  1090. turns: readonly ContinuedTurnReport[],
  1091. windowSize: number,
  1092. ): object[] {
  1093. const windows: object[] = []
  1094. for (let start = 0; start < turns.length; start += windowSize) {
  1095. const window = turns.slice(start, start + windowSize)
  1096. const streamWall = window.map(turn => turn.stream.wallMs)
  1097. const userEcho = window.map(turn => turn.stream.clickToUserEchoMs)
  1098. const firstChunk = window.map(turn => turn.stream.clickToFirstChunkMs)
  1099. windows.push({
  1100. turns: `${String(start + 1)}-${String(start + window.length)}`,
  1101. toolTurns: window.filter(turn => turn.kind === 'tool').length,
  1102. average: {
  1103. composerFillWallMs: average(window.map(turn => turn.composerFill.wallMs)),
  1104. streamWallMs: average(streamWall),
  1105. clickToUserEchoMs: average(userEcho),
  1106. clickToFirstChunkMs: average(firstChunk),
  1107. taskMs: average(window.map(turn => turn.stream.taskMs)),
  1108. scriptMs: average(window.map(turn => turn.stream.scriptMs)),
  1109. recalcStyleMs: average(window.map(turn => turn.stream.recalcStyleMs)),
  1110. mutationBatches: average(window.map(turn => turn.stream.batches)),
  1111. },
  1112. p95: {
  1113. streamWallMs: rounded(p95(streamWall)),
  1114. clickToUserEchoMs: rounded(p95(userEcho)),
  1115. clickToFirstChunkMs: rounded(p95(firstChunk)),
  1116. },
  1117. })
  1118. }
  1119. return windows
  1120. }
  1121. describe('manual web performance: complex workspace and history', () => {
  1122. let browser: Browser
  1123. beforeAll(async () => {
  1124. if (webSnapshotMode() === 'record') {
  1125. throw new Error('manual web performance runs only with deterministic replay')
  1126. }
  1127. browser = await chromium.launch()
  1128. })
  1129. afterAll(async () => {
  1130. await browser?.close()
  1131. })
  1132. it('reports workspace, history, and trajectory cardinality costs', async () => {
  1133. const world = await launchPerformanceWorld({
  1134. browser,
  1135. sidebarSessions: SIDEBAR_SESSION_COUNT,
  1136. seedLongHistory: true,
  1137. })
  1138. try {
  1139. const bootStarted = performance.now()
  1140. const group = await openPerformancePage(world, SIDEBAR_SESSION_COUNT + 1)
  1141. const bootReadyMs = performance.now() - bootStarted
  1142. const page = world.page
  1143. const cdp = await page.context().newCDPSession(page)
  1144. await cdp.send('Performance.enable')
  1145. const firstContentfulPaintMs = await page.evaluate(
  1146. () => globalThis.performance.getEntriesByName('first-contentful-paint')[0]?.startTime,
  1147. )
  1148. const sidebar = await measure(cdp, async () => {
  1149. await group.click()
  1150. return stableCount(
  1151. page.getByRole('treeitem'),
  1152. count => count === SIDEBAR_SESSION_COUNT + 2,
  1153. )
  1154. })
  1155. expect(sidebar.value).toBe(SIDEBAR_SESSION_COUNT + 2)
  1156. await group.click()
  1157. await expect.poll(() => page.getByRole('treeitem').count()).toBe(1)
  1158. const contentSearch = await measure(cdp, async () => {
  1159. await page.getByRole('textbox', { name: 'Search name, keywords...', exact: true })
  1160. .fill('LONG_PERF_SENTINEL')
  1161. const results = page.getByRole('tree', { name: 'Search results' }).getByRole('treeitem')
  1162. await expect.poll(() => results.count(), { timeout: 60_000 }).toBe(1)
  1163. await results.first().waitFor({ timeout: 60_000 })
  1164. return results.first()
  1165. })
  1166. const opened = await measure(cdp, async () => {
  1167. await contentSearch.value.click()
  1168. await page.getByRole('tab', { name: 'Trajectory', exact: true }).waitFor({ timeout: 30_000 })
  1169. return conversationTurns(page)
  1170. })
  1171. expect(opened.value).toBe(DEFAULT_HISTORY_TURNS)
  1172. const trajectoryRows = page.getByRole('row')
  1173. const coldTrajectory = await measure(cdp, async () => {
  1174. await page.getByRole('tab', { name: 'Trajectory', exact: true }).click()
  1175. return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS)
  1176. })
  1177. expect(coldTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS)
  1178. const collapseTurns = await measure(cdp, async () => {
  1179. await page.getByRole('button', { name: 'Collapse turns', exact: true }).click()
  1180. return stableCount(trajectoryRows, count => count > 0 && count < EXPECTED_TRAJECTORY_ROWS)
  1181. })
  1182. expect(collapseTurns.value).toBeLessThan(EXPECTED_TRAJECTORY_ROWS)
  1183. const trajectorySearch = await measure(cdp, async () => {
  1184. await page.getByRole('searchbox', { name: 'Search trajectory', exact: true }).fill('turn 499')
  1185. return stableCount(trajectoryRows, count => count > 0 && count < 20)
  1186. })
  1187. expect(trajectorySearch.value).toBeLessThan(20)
  1188. await page.getByRole('tab', { name: 'Chat', exact: true }).click()
  1189. const historyPages: { turns: number; measurement: Measurement }[] = []
  1190. let turns = await conversationTurns(page)
  1191. while (turns < LONG_HISTORY_TURNS) {
  1192. const previousTurns = turns
  1193. const older = await measure(cdp, async () => {
  1194. await page.getByRole('button', { name: 'Load earlier', exact: true }).click()
  1195. await expect.poll(() => conversationTurns(page), { timeout: 30_000 })
  1196. .toBeGreaterThan(previousTurns)
  1197. return conversationTurns(page)
  1198. })
  1199. turns = older.value
  1200. historyPages.push({ turns, measurement: older.measurement })
  1201. }
  1202. const warmTrajectory = await measure(cdp, async () => {
  1203. await page.getByRole('tab', { name: 'Trajectory', exact: true }).click()
  1204. return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS)
  1205. })
  1206. expect(warmTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS)
  1207. const warmConversation = await measure(cdp, async () => {
  1208. await page.getByRole('tab', { name: 'Chat', exact: true }).click()
  1209. return conversationTurns(page)
  1210. })
  1211. expect(warmConversation.value).toBe(LONG_HISTORY_TURNS)
  1212. console.info(`WEB_PERF_RESULT ${JSON.stringify({
  1213. scenario: 'workspace-history-trajectory',
  1214. fixture: {
  1215. sidebarSessions: SIDEBAR_SESSION_COUNT,
  1216. totalSessions: SIDEBAR_SESSION_COUNT + 1,
  1217. longHistoryTurns: LONG_HISTORY_TURNS,
  1218. toolCalls: EXPECTED_TOOL_CALLS,
  1219. trajectoryRows: EXPECTED_TRAJECTORY_ROWS,
  1220. },
  1221. setupMs: rounded(world.setupMs),
  1222. boot: {
  1223. readyMs: rounded(bootReadyMs),
  1224. firstContentfulPaintMs: firstContentfulPaintMs === undefined
  1225. ? null
  1226. : rounded(firstContentfulPaintMs),
  1227. },
  1228. sidebarExpand: sidebar.measurement,
  1229. contentSearch: contentSearch.measurement,
  1230. openLongHistory: { initialTurns: opened.value, ...opened.measurement },
  1231. coldTrajectory: { rows: coldTrajectory.value, ...coldTrajectory.measurement },
  1232. collapseTurns: { rows: collapseTurns.value, ...collapseTurns.measurement },
  1233. trajectorySearch: { rows: trajectorySearch.value, ...trajectorySearch.measurement },
  1234. historyPages,
  1235. warmTrajectory: { rows: warmTrajectory.value, ...warmTrajectory.measurement },
  1236. warmConversation: { turns: warmConversation.value, ...warmConversation.measurement },
  1237. }, null, 2)}`)
  1238. expect(world.tripwire.warnings).toEqual([])
  1239. expect(world.tripwire.pageErrors).toEqual([])
  1240. } finally {
  1241. await closePerformanceWorld(world)
  1242. }
  1243. })
  1244. it('reports default 24-turn history plus eight continued turns', async () => {
  1245. const world = await launchPerformanceWorld({
  1246. browser,
  1247. replay: performanceReplayOverride(COMPARISON_TURNS, comparisonTurn),
  1248. seedLongHistory: true,
  1249. })
  1250. try {
  1251. await openPerformancePage(world, 1)
  1252. const cdp = await world.page.context().newCDPSession(world.page)
  1253. await cdp.send('Performance.enable')
  1254. const opened = await measure(cdp, () => openLongHistory(world.page))
  1255. expect(opened.value).toBe(DEFAULT_HISTORY_TURNS)
  1256. const conversation = await continueConversation(world, cdp, {
  1257. startingTurns: DEFAULT_HISTORY_TURNS,
  1258. turnCount: COMPARISON_TURNS,
  1259. turnSpec: comparisonTurn,
  1260. expectedSessionId: SessionId(LONG_SESSION_ID),
  1261. })
  1262. expect(conversation.toolTurns).toBe(2)
  1263. console.info(`WEB_PERF_RESULT ${JSON.stringify({
  1264. scenario: 'default-resume-24-plus-8',
  1265. setupMs: rounded(world.setupMs),
  1266. replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
  1267. openLongHistory: { turns: opened.value, ...opened.measurement },
  1268. conversation,
  1269. }, null, 2)}`)
  1270. expect(world.tripwire.warnings).toEqual([])
  1271. expect(world.tripwire.pageErrors).toEqual([])
  1272. } finally {
  1273. await closePerformanceWorld(world)
  1274. }
  1275. })
  1276. it('reports fully expanded 500-turn history plus eight continued turns', async () => {
  1277. const world = await launchPerformanceWorld({
  1278. browser,
  1279. replay: performanceReplayOverride(COMPARISON_TURNS, comparisonTurn),
  1280. seedLongHistory: true,
  1281. })
  1282. try {
  1283. await openPerformancePage(world, 1)
  1284. const cdp = await world.page.context().newCDPSession(world.page)
  1285. await cdp.send('Performance.enable')
  1286. expect(await openLongHistory(world.page)).toBe(DEFAULT_HISTORY_TURNS)
  1287. const historyPages: { turns: number; measurement: Measurement }[] = []
  1288. let turns = DEFAULT_HISTORY_TURNS
  1289. while (turns < LONG_HISTORY_TURNS) {
  1290. const previousTurns = turns
  1291. const older = await measure(cdp, async () => {
  1292. await world.page.getByRole('button', { name: 'Load earlier', exact: true }).click()
  1293. await expect.poll(() => conversationTurns(world.page), { timeout: 30_000 })
  1294. .toBeGreaterThan(previousTurns)
  1295. return conversationTurns(world.page)
  1296. })
  1297. turns = older.value
  1298. historyPages.push({ turns, measurement: older.measurement })
  1299. }
  1300. expect(turns).toBe(LONG_HISTORY_TURNS)
  1301. const conversation = await continueConversation(world, cdp, {
  1302. startingTurns: LONG_HISTORY_TURNS,
  1303. turnCount: COMPARISON_TURNS,
  1304. turnSpec: comparisonTurn,
  1305. expectedSessionId: SessionId(LONG_SESSION_ID),
  1306. })
  1307. expect(conversation.toolTurns).toBe(2)
  1308. console.info(`WEB_PERF_RESULT ${JSON.stringify({
  1309. scenario: 'expanded-history-500-plus-8',
  1310. setupMs: rounded(world.setupMs),
  1311. replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
  1312. historyPages,
  1313. conversation,
  1314. }, null, 2)}`)
  1315. expect(world.tripwire.warnings).toEqual([])
  1316. expect(world.tripwire.pageErrors).toEqual([])
  1317. } finally {
  1318. await closePerformanceWorld(world)
  1319. }
  1320. })
  1321. it('reports one hundred generated turns and the next user-message paint', async () => {
  1322. const world = await launchPerformanceWorld({
  1323. browser,
  1324. replay: performanceReplayOverride(POST_SOAK_RENDER_TURN, soakTurn),
  1325. })
  1326. let testFailure: unknown
  1327. try {
  1328. await world.page.goto(world.scaffold.authenticatedUrl, { waitUntil: 'load' })
  1329. await world.page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
  1330. await connectFreshWorkspace(world.page, world.scaffold.workspaceCwd, 'continuous-conversation-perf')
  1331. const cdp = await world.page.context().newCDPSession(world.page)
  1332. await cdp.send('Performance.enable')
  1333. const conversation = await continueConversation(world, cdp, {
  1334. startingTurns: 0,
  1335. turnCount: SOAK_TURNS,
  1336. turnSpec: soakTurn,
  1337. checkpointInterval: SOAK_CHECKPOINT_INTERVAL,
  1338. })
  1339. expect(conversation.toolTurns).toBe(SOAK_TURNS / SOAK_TOOL_INTERVAL)
  1340. const postSoakUserRender = await measurePostSoakUserRender(world, cdp)
  1341. const { turns, ...retained } = conversation
  1342. console.info(`WEB_PERF_RESULT ${JSON.stringify({
  1343. scenario: 'continuous-100-turn-soak',
  1344. setupMs: rounded(world.setupMs),
  1345. replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
  1346. ...retained,
  1347. windows: summarizeTurnWindows(turns, SOAK_CHECKPOINT_INTERVAL),
  1348. postSoakUserRender,
  1349. }, null, 2)}`)
  1350. expect(world.tripwire.warnings).toEqual([])
  1351. expect(world.tripwire.pageErrors).toEqual([])
  1352. } catch (error) {
  1353. testFailure = error
  1354. throw error
  1355. } finally {
  1356. try {
  1357. await closePerformanceWorld(world)
  1358. } catch (cleanupError) {
  1359. if (testFailure === undefined) throw cleanupError
  1360. throw new AggregateError(
  1361. [testFailure, cleanupError],
  1362. 'continuous conversation performance test and teardown failed',
  1363. )
  1364. }
  1365. }
  1366. })
  1367. })