complex-history.perf.ts 52 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432
  1. // Opt-in browser benchmark for high-cardinality workspace and history
  2. // rendering. It reports measurements without timing assertions because host
  3. // speed is not a correctness contract; structural assertions keep the load
  4. // shape from silently shrinking.
  5. import { mkdtemp, rm, writeFile } from 'node:fs/promises'
  6. import { tmpdir } from 'node:os'
  7. import { join } from 'node:path'
  8. import { performance } from 'node:perf_hooks'
  9. import type { Browser, CDPSession, Locator, Page } from 'playwright'
  10. import { chromium } from 'playwright'
  11. import { afterAll, beforeAll, describe, expect, it } from 'vitest'
  12. import type { StreamChunk } from '@deepseek-ai/dsh-llm'
  13. import {
  14. CallId,
  15. createAssistantMessage,
  16. createToolResultMessage,
  17. createUserMessage,
  18. } from '@deepseek-ai/dsh-llm'
  19. import type { ReplayEntry, ReplayOverrideDoc } from '@deepseek-ai/dsh-llm-replay'
  20. import type { SessionEvent } from '@deepseek-ai/dsh-session'
  21. import {
  22. SESSION_FORMAT_VERSION,
  23. Session,
  24. SessionId,
  25. } from '@deepseek-ai/dsh-session'
  26. // Carries the session/title event declaration into the fixture builder.
  27. import type {} from '@deepseek-ai/dsh-session-title'
  28. import {
  29. launchWebScaffold,
  30. seedSession,
  31. watchConsole,
  32. webSnapshotMode,
  33. type WebScaffold,
  34. } from './scaffold.ts'
  35. import { connectFreshWorkspace, newEnglishPage } from './support.ts'
  36. const SIDEBAR_SESSION_COUNT = 1_000
  37. const LONG_SESSION_ID = 'perf-long-history'
  38. const LONG_SESSION_TITLE = 'LONG_PERF_SENTINEL 500-turn session'
  39. const LONG_HISTORY_TURNS = 500
  40. const TOOL_TURN_INTERVAL = 10
  41. const TOOLS_PER_TOOL_TURN = 10
  42. const EXPECTED_TOOL_CALLS = LONG_HISTORY_TURNS / TOOL_TURN_INTERVAL * TOOLS_PER_TOOL_TURN
  43. const EXPECTED_TRAJECTORY_ROWS = 2_100
  44. const DEFAULT_HISTORY_TURNS = 24
  45. const PERF_REPLAY_CONTEXT_WINDOW = 10_000_000
  46. const STREAM_PACE_MS = 8
  47. const STREAM_DELTA_COUNT = 120
  48. const COMPARISON_TURNS = 8
  49. const COMPARISON_DELTA_COUNT = 24
  50. const COMPARISON_TOOL_INTERVAL = 3
  51. const SOAK_TURNS = 100
  52. const POST_SOAK_RENDER_TURN = SOAK_TURNS + 1
  53. const SOAK_DELTA_COUNT = 8
  54. const SOAK_TOOL_INTERVAL = 10
  55. const SOAK_CHECKPOINT_INTERVAL = 10
  56. const LONG_CONTINUATION_USER_PREFIX = 'LONG_CONTINUATION_USER'
  57. const LONG_CONTINUATION_FIRST_PREFIX = 'LONG_CONTINUATION_FIRST'
  58. const LONG_CONTINUATION_DONE_PREFIX = 'LONG_CONTINUATION_DONE'
  59. const SOAK_USER_PREFIX = 'SOAK_CONVERSATION_USER'
  60. const SOAK_FIRST_PREFIX = 'SOAK_CONVERSATION_FIRST'
  61. const SOAK_DONE_PREFIX = 'SOAK_CONVERSATION_DONE'
  62. const LIVE_PROMPT_MARKER = 'STREAM_PERF_USER_INPUT'
  63. const STREAM_FIRST_MARKER = 'STREAM_PERF_FIRST'
  64. const STREAM_DONE_MARKER = 'STREAM_PERF_DONE'
  65. const LIVE_PROMPT = [
  66. LIVE_PROMPT_MARKER,
  67. 'Analyze the following mixed-language project context and return a concise diagnostic.',
  68. ...Array.from(
  69. { length: 48 },
  70. (_, index) =>
  71. `Context ${String(index + 1).padStart(2, '0')}: 用户正在检查长会话中的增量渲染性能。`
  72. + ` Preserve item ${String(index)} and compare ${'payload'.repeat(8)}.`,
  73. ),
  74. '```ts',
  75. ...Array.from(
  76. { length: 40 },
  77. (_, index) => `const sample_${String(index)} = ${JSON.stringify(`value-${String(index)}-${'x'.repeat(32)}`)}`,
  78. ),
  79. '```',
  80. ].join('\n')
  81. const STREAM_DELTAS = Array.from({ length: STREAM_DELTA_COUNT }, (_, index) => {
  82. if (index === 0) return `${STREAM_FIRST_MARKER} `
  83. if (index === STREAM_DELTA_COUNT - 1) return `${STREAM_DONE_MARKER}.`
  84. return `chunk-${String(index).padStart(3, '0')} ${'response'.repeat(3)} `
  85. })
  86. interface ChromiumMetrics {
  87. readonly [name: string]: number
  88. }
  89. interface Measurement {
  90. readonly wallMs: number
  91. readonly taskMs: number
  92. readonly scriptMs: number
  93. readonly layoutMs: number
  94. readonly recalcStyleMs: number
  95. readonly devtoolsMs: number
  96. readonly nodesDelta: number
  97. readonly listenersDelta: number
  98. readonly heapDeltaMb: number
  99. readonly totalNodes: number
  100. readonly heapMb: number
  101. }
  102. interface MutationProbeResult {
  103. readonly batches: number
  104. readonly records: number
  105. }
  106. interface UserRenderProbeResult {
  107. readonly trustedClick: boolean
  108. readonly sendToDomMs: number
  109. readonly sendToPaintMs: number
  110. readonly domToPaintMs: number
  111. readonly mutationBatches: number
  112. readonly mutationRecords: number
  113. }
  114. interface RetainedBrowserState {
  115. readonly domElements: number
  116. readonly nodes: number
  117. readonly listeners: number
  118. readonly heapMb: number
  119. }
  120. interface ContinuedTurnReport {
  121. readonly ordinal: number
  122. readonly resultingTurns: number
  123. readonly kind: 'text' | 'tool'
  124. readonly promptChars: number
  125. readonly composerFill: Measurement
  126. readonly stream: MutationProbeResult & Measurement & {
  127. readonly paceMs: number
  128. readonly deltaChunks: number
  129. readonly persistedChunks: number
  130. readonly toolCalls: number
  131. readonly toolResults: number
  132. readonly clickToUserEchoMs: number
  133. readonly clickToFirstChunkMs: number
  134. readonly firstChunkToSettledMs: number
  135. }
  136. }
  137. interface ConversationTurnSpec {
  138. readonly prompt: string
  139. readonly deltas: readonly string[]
  140. readonly userMarker: string
  141. readonly firstMarker: string
  142. readonly doneMarker: string
  143. readonly toolResultMarker?: string
  144. }
  145. interface RetainedCheckpoint {
  146. readonly turns: number
  147. readonly state: RetainedBrowserState
  148. }
  149. interface ConversationReport {
  150. readonly startingTurns: number
  151. readonly turnsAdded: number
  152. readonly toolTurns: number
  153. readonly retainedBefore: RetainedBrowserState
  154. readonly retainedAfter: RetainedBrowserState
  155. readonly retainedDelta: RetainedBrowserState
  156. readonly checkpoints: readonly RetainedCheckpoint[]
  157. readonly turns: readonly ContinuedTurnReport[]
  158. }
  159. interface PerformanceWorld {
  160. readonly scaffold: WebScaffold
  161. readonly page: Page
  162. readonly tripwire: ReturnType<typeof watchConsole>
  163. readonly sessionEvents: SessionEvent[]
  164. readonly setupMs: number
  165. readonly replayDir?: string
  166. }
  167. interface PerformanceWorldOptions {
  168. readonly browser: Browser
  169. readonly replay?: ReplayOverrideDoc
  170. readonly sidebarSessions?: number
  171. readonly seedLongHistory?: boolean
  172. }
  173. function text(value: string): { type: 'text'; text: string }[] {
  174. return [{ type: 'text', text: value }]
  175. }
  176. function appendTitle(session: Session, title: string, messageSeq: number): void {
  177. session.append('session/title', {
  178. title,
  179. messageSeqs: [messageSeq],
  180. source: { kind: 'fallback' },
  181. })
  182. }
  183. function appendRequestHeader(session: Session, turn: number, step: number): void {
  184. session.append('request/header', {
  185. header: {
  186. config: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
  187. system: `Synthetic performance system prompt for turn ${String(turn)}, step ${String(step)}.`,
  188. },
  189. reason: turn === 1 && step === 1 ? 'initial' : 'change',
  190. })
  191. }
  192. function appendAssistant(
  193. session: Session,
  194. turn: number,
  195. step: number,
  196. body: string,
  197. ): void {
  198. session.append('assistant/message', {
  199. turn,
  200. step,
  201. message: createAssistantMessage({
  202. content: text(body),
  203. source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
  204. }),
  205. usage: {
  206. inputTokens: 4_000 + turn * 10,
  207. outputTokens: 200 + step * 20,
  208. cacheReadTokens: turn % 2 === 0 ? 2_000 : 0,
  209. },
  210. }, { surfaceOp: 'append' })
  211. }
  212. function appendToolStep(
  213. session: Session,
  214. turn: number,
  215. step: number,
  216. toolCount: number,
  217. ): void {
  218. const calls = Array.from({ length: toolCount }, (_, index) => {
  219. const callId = CallId(`perf-call-${String(turn)}-${String(index)}`)
  220. const args = JSON.stringify({
  221. turn,
  222. index,
  223. payload: 'x'.repeat(120),
  224. })
  225. return { callId, index, args }
  226. })
  227. session.append('assistant/message', {
  228. turn,
  229. step,
  230. message: createAssistantMessage({
  231. content: [
  232. {
  233. type: 'reasoning',
  234. text: `Dispatching ${String(toolCount)} synthetic tools for turn ${String(turn)}.`,
  235. },
  236. ...calls.map(({ callId, args }) => ({
  237. type: 'tool-call' as const,
  238. id: callId,
  239. name: 'synthetic_tool',
  240. arguments: args,
  241. })),
  242. ],
  243. source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
  244. }),
  245. usage: {
  246. inputTokens: 6_000 + turn * 10,
  247. outputTokens: 500,
  248. cacheReadTokens: 3_000,
  249. reasoningTokens: 50,
  250. },
  251. }, { surfaceOp: 'append' })
  252. const callEvents = calls.map(({ callId, args }) =>
  253. session.append('tool/call', {
  254. turn,
  255. step,
  256. callId,
  257. name: 'synthetic_tool',
  258. arguments: args,
  259. }))
  260. for (const [index, call] of calls.entries()) {
  261. const source = callEvents[index]
  262. if (source === undefined) throw new Error(`missing synthetic tool call ${String(index)}`)
  263. session.append('tool/result', {
  264. turn,
  265. step,
  266. message: createToolResultMessage({
  267. callId: call.callId,
  268. content: text(
  269. `synthetic result turn=${String(turn)} index=${String(call.index)} ${'r'.repeat(400)}`,
  270. ),
  271. isError: false,
  272. }),
  273. }, { surfaceOp: 'append', sourceEventSeqs: [source.seq] })
  274. }
  275. }
  276. function fencedCode(turn: number): string {
  277. if (turn % 25 !== 0) return ''
  278. const lines = Array.from(
  279. { length: 80 },
  280. (_, index) => `const value_${String(index)} = ${String(turn + index)}`,
  281. )
  282. return `\n\n\`\`\`ts\n${lines.join('\n')}\n\`\`\``
  283. }
  284. function fixtureLog(session: Session): string {
  285. const header = {
  286. type: 'session',
  287. version: SESSION_FORMAT_VERSION,
  288. id: '{{sessionId}}',
  289. createdAt: Date.now() - 60_000,
  290. cwd: '{{cwd}}',
  291. }
  292. return [
  293. JSON.stringify(header),
  294. ...session.events.map(event => JSON.stringify(event)),
  295. '',
  296. ].join('\n')
  297. }
  298. function smallSidebarFixture(): string {
  299. const session = Session.create(SessionId('perf-small-template'))
  300. session.append('turn/start', {
  301. turn: 1,
  302. })
  303. const user = session.append('user/message', createUserMessage({
  304. content: text('Inspect this compact synthetic session.'),
  305. source: { kind: 'user' },
  306. }), { surfaceOp: 'append' })
  307. appendTitle(session, 'Synthetic sidebar session', user.seq)
  308. session.append('step/start', { turn: 1, step: 1 })
  309. appendRequestHeader(session, 1, 1)
  310. appendToolStep(session, 1, 1, 2)
  311. session.append('step/end', { turn: 1, step: 1 })
  312. session.append('step/start', { turn: 1, step: 2 })
  313. appendRequestHeader(session, 1, 2)
  314. appendAssistant(session, 1, 2, 'Synthetic sidebar fixture complete.')
  315. session.append('step/end', { turn: 1, step: 2 })
  316. session.append('turn/end', { turn: 1, reason: { kind: 'completed' } })
  317. return fixtureLog(session)
  318. }
  319. function longHistoryFixture(): string {
  320. const session = Session.create(SessionId(LONG_SESSION_ID))
  321. for (let turn = 1; turn <= LONG_HISTORY_TURNS; turn += 1) {
  322. session.append('turn/start', {
  323. turn,
  324. })
  325. const user = session.append('user/message', createUserMessage({
  326. content: text(
  327. `LONG_PERF_SENTINEL turn ${String(turn)}: analyze payload ${'u'.repeat(200)}`,
  328. ),
  329. source: { kind: 'user' },
  330. }), { surfaceOp: 'append' })
  331. if (turn === 1) appendTitle(session, LONG_SESSION_TITLE, user.seq)
  332. session.append('step/start', { turn, step: 1 })
  333. appendRequestHeader(session, turn, 1)
  334. if (turn % TOOL_TURN_INTERVAL === 0) {
  335. appendToolStep(session, turn, 1, TOOLS_PER_TOOL_TURN)
  336. session.append('step/end', { turn, step: 1 })
  337. session.append('step/start', { turn, step: 2 })
  338. appendRequestHeader(session, turn, 2)
  339. appendAssistant(
  340. session,
  341. turn,
  342. 2,
  343. `All synthetic tools completed for turn ${String(turn)}. ${'z'.repeat(320)}${fencedCode(turn)}`,
  344. )
  345. session.append('step/end', { turn, step: 2 })
  346. } else {
  347. appendAssistant(
  348. session,
  349. turn,
  350. 1,
  351. `Synthetic assistant response for turn ${String(turn)}. ${'a'.repeat(320)}${fencedCode(turn)}`,
  352. )
  353. session.append('step/end', { turn, step: 1 })
  354. }
  355. session.append('turn/end', { turn, reason: { kind: 'completed' } })
  356. }
  357. return fixtureLog(session)
  358. }
  359. function textStream(deltas: readonly string[], inputTokens: number): StreamChunk[] {
  360. const response = deltas.join('')
  361. return [
  362. { type: 'block-start', index: 0, blockType: 'text' },
  363. ...deltas.map(text => ({
  364. type: 'text-delta' as const,
  365. index: 0,
  366. text,
  367. })),
  368. {
  369. type: 'block-end',
  370. index: 0,
  371. block: { type: 'text', text: response },
  372. },
  373. {
  374. type: 'usage',
  375. usage: {
  376. inputTokens,
  377. outputTokens: Math.ceil(response.length / 4),
  378. },
  379. },
  380. { type: 'finish', reason: { kind: 'stop' } },
  381. ]
  382. }
  383. function comparisonPrompt(index: number): string {
  384. if (index === COMPARISON_TURNS) return LIVE_PROMPT
  385. return (`${LONG_CONTINUATION_USER_PREFIX}_${String(index)} `
  386. + `继续分析这个长会话的第 ${String(index)} 个增量问题,并保留当前滚动和输入响应。 `
  387. + 'context '.repeat(80)).trimEnd()
  388. }
  389. function comparisonDeltas(index: number): string[] {
  390. if (index === COMPARISON_TURNS) return STREAM_DELTAS
  391. return Array.from({ length: COMPARISON_DELTA_COUNT }, (_, chunkIndex) => {
  392. if (chunkIndex === 0) return `${LONG_CONTINUATION_FIRST_PREFIX}_${String(index)} `
  393. if (chunkIndex === COMPARISON_DELTA_COUNT - 1) {
  394. return `${LONG_CONTINUATION_DONE_PREFIX}_${String(index)}.`
  395. }
  396. return `turn-${String(index)}-chunk-${String(chunkIndex).padStart(2, '0')} ${'response'.repeat(2)} `
  397. })
  398. }
  399. function comparisonTurn(index: number): ConversationTurnSpec {
  400. const toolResultMarker = index % COMPARISON_TOOL_INTERVAL === 0
  401. ? `LONG_CONTINUATION_TOOL_RESULT_${String(index)}`
  402. : undefined
  403. return {
  404. prompt: comparisonPrompt(index),
  405. deltas: comparisonDeltas(index),
  406. userMarker: index === COMPARISON_TURNS
  407. ? LIVE_PROMPT_MARKER
  408. : `${LONG_CONTINUATION_USER_PREFIX}_${String(index)}`,
  409. firstMarker: index === COMPARISON_TURNS
  410. ? STREAM_FIRST_MARKER
  411. : `${LONG_CONTINUATION_FIRST_PREFIX}_${String(index)}`,
  412. doneMarker: index === COMPARISON_TURNS
  413. ? STREAM_DONE_MARKER
  414. : `${LONG_CONTINUATION_DONE_PREFIX}_${String(index)}`,
  415. ...toolResultMarker === undefined ? {} : { toolResultMarker },
  416. }
  417. }
  418. function soakTurn(index: number): ConversationTurnSpec {
  419. const suffix = String(index).padStart(3, '0')
  420. const toolResultMarker = index % SOAK_TOOL_INTERVAL === 0
  421. ? `SOAK_CONVERSATION_TOOL_RESULT_${suffix}`
  422. : undefined
  423. return {
  424. prompt: (`${SOAK_USER_PREFIX}_${suffix} `
  425. + `持续对话第 ${String(index)} 轮,检查增量渲染与保留状态。 `
  426. + 'context '.repeat(20)).trimEnd(),
  427. deltas: Array.from({ length: SOAK_DELTA_COUNT }, (_, chunkIndex) => {
  428. if (chunkIndex === 0) return `${SOAK_FIRST_PREFIX}_${suffix} `
  429. if (chunkIndex === SOAK_DELTA_COUNT - 1) return `${SOAK_DONE_PREFIX}_${suffix}.`
  430. return `soak-${suffix}-${String(chunkIndex).padStart(2, '0')} response `
  431. }),
  432. userMarker: `${SOAK_USER_PREFIX}_${suffix}`,
  433. firstMarker: `${SOAK_FIRST_PREFIX}_${suffix}`,
  434. doneMarker: `${SOAK_DONE_PREFIX}_${suffix}`,
  435. ...toolResultMarker === undefined ? {} : { toolResultMarker },
  436. }
  437. }
  438. function toolStream(index: number, marker: string): StreamChunk[] {
  439. const callId = CallId(`performance-tool-${marker.toLowerCase()}-${String(index)}`)
  440. const args = JSON.stringify({
  441. command: `printf '${marker}\\n'`,
  442. description: `Emit performance marker ${String(index)}`,
  443. })
  444. return [
  445. { type: 'block-start', index: 0, blockType: 'tool-call' },
  446. {
  447. type: 'tool-call-delta',
  448. index: 0,
  449. id: callId,
  450. name: 'bash',
  451. argumentsDelta: args,
  452. },
  453. {
  454. type: 'block-end',
  455. index: 0,
  456. block: { type: 'tool-call', id: callId, name: 'bash', arguments: args },
  457. },
  458. { type: 'usage', usage: { inputTokens: 256, outputTokens: 32 } },
  459. { type: 'finish', reason: { kind: 'tool-calls' } },
  460. ]
  461. }
  462. function performanceReplayOverride(
  463. turnCount: number,
  464. turnSpec: (index: number) => ConversationTurnSpec,
  465. ): ReplayOverrideDoc {
  466. const continuationEntries = Array.from(
  467. { length: turnCount },
  468. (_, offset): ReplayEntry[] => {
  469. const index = offset + 1
  470. const spec = turnSpec(index)
  471. const finalResponse: ReplayEntry = {
  472. kind: 'chunks',
  473. chunks: textStream(spec.deltas, Math.ceil(spec.prompt.length / 4)),
  474. }
  475. return spec.toolResultMarker === undefined
  476. ? [finalResponse]
  477. : [{ kind: 'chunks', chunks: toolStream(index, spec.toolResultMarker) }, finalResponse]
  478. },
  479. ).flat()
  480. return continuationEntries
  481. }
  482. function rounded(value: number): number {
  483. return Math.round(value * 1_000) / 1_000
  484. }
  485. async function chromiumMetrics(cdp: CDPSession): Promise<ChromiumMetrics> {
  486. const payload = await cdp.send('Performance.getMetrics')
  487. return Object.fromEntries(payload.metrics.map(metric => [metric.name, metric.value]))
  488. }
  489. async function retainedBrowserState(
  490. cdp: CDPSession,
  491. page: Page,
  492. ): Promise<RetainedBrowserState> {
  493. await cdp.send('HeapProfiler.collectGarbage')
  494. const metrics = await chromiumMetrics(cdp)
  495. return {
  496. domElements: await page.evaluate(() => document.querySelectorAll('*').length),
  497. nodes: requiredMetric(metrics, 'Nodes'),
  498. listeners: requiredMetric(metrics, 'JSEventListeners'),
  499. heapMb: rounded(requiredMetric(metrics, 'JSHeapUsedSize') / 1_048_576),
  500. }
  501. }
  502. function requiredMetric(metrics: ChromiumMetrics, name: string): number {
  503. const value = metrics[name]
  504. if (value === undefined) throw new Error(`Chromium performance metric ${name} is unavailable`)
  505. return value
  506. }
  507. function metricDelta(
  508. before: ChromiumMetrics,
  509. after: ChromiumMetrics,
  510. wallMs: number,
  511. ): Measurement {
  512. return {
  513. wallMs: rounded(wallMs),
  514. taskMs: rounded((requiredMetric(after, 'TaskDuration') - requiredMetric(before, 'TaskDuration')) * 1_000),
  515. scriptMs: rounded((requiredMetric(after, 'ScriptDuration') - requiredMetric(before, 'ScriptDuration')) * 1_000),
  516. layoutMs: rounded((requiredMetric(after, 'LayoutDuration') - requiredMetric(before, 'LayoutDuration')) * 1_000),
  517. recalcStyleMs: rounded(
  518. (requiredMetric(after, 'RecalcStyleDuration') - requiredMetric(before, 'RecalcStyleDuration')) * 1_000,
  519. ),
  520. devtoolsMs: rounded(
  521. (requiredMetric(after, 'DevToolsCommandDuration') - requiredMetric(before, 'DevToolsCommandDuration')) * 1_000,
  522. ),
  523. nodesDelta: requiredMetric(after, 'Nodes') - requiredMetric(before, 'Nodes'),
  524. listenersDelta: requiredMetric(after, 'JSEventListeners') - requiredMetric(before, 'JSEventListeners'),
  525. heapDeltaMb: rounded(
  526. (requiredMetric(after, 'JSHeapUsedSize') - requiredMetric(before, 'JSHeapUsedSize')) / 1_048_576,
  527. ),
  528. totalNodes: requiredMetric(after, 'Nodes'),
  529. heapMb: rounded(requiredMetric(after, 'JSHeapUsedSize') / 1_048_576),
  530. }
  531. }
  532. async function measure<T>(
  533. cdp: CDPSession,
  534. action: () => Promise<T>,
  535. ): Promise<{ measurement: Measurement; value: T }> {
  536. const before = await chromiumMetrics(cdp)
  537. const started = performance.now()
  538. const value = await action()
  539. const wallMs = performance.now() - started
  540. const after = await chromiumMetrics(cdp)
  541. return { measurement: metricDelta(before, after, wallMs), value }
  542. }
  543. async function startMutationProbe(page: Page): Promise<void> {
  544. await page.evaluate(() => {
  545. const target = document.querySelector('[class*="centerCol"]')
  546. if (target === null) throw new Error('stream mutation probe target is unavailable')
  547. const probe = {
  548. batches: 0,
  549. records: 0,
  550. observer: undefined as MutationObserver | undefined,
  551. }
  552. const observer = new MutationObserver((records) => {
  553. probe.batches += 1
  554. probe.records += records.length
  555. })
  556. probe.observer = observer
  557. observer.observe(target, {
  558. attributes: true,
  559. attributeFilter: ['data-streaming'],
  560. characterData: true,
  561. childList: true,
  562. subtree: true,
  563. })
  564. Reflect.set(globalThis, '__dshPerfMutationProbe', probe)
  565. })
  566. }
  567. async function stopMutationProbe(page: Page): Promise<MutationProbeResult> {
  568. return page.evaluate(() => {
  569. const probe = Reflect.get(globalThis, '__dshPerfMutationProbe') as
  570. | { batches: number; records: number; observer: MutationObserver }
  571. | undefined
  572. if (probe === undefined) throw new Error('stream mutation probe was not started')
  573. probe.observer.disconnect()
  574. Reflect.deleteProperty(globalThis, '__dshPerfMutationProbe')
  575. return { batches: probe.batches, records: probe.records }
  576. })
  577. }
  578. async function startUserRenderProbe(
  579. page: Page,
  580. marker: string,
  581. ): Promise<void> {
  582. const send = page.getByRole('button', { name: 'Send message', exact: true })
  583. await send.waitFor({ timeout: 15_000 })
  584. await expect.poll(() => send.isEnabled(), { timeout: 15_000 }).toBe(true)
  585. await page.evaluate((expectedMarker) => {
  586. const target = document.querySelector('[data-conversation-scroll]')
  587. if (target === null) throw new Error('user render probe target is unavailable')
  588. const probe: {
  589. sendAt?: number
  590. domAt?: number
  591. paintAt?: number
  592. trustedClick?: boolean
  593. batches: number
  594. records: number
  595. observer?: MutationObserver
  596. pollForMarker?: () => void
  597. } = { batches: 0, records: 0 }
  598. const transcriptContainsMarker = (): boolean => {
  599. // The composer projects the draft into a backdrop and hidden sizing
  600. // mirror; only a matching text node outside that card proves delivery.
  601. const walker = document.createTreeWalker(target, NodeFilter.SHOW_TEXT)
  602. for (let node = walker.nextNode(); node !== null; node = walker.nextNode()) {
  603. if (!node.textContent?.includes(expectedMarker)) continue
  604. const parent = node.parentElement
  605. if (parent !== null && parent.closest('[data-composer-card]') === null) return true
  606. }
  607. return false
  608. }
  609. const markDomVisible = (): void => {
  610. if (
  611. probe.sendAt === undefined
  612. || probe.domAt !== undefined
  613. || !transcriptContainsMarker()
  614. ) return
  615. probe.domAt = globalThis.performance.now()
  616. // The second callback runs only after a render opportunity for the DOM
  617. // insertion observed before the first callback.
  618. requestAnimationFrame(() => {
  619. requestAnimationFrame(() => {
  620. probe.paintAt = globalThis.performance.now()
  621. observer.disconnect()
  622. })
  623. })
  624. }
  625. const pollForMarker = (): void => {
  626. markDomVisible()
  627. if (probe.domAt === undefined) requestAnimationFrame(pollForMarker)
  628. }
  629. const observer = new MutationObserver((records) => {
  630. if (probe.sendAt === undefined) return
  631. probe.batches += 1
  632. probe.records += records.length
  633. markDomVisible()
  634. })
  635. probe.observer = observer
  636. probe.pollForMarker = pollForMarker
  637. observer.observe(target, {
  638. attributes: true,
  639. attributeFilter: ['data-streaming'],
  640. characterData: true,
  641. childList: true,
  642. subtree: true,
  643. })
  644. Reflect.set(globalThis, '__dshPerfUserRenderProbe', probe)
  645. }, marker)
  646. await send.evaluate((button) => {
  647. button.addEventListener('click', (event) => {
  648. const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
  649. | { sendAt?: number; trustedClick?: boolean; pollForMarker?: () => void }
  650. | undefined
  651. if (probe?.pollForMarker === undefined) throw new Error('user render probe was not started')
  652. probe.trustedClick = event.isTrusted
  653. probe.sendAt = globalThis.performance.now()
  654. requestAnimationFrame(probe.pollForMarker)
  655. }, { capture: true, once: true })
  656. })
  657. }
  658. async function triggerUserRenderProbe(page: Page): Promise<void> {
  659. const send = page.getByRole('button', { name: 'Send message', exact: true })
  660. await send.click()
  661. }
  662. async function stopUserRenderProbe(
  663. page: Page,
  664. marker: string,
  665. ): Promise<UserRenderProbeResult> {
  666. try {
  667. await page.waitForFunction(() => {
  668. const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
  669. | { paintAt?: number }
  670. | undefined
  671. return probe?.paintAt !== undefined
  672. }, undefined, { timeout: 15_000 })
  673. } catch (error) {
  674. const diagnostic = await page.evaluate((expectedMarker) => {
  675. const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
  676. | {
  677. sendAt?: number
  678. domAt?: number
  679. paintAt?: number
  680. trustedClick?: boolean
  681. batches: number
  682. records: number
  683. }
  684. | undefined
  685. const target = document.querySelector('[data-conversation-scroll]')
  686. let markerOutsideComposer = false
  687. if (target !== null) {
  688. const walker = document.createTreeWalker(target, NodeFilter.SHOW_TEXT)
  689. for (let node = walker.nextNode(); node !== null; node = walker.nextNode()) {
  690. if (!node.textContent?.includes(expectedMarker)) continue
  691. const parent = node.parentElement
  692. if (parent !== null && parent.closest('[data-composer-card]') === null) {
  693. markerOutsideComposer = true
  694. break
  695. }
  696. }
  697. }
  698. return {
  699. probe,
  700. markerOutsideComposer,
  701. markerInComposer: document.querySelector('[data-composer-card]')
  702. ?.textContent?.includes(expectedMarker) ?? false,
  703. }
  704. }, marker)
  705. throw new Error(`user render probe timed out: ${JSON.stringify(diagnostic)}`, { cause: error })
  706. }
  707. return page.evaluate(() => {
  708. const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
  709. | {
  710. sendAt?: number
  711. domAt?: number
  712. paintAt?: number
  713. trustedClick?: boolean
  714. batches: number
  715. records: number
  716. }
  717. | undefined
  718. Reflect.deleteProperty(globalThis, '__dshPerfUserRenderProbe')
  719. if (
  720. probe?.trustedClick !== true
  721. || probe.sendAt === undefined
  722. || probe.domAt === undefined
  723. || probe.paintAt === undefined
  724. ) {
  725. throw new Error('user render probe did not observe a trusted click, DOM insertion, and paint')
  726. }
  727. return {
  728. trustedClick: probe.trustedClick,
  729. sendToDomMs: probe.domAt - probe.sendAt,
  730. sendToPaintMs: probe.paintAt - probe.sendAt,
  731. domToPaintMs: probe.paintAt - probe.domAt,
  732. mutationBatches: probe.batches,
  733. mutationRecords: probe.records,
  734. }
  735. })
  736. }
  737. async function stableCount(
  738. locator: Locator,
  739. accepts: (count: number) => boolean,
  740. timeoutMs = 60_000,
  741. ): Promise<number> {
  742. const deadline = performance.now() + timeoutMs
  743. let previous = -1
  744. let stableReads = 0
  745. while (performance.now() < deadline) {
  746. const count = await locator.count()
  747. stableReads = accepts(count) && count === previous ? stableReads + 1 : 0
  748. if (stableReads >= 4) return count
  749. previous = count
  750. await new Promise(resolve => setTimeout(resolve, 50))
  751. }
  752. throw new Error(`browser row count did not stabilize; last count ${String(previous)}`)
  753. }
  754. async function conversationTurns(page: Page): Promise<number> {
  755. const stats = page.getByText(/\d+ turns · \d+ steps/, { exact: true }).last()
  756. await stats.waitFor({ timeout: 15_000 })
  757. const value = await stats.textContent()
  758. const match = value?.match(/^(\d+) turns · \d+ steps$/)
  759. if (match?.[1] === undefined) throw new Error(`unexpected conversation stats ${JSON.stringify(value)}`)
  760. return Number(match[1])
  761. }
  762. function retainedDelta(
  763. before: RetainedBrowserState,
  764. after: RetainedBrowserState,
  765. ): RetainedBrowserState {
  766. return {
  767. domElements: after.domElements - before.domElements,
  768. nodes: after.nodes - before.nodes,
  769. listeners: after.listeners - before.listeners,
  770. heapMb: rounded(after.heapMb - before.heapMb),
  771. }
  772. }
  773. async function launchPerformanceWorld(
  774. options: PerformanceWorldOptions,
  775. ): Promise<PerformanceWorld> {
  776. const setupStarted = performance.now()
  777. let replayDir: string | undefined
  778. let scaffold: WebScaffold | undefined
  779. let page: Page | undefined
  780. try {
  781. if (options.replay === undefined) {
  782. scaffold = await launchWebScaffold()
  783. } else {
  784. replayDir = await mkdtemp(join(tmpdir(), 'dsh-web-perf-replay-'))
  785. const replayOverride = join(replayDir, 'replay.override.json')
  786. await writeFile(replayOverride, JSON.stringify(options.replay))
  787. scaffold = await launchWebScaffold({
  788. // The override-only fixture supplies one positional script for this
  789. // world's single live session.
  790. replayFixture: join(replayDir, 'override-only.jsonl'),
  791. replayOverride,
  792. paceMs: STREAM_PACE_MS,
  793. replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
  794. })
  795. }
  796. const sessionEvents: SessionEvent[] = []
  797. scaffold.ctx.on('session/event', (_session, event: SessionEvent) => {
  798. sessionEvents.push(event)
  799. })
  800. if ((options.sidebarSessions ?? 0) > 0) {
  801. const small = smallSidebarFixture()
  802. for (let index = 0; index < (options.sidebarSessions ?? 0); index += 1) {
  803. await seedSession(scaffold, small, `perf-sidebar-${String(index).padStart(4, '0')}`)
  804. }
  805. }
  806. if (options.seedLongHistory === true) {
  807. await seedSession(scaffold, longHistoryFixture(), LONG_SESSION_ID)
  808. }
  809. const setupMs = performance.now() - setupStarted
  810. page = await newEnglishPage(options.browser)
  811. return {
  812. scaffold,
  813. page,
  814. tripwire: watchConsole(page),
  815. sessionEvents,
  816. setupMs,
  817. ...replayDir === undefined ? {} : { replayDir },
  818. }
  819. } catch (error) {
  820. const failures: unknown[] = [error]
  821. if (page !== undefined) {
  822. try {
  823. await page.close()
  824. } catch (cleanupError) {
  825. failures.push(cleanupError)
  826. }
  827. }
  828. if (scaffold !== undefined) {
  829. try {
  830. await scaffold.close()
  831. } catch (cleanupError) {
  832. failures.push(cleanupError)
  833. }
  834. }
  835. if (replayDir !== undefined) {
  836. try {
  837. await rm(replayDir, { recursive: true, force: true })
  838. } catch (cleanupError) {
  839. failures.push(cleanupError)
  840. }
  841. }
  842. if (failures.length === 1) throw error
  843. throw new AggregateError(failures, 'web performance setup and cleanup failed')
  844. }
  845. }
  846. async function closePerformanceWorld(world: PerformanceWorld): Promise<void> {
  847. const failures: unknown[] = []
  848. await world.page.close().catch((error: unknown) => failures.push(error))
  849. await world.scaffold.close().catch((error: unknown) => failures.push(error))
  850. if (world.replayDir !== undefined) {
  851. await rm(world.replayDir, { recursive: true, force: true })
  852. .catch((error: unknown) => failures.push(error))
  853. }
  854. if (failures.length === 1) throw failures[0]
  855. if (failures.length > 1) throw new AggregateError(failures, 'web performance teardown failed')
  856. }
  857. async function openPerformancePage(
  858. world: PerformanceWorld,
  859. expectedSessions: number,
  860. ): Promise<Locator> {
  861. await world.page.goto(world.scaffold.baseUrl, { waitUntil: 'load' })
  862. await world.page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
  863. const group = world.page.getByRole('treeitem').first()
  864. await expect.poll(() => group.textContent(), { timeout: 30_000 })
  865. .toContain(`${String(expectedSessions)} ${expectedSessions === 1 ? 'session' : 'sessions'}`)
  866. return group
  867. }
  868. async function openLongHistory(page: Page): Promise<number> {
  869. await page.getByRole('textbox', { name: 'Search name, keywords...', exact: true })
  870. .fill('LONG_PERF_SENTINEL')
  871. const results = page.getByRole('tree', { name: 'Search results' }).getByRole('treeitem')
  872. await expect.poll(() => results.count(), { timeout: 60_000 }).toBe(1)
  873. await results.first().click()
  874. await page.getByRole('tab', { name: 'Chat', exact: true }).waitFor({ timeout: 30_000 })
  875. return conversationTurns(page)
  876. }
  877. async function continueConversation(
  878. world: PerformanceWorld,
  879. cdp: CDPSession,
  880. options: {
  881. readonly startingTurns: number
  882. readonly turnCount: number
  883. readonly turnSpec: (index: number) => ConversationTurnSpec
  884. readonly expectedSessionId?: SessionId
  885. readonly checkpointInterval?: number
  886. },
  887. ): Promise<ConversationReport> {
  888. const composer = world.page.locator('textarea:enabled').last()
  889. await composer.waitFor({ timeout: 15_000 })
  890. const retainedBefore = await retainedBrowserState(cdp, world.page)
  891. const checkpoints: RetainedCheckpoint[] = [{ turns: options.startingTurns, state: retainedBefore }]
  892. const turns: ContinuedTurnReport[] = []
  893. let liveSessionId = options.expectedSessionId
  894. for (let index = 1; index <= options.turnCount; index += 1) {
  895. const spec = options.turnSpec(index)
  896. const composerFill = await measure(cdp, async () => {
  897. await composer.fill(spec.prompt)
  898. await expect.poll(() => composer.inputValue()).toBe(spec.prompt)
  899. return (await composer.inputValue()).length
  900. })
  901. expect(composerFill.value).toBe(spec.prompt.length)
  902. const eventStart = world.sessionEvents.length
  903. await startMutationProbe(world.page)
  904. const streamBefore = await chromiumMetrics(cdp)
  905. const streamStarted = performance.now()
  906. const settled = world.scaffold.whenTurnSettled(60_000)
  907. await world.page.getByRole('button', { name: 'Send message', exact: true }).click()
  908. await world.page.getByText(spec.userMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  909. const clickToUserEchoMs = performance.now() - streamStarted
  910. await world.page.getByText(spec.firstMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  911. const clickToFirstChunkMs = performance.now() - streamStarted
  912. const settledSessionId = await settled
  913. if (liveSessionId === undefined) {
  914. liveSessionId = settledSessionId
  915. } else {
  916. expect(settledSessionId).toBe(liveSessionId)
  917. }
  918. await expect.poll(
  919. () => world.page.locator('[data-streaming="true"]').count(),
  920. { timeout: 15_000 },
  921. ).toBe(0)
  922. await world.page.getByText(spec.doneMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  923. const clickToSettledMs = performance.now() - streamStarted
  924. const streamAfter = await chromiumMetrics(cdp)
  925. const mutations = await stopMutationProbe(world.page)
  926. const turnEvents = world.sessionEvents.slice(eventStart)
  927. const chunks = turnEvents.filter(event => event.type === 'assistant/chunk')
  928. const toolCalls = turnEvents.filter(event => event.type === 'tool/call')
  929. const toolResults = turnEvents.filter(event => event.type === 'tool/result')
  930. const toolTurn = spec.toolResultMarker !== undefined
  931. expect(chunks).toHaveLength(spec.deltas.length + (toolTurn ? 9 : 4))
  932. expect(toolCalls).toHaveLength(toolTurn ? 1 : 0)
  933. expect(toolResults).toHaveLength(toolTurn ? 1 : 0)
  934. if (spec.toolResultMarker !== undefined) {
  935. expect(toolCalls[0]?.data.name).toBe('bash')
  936. const toolResult = toolResults[0]
  937. if (toolResult?.type !== 'tool/result') {
  938. throw new Error(`continued turn ${String(index)} did not log its tool result`)
  939. }
  940. const resultBlock = toolResult.data.message.content.find(block => block.type === 'tool-result')
  941. expect(resultBlock?.isError).toBe(false)
  942. expect(resultBlock?.content
  943. .filter(block => block.type === 'text')
  944. .map(block => block.text)
  945. .join('')).toContain(spec.toolResultMarker)
  946. }
  947. const user = turnEvents.find(
  948. event => event.type === 'user/message' && event.data.source.kind === 'user',
  949. )
  950. if (user?.type !== 'user/message') {
  951. throw new Error(`continued turn ${String(index)} did not log its user message`)
  952. }
  953. expect(user.data.content
  954. .filter(block => block.type === 'text')
  955. .map(block => block.text)
  956. .join('')).toBe(spec.prompt)
  957. const resultingTurns = await conversationTurns(world.page)
  958. expect(resultingTurns).toBe(options.startingTurns + index)
  959. turns.push({
  960. ordinal: index,
  961. resultingTurns,
  962. kind: toolTurn ? 'tool' : 'text',
  963. promptChars: spec.prompt.length,
  964. composerFill: composerFill.measurement,
  965. stream: {
  966. paceMs: STREAM_PACE_MS,
  967. deltaChunks: spec.deltas.length,
  968. persistedChunks: chunks.length,
  969. toolCalls: toolCalls.length,
  970. toolResults: toolResults.length,
  971. clickToUserEchoMs: rounded(clickToUserEchoMs),
  972. clickToFirstChunkMs: rounded(clickToFirstChunkMs),
  973. firstChunkToSettledMs: rounded(clickToSettledMs - clickToFirstChunkMs),
  974. ...mutations,
  975. ...metricDelta(streamBefore, streamAfter, clickToSettledMs),
  976. },
  977. })
  978. if (options.checkpointInterval !== undefined && index % options.checkpointInterval === 0) {
  979. checkpoints.push({
  980. turns: options.startingTurns + index,
  981. state: await retainedBrowserState(cdp, world.page),
  982. })
  983. }
  984. }
  985. const lastCheckpoint = checkpoints.at(-1)
  986. const retainedAfter = lastCheckpoint?.turns === options.startingTurns + options.turnCount
  987. ? lastCheckpoint.state
  988. : await retainedBrowserState(cdp, world.page)
  989. if (lastCheckpoint?.turns !== options.startingTurns + options.turnCount) {
  990. checkpoints.push({ turns: options.startingTurns + options.turnCount, state: retainedAfter })
  991. }
  992. return {
  993. startingTurns: options.startingTurns,
  994. turnsAdded: options.turnCount,
  995. toolTurns: turns.filter(turn => turn.kind === 'tool').length,
  996. retainedBefore,
  997. retainedAfter,
  998. retainedDelta: retainedDelta(retainedBefore, retainedAfter),
  999. checkpoints,
  1000. turns,
  1001. }
  1002. }
  1003. async function measurePostSoakUserRender(
  1004. world: PerformanceWorld,
  1005. cdp: CDPSession,
  1006. ): Promise<object> {
  1007. const spec = soakTurn(POST_SOAK_RENDER_TURN)
  1008. if (spec.toolResultMarker !== undefined) {
  1009. throw new Error('post-soak render probe must remain a text-only turn')
  1010. }
  1011. const composer = world.page.locator('textarea:enabled').last()
  1012. const composerFill = await measure(cdp, async () => {
  1013. await composer.fill(spec.prompt)
  1014. await expect.poll(() => composer.inputValue()).toBe(spec.prompt)
  1015. return (await composer.inputValue()).length
  1016. })
  1017. expect(composerFill.value).toBe(spec.prompt.length)
  1018. const eventStart = world.sessionEvents.length
  1019. await startUserRenderProbe(world.page, spec.userMarker)
  1020. const browserBefore = await chromiumMetrics(cdp)
  1021. const fullTurnStarted = performance.now()
  1022. const settled = world.scaffold.whenTurnSettled(60_000)
  1023. await triggerUserRenderProbe(world.page)
  1024. const browserTiming = await stopUserRenderProbe(world.page, spec.userMarker)
  1025. const browserAfter = await chromiumMetrics(cdp)
  1026. const browserAfterPaintSample = metricDelta(
  1027. browserBefore,
  1028. browserAfter,
  1029. performance.now() - fullTurnStarted,
  1030. )
  1031. await world.page.getByText(spec.firstMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  1032. const settledSessionId = await settled
  1033. await expect.poll(
  1034. () => world.page.locator('[data-streaming="true"]').count(),
  1035. { timeout: 15_000 },
  1036. ).toBe(0)
  1037. await world.page.getByText(spec.doneMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  1038. const fullTurnMs = performance.now() - fullTurnStarted
  1039. const turnEvents = world.sessionEvents.slice(eventStart)
  1040. const chunks = turnEvents.filter(event => event.type === 'assistant/chunk')
  1041. const user = turnEvents.find(
  1042. event => event.type === 'user/message' && event.data.source.kind === 'user',
  1043. )
  1044. if (user?.type !== 'user/message') throw new Error('post-soak render probe did not log its user message')
  1045. expect(user.data.content
  1046. .filter(block => block.type === 'text')
  1047. .map(block => block.text)
  1048. .join('')).toBe(spec.prompt)
  1049. expect(chunks).toHaveLength(spec.deltas.length + 4)
  1050. expect(turnEvents.filter(event => event.type === 'tool/call')).toHaveLength(0)
  1051. expect(turnEvents.filter(event => event.type === 'tool/result')).toHaveLength(0)
  1052. expect(world.scaffold.ctx.agents.get(settledSessionId)).toBeDefined()
  1053. expect(await conversationTurns(world.page)).toBe(POST_SOAK_RENDER_TURN)
  1054. return {
  1055. ordinal: POST_SOAK_RENDER_TURN,
  1056. resultingTurns: POST_SOAK_RENDER_TURN,
  1057. promptChars: spec.prompt.length,
  1058. composerFill: composerFill.measurement,
  1059. trustedClick: browserTiming.trustedClick,
  1060. sendToDomMs: rounded(browserTiming.sendToDomMs),
  1061. sendToPaintMs: rounded(browserTiming.sendToPaintMs),
  1062. domToPaintMs: rounded(browserTiming.domToPaintMs),
  1063. mutationBatchesThroughPaint: browserTiming.mutationBatches,
  1064. mutationRecordsThroughPaint: browserTiming.mutationRecords,
  1065. browserAfterPaintSample,
  1066. fullTurnMs: rounded(fullTurnMs),
  1067. persistedChunks: chunks.length,
  1068. }
  1069. }
  1070. function average(values: readonly number[]): number {
  1071. return rounded(values.reduce((sum, value) => sum + value, 0) / values.length)
  1072. }
  1073. function p95(values: readonly number[]): number {
  1074. const sorted = [...values].sort((left, right) => left - right)
  1075. return sorted[Math.max(0, Math.ceil(sorted.length * 0.95) - 1)] ?? 0
  1076. }
  1077. function summarizeTurnWindows(
  1078. turns: readonly ContinuedTurnReport[],
  1079. windowSize: number,
  1080. ): object[] {
  1081. const windows: object[] = []
  1082. for (let start = 0; start < turns.length; start += windowSize) {
  1083. const window = turns.slice(start, start + windowSize)
  1084. const streamWall = window.map(turn => turn.stream.wallMs)
  1085. const userEcho = window.map(turn => turn.stream.clickToUserEchoMs)
  1086. const firstChunk = window.map(turn => turn.stream.clickToFirstChunkMs)
  1087. windows.push({
  1088. turns: `${String(start + 1)}-${String(start + window.length)}`,
  1089. toolTurns: window.filter(turn => turn.kind === 'tool').length,
  1090. average: {
  1091. composerFillWallMs: average(window.map(turn => turn.composerFill.wallMs)),
  1092. streamWallMs: average(streamWall),
  1093. clickToUserEchoMs: average(userEcho),
  1094. clickToFirstChunkMs: average(firstChunk),
  1095. taskMs: average(window.map(turn => turn.stream.taskMs)),
  1096. scriptMs: average(window.map(turn => turn.stream.scriptMs)),
  1097. recalcStyleMs: average(window.map(turn => turn.stream.recalcStyleMs)),
  1098. mutationBatches: average(window.map(turn => turn.stream.batches)),
  1099. },
  1100. p95: {
  1101. streamWallMs: rounded(p95(streamWall)),
  1102. clickToUserEchoMs: rounded(p95(userEcho)),
  1103. clickToFirstChunkMs: rounded(p95(firstChunk)),
  1104. },
  1105. })
  1106. }
  1107. return windows
  1108. }
  1109. describe('manual web performance: complex workspace and history', () => {
  1110. let browser: Browser
  1111. beforeAll(async () => {
  1112. if (webSnapshotMode() === 'record') {
  1113. throw new Error('manual web performance runs only with deterministic replay')
  1114. }
  1115. browser = await chromium.launch()
  1116. })
  1117. afterAll(async () => {
  1118. await browser?.close()
  1119. })
  1120. it('reports workspace, history, and trajectory cardinality costs', async () => {
  1121. const world = await launchPerformanceWorld({
  1122. browser,
  1123. sidebarSessions: SIDEBAR_SESSION_COUNT,
  1124. seedLongHistory: true,
  1125. })
  1126. try {
  1127. const bootStarted = performance.now()
  1128. const group = await openPerformancePage(world, SIDEBAR_SESSION_COUNT + 1)
  1129. const bootReadyMs = performance.now() - bootStarted
  1130. const page = world.page
  1131. const cdp = await page.context().newCDPSession(page)
  1132. await cdp.send('Performance.enable')
  1133. const firstContentfulPaintMs = await page.evaluate(
  1134. () => globalThis.performance.getEntriesByName('first-contentful-paint')[0]?.startTime,
  1135. )
  1136. const sidebar = await measure(cdp, async () => {
  1137. await group.click()
  1138. return stableCount(
  1139. page.getByRole('treeitem'),
  1140. count => count === SIDEBAR_SESSION_COUNT + 2,
  1141. )
  1142. })
  1143. expect(sidebar.value).toBe(SIDEBAR_SESSION_COUNT + 2)
  1144. await group.click()
  1145. await expect.poll(() => page.getByRole('treeitem').count()).toBe(1)
  1146. const contentSearch = await measure(cdp, async () => {
  1147. await page.getByRole('textbox', { name: 'Search name, keywords...', exact: true })
  1148. .fill('LONG_PERF_SENTINEL')
  1149. const results = page.getByRole('tree', { name: 'Search results' }).getByRole('treeitem')
  1150. await expect.poll(() => results.count(), { timeout: 60_000 }).toBe(1)
  1151. await results.first().waitFor({ timeout: 60_000 })
  1152. return results.first()
  1153. })
  1154. const opened = await measure(cdp, async () => {
  1155. await contentSearch.value.click()
  1156. await page.getByRole('tab', { name: 'Trajectory', exact: true }).waitFor({ timeout: 30_000 })
  1157. return conversationTurns(page)
  1158. })
  1159. expect(opened.value).toBe(DEFAULT_HISTORY_TURNS)
  1160. const trajectoryRows = page.getByRole('row')
  1161. const coldTrajectory = await measure(cdp, async () => {
  1162. await page.getByRole('tab', { name: 'Trajectory', exact: true }).click()
  1163. return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS)
  1164. })
  1165. expect(coldTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS)
  1166. const collapseTurns = await measure(cdp, async () => {
  1167. await page.getByRole('button', { name: 'Collapse turns', exact: true }).click()
  1168. return stableCount(trajectoryRows, count => count > 0 && count < EXPECTED_TRAJECTORY_ROWS)
  1169. })
  1170. expect(collapseTurns.value).toBeLessThan(EXPECTED_TRAJECTORY_ROWS)
  1171. const trajectorySearch = await measure(cdp, async () => {
  1172. await page.getByRole('searchbox', { name: 'Search trajectory', exact: true }).fill('turn 499')
  1173. return stableCount(trajectoryRows, count => count > 0 && count < 20)
  1174. })
  1175. expect(trajectorySearch.value).toBeLessThan(20)
  1176. await page.getByRole('tab', { name: 'Chat', exact: true }).click()
  1177. const historyPages: { turns: number; measurement: Measurement }[] = []
  1178. let turns = await conversationTurns(page)
  1179. while (turns < LONG_HISTORY_TURNS) {
  1180. const previousTurns = turns
  1181. const older = await measure(cdp, async () => {
  1182. await page.getByRole('button', { name: 'Load earlier', exact: true }).click()
  1183. await expect.poll(() => conversationTurns(page), { timeout: 30_000 })
  1184. .toBeGreaterThan(previousTurns)
  1185. return conversationTurns(page)
  1186. })
  1187. turns = older.value
  1188. historyPages.push({ turns, measurement: older.measurement })
  1189. }
  1190. const warmTrajectory = await measure(cdp, async () => {
  1191. await page.getByRole('tab', { name: 'Trajectory', exact: true }).click()
  1192. return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS)
  1193. })
  1194. expect(warmTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS)
  1195. const warmConversation = await measure(cdp, async () => {
  1196. await page.getByRole('tab', { name: 'Chat', exact: true }).click()
  1197. return conversationTurns(page)
  1198. })
  1199. expect(warmConversation.value).toBe(LONG_HISTORY_TURNS)
  1200. console.info(`WEB_PERF_RESULT ${JSON.stringify({
  1201. scenario: 'workspace-history-trajectory',
  1202. fixture: {
  1203. sidebarSessions: SIDEBAR_SESSION_COUNT,
  1204. totalSessions: SIDEBAR_SESSION_COUNT + 1,
  1205. longHistoryTurns: LONG_HISTORY_TURNS,
  1206. toolCalls: EXPECTED_TOOL_CALLS,
  1207. trajectoryRows: EXPECTED_TRAJECTORY_ROWS,
  1208. },
  1209. setupMs: rounded(world.setupMs),
  1210. boot: {
  1211. readyMs: rounded(bootReadyMs),
  1212. firstContentfulPaintMs: firstContentfulPaintMs === undefined
  1213. ? null
  1214. : rounded(firstContentfulPaintMs),
  1215. },
  1216. sidebarExpand: sidebar.measurement,
  1217. contentSearch: contentSearch.measurement,
  1218. openLongHistory: { initialTurns: opened.value, ...opened.measurement },
  1219. coldTrajectory: { rows: coldTrajectory.value, ...coldTrajectory.measurement },
  1220. collapseTurns: { rows: collapseTurns.value, ...collapseTurns.measurement },
  1221. trajectorySearch: { rows: trajectorySearch.value, ...trajectorySearch.measurement },
  1222. historyPages,
  1223. warmTrajectory: { rows: warmTrajectory.value, ...warmTrajectory.measurement },
  1224. warmConversation: { turns: warmConversation.value, ...warmConversation.measurement },
  1225. }, null, 2)}`)
  1226. expect(world.tripwire.warnings).toEqual([])
  1227. expect(world.tripwire.pageErrors).toEqual([])
  1228. } finally {
  1229. await closePerformanceWorld(world)
  1230. }
  1231. })
  1232. it('reports default 24-turn history plus eight continued turns', async () => {
  1233. const world = await launchPerformanceWorld({
  1234. browser,
  1235. replay: performanceReplayOverride(COMPARISON_TURNS, comparisonTurn),
  1236. seedLongHistory: true,
  1237. })
  1238. try {
  1239. await openPerformancePage(world, 1)
  1240. const cdp = await world.page.context().newCDPSession(world.page)
  1241. await cdp.send('Performance.enable')
  1242. const opened = await measure(cdp, () => openLongHistory(world.page))
  1243. expect(opened.value).toBe(DEFAULT_HISTORY_TURNS)
  1244. const conversation = await continueConversation(world, cdp, {
  1245. startingTurns: DEFAULT_HISTORY_TURNS,
  1246. turnCount: COMPARISON_TURNS,
  1247. turnSpec: comparisonTurn,
  1248. expectedSessionId: SessionId(LONG_SESSION_ID),
  1249. })
  1250. expect(conversation.toolTurns).toBe(2)
  1251. console.info(`WEB_PERF_RESULT ${JSON.stringify({
  1252. scenario: 'default-resume-24-plus-8',
  1253. setupMs: rounded(world.setupMs),
  1254. replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
  1255. openLongHistory: { turns: opened.value, ...opened.measurement },
  1256. conversation,
  1257. }, null, 2)}`)
  1258. expect(world.tripwire.warnings).toEqual([])
  1259. expect(world.tripwire.pageErrors).toEqual([])
  1260. } finally {
  1261. await closePerformanceWorld(world)
  1262. }
  1263. })
  1264. it('reports fully expanded 500-turn history plus eight continued turns', async () => {
  1265. const world = await launchPerformanceWorld({
  1266. browser,
  1267. replay: performanceReplayOverride(COMPARISON_TURNS, comparisonTurn),
  1268. seedLongHistory: true,
  1269. })
  1270. try {
  1271. await openPerformancePage(world, 1)
  1272. const cdp = await world.page.context().newCDPSession(world.page)
  1273. await cdp.send('Performance.enable')
  1274. expect(await openLongHistory(world.page)).toBe(DEFAULT_HISTORY_TURNS)
  1275. const historyPages: { turns: number; measurement: Measurement }[] = []
  1276. let turns = DEFAULT_HISTORY_TURNS
  1277. while (turns < LONG_HISTORY_TURNS) {
  1278. const previousTurns = turns
  1279. const older = await measure(cdp, async () => {
  1280. await world.page.getByRole('button', { name: 'Load earlier', exact: true }).click()
  1281. await expect.poll(() => conversationTurns(world.page), { timeout: 30_000 })
  1282. .toBeGreaterThan(previousTurns)
  1283. return conversationTurns(world.page)
  1284. })
  1285. turns = older.value
  1286. historyPages.push({ turns, measurement: older.measurement })
  1287. }
  1288. expect(turns).toBe(LONG_HISTORY_TURNS)
  1289. const conversation = await continueConversation(world, cdp, {
  1290. startingTurns: LONG_HISTORY_TURNS,
  1291. turnCount: COMPARISON_TURNS,
  1292. turnSpec: comparisonTurn,
  1293. expectedSessionId: SessionId(LONG_SESSION_ID),
  1294. })
  1295. expect(conversation.toolTurns).toBe(2)
  1296. console.info(`WEB_PERF_RESULT ${JSON.stringify({
  1297. scenario: 'expanded-history-500-plus-8',
  1298. setupMs: rounded(world.setupMs),
  1299. replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
  1300. historyPages,
  1301. conversation,
  1302. }, null, 2)}`)
  1303. expect(world.tripwire.warnings).toEqual([])
  1304. expect(world.tripwire.pageErrors).toEqual([])
  1305. } finally {
  1306. await closePerformanceWorld(world)
  1307. }
  1308. })
  1309. it('reports one hundred generated turns and the next user-message paint', async () => {
  1310. const world = await launchPerformanceWorld({
  1311. browser,
  1312. replay: performanceReplayOverride(POST_SOAK_RENDER_TURN, soakTurn),
  1313. })
  1314. let testFailure: unknown
  1315. try {
  1316. await world.page.goto(world.scaffold.baseUrl, { waitUntil: 'load' })
  1317. await world.page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
  1318. await connectFreshWorkspace(world.page, world.scaffold.workspaceCwd, 'continuous-conversation-perf')
  1319. const cdp = await world.page.context().newCDPSession(world.page)
  1320. await cdp.send('Performance.enable')
  1321. const conversation = await continueConversation(world, cdp, {
  1322. startingTurns: 0,
  1323. turnCount: SOAK_TURNS,
  1324. turnSpec: soakTurn,
  1325. checkpointInterval: SOAK_CHECKPOINT_INTERVAL,
  1326. })
  1327. expect(conversation.toolTurns).toBe(SOAK_TURNS / SOAK_TOOL_INTERVAL)
  1328. const postSoakUserRender = await measurePostSoakUserRender(world, cdp)
  1329. const { turns, ...retained } = conversation
  1330. console.info(`WEB_PERF_RESULT ${JSON.stringify({
  1331. scenario: 'continuous-100-turn-soak',
  1332. setupMs: rounded(world.setupMs),
  1333. replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
  1334. ...retained,
  1335. windows: summarizeTurnWindows(turns, SOAK_CHECKPOINT_INTERVAL),
  1336. postSoakUserRender,
  1337. }, null, 2)}`)
  1338. expect(world.tripwire.warnings).toEqual([])
  1339. expect(world.tripwire.pageErrors).toEqual([])
  1340. } catch (error) {
  1341. testFailure = error
  1342. throw error
  1343. } finally {
  1344. try {
  1345. await closePerformanceWorld(world)
  1346. } catch (cleanupError) {
  1347. if (testFailure === undefined) throw cleanupError
  1348. throw new AggregateError(
  1349. [testFailure, cleanupError],
  1350. 'continuous conversation performance test and teardown failed',
  1351. )
  1352. }
  1353. }
  1354. })
  1355. })