complex-history.perf.ts 53 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457
  1. // Opt-in browser benchmark for high-cardinality workspace and history
  2. // rendering. It reports measurements without timing assertions because host
  3. // speed is not a correctness contract; structural assertions keep the number
  4. // of workspaces and history entries from silently shrinking.
  5. import { mkdtemp, rm, writeFile } from 'node:fs/promises'
  6. import { tmpdir } from 'node:os'
  7. import { join } from 'node:path'
  8. import { performance } from 'node:perf_hooks'
  9. import type { Browser, CDPSession, Locator, Page } from 'playwright'
  10. import { chromium } from 'playwright'
  11. import { afterAll, beforeAll, describe, expect, it } from 'vitest'
  12. import type { StreamChunk } from '@deepseek-ai/dsh-llm'
  13. import {
  14. ToolCallId,
  15. createAssistantMessage,
  16. createSystemMessage,
  17. createToolResultMessage,
  18. createUserMessage,
  19. expandAssistantStream,
  20. } from '@deepseek-ai/dsh-llm'
  21. import type { ReplayEntry, ReplayOverrideDoc } from '@deepseek-ai/dsh-llm-replay'
  22. import type { SessionEvent, SessionSeq } from '@deepseek-ai/dsh-session'
  23. import {
  24. SESSION_FORMAT_VERSION,
  25. Session,
  26. SessionId,
  27. } from '@deepseek-ai/dsh-session'
  28. // Carries the session/title event declaration into the fixture builder.
  29. import type {} from '@deepseek-ai/dsh-session-title'
  30. import {
  31. launchWebScaffold,
  32. seedSession,
  33. watchConsole,
  34. webSnapshotMode,
  35. type WebScaffold,
  36. } from './scaffold.ts'
  37. import { connectFreshWorkspace, newEnglishPage } from './support.ts'
  38. const SIDEBAR_SESSION_COUNT = 1_000
  39. const LONG_SESSION_ID = 'perf-long-history'
  40. const LONG_SESSION_TITLE = 'LONG_PERF_SENTINEL 500-turn session'
  41. const LONG_HISTORY_TURNS = 500
  42. const TOOL_TURN_INTERVAL = 10
  43. const TOOLS_PER_TOOL_TURN = 10
  44. const EXPECTED_TOOL_CALLS = LONG_HISTORY_TURNS / TOOL_TURN_INTERVAL * TOOLS_PER_TOOL_TURN
  45. const EXPECTED_TRAJECTORY_ROWS = 2_100
  46. const DEFAULT_HISTORY_TURNS = 24
  47. const PERF_REPLAY_CONTEXT_WINDOW = 10_000_000
  48. const STREAM_PACE_MS = 8
  49. const STREAM_DELTA_COUNT = 120
  50. const COMPARISON_TURNS = 8
  51. const COMPARISON_DELTA_COUNT = 24
  52. const COMPARISON_TOOL_INTERVAL = 3
  53. const SOAK_TURNS = 100
  54. const POST_SOAK_RENDER_TURN = SOAK_TURNS + 1
  55. const SOAK_DELTA_COUNT = 8
  56. const SOAK_TOOL_INTERVAL = 10
  57. const SOAK_CHECKPOINT_INTERVAL = 10
  58. const LONG_CONTINUATION_USER_PREFIX = 'LONG_CONTINUATION_USER'
  59. const LONG_CONTINUATION_FIRST_PREFIX = 'LONG_CONTINUATION_FIRST'
  60. const LONG_CONTINUATION_DONE_PREFIX = 'LONG_CONTINUATION_DONE'
  61. const SOAK_USER_PREFIX = 'SOAK_CONVERSATION_USER'
  62. const SOAK_FIRST_PREFIX = 'SOAK_CONVERSATION_FIRST'
  63. const SOAK_DONE_PREFIX = 'SOAK_CONVERSATION_DONE'
  64. const LIVE_PROMPT_MARKER = 'STREAM_PERF_USER_INPUT'
  65. const STREAM_FIRST_MARKER = 'STREAM_PERF_FIRST'
  66. const STREAM_DONE_MARKER = 'STREAM_PERF_DONE'
  67. const LIVE_PROMPT = [
  68. LIVE_PROMPT_MARKER,
  69. 'Analyze the following mixed-language project context and return a concise diagnostic.',
  70. ...Array.from(
  71. { length: 48 },
  72. (_, index) =>
  73. `Context ${String(index + 1).padStart(2, '0')}: 用户正在检查长会话中的增量渲染性能。`
  74. + ` Preserve item ${String(index)} and compare ${'payload'.repeat(8)}.`,
  75. ),
  76. '```ts',
  77. ...Array.from(
  78. { length: 40 },
  79. (_, index) => `const sample_${String(index)} = ${JSON.stringify(`value-${String(index)}-${'x'.repeat(32)}`)}`,
  80. ),
  81. '```',
  82. ].join('\n')
  83. const STREAM_DELTAS = Array.from({ length: STREAM_DELTA_COUNT }, (_, index) => {
  84. if (index === 0) return `${STREAM_FIRST_MARKER} `
  85. if (index === STREAM_DELTA_COUNT - 1) return `${STREAM_DONE_MARKER}.`
  86. return `chunk-${String(index).padStart(3, '0')} ${'response'.repeat(3)} `
  87. })
  88. interface ChromiumMetrics {
  89. readonly [name: string]: number
  90. }
  91. interface Measurement {
  92. readonly wallMs: number
  93. readonly taskMs: number
  94. readonly scriptMs: number
  95. readonly layoutMs: number
  96. readonly recalcStyleMs: number
  97. readonly devtoolsMs: number
  98. readonly nodesDelta: number
  99. readonly listenersDelta: number
  100. readonly heapDeltaMb: number
  101. readonly totalNodes: number
  102. readonly heapMb: number
  103. }
  104. interface MutationProbeResult {
  105. readonly batches: number
  106. readonly records: number
  107. }
  108. interface UserRenderProbeResult {
  109. readonly trustedClick: boolean
  110. readonly sendToDomMs: number
  111. readonly sendToPaintMs: number
  112. readonly domToPaintMs: number
  113. readonly mutationBatches: number
  114. readonly mutationRecords: number
  115. }
  116. interface RetainedBrowserState {
  117. readonly domElements: number
  118. readonly nodes: number
  119. readonly listeners: number
  120. readonly heapMb: number
  121. }
  122. interface ContinuedTurnReport {
  123. readonly ordinal: number
  124. readonly resultingTurns: number
  125. readonly kind: 'text' | 'tool'
  126. readonly promptChars: number
  127. readonly composerFill: Measurement
  128. readonly stream: MutationProbeResult & Measurement & {
  129. readonly paceMs: number
  130. readonly deltaChunks: number
  131. readonly persistedChunks: number
  132. readonly toolCalls: number
  133. readonly toolResults: number
  134. readonly clickToUserEchoMs: number
  135. readonly clickToFirstChunkMs: number
  136. readonly firstChunkToSettledMs: number
  137. }
  138. }
  139. interface ConversationTurnSpec {
  140. readonly prompt: string
  141. readonly deltas: readonly string[]
  142. readonly userMarker: string
  143. readonly firstMarker: string
  144. readonly doneMarker: string
  145. readonly toolResultMarker?: string
  146. }
  147. interface RetainedCheckpoint {
  148. readonly turns: number
  149. readonly state: RetainedBrowserState
  150. }
  151. interface ConversationReport {
  152. readonly startingTurns: number
  153. readonly turnsAdded: number
  154. readonly toolTurns: number
  155. readonly retainedBefore: RetainedBrowserState
  156. readonly retainedAfter: RetainedBrowserState
  157. readonly retainedDelta: RetainedBrowserState
  158. readonly checkpoints: readonly RetainedCheckpoint[]
  159. readonly turns: readonly ContinuedTurnReport[]
  160. }
  161. interface PerformanceWorld {
  162. readonly scaffold: WebScaffold
  163. readonly page: Page
  164. readonly tripwire: ReturnType<typeof watchConsole>
  165. readonly sessionEvents: SessionEvent[]
  166. readonly setupMs: number
  167. readonly replayDir?: string
  168. }
  169. interface PerformanceWorldOptions {
  170. readonly browser: Browser
  171. readonly replay?: ReplayOverrideDoc
  172. readonly sidebarSessions?: number
  173. readonly seedLongHistory?: boolean
  174. }
  175. function text(value: string): { type: 'text'; text: string }[] {
  176. return [{ type: 'text', text: value }]
  177. }
  178. function appendTitle(session: Session, title: string, messageSeq: SessionSeq): void {
  179. session.append('session/title', {
  180. title,
  181. messageSeqs: [messageSeq],
  182. source: { kind: 'fallback' },
  183. })
  184. }
  185. function appendSystemPrompt(session: Session, turn: number, step: number): void {
  186. session.append('system/message', {
  187. turn,
  188. step,
  189. message: createSystemMessage(
  190. 'Synthetic performance system prompt.',
  191. '@deepseek-ai/dsh-system-prompt',
  192. ),
  193. }, { surfaceOp: 'append' })
  194. }
  195. function appendRequestHeader(session: Session, turn: number, step: number): void {
  196. session.append('request/header', {
  197. header: {
  198. config: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
  199. },
  200. reason: turn === 1 && step === 1 ? 'initial' : 'change',
  201. })
  202. }
  203. function appendAssistant(
  204. session: Session,
  205. turn: number,
  206. step: number,
  207. body: string,
  208. ): void {
  209. session.append('assistant/message', {
  210. stream: [],
  211. turn,
  212. step,
  213. message: createAssistantMessage({
  214. content: text(body),
  215. source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
  216. }),
  217. usage: {
  218. inputTokens: 4_000 + turn * 10,
  219. outputTokens: 200 + step * 20,
  220. cacheReadTokens: turn % 2 === 0 ? 2_000 : 0,
  221. },
  222. }, { surfaceOp: 'append' })
  223. }
  224. function appendToolStep(
  225. session: Session,
  226. turn: number,
  227. step: number,
  228. toolCount: number,
  229. ): void {
  230. const calls = Array.from({ length: toolCount }, (_, index) => {
  231. const callId = ToolCallId(`perf-call-${String(turn)}-${String(index)}`)
  232. const args = JSON.stringify({
  233. turn,
  234. index,
  235. payload: 'x'.repeat(120),
  236. })
  237. return { callId, index, args }
  238. })
  239. session.append('assistant/message', {
  240. stream: [],
  241. turn,
  242. step,
  243. message: createAssistantMessage({
  244. content: [
  245. {
  246. type: 'reasoning',
  247. text: `Dispatching ${String(toolCount)} synthetic tools for turn ${String(turn)}.`,
  248. },
  249. ...calls.map(({ callId, args }) => ({
  250. type: 'tool-call' as const,
  251. id: callId,
  252. name: 'synthetic_tool',
  253. arguments: args,
  254. })),
  255. ],
  256. source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
  257. }),
  258. usage: {
  259. inputTokens: 6_000 + turn * 10,
  260. outputTokens: 500,
  261. cacheReadTokens: 3_000,
  262. reasoningTokens: 50,
  263. },
  264. }, { surfaceOp: 'append' })
  265. const callEvents = calls.map(({ callId, args }) =>
  266. session.append('tool/call', {
  267. turn,
  268. step,
  269. callId,
  270. name: 'synthetic_tool',
  271. arguments: args,
  272. }))
  273. for (const [index, call] of calls.entries()) {
  274. const source = callEvents[index]
  275. if (source === undefined) throw new Error(`missing synthetic tool call ${String(index)}`)
  276. session.append('tool/result', {
  277. turn,
  278. step,
  279. message: createToolResultMessage({
  280. callId: call.callId,
  281. content: text(
  282. `synthetic result turn=${String(turn)} index=${String(call.index)} ${'r'.repeat(400)}`,
  283. ),
  284. isError: false,
  285. }),
  286. }, { surfaceOp: 'append', sourceEventSeqs: [source.seq] })
  287. }
  288. }
  289. function fencedCode(turn: number): string {
  290. if (turn % 25 !== 0) return ''
  291. const lines = Array.from(
  292. { length: 80 },
  293. (_, index) => `const value_${String(index)} = ${String(turn + index)}`,
  294. )
  295. return `\n\n\`\`\`ts\n${lines.join('\n')}\n\`\`\``
  296. }
  297. function fixtureLog(session: Session): string {
  298. const header = {
  299. type: 'session',
  300. version: SESSION_FORMAT_VERSION,
  301. id: '{{sessionId}}',
  302. createdAt: Date.now() - 60_000,
  303. cwd: '{{cwd}}',
  304. isSeeded: false,
  305. delegationDepth: 0,
  306. }
  307. return [
  308. JSON.stringify(header),
  309. ...session.snapshotEvents().map(event => JSON.stringify(event)),
  310. '',
  311. ].join('\n')
  312. }
  313. function smallSidebarFixture(): string {
  314. const session = Session.create(SessionId('perf-small-template'))
  315. session.append('turn/start', {
  316. turn: 1,
  317. })
  318. const user = session.append('user/message', createUserMessage({
  319. content: text('Inspect this compact synthetic session.'),
  320. source: { kind: 'user' },
  321. }), { surfaceOp: 'append' })
  322. appendTitle(session, 'Synthetic sidebar session', user.seq)
  323. session.append('step/start', { turn: 1, step: 1 })
  324. appendSystemPrompt(session, 1, 1)
  325. appendRequestHeader(session, 1, 1)
  326. appendToolStep(session, 1, 1, 2)
  327. session.append('step/end', { turn: 1, step: 1 })
  328. session.append('step/start', { turn: 1, step: 2 })
  329. appendRequestHeader(session, 1, 2)
  330. appendAssistant(session, 1, 2, 'Synthetic sidebar fixture complete.')
  331. session.append('step/end', { turn: 1, step: 2 })
  332. session.append('turn/end', { turn: 1, reason: { kind: 'completed' } })
  333. return fixtureLog(session)
  334. }
  335. function longHistoryFixture(): string {
  336. const session = Session.create(SessionId(LONG_SESSION_ID))
  337. for (let turn = 1; turn <= LONG_HISTORY_TURNS; turn += 1) {
  338. session.append('turn/start', {
  339. turn,
  340. })
  341. const user = session.append('user/message', createUserMessage({
  342. content: text(
  343. `LONG_PERF_SENTINEL turn ${String(turn)}: analyze payload ${'u'.repeat(200)}`,
  344. ),
  345. source: { kind: 'user' },
  346. }), { surfaceOp: 'append' })
  347. if (turn === 1) appendTitle(session, LONG_SESSION_TITLE, user.seq)
  348. session.append('step/start', { turn, step: 1 })
  349. if (turn === 1) appendSystemPrompt(session, turn, 1)
  350. appendRequestHeader(session, turn, 1)
  351. if (turn % TOOL_TURN_INTERVAL === 0) {
  352. appendToolStep(session, turn, 1, TOOLS_PER_TOOL_TURN)
  353. session.append('step/end', { turn, step: 1 })
  354. session.append('step/start', { turn, step: 2 })
  355. appendRequestHeader(session, turn, 2)
  356. appendAssistant(
  357. session,
  358. turn,
  359. 2,
  360. `All synthetic tools completed for turn ${String(turn)}. ${'z'.repeat(320)}${fencedCode(turn)}`,
  361. )
  362. session.append('step/end', { turn, step: 2 })
  363. } else {
  364. appendAssistant(
  365. session,
  366. turn,
  367. 1,
  368. `Synthetic assistant response for turn ${String(turn)}. ${'a'.repeat(320)}${fencedCode(turn)}`,
  369. )
  370. session.append('step/end', { turn, step: 1 })
  371. }
  372. session.append('turn/end', { turn, reason: { kind: 'completed' } })
  373. }
  374. return fixtureLog(session)
  375. }
  376. function textStream(deltas: readonly string[], inputTokens: number): StreamChunk[] {
  377. const response = deltas.join('')
  378. return [
  379. { type: 'block-start', index: 0, blockType: 'text' },
  380. ...deltas.map(text => ({
  381. type: 'text-delta' as const,
  382. index: 0,
  383. text,
  384. })),
  385. {
  386. type: 'block-end',
  387. index: 0,
  388. block: { type: 'text', text: response },
  389. },
  390. {
  391. type: 'usage',
  392. usage: {
  393. inputTokens,
  394. outputTokens: Math.ceil(response.length / 4),
  395. },
  396. },
  397. { type: 'finish', reason: { kind: 'stop' } },
  398. ]
  399. }
  400. function comparisonPrompt(index: number): string {
  401. if (index === COMPARISON_TURNS) return LIVE_PROMPT
  402. return (`${LONG_CONTINUATION_USER_PREFIX}_${String(index)} `
  403. + `继续分析这个长会话的第 ${String(index)} 个增量问题,并保留当前滚动和输入响应。 `
  404. + 'context '.repeat(80)).trimEnd()
  405. }
  406. function comparisonDeltas(index: number): string[] {
  407. if (index === COMPARISON_TURNS) return STREAM_DELTAS
  408. return Array.from({ length: COMPARISON_DELTA_COUNT }, (_, chunkIndex) => {
  409. if (chunkIndex === 0) return `${LONG_CONTINUATION_FIRST_PREFIX}_${String(index)} `
  410. if (chunkIndex === COMPARISON_DELTA_COUNT - 1) {
  411. return `${LONG_CONTINUATION_DONE_PREFIX}_${String(index)}.`
  412. }
  413. return `turn-${String(index)}-chunk-${String(chunkIndex).padStart(2, '0')} ${'response'.repeat(2)} `
  414. })
  415. }
  416. function comparisonTurn(index: number): ConversationTurnSpec {
  417. const toolResultMarker = index % COMPARISON_TOOL_INTERVAL === 0
  418. ? `LONG_CONTINUATION_TOOL_RESULT_${String(index)}`
  419. : undefined
  420. return {
  421. prompt: comparisonPrompt(index),
  422. deltas: comparisonDeltas(index),
  423. userMarker: index === COMPARISON_TURNS
  424. ? LIVE_PROMPT_MARKER
  425. : `${LONG_CONTINUATION_USER_PREFIX}_${String(index)}`,
  426. firstMarker: index === COMPARISON_TURNS
  427. ? STREAM_FIRST_MARKER
  428. : `${LONG_CONTINUATION_FIRST_PREFIX}_${String(index)}`,
  429. doneMarker: index === COMPARISON_TURNS
  430. ? STREAM_DONE_MARKER
  431. : `${LONG_CONTINUATION_DONE_PREFIX}_${String(index)}`,
  432. ...toolResultMarker === undefined ? {} : { toolResultMarker },
  433. }
  434. }
  435. function soakTurn(index: number): ConversationTurnSpec {
  436. const suffix = String(index).padStart(3, '0')
  437. const toolResultMarker = index % SOAK_TOOL_INTERVAL === 0
  438. ? `SOAK_CONVERSATION_TOOL_RESULT_${suffix}`
  439. : undefined
  440. return {
  441. prompt: (`${SOAK_USER_PREFIX}_${suffix} `
  442. + `持续对话第 ${String(index)} 轮,检查增量渲染与保留状态。 `
  443. + 'context '.repeat(20)).trimEnd(),
  444. deltas: Array.from({ length: SOAK_DELTA_COUNT }, (_, chunkIndex) => {
  445. if (chunkIndex === 0) return `${SOAK_FIRST_PREFIX}_${suffix} `
  446. if (chunkIndex === SOAK_DELTA_COUNT - 1) return `${SOAK_DONE_PREFIX}_${suffix}.`
  447. return `soak-${suffix}-${String(chunkIndex).padStart(2, '0')} response `
  448. }),
  449. userMarker: `${SOAK_USER_PREFIX}_${suffix}`,
  450. firstMarker: `${SOAK_FIRST_PREFIX}_${suffix}`,
  451. doneMarker: `${SOAK_DONE_PREFIX}_${suffix}`,
  452. ...toolResultMarker === undefined ? {} : { toolResultMarker },
  453. }
  454. }
  455. function toolStream(index: number, marker: string): StreamChunk[] {
  456. const callId = ToolCallId(`performance-tool-${marker.toLowerCase()}-${String(index)}`)
  457. const args = JSON.stringify({
  458. command: `printf '${marker}\\n'`,
  459. description: `Emit performance marker ${String(index)}`,
  460. })
  461. return [
  462. { type: 'block-start', index: 0, blockType: 'tool-call' },
  463. {
  464. type: 'tool-call-delta',
  465. index: 0,
  466. id: callId,
  467. name: 'bash',
  468. argumentsDelta: args,
  469. },
  470. {
  471. type: 'block-end',
  472. index: 0,
  473. block: { type: 'tool-call', id: callId, name: 'bash', arguments: args },
  474. },
  475. { type: 'usage', usage: { inputTokens: 256, outputTokens: 32 } },
  476. { type: 'finish', reason: { kind: 'tool-calls' } },
  477. ]
  478. }
  479. function performanceReplayOverride(
  480. turnCount: number,
  481. turnSpec: (index: number) => ConversationTurnSpec,
  482. ): ReplayOverrideDoc {
  483. const continuationEntries = Array.from(
  484. { length: turnCount },
  485. (_, offset): ReplayEntry[] => {
  486. const index = offset + 1
  487. const spec = turnSpec(index)
  488. const finalResponse: ReplayEntry = {
  489. kind: 'chunks',
  490. chunks: textStream(spec.deltas, Math.ceil(spec.prompt.length / 4)),
  491. }
  492. return spec.toolResultMarker === undefined
  493. ? [finalResponse]
  494. : [{ kind: 'chunks', chunks: toolStream(index, spec.toolResultMarker) }, finalResponse]
  495. },
  496. ).flat()
  497. return continuationEntries
  498. }
  499. function rounded(value: number): number {
  500. return Math.round(value * 1_000) / 1_000
  501. }
  502. async function chromiumMetrics(cdp: CDPSession): Promise<ChromiumMetrics> {
  503. const payload = await cdp.send('Performance.getMetrics')
  504. return Object.fromEntries(payload.metrics.map(metric => [metric.name, metric.value]))
  505. }
  506. async function retainedBrowserState(
  507. cdp: CDPSession,
  508. page: Page,
  509. ): Promise<RetainedBrowserState> {
  510. await cdp.send('HeapProfiler.collectGarbage')
  511. const metrics = await chromiumMetrics(cdp)
  512. return {
  513. domElements: await page.evaluate(() => document.querySelectorAll('*').length),
  514. nodes: requiredMetric(metrics, 'Nodes'),
  515. listeners: requiredMetric(metrics, 'JSEventListeners'),
  516. heapMb: rounded(requiredMetric(metrics, 'JSHeapUsedSize') / 1_048_576),
  517. }
  518. }
  519. function requiredMetric(metrics: ChromiumMetrics, name: string): number {
  520. const value = metrics[name]
  521. if (value === undefined) throw new Error(`Chromium performance metric ${name} is unavailable`)
  522. return value
  523. }
  524. function metricDelta(
  525. before: ChromiumMetrics,
  526. after: ChromiumMetrics,
  527. wallMs: number,
  528. ): Measurement {
  529. return {
  530. wallMs: rounded(wallMs),
  531. taskMs: rounded((requiredMetric(after, 'TaskDuration') - requiredMetric(before, 'TaskDuration')) * 1_000),
  532. scriptMs: rounded((requiredMetric(after, 'ScriptDuration') - requiredMetric(before, 'ScriptDuration')) * 1_000),
  533. layoutMs: rounded((requiredMetric(after, 'LayoutDuration') - requiredMetric(before, 'LayoutDuration')) * 1_000),
  534. recalcStyleMs: rounded(
  535. (requiredMetric(after, 'RecalcStyleDuration') - requiredMetric(before, 'RecalcStyleDuration')) * 1_000,
  536. ),
  537. devtoolsMs: rounded(
  538. (requiredMetric(after, 'DevToolsCommandDuration') - requiredMetric(before, 'DevToolsCommandDuration')) * 1_000,
  539. ),
  540. nodesDelta: requiredMetric(after, 'Nodes') - requiredMetric(before, 'Nodes'),
  541. listenersDelta: requiredMetric(after, 'JSEventListeners') - requiredMetric(before, 'JSEventListeners'),
  542. heapDeltaMb: rounded(
  543. (requiredMetric(after, 'JSHeapUsedSize') - requiredMetric(before, 'JSHeapUsedSize')) / 1_048_576,
  544. ),
  545. totalNodes: requiredMetric(after, 'Nodes'),
  546. heapMb: rounded(requiredMetric(after, 'JSHeapUsedSize') / 1_048_576),
  547. }
  548. }
  549. async function measure<T>(
  550. cdp: CDPSession,
  551. action: () => Promise<T>,
  552. ): Promise<{ measurement: Measurement; value: T }> {
  553. const before = await chromiumMetrics(cdp)
  554. const started = performance.now()
  555. const value = await action()
  556. const wallMs = performance.now() - started
  557. const after = await chromiumMetrics(cdp)
  558. return { measurement: metricDelta(before, after, wallMs), value }
  559. }
  560. async function startMutationProbe(page: Page): Promise<void> {
  561. await page.evaluate(() => {
  562. const target = document.querySelector('[class*="centerCol"]')
  563. if (target === null) throw new Error('stream mutation probe target is unavailable')
  564. const probe = {
  565. batches: 0,
  566. records: 0,
  567. observer: undefined as MutationObserver | undefined,
  568. }
  569. const observer = new MutationObserver((records) => {
  570. probe.batches += 1
  571. probe.records += records.length
  572. })
  573. probe.observer = observer
  574. observer.observe(target, {
  575. attributes: true,
  576. attributeFilter: ['data-streaming'],
  577. characterData: true,
  578. childList: true,
  579. subtree: true,
  580. })
  581. Reflect.set(globalThis, '__dshPerfMutationProbe', probe)
  582. })
  583. }
  584. async function stopMutationProbe(page: Page): Promise<MutationProbeResult> {
  585. return page.evaluate(() => {
  586. const probe = Reflect.get(globalThis, '__dshPerfMutationProbe') as
  587. | { batches: number; records: number; observer: MutationObserver }
  588. | undefined
  589. if (probe === undefined) throw new Error('stream mutation probe was not started')
  590. probe.observer.disconnect()
  591. Reflect.deleteProperty(globalThis, '__dshPerfMutationProbe')
  592. return { batches: probe.batches, records: probe.records }
  593. })
  594. }
  595. async function startUserRenderProbe(
  596. page: Page,
  597. marker: string,
  598. ): Promise<void> {
  599. const send = page.getByRole('button', { name: 'Send message', exact: true })
  600. await send.waitFor({ timeout: 15_000 })
  601. await expect.poll(() => send.isEnabled(), { timeout: 15_000 }).toBe(true)
  602. await page.evaluate((expectedMarker) => {
  603. const target = document.querySelector('[data-conversation-scroll]')
  604. if (target === null) throw new Error('user render probe target is unavailable')
  605. const probe: {
  606. sendAt?: number
  607. domAt?: number
  608. paintAt?: number
  609. trustedClick?: boolean
  610. batches: number
  611. records: number
  612. observer?: MutationObserver
  613. pollForMarker?: () => void
  614. } = { batches: 0, records: 0 }
  615. const transcriptContainsMarker = (): boolean => {
  616. // The composer projects the draft into a backdrop and hidden sizing
  617. // mirror; only a matching text node outside that card proves delivery.
  618. const walker = document.createTreeWalker(target, NodeFilter.SHOW_TEXT)
  619. for (let node = walker.nextNode(); node !== null; node = walker.nextNode()) {
  620. if (!node.textContent?.includes(expectedMarker)) continue
  621. const parent = node.parentElement
  622. if (parent !== null && parent.closest('[data-composer-card]') === null) return true
  623. }
  624. return false
  625. }
  626. const markDomVisible = (): void => {
  627. if (
  628. probe.sendAt === undefined
  629. || probe.domAt !== undefined
  630. || !transcriptContainsMarker()
  631. ) return
  632. probe.domAt = globalThis.performance.now()
  633. // The second callback runs only after a render opportunity for the DOM
  634. // insertion observed before the first callback.
  635. requestAnimationFrame(() => {
  636. requestAnimationFrame(() => {
  637. probe.paintAt = globalThis.performance.now()
  638. observer.disconnect()
  639. })
  640. })
  641. }
  642. const pollForMarker = (): void => {
  643. markDomVisible()
  644. if (probe.domAt === undefined) requestAnimationFrame(pollForMarker)
  645. }
  646. const observer = new MutationObserver((records) => {
  647. if (probe.sendAt === undefined) return
  648. probe.batches += 1
  649. probe.records += records.length
  650. markDomVisible()
  651. })
  652. probe.observer = observer
  653. probe.pollForMarker = pollForMarker
  654. observer.observe(target, {
  655. attributes: true,
  656. attributeFilter: ['data-streaming'],
  657. characterData: true,
  658. childList: true,
  659. subtree: true,
  660. })
  661. Reflect.set(globalThis, '__dshPerfUserRenderProbe', probe)
  662. }, marker)
  663. await send.evaluate((button) => {
  664. button.addEventListener('click', (event) => {
  665. const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
  666. | { sendAt?: number; trustedClick?: boolean; pollForMarker?: () => void }
  667. | undefined
  668. if (probe?.pollForMarker === undefined) throw new Error('user render probe was not started')
  669. probe.trustedClick = event.isTrusted
  670. probe.sendAt = globalThis.performance.now()
  671. requestAnimationFrame(probe.pollForMarker)
  672. }, { capture: true, once: true })
  673. })
  674. }
  675. async function triggerUserRenderProbe(page: Page): Promise<void> {
  676. const send = page.getByRole('button', { name: 'Send message', exact: true })
  677. await send.click()
  678. }
  679. async function stopUserRenderProbe(
  680. page: Page,
  681. marker: string,
  682. ): Promise<UserRenderProbeResult> {
  683. try {
  684. await page.waitForFunction(() => {
  685. const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
  686. | { paintAt?: number }
  687. | undefined
  688. return probe?.paintAt !== undefined
  689. }, undefined, { timeout: 15_000 })
  690. } catch (error) {
  691. const diagnostic = await page.evaluate((expectedMarker) => {
  692. const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
  693. | {
  694. sendAt?: number
  695. domAt?: number
  696. paintAt?: number
  697. trustedClick?: boolean
  698. batches: number
  699. records: number
  700. }
  701. | undefined
  702. const target = document.querySelector('[data-conversation-scroll]')
  703. let markerOutsideComposer = false
  704. if (target !== null) {
  705. const walker = document.createTreeWalker(target, NodeFilter.SHOW_TEXT)
  706. for (let node = walker.nextNode(); node !== null; node = walker.nextNode()) {
  707. if (!node.textContent?.includes(expectedMarker)) continue
  708. const parent = node.parentElement
  709. if (parent !== null && parent.closest('[data-composer-card]') === null) {
  710. markerOutsideComposer = true
  711. break
  712. }
  713. }
  714. }
  715. return {
  716. probe,
  717. markerOutsideComposer,
  718. markerInComposer: document.querySelector('[data-composer-card]')
  719. ?.textContent?.includes(expectedMarker) ?? false,
  720. }
  721. }, marker)
  722. throw new Error(`user render probe timed out: ${JSON.stringify(diagnostic)}`, { cause: error })
  723. }
  724. return page.evaluate(() => {
  725. const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
  726. | {
  727. sendAt?: number
  728. domAt?: number
  729. paintAt?: number
  730. trustedClick?: boolean
  731. batches: number
  732. records: number
  733. }
  734. | undefined
  735. Reflect.deleteProperty(globalThis, '__dshPerfUserRenderProbe')
  736. if (
  737. probe?.trustedClick !== true
  738. || probe.sendAt === undefined
  739. || probe.domAt === undefined
  740. || probe.paintAt === undefined
  741. ) {
  742. throw new Error('user render probe did not observe a trusted click, DOM insertion, and paint')
  743. }
  744. return {
  745. trustedClick: probe.trustedClick,
  746. sendToDomMs: probe.domAt - probe.sendAt,
  747. sendToPaintMs: probe.paintAt - probe.sendAt,
  748. domToPaintMs: probe.paintAt - probe.domAt,
  749. mutationBatches: probe.batches,
  750. mutationRecords: probe.records,
  751. }
  752. })
  753. }
  754. async function stableCount(
  755. locator: Locator,
  756. accepts: (count: number) => boolean,
  757. timeoutMs = 60_000,
  758. ): Promise<number> {
  759. const deadline = performance.now() + timeoutMs
  760. let previous = -1
  761. let stableReads = 0
  762. while (performance.now() < deadline) {
  763. const count = await locator.count()
  764. stableReads = accepts(count) && count === previous ? stableReads + 1 : 0
  765. if (stableReads >= 4) return count
  766. previous = count
  767. await new Promise(resolve => setTimeout(resolve, 50))
  768. }
  769. throw new Error(`browser row count did not stabilize; last count ${String(previous)}`)
  770. }
  771. async function conversationTurns(page: Page): Promise<number> {
  772. // Loaded-window turn count: one mounted turn-tail footer per settled turn in
  773. // the window (context keys are `${kind.length}:${kind}${id}`). The stats
  774. // strip cannot serve as this probe: its counts ride the whole-log
  775. // sessionStats projection and stay fixed across paging by design.
  776. return stableCount(page.locator('[data-chat-flow-key^="9:turn-tail"]'), count => count > 0)
  777. }
  778. function retainedDelta(
  779. before: RetainedBrowserState,
  780. after: RetainedBrowserState,
  781. ): RetainedBrowserState {
  782. return {
  783. domElements: after.domElements - before.domElements,
  784. nodes: after.nodes - before.nodes,
  785. listeners: after.listeners - before.listeners,
  786. heapMb: rounded(after.heapMb - before.heapMb),
  787. }
  788. }
  789. async function launchPerformanceWorld(
  790. options: PerformanceWorldOptions,
  791. ): Promise<PerformanceWorld> {
  792. const setupStarted = performance.now()
  793. let replayDir: string | undefined
  794. let scaffold: WebScaffold | undefined
  795. let page: Page | undefined
  796. try {
  797. if (options.replay === undefined) {
  798. scaffold = await launchWebScaffold()
  799. } else {
  800. replayDir = await mkdtemp(join(tmpdir(), 'dsh-web-perf-replay-'))
  801. const replayOverride = join(replayDir, 'replay.override.json')
  802. await writeFile(replayOverride, JSON.stringify(options.replay))
  803. scaffold = await launchWebScaffold({
  804. // The override-only fixture supplies one positional script for this
  805. // world's single live session.
  806. replayFixture: join(replayDir, 'override-only.jsonl'),
  807. replayOverride,
  808. paceMs: STREAM_PACE_MS,
  809. replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
  810. })
  811. }
  812. const sessionEvents: SessionEvent[] = []
  813. scaffold.ctx.on('session/event', (_session, event: SessionEvent) => {
  814. sessionEvents.push(event)
  815. })
  816. if ((options.sidebarSessions ?? 0) > 0) {
  817. const small = smallSidebarFixture()
  818. for (let index = 0; index < (options.sidebarSessions ?? 0); index += 1) {
  819. await seedSession(scaffold, small, `perf-sidebar-${String(index).padStart(4, '0')}`)
  820. }
  821. }
  822. if (options.seedLongHistory === true) {
  823. await seedSession(scaffold, longHistoryFixture(), LONG_SESSION_ID)
  824. }
  825. const setupMs = performance.now() - setupStarted
  826. page = await newEnglishPage(options.browser)
  827. return {
  828. scaffold,
  829. page,
  830. tripwire: watchConsole(page),
  831. sessionEvents,
  832. setupMs,
  833. ...replayDir === undefined ? {} : { replayDir },
  834. }
  835. } catch (error) {
  836. const failures: unknown[] = [error]
  837. if (page !== undefined) {
  838. try {
  839. await page.close()
  840. } catch (cleanupError) {
  841. failures.push(cleanupError)
  842. }
  843. }
  844. if (scaffold !== undefined) {
  845. try {
  846. await scaffold.close()
  847. } catch (cleanupError) {
  848. failures.push(cleanupError)
  849. }
  850. }
  851. if (replayDir !== undefined) {
  852. try {
  853. await rm(replayDir, { recursive: true, force: true })
  854. } catch (cleanupError) {
  855. failures.push(cleanupError)
  856. }
  857. }
  858. if (failures.length === 1) throw error
  859. throw new AggregateError(failures, 'web performance setup and cleanup failed')
  860. }
  861. }
  862. async function closePerformanceWorld(world: PerformanceWorld): Promise<void> {
  863. const failures: unknown[] = []
  864. await world.page.close().catch((error: unknown) => failures.push(error))
  865. await world.scaffold.close().catch((error: unknown) => failures.push(error))
  866. if (world.replayDir !== undefined) {
  867. await rm(world.replayDir, { recursive: true, force: true })
  868. .catch((error: unknown) => failures.push(error))
  869. }
  870. if (failures.length === 1) throw failures[0]
  871. if (failures.length > 1) throw new AggregateError(failures, 'web performance teardown failed')
  872. }
  873. async function openPerformancePage(
  874. world: PerformanceWorld,
  875. expectedSessions: number,
  876. ): Promise<Locator> {
  877. await world.page.goto(world.scaffold.authenticatedUrl, { waitUntil: 'load' })
  878. await world.page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
  879. const group = world.page.getByRole('treeitem').first()
  880. await expect.poll(() => group.textContent(), { timeout: 30_000 })
  881. .toContain(`${String(expectedSessions)} ${expectedSessions === 1 ? 'session' : 'sessions'}`)
  882. return group
  883. }
  884. async function openLongHistory(page: Page): Promise<number> {
  885. await page.getByRole('textbox', { name: 'Search name, keywords...', exact: true })
  886. .fill('LONG_PERF_SENTINEL')
  887. const results = page.getByRole('tree', { name: 'Search results' }).getByRole('treeitem')
  888. await expect.poll(() => results.count(), { timeout: 60_000 }).toBe(1)
  889. await results.first().click()
  890. await page.getByRole('tab', { name: 'Chat', exact: true }).waitFor({ timeout: 30_000 })
  891. return conversationTurns(page)
  892. }
  893. async function continueConversation(
  894. world: PerformanceWorld,
  895. cdp: CDPSession,
  896. options: {
  897. readonly startingTurns: number
  898. readonly turnCount: number
  899. readonly turnSpec: (index: number) => ConversationTurnSpec
  900. readonly expectedSessionId?: SessionId
  901. readonly checkpointInterval?: number
  902. },
  903. ): Promise<ConversationReport> {
  904. const composer = world.page.locator('[data-composer-input][contenteditable="true"]').last()
  905. await composer.waitFor({ timeout: 15_000 })
  906. const retainedBefore = await retainedBrowserState(cdp, world.page)
  907. const checkpoints: RetainedCheckpoint[] = [{ turns: options.startingTurns, state: retainedBefore }]
  908. const turns: ContinuedTurnReport[] = []
  909. let liveSessionId = options.expectedSessionId
  910. for (let index = 1; index <= options.turnCount; index += 1) {
  911. const spec = options.turnSpec(index)
  912. const composerFill = await measure(cdp, async () => {
  913. await composer.fill(spec.prompt)
  914. await expect.poll(() => composer.textContent()).toBe(spec.prompt)
  915. return ((await composer.textContent()) ?? '').length
  916. })
  917. expect(composerFill.value).toBe(spec.prompt.length)
  918. const eventStart = world.sessionEvents.length
  919. await startMutationProbe(world.page)
  920. const streamBefore = await chromiumMetrics(cdp)
  921. const streamStarted = performance.now()
  922. const settled = world.scaffold.whenTurnSettled(60_000)
  923. await world.page.getByRole('button', { name: 'Send message', exact: true }).click()
  924. await world.page.getByText(spec.userMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  925. const clickToUserEchoMs = performance.now() - streamStarted
  926. await world.page.getByText(spec.firstMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  927. const clickToFirstChunkMs = performance.now() - streamStarted
  928. const settledSessionId = await settled
  929. if (liveSessionId === undefined) {
  930. liveSessionId = settledSessionId
  931. } else {
  932. expect(settledSessionId).toBe(liveSessionId)
  933. }
  934. await expect.poll(
  935. () => world.page.locator('[data-streaming="true"]').count(),
  936. { timeout: 15_000 },
  937. ).toBe(0)
  938. await world.page.getByText(spec.doneMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  939. const clickToSettledMs = performance.now() - streamStarted
  940. const streamAfter = await chromiumMetrics(cdp)
  941. const mutations = await stopMutationProbe(world.page)
  942. const turnEvents = world.sessionEvents.slice(eventStart)
  943. const chunks = turnEvents.flatMap(event => (
  944. event.type === 'assistant/message' || event.type === 'assistant/attempt'
  945. ? expandAssistantStream(event.data.stream)
  946. : []
  947. ))
  948. const toolCalls = turnEvents.filter(event => event.type === 'tool/call')
  949. const toolResults = turnEvents.filter(event => event.type === 'tool/result')
  950. const toolTurn = spec.toolResultMarker !== undefined
  951. expect(chunks).toHaveLength(spec.deltas.length + (toolTurn ? 9 : 4))
  952. expect(toolCalls).toHaveLength(toolTurn ? 1 : 0)
  953. expect(toolResults).toHaveLength(toolTurn ? 1 : 0)
  954. if (spec.toolResultMarker !== undefined) {
  955. expect(toolCalls[0]?.data.name).toBe('bash')
  956. const toolResult = toolResults[0]
  957. if (toolResult?.type !== 'tool/result') {
  958. throw new Error(`continued turn ${String(index)} did not log its tool result`)
  959. }
  960. const resultBlock = toolResult.data.message.content.find(block => block.type === 'tool-result')
  961. expect(resultBlock?.isError).toBe(false)
  962. expect(resultBlock?.content
  963. .filter(block => block.type === 'text')
  964. .map(block => block.text)
  965. .join('')).toContain(spec.toolResultMarker)
  966. }
  967. const user = turnEvents.find(
  968. event => event.type === 'user/message' && event.data.source.kind === 'user',
  969. )
  970. if (user?.type !== 'user/message') {
  971. throw new Error(`continued turn ${String(index)} did not log its user message`)
  972. }
  973. expect(user.data.content
  974. .filter(block => block.type === 'text')
  975. .map(block => block.text)
  976. .join('')).toBe(spec.prompt)
  977. const resultingTurns = await conversationTurns(world.page)
  978. expect(resultingTurns).toBe(options.startingTurns + index)
  979. turns.push({
  980. ordinal: index,
  981. resultingTurns,
  982. kind: toolTurn ? 'tool' : 'text',
  983. promptChars: spec.prompt.length,
  984. composerFill: composerFill.measurement,
  985. stream: {
  986. paceMs: STREAM_PACE_MS,
  987. deltaChunks: spec.deltas.length,
  988. persistedChunks: chunks.length,
  989. toolCalls: toolCalls.length,
  990. toolResults: toolResults.length,
  991. clickToUserEchoMs: rounded(clickToUserEchoMs),
  992. clickToFirstChunkMs: rounded(clickToFirstChunkMs),
  993. firstChunkToSettledMs: rounded(clickToSettledMs - clickToFirstChunkMs),
  994. ...mutations,
  995. ...metricDelta(streamBefore, streamAfter, clickToSettledMs),
  996. },
  997. })
  998. if (options.checkpointInterval !== undefined && index % options.checkpointInterval === 0) {
  999. checkpoints.push({
  1000. turns: options.startingTurns + index,
  1001. state: await retainedBrowserState(cdp, world.page),
  1002. })
  1003. }
  1004. }
  1005. const lastCheckpoint = checkpoints.at(-1)
  1006. const retainedAfter = lastCheckpoint?.turns === options.startingTurns + options.turnCount
  1007. ? lastCheckpoint.state
  1008. : await retainedBrowserState(cdp, world.page)
  1009. if (lastCheckpoint?.turns !== options.startingTurns + options.turnCount) {
  1010. checkpoints.push({ turns: options.startingTurns + options.turnCount, state: retainedAfter })
  1011. }
  1012. return {
  1013. startingTurns: options.startingTurns,
  1014. turnsAdded: options.turnCount,
  1015. toolTurns: turns.filter(turn => turn.kind === 'tool').length,
  1016. retainedBefore,
  1017. retainedAfter,
  1018. retainedDelta: retainedDelta(retainedBefore, retainedAfter),
  1019. checkpoints,
  1020. turns,
  1021. }
  1022. }
  1023. async function measurePostSoakUserRender(
  1024. world: PerformanceWorld,
  1025. cdp: CDPSession,
  1026. ): Promise<object> {
  1027. const spec = soakTurn(POST_SOAK_RENDER_TURN)
  1028. if (spec.toolResultMarker !== undefined) {
  1029. throw new Error('post-soak render probe must remain a text-only turn')
  1030. }
  1031. const composer = world.page.locator('[data-composer-input][contenteditable="true"]').last()
  1032. const composerFill = await measure(cdp, async () => {
  1033. await composer.fill(spec.prompt)
  1034. await expect.poll(() => composer.textContent()).toBe(spec.prompt)
  1035. return ((await composer.textContent()) ?? '').length
  1036. })
  1037. expect(composerFill.value).toBe(spec.prompt.length)
  1038. const eventStart = world.sessionEvents.length
  1039. await startUserRenderProbe(world.page, spec.userMarker)
  1040. const browserBefore = await chromiumMetrics(cdp)
  1041. const fullTurnStarted = performance.now()
  1042. const settled = world.scaffold.whenTurnSettled(60_000)
  1043. await triggerUserRenderProbe(world.page)
  1044. const browserTiming = await stopUserRenderProbe(world.page, spec.userMarker)
  1045. const browserAfter = await chromiumMetrics(cdp)
  1046. const browserAfterPaintSample = metricDelta(
  1047. browserBefore,
  1048. browserAfter,
  1049. performance.now() - fullTurnStarted,
  1050. )
  1051. await world.page.getByText(spec.firstMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  1052. const settledSessionId = await settled
  1053. await expect.poll(
  1054. () => world.page.locator('[data-streaming="true"]').count(),
  1055. { timeout: 15_000 },
  1056. ).toBe(0)
  1057. await world.page.getByText(spec.doneMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  1058. const fullTurnMs = performance.now() - fullTurnStarted
  1059. const turnEvents = world.sessionEvents.slice(eventStart)
  1060. const chunks = turnEvents.flatMap(event => (
  1061. event.type === 'assistant/message' || event.type === 'assistant/attempt'
  1062. ? expandAssistantStream(event.data.stream)
  1063. : []
  1064. ))
  1065. const user = turnEvents.find(
  1066. event => event.type === 'user/message' && event.data.source.kind === 'user',
  1067. )
  1068. if (user?.type !== 'user/message') throw new Error('post-soak render probe did not log its user message')
  1069. expect(user.data.content
  1070. .filter(block => block.type === 'text')
  1071. .map(block => block.text)
  1072. .join('')).toBe(spec.prompt)
  1073. expect(chunks).toHaveLength(spec.deltas.length + 4)
  1074. expect(turnEvents.filter(event => event.type === 'tool/call')).toHaveLength(0)
  1075. expect(turnEvents.filter(event => event.type === 'tool/result')).toHaveLength(0)
  1076. expect(world.scaffold.ctx.agents.get(settledSessionId)).toBeDefined()
  1077. expect(await conversationTurns(world.page)).toBe(POST_SOAK_RENDER_TURN)
  1078. return {
  1079. ordinal: POST_SOAK_RENDER_TURN,
  1080. resultingTurns: POST_SOAK_RENDER_TURN,
  1081. promptChars: spec.prompt.length,
  1082. composerFill: composerFill.measurement,
  1083. trustedClick: browserTiming.trustedClick,
  1084. sendToDomMs: rounded(browserTiming.sendToDomMs),
  1085. sendToPaintMs: rounded(browserTiming.sendToPaintMs),
  1086. domToPaintMs: rounded(browserTiming.domToPaintMs),
  1087. mutationBatchesThroughPaint: browserTiming.mutationBatches,
  1088. mutationRecordsThroughPaint: browserTiming.mutationRecords,
  1089. browserAfterPaintSample,
  1090. fullTurnMs: rounded(fullTurnMs),
  1091. persistedChunks: chunks.length,
  1092. }
  1093. }
  1094. function average(values: readonly number[]): number {
  1095. return rounded(values.reduce((sum, value) => sum + value, 0) / values.length)
  1096. }
  1097. function p95(values: readonly number[]): number {
  1098. const sorted = [...values].sort((left, right) => left - right)
  1099. return sorted[Math.max(0, Math.ceil(sorted.length * 0.95) - 1)] ?? 0
  1100. }
  1101. function summarizeTurnWindows(
  1102. turns: readonly ContinuedTurnReport[],
  1103. windowSize: number,
  1104. ): object[] {
  1105. const windows: object[] = []
  1106. for (let start = 0; start < turns.length; start += windowSize) {
  1107. const window = turns.slice(start, start + windowSize)
  1108. const streamWall = window.map(turn => turn.stream.wallMs)
  1109. const userEcho = window.map(turn => turn.stream.clickToUserEchoMs)
  1110. const firstChunk = window.map(turn => turn.stream.clickToFirstChunkMs)
  1111. windows.push({
  1112. turns: `${String(start + 1)}-${String(start + window.length)}`,
  1113. toolTurns: window.filter(turn => turn.kind === 'tool').length,
  1114. average: {
  1115. composerFillWallMs: average(window.map(turn => turn.composerFill.wallMs)),
  1116. streamWallMs: average(streamWall),
  1117. clickToUserEchoMs: average(userEcho),
  1118. clickToFirstChunkMs: average(firstChunk),
  1119. taskMs: average(window.map(turn => turn.stream.taskMs)),
  1120. scriptMs: average(window.map(turn => turn.stream.scriptMs)),
  1121. recalcStyleMs: average(window.map(turn => turn.stream.recalcStyleMs)),
  1122. mutationBatches: average(window.map(turn => turn.stream.batches)),
  1123. },
  1124. p95: {
  1125. streamWallMs: rounded(p95(streamWall)),
  1126. clickToUserEchoMs: rounded(p95(userEcho)),
  1127. clickToFirstChunkMs: rounded(p95(firstChunk)),
  1128. },
  1129. })
  1130. }
  1131. return windows
  1132. }
  1133. describe('manual web performance: complex workspace and history', () => {
  1134. let browser: Browser
  1135. beforeAll(async () => {
  1136. if (webSnapshotMode() === 'record') {
  1137. throw new Error('manual web performance runs only with deterministic replay')
  1138. }
  1139. browser = await chromium.launch()
  1140. })
  1141. afterAll(async () => {
  1142. await browser?.close()
  1143. })
  1144. it('reports workspace, history, and trajectory cardinality costs', async () => {
  1145. const world = await launchPerformanceWorld({
  1146. browser,
  1147. sidebarSessions: SIDEBAR_SESSION_COUNT,
  1148. seedLongHistory: true,
  1149. })
  1150. try {
  1151. const bootStarted = performance.now()
  1152. const group = await openPerformancePage(world, SIDEBAR_SESSION_COUNT + 1)
  1153. const bootReadyMs = performance.now() - bootStarted
  1154. const page = world.page
  1155. const cdp = await page.context().newCDPSession(page)
  1156. await cdp.send('Performance.enable')
  1157. const firstContentfulPaintMs = await page.evaluate(
  1158. () => globalThis.performance.getEntriesByName('first-contentful-paint')[0]?.startTime,
  1159. )
  1160. const sidebar = await measure(cdp, async () => {
  1161. await group.click()
  1162. return stableCount(
  1163. page.getByRole('treeitem'),
  1164. count => count === SIDEBAR_SESSION_COUNT + 2,
  1165. )
  1166. })
  1167. expect(sidebar.value).toBe(SIDEBAR_SESSION_COUNT + 2)
  1168. await group.click()
  1169. await expect.poll(() => page.getByRole('treeitem').count()).toBe(1)
  1170. const contentSearch = await measure(cdp, async () => {
  1171. await page.getByRole('textbox', { name: 'Search name, keywords...', exact: true })
  1172. .fill('LONG_PERF_SENTINEL')
  1173. const results = page.getByRole('tree', { name: 'Search results' }).getByRole('treeitem')
  1174. await expect.poll(() => results.count(), { timeout: 60_000 }).toBe(1)
  1175. await results.first().waitFor({ timeout: 60_000 })
  1176. return results.first()
  1177. })
  1178. const opened = await measure(cdp, async () => {
  1179. await contentSearch.value.click()
  1180. await page.getByRole('tab', { name: 'Trajectory', exact: true }).waitFor({ timeout: 30_000 })
  1181. return conversationTurns(page)
  1182. })
  1183. expect(opened.value).toBe(DEFAULT_HISTORY_TURNS)
  1184. const trajectoryRows = page.getByRole('row')
  1185. const coldTrajectory = await measure(cdp, async () => {
  1186. await page.getByRole('tab', { name: 'Trajectory', exact: true }).click()
  1187. return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS)
  1188. })
  1189. expect(coldTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS)
  1190. const collapseTurns = await measure(cdp, async () => {
  1191. await page.getByRole('button', { name: 'Collapse turns', exact: true }).click()
  1192. return stableCount(trajectoryRows, count => count > 0 && count < EXPECTED_TRAJECTORY_ROWS)
  1193. })
  1194. expect(collapseTurns.value).toBeLessThan(EXPECTED_TRAJECTORY_ROWS)
  1195. const trajectorySearch = await measure(cdp, async () => {
  1196. await page.getByRole('searchbox', { name: 'Search trajectory', exact: true }).fill('turn 499')
  1197. return stableCount(trajectoryRows, count => count > 0 && count < 20)
  1198. })
  1199. expect(trajectorySearch.value).toBeLessThan(20)
  1200. await page.getByRole('tab', { name: 'Chat', exact: true }).click()
  1201. const historyPages: { turns: number; measurement: Measurement }[] = []
  1202. let turns = await conversationTurns(page)
  1203. while (turns < LONG_HISTORY_TURNS) {
  1204. const previousTurns = turns
  1205. const older = await measure(cdp, async () => {
  1206. await page.getByRole('button', { name: 'Load earlier', exact: true }).click()
  1207. await expect.poll(() => conversationTurns(page), { timeout: 30_000 })
  1208. .toBeGreaterThan(previousTurns)
  1209. return conversationTurns(page)
  1210. })
  1211. turns = older.value
  1212. historyPages.push({ turns, measurement: older.measurement })
  1213. }
  1214. const warmTrajectory = await measure(cdp, async () => {
  1215. await page.getByRole('tab', { name: 'Trajectory', exact: true }).click()
  1216. return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS)
  1217. })
  1218. expect(warmTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS)
  1219. const warmConversation = await measure(cdp, async () => {
  1220. await page.getByRole('tab', { name: 'Chat', exact: true }).click()
  1221. return conversationTurns(page)
  1222. })
  1223. expect(warmConversation.value).toBe(LONG_HISTORY_TURNS)
  1224. console.info(`WEB_PERF_RESULT ${JSON.stringify({
  1225. scenario: 'workspace-history-trajectory',
  1226. fixture: {
  1227. sidebarSessions: SIDEBAR_SESSION_COUNT,
  1228. totalSessions: SIDEBAR_SESSION_COUNT + 1,
  1229. longHistoryTurns: LONG_HISTORY_TURNS,
  1230. toolCalls: EXPECTED_TOOL_CALLS,
  1231. trajectoryRows: EXPECTED_TRAJECTORY_ROWS,
  1232. },
  1233. setupMs: rounded(world.setupMs),
  1234. boot: {
  1235. readyMs: rounded(bootReadyMs),
  1236. firstContentfulPaintMs: firstContentfulPaintMs === undefined
  1237. ? null
  1238. : rounded(firstContentfulPaintMs),
  1239. },
  1240. sidebarExpand: sidebar.measurement,
  1241. contentSearch: contentSearch.measurement,
  1242. openLongHistory: { initialTurns: opened.value, ...opened.measurement },
  1243. coldTrajectory: { rows: coldTrajectory.value, ...coldTrajectory.measurement },
  1244. collapseTurns: { rows: collapseTurns.value, ...collapseTurns.measurement },
  1245. trajectorySearch: { rows: trajectorySearch.value, ...trajectorySearch.measurement },
  1246. historyPages,
  1247. warmTrajectory: { rows: warmTrajectory.value, ...warmTrajectory.measurement },
  1248. warmConversation: { turns: warmConversation.value, ...warmConversation.measurement },
  1249. }, null, 2)}`)
  1250. expect(world.tripwire.warnings).toEqual([])
  1251. expect(world.tripwire.pageErrors).toEqual([])
  1252. } finally {
  1253. await closePerformanceWorld(world)
  1254. }
  1255. })
  1256. it('reports default 24-turn history plus eight continued turns', async () => {
  1257. const world = await launchPerformanceWorld({
  1258. browser,
  1259. replay: performanceReplayOverride(COMPARISON_TURNS, comparisonTurn),
  1260. seedLongHistory: true,
  1261. })
  1262. try {
  1263. await openPerformancePage(world, 1)
  1264. const cdp = await world.page.context().newCDPSession(world.page)
  1265. await cdp.send('Performance.enable')
  1266. const opened = await measure(cdp, () => openLongHistory(world.page))
  1267. expect(opened.value).toBe(DEFAULT_HISTORY_TURNS)
  1268. const conversation = await continueConversation(world, cdp, {
  1269. startingTurns: DEFAULT_HISTORY_TURNS,
  1270. turnCount: COMPARISON_TURNS,
  1271. turnSpec: comparisonTurn,
  1272. expectedSessionId: SessionId(LONG_SESSION_ID),
  1273. })
  1274. expect(conversation.toolTurns).toBe(2)
  1275. console.info(`WEB_PERF_RESULT ${JSON.stringify({
  1276. scenario: 'default-resume-24-plus-8',
  1277. setupMs: rounded(world.setupMs),
  1278. replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
  1279. openLongHistory: { turns: opened.value, ...opened.measurement },
  1280. conversation,
  1281. }, null, 2)}`)
  1282. expect(world.tripwire.warnings).toEqual([])
  1283. expect(world.tripwire.pageErrors).toEqual([])
  1284. } finally {
  1285. await closePerformanceWorld(world)
  1286. }
  1287. })
  1288. it('reports fully expanded 500-turn history plus eight continued turns', async () => {
  1289. const world = await launchPerformanceWorld({
  1290. browser,
  1291. replay: performanceReplayOverride(COMPARISON_TURNS, comparisonTurn),
  1292. seedLongHistory: true,
  1293. })
  1294. try {
  1295. await openPerformancePage(world, 1)
  1296. const cdp = await world.page.context().newCDPSession(world.page)
  1297. await cdp.send('Performance.enable')
  1298. expect(await openLongHistory(world.page)).toBe(DEFAULT_HISTORY_TURNS)
  1299. const historyPages: { turns: number; measurement: Measurement }[] = []
  1300. let turns = DEFAULT_HISTORY_TURNS
  1301. while (turns < LONG_HISTORY_TURNS) {
  1302. const previousTurns = turns
  1303. const older = await measure(cdp, async () => {
  1304. await world.page.getByRole('button', { name: 'Load earlier', exact: true }).click()
  1305. await expect.poll(() => conversationTurns(world.page), { timeout: 30_000 })
  1306. .toBeGreaterThan(previousTurns)
  1307. return conversationTurns(world.page)
  1308. })
  1309. turns = older.value
  1310. historyPages.push({ turns, measurement: older.measurement })
  1311. }
  1312. expect(turns).toBe(LONG_HISTORY_TURNS)
  1313. const conversation = await continueConversation(world, cdp, {
  1314. startingTurns: LONG_HISTORY_TURNS,
  1315. turnCount: COMPARISON_TURNS,
  1316. turnSpec: comparisonTurn,
  1317. expectedSessionId: SessionId(LONG_SESSION_ID),
  1318. })
  1319. expect(conversation.toolTurns).toBe(2)
  1320. console.info(`WEB_PERF_RESULT ${JSON.stringify({
  1321. scenario: 'expanded-history-500-plus-8',
  1322. setupMs: rounded(world.setupMs),
  1323. replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
  1324. historyPages,
  1325. conversation,
  1326. }, null, 2)}`)
  1327. expect(world.tripwire.warnings).toEqual([])
  1328. expect(world.tripwire.pageErrors).toEqual([])
  1329. } finally {
  1330. await closePerformanceWorld(world)
  1331. }
  1332. })
  1333. it('reports one hundred generated turns and the next user-message paint', async () => {
  1334. const world = await launchPerformanceWorld({
  1335. browser,
  1336. replay: performanceReplayOverride(POST_SOAK_RENDER_TURN, soakTurn),
  1337. })
  1338. let testFailure: unknown
  1339. try {
  1340. await world.page.goto(world.scaffold.authenticatedUrl, { waitUntil: 'load' })
  1341. await world.page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
  1342. await connectFreshWorkspace(world.page, world.scaffold.workspaceCwd, 'continuous-conversation-perf')
  1343. const cdp = await world.page.context().newCDPSession(world.page)
  1344. await cdp.send('Performance.enable')
  1345. const conversation = await continueConversation(world, cdp, {
  1346. startingTurns: 0,
  1347. turnCount: SOAK_TURNS,
  1348. turnSpec: soakTurn,
  1349. checkpointInterval: SOAK_CHECKPOINT_INTERVAL,
  1350. })
  1351. expect(conversation.toolTurns).toBe(SOAK_TURNS / SOAK_TOOL_INTERVAL)
  1352. const postSoakUserRender = await measurePostSoakUserRender(world, cdp)
  1353. const { turns, ...retained } = conversation
  1354. console.info(`WEB_PERF_RESULT ${JSON.stringify({
  1355. scenario: 'continuous-100-turn-soak',
  1356. setupMs: rounded(world.setupMs),
  1357. replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
  1358. ...retained,
  1359. windows: summarizeTurnWindows(turns, SOAK_CHECKPOINT_INTERVAL),
  1360. postSoakUserRender,
  1361. }, null, 2)}`)
  1362. expect(world.tripwire.warnings).toEqual([])
  1363. expect(world.tripwire.pageErrors).toEqual([])
  1364. } catch (error) {
  1365. testFailure = error
  1366. throw error
  1367. } finally {
  1368. try {
  1369. await closePerformanceWorld(world)
  1370. } catch (cleanupError) {
  1371. if (testFailure === undefined) throw cleanupError
  1372. throw new AggregateError(
  1373. [testFailure, cleanupError],
  1374. 'continuous conversation performance test and teardown failed',
  1375. )
  1376. }
  1377. }
  1378. })
  1379. })