complex-history.perf.ts 52 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431
  1. // Opt-in browser benchmark for high-cardinality workspace and history
  2. // rendering. It reports measurements without timing assertions because host
  3. // speed is not a correctness contract; structural assertions keep the number
  4. // of workspaces and history entries from silently shrinking.
  5. import { mkdtemp, rm, writeFile } from 'node:fs/promises'
  6. import { tmpdir } from 'node:os'
  7. import { join } from 'node:path'
  8. import { performance } from 'node:perf_hooks'
  9. import type { Browser, CDPSession, Locator, Page } from 'playwright'
  10. import { chromium } from 'playwright'
  11. import { afterAll, beforeAll, describe, expect, it } from 'vitest'
  12. import type { StreamChunk } from '@deepseek-ai/dsh-llm'
  13. import {
  14. CallId,
  15. createAssistantMessage,
  16. createToolResultMessage,
  17. createUserMessage,
  18. } from '@deepseek-ai/dsh-llm'
  19. import type { ReplayEntry, ReplayOverrideDoc } from '@deepseek-ai/dsh-llm-replay'
  20. import type { SessionEvent } from '@deepseek-ai/dsh-session'
  21. import {
  22. SESSION_FORMAT_VERSION,
  23. Session,
  24. SessionId,
  25. } from '@deepseek-ai/dsh-session'
  26. // Carries the session/title event declaration into the fixture builder.
  27. import type {} from '@deepseek-ai/dsh-session-title'
  28. import {
  29. launchWebScaffold,
  30. seedSession,
  31. watchConsole,
  32. webSnapshotMode,
  33. type WebScaffold,
  34. } from './scaffold.ts'
  35. import { connectFreshWorkspace, newEnglishPage } from './support.ts'
  36. const SIDEBAR_SESSION_COUNT = 1_000
  37. const LONG_SESSION_ID = 'perf-long-history'
  38. const LONG_SESSION_TITLE = 'LONG_PERF_SENTINEL 500-turn session'
  39. const LONG_HISTORY_TURNS = 500
  40. const TOOL_TURN_INTERVAL = 10
  41. const TOOLS_PER_TOOL_TURN = 10
  42. const EXPECTED_TOOL_CALLS = LONG_HISTORY_TURNS / TOOL_TURN_INTERVAL * TOOLS_PER_TOOL_TURN
  43. const EXPECTED_TRAJECTORY_ROWS = 2_100
  44. const DEFAULT_HISTORY_TURNS = 24
  45. const PERF_REPLAY_CONTEXT_WINDOW = 10_000_000
  46. const STREAM_PACE_MS = 8
  47. const STREAM_DELTA_COUNT = 120
  48. const COMPARISON_TURNS = 8
  49. const COMPARISON_DELTA_COUNT = 24
  50. const COMPARISON_TOOL_INTERVAL = 3
  51. const SOAK_TURNS = 100
  52. const POST_SOAK_RENDER_TURN = SOAK_TURNS + 1
  53. const SOAK_DELTA_COUNT = 8
  54. const SOAK_TOOL_INTERVAL = 10
  55. const SOAK_CHECKPOINT_INTERVAL = 10
  56. const LONG_CONTINUATION_USER_PREFIX = 'LONG_CONTINUATION_USER'
  57. const LONG_CONTINUATION_FIRST_PREFIX = 'LONG_CONTINUATION_FIRST'
  58. const LONG_CONTINUATION_DONE_PREFIX = 'LONG_CONTINUATION_DONE'
  59. const SOAK_USER_PREFIX = 'SOAK_CONVERSATION_USER'
  60. const SOAK_FIRST_PREFIX = 'SOAK_CONVERSATION_FIRST'
  61. const SOAK_DONE_PREFIX = 'SOAK_CONVERSATION_DONE'
  62. const LIVE_PROMPT_MARKER = 'STREAM_PERF_USER_INPUT'
  63. const STREAM_FIRST_MARKER = 'STREAM_PERF_FIRST'
  64. const STREAM_DONE_MARKER = 'STREAM_PERF_DONE'
  65. const LIVE_PROMPT = [
  66. LIVE_PROMPT_MARKER,
  67. 'Analyze the following mixed-language project context and return a concise diagnostic.',
  68. ...Array.from(
  69. { length: 48 },
  70. (_, index) =>
  71. `Context ${String(index + 1).padStart(2, '0')}: 用户正在检查长会话中的增量渲染性能。`
  72. + ` Preserve item ${String(index)} and compare ${'payload'.repeat(8)}.`,
  73. ),
  74. '```ts',
  75. ...Array.from(
  76. { length: 40 },
  77. (_, index) => `const sample_${String(index)} = ${JSON.stringify(`value-${String(index)}-${'x'.repeat(32)}`)}`,
  78. ),
  79. '```',
  80. ].join('\n')
  81. const STREAM_DELTAS = Array.from({ length: STREAM_DELTA_COUNT }, (_, index) => {
  82. if (index === 0) return `${STREAM_FIRST_MARKER} `
  83. if (index === STREAM_DELTA_COUNT - 1) return `${STREAM_DONE_MARKER}.`
  84. return `chunk-${String(index).padStart(3, '0')} ${'response'.repeat(3)} `
  85. })
  86. interface ChromiumMetrics {
  87. readonly [name: string]: number
  88. }
  89. interface Measurement {
  90. readonly wallMs: number
  91. readonly taskMs: number
  92. readonly scriptMs: number
  93. readonly layoutMs: number
  94. readonly recalcStyleMs: number
  95. readonly devtoolsMs: number
  96. readonly nodesDelta: number
  97. readonly listenersDelta: number
  98. readonly heapDeltaMb: number
  99. readonly totalNodes: number
  100. readonly heapMb: number
  101. }
  102. interface MutationProbeResult {
  103. readonly batches: number
  104. readonly records: number
  105. }
  106. interface UserRenderProbeResult {
  107. readonly trustedClick: boolean
  108. readonly sendToDomMs: number
  109. readonly sendToPaintMs: number
  110. readonly domToPaintMs: number
  111. readonly mutationBatches: number
  112. readonly mutationRecords: number
  113. }
  114. interface RetainedBrowserState {
  115. readonly domElements: number
  116. readonly nodes: number
  117. readonly listeners: number
  118. readonly heapMb: number
  119. }
  120. interface ContinuedTurnReport {
  121. readonly ordinal: number
  122. readonly resultingTurns: number
  123. readonly kind: 'text' | 'tool'
  124. readonly promptChars: number
  125. readonly composerFill: Measurement
  126. readonly stream: MutationProbeResult & Measurement & {
  127. readonly paceMs: number
  128. readonly deltaChunks: number
  129. readonly persistedChunks: number
  130. readonly toolCalls: number
  131. readonly toolResults: number
  132. readonly clickToUserEchoMs: number
  133. readonly clickToFirstChunkMs: number
  134. readonly firstChunkToSettledMs: number
  135. }
  136. }
  137. interface ConversationTurnSpec {
  138. readonly prompt: string
  139. readonly deltas: readonly string[]
  140. readonly userMarker: string
  141. readonly firstMarker: string
  142. readonly doneMarker: string
  143. readonly toolResultMarker?: string
  144. }
  145. interface RetainedCheckpoint {
  146. readonly turns: number
  147. readonly state: RetainedBrowserState
  148. }
  149. interface ConversationReport {
  150. readonly startingTurns: number
  151. readonly turnsAdded: number
  152. readonly toolTurns: number
  153. readonly retainedBefore: RetainedBrowserState
  154. readonly retainedAfter: RetainedBrowserState
  155. readonly retainedDelta: RetainedBrowserState
  156. readonly checkpoints: readonly RetainedCheckpoint[]
  157. readonly turns: readonly ContinuedTurnReport[]
  158. }
  159. interface PerformanceWorld {
  160. readonly scaffold: WebScaffold
  161. readonly page: Page
  162. readonly tripwire: ReturnType<typeof watchConsole>
  163. readonly sessionEvents: SessionEvent[]
  164. readonly setupMs: number
  165. readonly replayDir?: string
  166. }
  167. interface PerformanceWorldOptions {
  168. readonly browser: Browser
  169. readonly replay?: ReplayOverrideDoc
  170. readonly sidebarSessions?: number
  171. readonly seedLongHistory?: boolean
  172. }
  173. function text(value: string): { type: 'text'; text: string }[] {
  174. return [{ type: 'text', text: value }]
  175. }
  176. function appendTitle(session: Session, title: string, messageSeq: number): void {
  177. session.append('session/title', {
  178. title,
  179. messageSeqs: [messageSeq],
  180. source: { kind: 'fallback' },
  181. })
  182. }
  183. function appendRequestHeader(session: Session, turn: number, step: number): void {
  184. session.append('request/header', {
  185. header: {
  186. config: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
  187. system: `Synthetic performance system prompt for turn ${String(turn)}, step ${String(step)}.`,
  188. },
  189. reason: turn === 1 && step === 1 ? 'initial' : 'change',
  190. })
  191. }
  192. function appendAssistant(
  193. session: Session,
  194. turn: number,
  195. step: number,
  196. body: string,
  197. ): void {
  198. session.append('assistant/message', {
  199. turn,
  200. step,
  201. message: createAssistantMessage({
  202. content: text(body),
  203. source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
  204. }),
  205. usage: {
  206. inputTokens: 4_000 + turn * 10,
  207. outputTokens: 200 + step * 20,
  208. cacheReadTokens: turn % 2 === 0 ? 2_000 : 0,
  209. },
  210. }, { surfaceOp: 'append' })
  211. }
  212. function appendToolStep(
  213. session: Session,
  214. turn: number,
  215. step: number,
  216. toolCount: number,
  217. ): void {
  218. const calls = Array.from({ length: toolCount }, (_, index) => {
  219. const callId = CallId(`perf-call-${String(turn)}-${String(index)}`)
  220. const args = JSON.stringify({
  221. turn,
  222. index,
  223. payload: 'x'.repeat(120),
  224. })
  225. return { callId, index, args }
  226. })
  227. session.append('assistant/message', {
  228. turn,
  229. step,
  230. message: createAssistantMessage({
  231. content: [
  232. {
  233. type: 'reasoning',
  234. text: `Dispatching ${String(toolCount)} synthetic tools for turn ${String(turn)}.`,
  235. },
  236. ...calls.map(({ callId, args }) => ({
  237. type: 'tool-call' as const,
  238. id: callId,
  239. name: 'synthetic_tool',
  240. arguments: args,
  241. })),
  242. ],
  243. source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
  244. }),
  245. usage: {
  246. inputTokens: 6_000 + turn * 10,
  247. outputTokens: 500,
  248. cacheReadTokens: 3_000,
  249. reasoningTokens: 50,
  250. },
  251. }, { surfaceOp: 'append' })
  252. const callEvents = calls.map(({ callId, args }) =>
  253. session.append('tool/call', {
  254. turn,
  255. step,
  256. callId,
  257. name: 'synthetic_tool',
  258. arguments: args,
  259. }))
  260. for (const [index, call] of calls.entries()) {
  261. const source = callEvents[index]
  262. if (source === undefined) throw new Error(`missing synthetic tool call ${String(index)}`)
  263. session.append('tool/result', {
  264. turn,
  265. step,
  266. message: createToolResultMessage({
  267. callId: call.callId,
  268. content: text(
  269. `synthetic result turn=${String(turn)} index=${String(call.index)} ${'r'.repeat(400)}`,
  270. ),
  271. isError: false,
  272. }),
  273. }, { surfaceOp: 'append', sourceEventSeqs: [source.seq] })
  274. }
  275. }
  276. function fencedCode(turn: number): string {
  277. if (turn % 25 !== 0) return ''
  278. const lines = Array.from(
  279. { length: 80 },
  280. (_, index) => `const value_${String(index)} = ${String(turn + index)}`,
  281. )
  282. return `\n\n\`\`\`ts\n${lines.join('\n')}\n\`\`\``
  283. }
  284. function fixtureLog(session: Session): string {
  285. const header = {
  286. type: 'session',
  287. version: SESSION_FORMAT_VERSION,
  288. id: '{{sessionId}}',
  289. createdAt: Date.now() - 60_000,
  290. cwd: '{{cwd}}',
  291. }
  292. return [
  293. JSON.stringify(header),
  294. ...session.events.map(event => JSON.stringify(event)),
  295. '',
  296. ].join('\n')
  297. }
  298. function smallSidebarFixture(): string {
  299. const session = Session.create(SessionId('perf-small-template'))
  300. session.append('turn/start', {
  301. turn: 1,
  302. })
  303. const user = session.append('user/message', createUserMessage({
  304. content: text('Inspect this compact synthetic session.'),
  305. source: { kind: 'user' },
  306. }), { surfaceOp: 'append' })
  307. appendTitle(session, 'Synthetic sidebar session', user.seq)
  308. session.append('step/start', { turn: 1, step: 1 })
  309. appendRequestHeader(session, 1, 1)
  310. appendToolStep(session, 1, 1, 2)
  311. session.append('step/end', { turn: 1, step: 1 })
  312. session.append('step/start', { turn: 1, step: 2 })
  313. appendRequestHeader(session, 1, 2)
  314. appendAssistant(session, 1, 2, 'Synthetic sidebar fixture complete.')
  315. session.append('step/end', { turn: 1, step: 2 })
  316. session.append('turn/end', { turn: 1, reason: { kind: 'completed' } })
  317. return fixtureLog(session)
  318. }
  319. function longHistoryFixture(): string {
  320. const session = Session.create(SessionId(LONG_SESSION_ID))
  321. for (let turn = 1; turn <= LONG_HISTORY_TURNS; turn += 1) {
  322. session.append('turn/start', {
  323. turn,
  324. })
  325. const user = session.append('user/message', createUserMessage({
  326. content: text(
  327. `LONG_PERF_SENTINEL turn ${String(turn)}: analyze payload ${'u'.repeat(200)}`,
  328. ),
  329. source: { kind: 'user' },
  330. }), { surfaceOp: 'append' })
  331. if (turn === 1) appendTitle(session, LONG_SESSION_TITLE, user.seq)
  332. session.append('step/start', { turn, step: 1 })
  333. appendRequestHeader(session, turn, 1)
  334. if (turn % TOOL_TURN_INTERVAL === 0) {
  335. appendToolStep(session, turn, 1, TOOLS_PER_TOOL_TURN)
  336. session.append('step/end', { turn, step: 1 })
  337. session.append('step/start', { turn, step: 2 })
  338. appendRequestHeader(session, turn, 2)
  339. appendAssistant(
  340. session,
  341. turn,
  342. 2,
  343. `All synthetic tools completed for turn ${String(turn)}. ${'z'.repeat(320)}${fencedCode(turn)}`,
  344. )
  345. session.append('step/end', { turn, step: 2 })
  346. } else {
  347. appendAssistant(
  348. session,
  349. turn,
  350. 1,
  351. `Synthetic assistant response for turn ${String(turn)}. ${'a'.repeat(320)}${fencedCode(turn)}`,
  352. )
  353. session.append('step/end', { turn, step: 1 })
  354. }
  355. session.append('turn/end', { turn, reason: { kind: 'completed' } })
  356. }
  357. return fixtureLog(session)
  358. }
  359. function textStream(deltas: readonly string[], inputTokens: number): StreamChunk[] {
  360. const response = deltas.join('')
  361. return [
  362. { type: 'block-start', index: 0, blockType: 'text' },
  363. ...deltas.map(text => ({
  364. type: 'text-delta' as const,
  365. index: 0,
  366. text,
  367. })),
  368. {
  369. type: 'block-end',
  370. index: 0,
  371. block: { type: 'text', text: response },
  372. },
  373. {
  374. type: 'usage',
  375. usage: {
  376. inputTokens,
  377. outputTokens: Math.ceil(response.length / 4),
  378. },
  379. },
  380. { type: 'finish', reason: { kind: 'stop' } },
  381. ]
  382. }
  383. function comparisonPrompt(index: number): string {
  384. if (index === COMPARISON_TURNS) return LIVE_PROMPT
  385. return (`${LONG_CONTINUATION_USER_PREFIX}_${String(index)} `
  386. + `继续分析这个长会话的第 ${String(index)} 个增量问题,并保留当前滚动和输入响应。 `
  387. + 'context '.repeat(80)).trimEnd()
  388. }
  389. function comparisonDeltas(index: number): string[] {
  390. if (index === COMPARISON_TURNS) return STREAM_DELTAS
  391. return Array.from({ length: COMPARISON_DELTA_COUNT }, (_, chunkIndex) => {
  392. if (chunkIndex === 0) return `${LONG_CONTINUATION_FIRST_PREFIX}_${String(index)} `
  393. if (chunkIndex === COMPARISON_DELTA_COUNT - 1) {
  394. return `${LONG_CONTINUATION_DONE_PREFIX}_${String(index)}.`
  395. }
  396. return `turn-${String(index)}-chunk-${String(chunkIndex).padStart(2, '0')} ${'response'.repeat(2)} `
  397. })
  398. }
  399. function comparisonTurn(index: number): ConversationTurnSpec {
  400. const toolResultMarker = index % COMPARISON_TOOL_INTERVAL === 0
  401. ? `LONG_CONTINUATION_TOOL_RESULT_${String(index)}`
  402. : undefined
  403. return {
  404. prompt: comparisonPrompt(index),
  405. deltas: comparisonDeltas(index),
  406. userMarker: index === COMPARISON_TURNS
  407. ? LIVE_PROMPT_MARKER
  408. : `${LONG_CONTINUATION_USER_PREFIX}_${String(index)}`,
  409. firstMarker: index === COMPARISON_TURNS
  410. ? STREAM_FIRST_MARKER
  411. : `${LONG_CONTINUATION_FIRST_PREFIX}_${String(index)}`,
  412. doneMarker: index === COMPARISON_TURNS
  413. ? STREAM_DONE_MARKER
  414. : `${LONG_CONTINUATION_DONE_PREFIX}_${String(index)}`,
  415. ...toolResultMarker === undefined ? {} : { toolResultMarker },
  416. }
  417. }
  418. function soakTurn(index: number): ConversationTurnSpec {
  419. const suffix = String(index).padStart(3, '0')
  420. const toolResultMarker = index % SOAK_TOOL_INTERVAL === 0
  421. ? `SOAK_CONVERSATION_TOOL_RESULT_${suffix}`
  422. : undefined
  423. return {
  424. prompt: (`${SOAK_USER_PREFIX}_${suffix} `
  425. + `持续对话第 ${String(index)} 轮,检查增量渲染与保留状态。 `
  426. + 'context '.repeat(20)).trimEnd(),
  427. deltas: Array.from({ length: SOAK_DELTA_COUNT }, (_, chunkIndex) => {
  428. if (chunkIndex === 0) return `${SOAK_FIRST_PREFIX}_${suffix} `
  429. if (chunkIndex === SOAK_DELTA_COUNT - 1) return `${SOAK_DONE_PREFIX}_${suffix}.`
  430. return `soak-${suffix}-${String(chunkIndex).padStart(2, '0')} response `
  431. }),
  432. userMarker: `${SOAK_USER_PREFIX}_${suffix}`,
  433. firstMarker: `${SOAK_FIRST_PREFIX}_${suffix}`,
  434. doneMarker: `${SOAK_DONE_PREFIX}_${suffix}`,
  435. ...toolResultMarker === undefined ? {} : { toolResultMarker },
  436. }
  437. }
  438. function toolStream(index: number, marker: string): StreamChunk[] {
  439. const callId = CallId(`performance-tool-${marker.toLowerCase()}-${String(index)}`)
  440. const args = JSON.stringify({
  441. command: `printf '${marker}\\n'`,
  442. description: `Emit performance marker ${String(index)}`,
  443. })
  444. return [
  445. { type: 'block-start', index: 0, blockType: 'tool-call' },
  446. {
  447. type: 'tool-call-delta',
  448. index: 0,
  449. id: callId,
  450. name: 'bash',
  451. argumentsDelta: args,
  452. },
  453. {
  454. type: 'block-end',
  455. index: 0,
  456. block: { type: 'tool-call', id: callId, name: 'bash', arguments: args },
  457. },
  458. { type: 'usage', usage: { inputTokens: 256, outputTokens: 32 } },
  459. { type: 'finish', reason: { kind: 'tool-calls' } },
  460. ]
  461. }
  462. function performanceReplayOverride(
  463. turnCount: number,
  464. turnSpec: (index: number) => ConversationTurnSpec,
  465. ): ReplayOverrideDoc {
  466. const continuationEntries = Array.from(
  467. { length: turnCount },
  468. (_, offset): ReplayEntry[] => {
  469. const index = offset + 1
  470. const spec = turnSpec(index)
  471. const finalResponse: ReplayEntry = {
  472. kind: 'chunks',
  473. chunks: textStream(spec.deltas, Math.ceil(spec.prompt.length / 4)),
  474. }
  475. return spec.toolResultMarker === undefined
  476. ? [finalResponse]
  477. : [{ kind: 'chunks', chunks: toolStream(index, spec.toolResultMarker) }, finalResponse]
  478. },
  479. ).flat()
  480. return continuationEntries
  481. }
  482. function rounded(value: number): number {
  483. return Math.round(value * 1_000) / 1_000
  484. }
  485. async function chromiumMetrics(cdp: CDPSession): Promise<ChromiumMetrics> {
  486. const payload = await cdp.send('Performance.getMetrics')
  487. return Object.fromEntries(payload.metrics.map(metric => [metric.name, metric.value]))
  488. }
  489. async function retainedBrowserState(
  490. cdp: CDPSession,
  491. page: Page,
  492. ): Promise<RetainedBrowserState> {
  493. await cdp.send('HeapProfiler.collectGarbage')
  494. const metrics = await chromiumMetrics(cdp)
  495. return {
  496. domElements: await page.evaluate(() => document.querySelectorAll('*').length),
  497. nodes: requiredMetric(metrics, 'Nodes'),
  498. listeners: requiredMetric(metrics, 'JSEventListeners'),
  499. heapMb: rounded(requiredMetric(metrics, 'JSHeapUsedSize') / 1_048_576),
  500. }
  501. }
  502. function requiredMetric(metrics: ChromiumMetrics, name: string): number {
  503. const value = metrics[name]
  504. if (value === undefined) throw new Error(`Chromium performance metric ${name} is unavailable`)
  505. return value
  506. }
  507. function metricDelta(
  508. before: ChromiumMetrics,
  509. after: ChromiumMetrics,
  510. wallMs: number,
  511. ): Measurement {
  512. return {
  513. wallMs: rounded(wallMs),
  514. taskMs: rounded((requiredMetric(after, 'TaskDuration') - requiredMetric(before, 'TaskDuration')) * 1_000),
  515. scriptMs: rounded((requiredMetric(after, 'ScriptDuration') - requiredMetric(before, 'ScriptDuration')) * 1_000),
  516. layoutMs: rounded((requiredMetric(after, 'LayoutDuration') - requiredMetric(before, 'LayoutDuration')) * 1_000),
  517. recalcStyleMs: rounded(
  518. (requiredMetric(after, 'RecalcStyleDuration') - requiredMetric(before, 'RecalcStyleDuration')) * 1_000,
  519. ),
  520. devtoolsMs: rounded(
  521. (requiredMetric(after, 'DevToolsCommandDuration') - requiredMetric(before, 'DevToolsCommandDuration')) * 1_000,
  522. ),
  523. nodesDelta: requiredMetric(after, 'Nodes') - requiredMetric(before, 'Nodes'),
  524. listenersDelta: requiredMetric(after, 'JSEventListeners') - requiredMetric(before, 'JSEventListeners'),
  525. heapDeltaMb: rounded(
  526. (requiredMetric(after, 'JSHeapUsedSize') - requiredMetric(before, 'JSHeapUsedSize')) / 1_048_576,
  527. ),
  528. totalNodes: requiredMetric(after, 'Nodes'),
  529. heapMb: rounded(requiredMetric(after, 'JSHeapUsedSize') / 1_048_576),
  530. }
  531. }
  532. async function measure<T>(
  533. cdp: CDPSession,
  534. action: () => Promise<T>,
  535. ): Promise<{ measurement: Measurement; value: T }> {
  536. const before = await chromiumMetrics(cdp)
  537. const started = performance.now()
  538. const value = await action()
  539. const wallMs = performance.now() - started
  540. const after = await chromiumMetrics(cdp)
  541. return { measurement: metricDelta(before, after, wallMs), value }
  542. }
  543. async function startMutationProbe(page: Page): Promise<void> {
  544. await page.evaluate(() => {
  545. const target = document.querySelector('[class*="centerCol"]')
  546. if (target === null) throw new Error('stream mutation probe target is unavailable')
  547. const probe = {
  548. batches: 0,
  549. records: 0,
  550. observer: undefined as MutationObserver | undefined,
  551. }
  552. const observer = new MutationObserver((records) => {
  553. probe.batches += 1
  554. probe.records += records.length
  555. })
  556. probe.observer = observer
  557. observer.observe(target, {
  558. attributes: true,
  559. attributeFilter: ['data-streaming'],
  560. characterData: true,
  561. childList: true,
  562. subtree: true,
  563. })
  564. Reflect.set(globalThis, '__dshPerfMutationProbe', probe)
  565. })
  566. }
  567. async function stopMutationProbe(page: Page): Promise<MutationProbeResult> {
  568. return page.evaluate(() => {
  569. const probe = Reflect.get(globalThis, '__dshPerfMutationProbe') as
  570. | { batches: number; records: number; observer: MutationObserver }
  571. | undefined
  572. if (probe === undefined) throw new Error('stream mutation probe was not started')
  573. probe.observer.disconnect()
  574. Reflect.deleteProperty(globalThis, '__dshPerfMutationProbe')
  575. return { batches: probe.batches, records: probe.records }
  576. })
  577. }
  578. async function startUserRenderProbe(
  579. page: Page,
  580. marker: string,
  581. ): Promise<void> {
  582. const send = page.getByRole('button', { name: 'Send message', exact: true })
  583. await send.waitFor({ timeout: 15_000 })
  584. await expect.poll(() => send.isEnabled(), { timeout: 15_000 }).toBe(true)
  585. await page.evaluate((expectedMarker) => {
  586. const target = document.querySelector('[data-conversation-scroll]')
  587. if (target === null) throw new Error('user render probe target is unavailable')
  588. const probe: {
  589. sendAt?: number
  590. domAt?: number
  591. paintAt?: number
  592. trustedClick?: boolean
  593. batches: number
  594. records: number
  595. observer?: MutationObserver
  596. pollForMarker?: () => void
  597. } = { batches: 0, records: 0 }
  598. const transcriptContainsMarker = (): boolean => {
  599. // The composer projects the draft into a backdrop and hidden sizing
  600. // mirror; only a matching text node outside that card proves delivery.
  601. const walker = document.createTreeWalker(target, NodeFilter.SHOW_TEXT)
  602. for (let node = walker.nextNode(); node !== null; node = walker.nextNode()) {
  603. if (!node.textContent?.includes(expectedMarker)) continue
  604. const parent = node.parentElement
  605. if (parent !== null && parent.closest('[data-composer-card]') === null) return true
  606. }
  607. return false
  608. }
  609. const markDomVisible = (): void => {
  610. if (
  611. probe.sendAt === undefined
  612. || probe.domAt !== undefined
  613. || !transcriptContainsMarker()
  614. ) return
  615. probe.domAt = globalThis.performance.now()
  616. // The second callback runs only after a render opportunity for the DOM
  617. // insertion observed before the first callback.
  618. requestAnimationFrame(() => {
  619. requestAnimationFrame(() => {
  620. probe.paintAt = globalThis.performance.now()
  621. observer.disconnect()
  622. })
  623. })
  624. }
  625. const pollForMarker = (): void => {
  626. markDomVisible()
  627. if (probe.domAt === undefined) requestAnimationFrame(pollForMarker)
  628. }
  629. const observer = new MutationObserver((records) => {
  630. if (probe.sendAt === undefined) return
  631. probe.batches += 1
  632. probe.records += records.length
  633. markDomVisible()
  634. })
  635. probe.observer = observer
  636. probe.pollForMarker = pollForMarker
  637. observer.observe(target, {
  638. attributes: true,
  639. attributeFilter: ['data-streaming'],
  640. characterData: true,
  641. childList: true,
  642. subtree: true,
  643. })
  644. Reflect.set(globalThis, '__dshPerfUserRenderProbe', probe)
  645. }, marker)
  646. await send.evaluate((button) => {
  647. button.addEventListener('click', (event) => {
  648. const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
  649. | { sendAt?: number; trustedClick?: boolean; pollForMarker?: () => void }
  650. | undefined
  651. if (probe?.pollForMarker === undefined) throw new Error('user render probe was not started')
  652. probe.trustedClick = event.isTrusted
  653. probe.sendAt = globalThis.performance.now()
  654. requestAnimationFrame(probe.pollForMarker)
  655. }, { capture: true, once: true })
  656. })
  657. }
  658. async function triggerUserRenderProbe(page: Page): Promise<void> {
  659. const send = page.getByRole('button', { name: 'Send message', exact: true })
  660. await send.click()
  661. }
  662. async function stopUserRenderProbe(
  663. page: Page,
  664. marker: string,
  665. ): Promise<UserRenderProbeResult> {
  666. try {
  667. await page.waitForFunction(() => {
  668. const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
  669. | { paintAt?: number }
  670. | undefined
  671. return probe?.paintAt !== undefined
  672. }, undefined, { timeout: 15_000 })
  673. } catch (error) {
  674. const diagnostic = await page.evaluate((expectedMarker) => {
  675. const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
  676. | {
  677. sendAt?: number
  678. domAt?: number
  679. paintAt?: number
  680. trustedClick?: boolean
  681. batches: number
  682. records: number
  683. }
  684. | undefined
  685. const target = document.querySelector('[data-conversation-scroll]')
  686. let markerOutsideComposer = false
  687. if (target !== null) {
  688. const walker = document.createTreeWalker(target, NodeFilter.SHOW_TEXT)
  689. for (let node = walker.nextNode(); node !== null; node = walker.nextNode()) {
  690. if (!node.textContent?.includes(expectedMarker)) continue
  691. const parent = node.parentElement
  692. if (parent !== null && parent.closest('[data-composer-card]') === null) {
  693. markerOutsideComposer = true
  694. break
  695. }
  696. }
  697. }
  698. return {
  699. probe,
  700. markerOutsideComposer,
  701. markerInComposer: document.querySelector('[data-composer-card]')
  702. ?.textContent?.includes(expectedMarker) ?? false,
  703. }
  704. }, marker)
  705. throw new Error(`user render probe timed out: ${JSON.stringify(diagnostic)}`, { cause: error })
  706. }
  707. return page.evaluate(() => {
  708. const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
  709. | {
  710. sendAt?: number
  711. domAt?: number
  712. paintAt?: number
  713. trustedClick?: boolean
  714. batches: number
  715. records: number
  716. }
  717. | undefined
  718. Reflect.deleteProperty(globalThis, '__dshPerfUserRenderProbe')
  719. if (
  720. probe?.trustedClick !== true
  721. || probe.sendAt === undefined
  722. || probe.domAt === undefined
  723. || probe.paintAt === undefined
  724. ) {
  725. throw new Error('user render probe did not observe a trusted click, DOM insertion, and paint')
  726. }
  727. return {
  728. trustedClick: probe.trustedClick,
  729. sendToDomMs: probe.domAt - probe.sendAt,
  730. sendToPaintMs: probe.paintAt - probe.sendAt,
  731. domToPaintMs: probe.paintAt - probe.domAt,
  732. mutationBatches: probe.batches,
  733. mutationRecords: probe.records,
  734. }
  735. })
  736. }
  737. async function stableCount(
  738. locator: Locator,
  739. accepts: (count: number) => boolean,
  740. timeoutMs = 60_000,
  741. ): Promise<number> {
  742. const deadline = performance.now() + timeoutMs
  743. let previous = -1
  744. let stableReads = 0
  745. while (performance.now() < deadline) {
  746. const count = await locator.count()
  747. stableReads = accepts(count) && count === previous ? stableReads + 1 : 0
  748. if (stableReads >= 4) return count
  749. previous = count
  750. await new Promise(resolve => setTimeout(resolve, 50))
  751. }
  752. throw new Error(`browser row count did not stabilize; last count ${String(previous)}`)
  753. }
  754. async function conversationTurns(page: Page): Promise<number> {
  755. // Loaded-window turn count: one mounted turn-tail footer per settled turn in
  756. // the window (context keys are `${kind.length}:${kind}${id}`). The stats
  757. // strip cannot serve as this probe: its counts ride the whole-log
  758. // sessionStats projection and stay fixed across paging by design.
  759. return stableCount(page.locator('[data-chat-flow-key^="9:turn-tail"]'), count => count > 0)
  760. }
  761. function retainedDelta(
  762. before: RetainedBrowserState,
  763. after: RetainedBrowserState,
  764. ): RetainedBrowserState {
  765. return {
  766. domElements: after.domElements - before.domElements,
  767. nodes: after.nodes - before.nodes,
  768. listeners: after.listeners - before.listeners,
  769. heapMb: rounded(after.heapMb - before.heapMb),
  770. }
  771. }
  772. async function launchPerformanceWorld(
  773. options: PerformanceWorldOptions,
  774. ): Promise<PerformanceWorld> {
  775. const setupStarted = performance.now()
  776. let replayDir: string | undefined
  777. let scaffold: WebScaffold | undefined
  778. let page: Page | undefined
  779. try {
  780. if (options.replay === undefined) {
  781. scaffold = await launchWebScaffold()
  782. } else {
  783. replayDir = await mkdtemp(join(tmpdir(), 'dsh-web-perf-replay-'))
  784. const replayOverride = join(replayDir, 'replay.override.json')
  785. await writeFile(replayOverride, JSON.stringify(options.replay))
  786. scaffold = await launchWebScaffold({
  787. // The override-only fixture supplies one positional script for this
  788. // world's single live session.
  789. replayFixture: join(replayDir, 'override-only.jsonl'),
  790. replayOverride,
  791. paceMs: STREAM_PACE_MS,
  792. replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
  793. })
  794. }
  795. const sessionEvents: SessionEvent[] = []
  796. scaffold.ctx.on('session/event', (_session, event: SessionEvent) => {
  797. sessionEvents.push(event)
  798. })
  799. if ((options.sidebarSessions ?? 0) > 0) {
  800. const small = smallSidebarFixture()
  801. for (let index = 0; index < (options.sidebarSessions ?? 0); index += 1) {
  802. await seedSession(scaffold, small, `perf-sidebar-${String(index).padStart(4, '0')}`)
  803. }
  804. }
  805. if (options.seedLongHistory === true) {
  806. await seedSession(scaffold, longHistoryFixture(), LONG_SESSION_ID)
  807. }
  808. const setupMs = performance.now() - setupStarted
  809. page = await newEnglishPage(options.browser)
  810. return {
  811. scaffold,
  812. page,
  813. tripwire: watchConsole(page),
  814. sessionEvents,
  815. setupMs,
  816. ...replayDir === undefined ? {} : { replayDir },
  817. }
  818. } catch (error) {
  819. const failures: unknown[] = [error]
  820. if (page !== undefined) {
  821. try {
  822. await page.close()
  823. } catch (cleanupError) {
  824. failures.push(cleanupError)
  825. }
  826. }
  827. if (scaffold !== undefined) {
  828. try {
  829. await scaffold.close()
  830. } catch (cleanupError) {
  831. failures.push(cleanupError)
  832. }
  833. }
  834. if (replayDir !== undefined) {
  835. try {
  836. await rm(replayDir, { recursive: true, force: true })
  837. } catch (cleanupError) {
  838. failures.push(cleanupError)
  839. }
  840. }
  841. if (failures.length === 1) throw error
  842. throw new AggregateError(failures, 'web performance setup and cleanup failed')
  843. }
  844. }
  845. async function closePerformanceWorld(world: PerformanceWorld): Promise<void> {
  846. const failures: unknown[] = []
  847. await world.page.close().catch((error: unknown) => failures.push(error))
  848. await world.scaffold.close().catch((error: unknown) => failures.push(error))
  849. if (world.replayDir !== undefined) {
  850. await rm(world.replayDir, { recursive: true, force: true })
  851. .catch((error: unknown) => failures.push(error))
  852. }
  853. if (failures.length === 1) throw failures[0]
  854. if (failures.length > 1) throw new AggregateError(failures, 'web performance teardown failed')
  855. }
  856. async function openPerformancePage(
  857. world: PerformanceWorld,
  858. expectedSessions: number,
  859. ): Promise<Locator> {
  860. await world.page.goto(world.scaffold.baseUrl, { waitUntil: 'load' })
  861. await world.page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
  862. const group = world.page.getByRole('treeitem').first()
  863. await expect.poll(() => group.textContent(), { timeout: 30_000 })
  864. .toContain(`${String(expectedSessions)} ${expectedSessions === 1 ? 'session' : 'sessions'}`)
  865. return group
  866. }
  867. async function openLongHistory(page: Page): Promise<number> {
  868. await page.getByRole('textbox', { name: 'Search name, keywords...', exact: true })
  869. .fill('LONG_PERF_SENTINEL')
  870. const results = page.getByRole('tree', { name: 'Search results' }).getByRole('treeitem')
  871. await expect.poll(() => results.count(), { timeout: 60_000 }).toBe(1)
  872. await results.first().click()
  873. await page.getByRole('tab', { name: 'Chat', exact: true }).waitFor({ timeout: 30_000 })
  874. return conversationTurns(page)
  875. }
  876. async function continueConversation(
  877. world: PerformanceWorld,
  878. cdp: CDPSession,
  879. options: {
  880. readonly startingTurns: number
  881. readonly turnCount: number
  882. readonly turnSpec: (index: number) => ConversationTurnSpec
  883. readonly expectedSessionId?: SessionId
  884. readonly checkpointInterval?: number
  885. },
  886. ): Promise<ConversationReport> {
  887. const composer = world.page.locator('textarea:enabled').last()
  888. await composer.waitFor({ timeout: 15_000 })
  889. const retainedBefore = await retainedBrowserState(cdp, world.page)
  890. const checkpoints: RetainedCheckpoint[] = [{ turns: options.startingTurns, state: retainedBefore }]
  891. const turns: ContinuedTurnReport[] = []
  892. let liveSessionId = options.expectedSessionId
  893. for (let index = 1; index <= options.turnCount; index += 1) {
  894. const spec = options.turnSpec(index)
  895. const composerFill = await measure(cdp, async () => {
  896. await composer.fill(spec.prompt)
  897. await expect.poll(() => composer.inputValue()).toBe(spec.prompt)
  898. return (await composer.inputValue()).length
  899. })
  900. expect(composerFill.value).toBe(spec.prompt.length)
  901. const eventStart = world.sessionEvents.length
  902. await startMutationProbe(world.page)
  903. const streamBefore = await chromiumMetrics(cdp)
  904. const streamStarted = performance.now()
  905. const settled = world.scaffold.whenTurnSettled(60_000)
  906. await world.page.getByRole('button', { name: 'Send message', exact: true }).click()
  907. await world.page.getByText(spec.userMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  908. const clickToUserEchoMs = performance.now() - streamStarted
  909. await world.page.getByText(spec.firstMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  910. const clickToFirstChunkMs = performance.now() - streamStarted
  911. const settledSessionId = await settled
  912. if (liveSessionId === undefined) {
  913. liveSessionId = settledSessionId
  914. } else {
  915. expect(settledSessionId).toBe(liveSessionId)
  916. }
  917. await expect.poll(
  918. () => world.page.locator('[data-streaming="true"]').count(),
  919. { timeout: 15_000 },
  920. ).toBe(0)
  921. await world.page.getByText(spec.doneMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  922. const clickToSettledMs = performance.now() - streamStarted
  923. const streamAfter = await chromiumMetrics(cdp)
  924. const mutations = await stopMutationProbe(world.page)
  925. const turnEvents = world.sessionEvents.slice(eventStart)
  926. const chunks = turnEvents.filter(event => event.type === 'assistant/chunk')
  927. const toolCalls = turnEvents.filter(event => event.type === 'tool/call')
  928. const toolResults = turnEvents.filter(event => event.type === 'tool/result')
  929. const toolTurn = spec.toolResultMarker !== undefined
  930. expect(chunks).toHaveLength(spec.deltas.length + (toolTurn ? 9 : 4))
  931. expect(toolCalls).toHaveLength(toolTurn ? 1 : 0)
  932. expect(toolResults).toHaveLength(toolTurn ? 1 : 0)
  933. if (spec.toolResultMarker !== undefined) {
  934. expect(toolCalls[0]?.data.name).toBe('bash')
  935. const toolResult = toolResults[0]
  936. if (toolResult?.type !== 'tool/result') {
  937. throw new Error(`continued turn ${String(index)} did not log its tool result`)
  938. }
  939. const resultBlock = toolResult.data.message.content.find(block => block.type === 'tool-result')
  940. expect(resultBlock?.isError).toBe(false)
  941. expect(resultBlock?.content
  942. .filter(block => block.type === 'text')
  943. .map(block => block.text)
  944. .join('')).toContain(spec.toolResultMarker)
  945. }
  946. const user = turnEvents.find(
  947. event => event.type === 'user/message' && event.data.source.kind === 'user',
  948. )
  949. if (user?.type !== 'user/message') {
  950. throw new Error(`continued turn ${String(index)} did not log its user message`)
  951. }
  952. expect(user.data.content
  953. .filter(block => block.type === 'text')
  954. .map(block => block.text)
  955. .join('')).toBe(spec.prompt)
  956. const resultingTurns = await conversationTurns(world.page)
  957. expect(resultingTurns).toBe(options.startingTurns + index)
  958. turns.push({
  959. ordinal: index,
  960. resultingTurns,
  961. kind: toolTurn ? 'tool' : 'text',
  962. promptChars: spec.prompt.length,
  963. composerFill: composerFill.measurement,
  964. stream: {
  965. paceMs: STREAM_PACE_MS,
  966. deltaChunks: spec.deltas.length,
  967. persistedChunks: chunks.length,
  968. toolCalls: toolCalls.length,
  969. toolResults: toolResults.length,
  970. clickToUserEchoMs: rounded(clickToUserEchoMs),
  971. clickToFirstChunkMs: rounded(clickToFirstChunkMs),
  972. firstChunkToSettledMs: rounded(clickToSettledMs - clickToFirstChunkMs),
  973. ...mutations,
  974. ...metricDelta(streamBefore, streamAfter, clickToSettledMs),
  975. },
  976. })
  977. if (options.checkpointInterval !== undefined && index % options.checkpointInterval === 0) {
  978. checkpoints.push({
  979. turns: options.startingTurns + index,
  980. state: await retainedBrowserState(cdp, world.page),
  981. })
  982. }
  983. }
  984. const lastCheckpoint = checkpoints.at(-1)
  985. const retainedAfter = lastCheckpoint?.turns === options.startingTurns + options.turnCount
  986. ? lastCheckpoint.state
  987. : await retainedBrowserState(cdp, world.page)
  988. if (lastCheckpoint?.turns !== options.startingTurns + options.turnCount) {
  989. checkpoints.push({ turns: options.startingTurns + options.turnCount, state: retainedAfter })
  990. }
  991. return {
  992. startingTurns: options.startingTurns,
  993. turnsAdded: options.turnCount,
  994. toolTurns: turns.filter(turn => turn.kind === 'tool').length,
  995. retainedBefore,
  996. retainedAfter,
  997. retainedDelta: retainedDelta(retainedBefore, retainedAfter),
  998. checkpoints,
  999. turns,
  1000. }
  1001. }
  1002. async function measurePostSoakUserRender(
  1003. world: PerformanceWorld,
  1004. cdp: CDPSession,
  1005. ): Promise<object> {
  1006. const spec = soakTurn(POST_SOAK_RENDER_TURN)
  1007. if (spec.toolResultMarker !== undefined) {
  1008. throw new Error('post-soak render probe must remain a text-only turn')
  1009. }
  1010. const composer = world.page.locator('textarea:enabled').last()
  1011. const composerFill = await measure(cdp, async () => {
  1012. await composer.fill(spec.prompt)
  1013. await expect.poll(() => composer.inputValue()).toBe(spec.prompt)
  1014. return (await composer.inputValue()).length
  1015. })
  1016. expect(composerFill.value).toBe(spec.prompt.length)
  1017. const eventStart = world.sessionEvents.length
  1018. await startUserRenderProbe(world.page, spec.userMarker)
  1019. const browserBefore = await chromiumMetrics(cdp)
  1020. const fullTurnStarted = performance.now()
  1021. const settled = world.scaffold.whenTurnSettled(60_000)
  1022. await triggerUserRenderProbe(world.page)
  1023. const browserTiming = await stopUserRenderProbe(world.page, spec.userMarker)
  1024. const browserAfter = await chromiumMetrics(cdp)
  1025. const browserAfterPaintSample = metricDelta(
  1026. browserBefore,
  1027. browserAfter,
  1028. performance.now() - fullTurnStarted,
  1029. )
  1030. await world.page.getByText(spec.firstMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  1031. const settledSessionId = await settled
  1032. await expect.poll(
  1033. () => world.page.locator('[data-streaming="true"]').count(),
  1034. { timeout: 15_000 },
  1035. ).toBe(0)
  1036. await world.page.getByText(spec.doneMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
  1037. const fullTurnMs = performance.now() - fullTurnStarted
  1038. const turnEvents = world.sessionEvents.slice(eventStart)
  1039. const chunks = turnEvents.filter(event => event.type === 'assistant/chunk')
  1040. const user = turnEvents.find(
  1041. event => event.type === 'user/message' && event.data.source.kind === 'user',
  1042. )
  1043. if (user?.type !== 'user/message') throw new Error('post-soak render probe did not log its user message')
  1044. expect(user.data.content
  1045. .filter(block => block.type === 'text')
  1046. .map(block => block.text)
  1047. .join('')).toBe(spec.prompt)
  1048. expect(chunks).toHaveLength(spec.deltas.length + 4)
  1049. expect(turnEvents.filter(event => event.type === 'tool/call')).toHaveLength(0)
  1050. expect(turnEvents.filter(event => event.type === 'tool/result')).toHaveLength(0)
  1051. expect(world.scaffold.ctx.agents.get(settledSessionId)).toBeDefined()
  1052. expect(await conversationTurns(world.page)).toBe(POST_SOAK_RENDER_TURN)
  1053. return {
  1054. ordinal: POST_SOAK_RENDER_TURN,
  1055. resultingTurns: POST_SOAK_RENDER_TURN,
  1056. promptChars: spec.prompt.length,
  1057. composerFill: composerFill.measurement,
  1058. trustedClick: browserTiming.trustedClick,
  1059. sendToDomMs: rounded(browserTiming.sendToDomMs),
  1060. sendToPaintMs: rounded(browserTiming.sendToPaintMs),
  1061. domToPaintMs: rounded(browserTiming.domToPaintMs),
  1062. mutationBatchesThroughPaint: browserTiming.mutationBatches,
  1063. mutationRecordsThroughPaint: browserTiming.mutationRecords,
  1064. browserAfterPaintSample,
  1065. fullTurnMs: rounded(fullTurnMs),
  1066. persistedChunks: chunks.length,
  1067. }
  1068. }
  1069. function average(values: readonly number[]): number {
  1070. return rounded(values.reduce((sum, value) => sum + value, 0) / values.length)
  1071. }
  1072. function p95(values: readonly number[]): number {
  1073. const sorted = [...values].sort((left, right) => left - right)
  1074. return sorted[Math.max(0, Math.ceil(sorted.length * 0.95) - 1)] ?? 0
  1075. }
  1076. function summarizeTurnWindows(
  1077. turns: readonly ContinuedTurnReport[],
  1078. windowSize: number,
  1079. ): object[] {
  1080. const windows: object[] = []
  1081. for (let start = 0; start < turns.length; start += windowSize) {
  1082. const window = turns.slice(start, start + windowSize)
  1083. const streamWall = window.map(turn => turn.stream.wallMs)
  1084. const userEcho = window.map(turn => turn.stream.clickToUserEchoMs)
  1085. const firstChunk = window.map(turn => turn.stream.clickToFirstChunkMs)
  1086. windows.push({
  1087. turns: `${String(start + 1)}-${String(start + window.length)}`,
  1088. toolTurns: window.filter(turn => turn.kind === 'tool').length,
  1089. average: {
  1090. composerFillWallMs: average(window.map(turn => turn.composerFill.wallMs)),
  1091. streamWallMs: average(streamWall),
  1092. clickToUserEchoMs: average(userEcho),
  1093. clickToFirstChunkMs: average(firstChunk),
  1094. taskMs: average(window.map(turn => turn.stream.taskMs)),
  1095. scriptMs: average(window.map(turn => turn.stream.scriptMs)),
  1096. recalcStyleMs: average(window.map(turn => turn.stream.recalcStyleMs)),
  1097. mutationBatches: average(window.map(turn => turn.stream.batches)),
  1098. },
  1099. p95: {
  1100. streamWallMs: rounded(p95(streamWall)),
  1101. clickToUserEchoMs: rounded(p95(userEcho)),
  1102. clickToFirstChunkMs: rounded(p95(firstChunk)),
  1103. },
  1104. })
  1105. }
  1106. return windows
  1107. }
  1108. describe('manual web performance: complex workspace and history', () => {
  1109. let browser: Browser
  1110. beforeAll(async () => {
  1111. if (webSnapshotMode() === 'record') {
  1112. throw new Error('manual web performance runs only with deterministic replay')
  1113. }
  1114. browser = await chromium.launch()
  1115. })
  1116. afterAll(async () => {
  1117. await browser?.close()
  1118. })
  1119. it('reports workspace, history, and trajectory cardinality costs', async () => {
  1120. const world = await launchPerformanceWorld({
  1121. browser,
  1122. sidebarSessions: SIDEBAR_SESSION_COUNT,
  1123. seedLongHistory: true,
  1124. })
  1125. try {
  1126. const bootStarted = performance.now()
  1127. const group = await openPerformancePage(world, SIDEBAR_SESSION_COUNT + 1)
  1128. const bootReadyMs = performance.now() - bootStarted
  1129. const page = world.page
  1130. const cdp = await page.context().newCDPSession(page)
  1131. await cdp.send('Performance.enable')
  1132. const firstContentfulPaintMs = await page.evaluate(
  1133. () => globalThis.performance.getEntriesByName('first-contentful-paint')[0]?.startTime,
  1134. )
  1135. const sidebar = await measure(cdp, async () => {
  1136. await group.click()
  1137. return stableCount(
  1138. page.getByRole('treeitem'),
  1139. count => count === SIDEBAR_SESSION_COUNT + 2,
  1140. )
  1141. })
  1142. expect(sidebar.value).toBe(SIDEBAR_SESSION_COUNT + 2)
  1143. await group.click()
  1144. await expect.poll(() => page.getByRole('treeitem').count()).toBe(1)
  1145. const contentSearch = await measure(cdp, async () => {
  1146. await page.getByRole('textbox', { name: 'Search name, keywords...', exact: true })
  1147. .fill('LONG_PERF_SENTINEL')
  1148. const results = page.getByRole('tree', { name: 'Search results' }).getByRole('treeitem')
  1149. await expect.poll(() => results.count(), { timeout: 60_000 }).toBe(1)
  1150. await results.first().waitFor({ timeout: 60_000 })
  1151. return results.first()
  1152. })
  1153. const opened = await measure(cdp, async () => {
  1154. await contentSearch.value.click()
  1155. await page.getByRole('tab', { name: 'Trajectory', exact: true }).waitFor({ timeout: 30_000 })
  1156. return conversationTurns(page)
  1157. })
  1158. expect(opened.value).toBe(DEFAULT_HISTORY_TURNS)
  1159. const trajectoryRows = page.getByRole('row')
  1160. const coldTrajectory = await measure(cdp, async () => {
  1161. await page.getByRole('tab', { name: 'Trajectory', exact: true }).click()
  1162. return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS)
  1163. })
  1164. expect(coldTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS)
  1165. const collapseTurns = await measure(cdp, async () => {
  1166. await page.getByRole('button', { name: 'Collapse turns', exact: true }).click()
  1167. return stableCount(trajectoryRows, count => count > 0 && count < EXPECTED_TRAJECTORY_ROWS)
  1168. })
  1169. expect(collapseTurns.value).toBeLessThan(EXPECTED_TRAJECTORY_ROWS)
  1170. const trajectorySearch = await measure(cdp, async () => {
  1171. await page.getByRole('searchbox', { name: 'Search trajectory', exact: true }).fill('turn 499')
  1172. return stableCount(trajectoryRows, count => count > 0 && count < 20)
  1173. })
  1174. expect(trajectorySearch.value).toBeLessThan(20)
  1175. await page.getByRole('tab', { name: 'Chat', exact: true }).click()
  1176. const historyPages: { turns: number; measurement: Measurement }[] = []
  1177. let turns = await conversationTurns(page)
  1178. while (turns < LONG_HISTORY_TURNS) {
  1179. const previousTurns = turns
  1180. const older = await measure(cdp, async () => {
  1181. await page.getByRole('button', { name: 'Load earlier', exact: true }).click()
  1182. await expect.poll(() => conversationTurns(page), { timeout: 30_000 })
  1183. .toBeGreaterThan(previousTurns)
  1184. return conversationTurns(page)
  1185. })
  1186. turns = older.value
  1187. historyPages.push({ turns, measurement: older.measurement })
  1188. }
  1189. const warmTrajectory = await measure(cdp, async () => {
  1190. await page.getByRole('tab', { name: 'Trajectory', exact: true }).click()
  1191. return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS)
  1192. })
  1193. expect(warmTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS)
  1194. const warmConversation = await measure(cdp, async () => {
  1195. await page.getByRole('tab', { name: 'Chat', exact: true }).click()
  1196. return conversationTurns(page)
  1197. })
  1198. expect(warmConversation.value).toBe(LONG_HISTORY_TURNS)
  1199. console.info(`WEB_PERF_RESULT ${JSON.stringify({
  1200. scenario: 'workspace-history-trajectory',
  1201. fixture: {
  1202. sidebarSessions: SIDEBAR_SESSION_COUNT,
  1203. totalSessions: SIDEBAR_SESSION_COUNT + 1,
  1204. longHistoryTurns: LONG_HISTORY_TURNS,
  1205. toolCalls: EXPECTED_TOOL_CALLS,
  1206. trajectoryRows: EXPECTED_TRAJECTORY_ROWS,
  1207. },
  1208. setupMs: rounded(world.setupMs),
  1209. boot: {
  1210. readyMs: rounded(bootReadyMs),
  1211. firstContentfulPaintMs: firstContentfulPaintMs === undefined
  1212. ? null
  1213. : rounded(firstContentfulPaintMs),
  1214. },
  1215. sidebarExpand: sidebar.measurement,
  1216. contentSearch: contentSearch.measurement,
  1217. openLongHistory: { initialTurns: opened.value, ...opened.measurement },
  1218. coldTrajectory: { rows: coldTrajectory.value, ...coldTrajectory.measurement },
  1219. collapseTurns: { rows: collapseTurns.value, ...collapseTurns.measurement },
  1220. trajectorySearch: { rows: trajectorySearch.value, ...trajectorySearch.measurement },
  1221. historyPages,
  1222. warmTrajectory: { rows: warmTrajectory.value, ...warmTrajectory.measurement },
  1223. warmConversation: { turns: warmConversation.value, ...warmConversation.measurement },
  1224. }, null, 2)}`)
  1225. expect(world.tripwire.warnings).toEqual([])
  1226. expect(world.tripwire.pageErrors).toEqual([])
  1227. } finally {
  1228. await closePerformanceWorld(world)
  1229. }
  1230. })
  1231. it('reports default 24-turn history plus eight continued turns', async () => {
  1232. const world = await launchPerformanceWorld({
  1233. browser,
  1234. replay: performanceReplayOverride(COMPARISON_TURNS, comparisonTurn),
  1235. seedLongHistory: true,
  1236. })
  1237. try {
  1238. await openPerformancePage(world, 1)
  1239. const cdp = await world.page.context().newCDPSession(world.page)
  1240. await cdp.send('Performance.enable')
  1241. const opened = await measure(cdp, () => openLongHistory(world.page))
  1242. expect(opened.value).toBe(DEFAULT_HISTORY_TURNS)
  1243. const conversation = await continueConversation(world, cdp, {
  1244. startingTurns: DEFAULT_HISTORY_TURNS,
  1245. turnCount: COMPARISON_TURNS,
  1246. turnSpec: comparisonTurn,
  1247. expectedSessionId: SessionId(LONG_SESSION_ID),
  1248. })
  1249. expect(conversation.toolTurns).toBe(2)
  1250. console.info(`WEB_PERF_RESULT ${JSON.stringify({
  1251. scenario: 'default-resume-24-plus-8',
  1252. setupMs: rounded(world.setupMs),
  1253. replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
  1254. openLongHistory: { turns: opened.value, ...opened.measurement },
  1255. conversation,
  1256. }, null, 2)}`)
  1257. expect(world.tripwire.warnings).toEqual([])
  1258. expect(world.tripwire.pageErrors).toEqual([])
  1259. } finally {
  1260. await closePerformanceWorld(world)
  1261. }
  1262. })
  1263. it('reports fully expanded 500-turn history plus eight continued turns', async () => {
  1264. const world = await launchPerformanceWorld({
  1265. browser,
  1266. replay: performanceReplayOverride(COMPARISON_TURNS, comparisonTurn),
  1267. seedLongHistory: true,
  1268. })
  1269. try {
  1270. await openPerformancePage(world, 1)
  1271. const cdp = await world.page.context().newCDPSession(world.page)
  1272. await cdp.send('Performance.enable')
  1273. expect(await openLongHistory(world.page)).toBe(DEFAULT_HISTORY_TURNS)
  1274. const historyPages: { turns: number; measurement: Measurement }[] = []
  1275. let turns = DEFAULT_HISTORY_TURNS
  1276. while (turns < LONG_HISTORY_TURNS) {
  1277. const previousTurns = turns
  1278. const older = await measure(cdp, async () => {
  1279. await world.page.getByRole('button', { name: 'Load earlier', exact: true }).click()
  1280. await expect.poll(() => conversationTurns(world.page), { timeout: 30_000 })
  1281. .toBeGreaterThan(previousTurns)
  1282. return conversationTurns(world.page)
  1283. })
  1284. turns = older.value
  1285. historyPages.push({ turns, measurement: older.measurement })
  1286. }
  1287. expect(turns).toBe(LONG_HISTORY_TURNS)
  1288. const conversation = await continueConversation(world, cdp, {
  1289. startingTurns: LONG_HISTORY_TURNS,
  1290. turnCount: COMPARISON_TURNS,
  1291. turnSpec: comparisonTurn,
  1292. expectedSessionId: SessionId(LONG_SESSION_ID),
  1293. })
  1294. expect(conversation.toolTurns).toBe(2)
  1295. console.info(`WEB_PERF_RESULT ${JSON.stringify({
  1296. scenario: 'expanded-history-500-plus-8',
  1297. setupMs: rounded(world.setupMs),
  1298. replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
  1299. historyPages,
  1300. conversation,
  1301. }, null, 2)}`)
  1302. expect(world.tripwire.warnings).toEqual([])
  1303. expect(world.tripwire.pageErrors).toEqual([])
  1304. } finally {
  1305. await closePerformanceWorld(world)
  1306. }
  1307. })
  1308. it('reports one hundred generated turns and the next user-message paint', async () => {
  1309. const world = await launchPerformanceWorld({
  1310. browser,
  1311. replay: performanceReplayOverride(POST_SOAK_RENDER_TURN, soakTurn),
  1312. })
  1313. let testFailure: unknown
  1314. try {
  1315. await world.page.goto(world.scaffold.baseUrl, { waitUntil: 'load' })
  1316. await world.page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
  1317. await connectFreshWorkspace(world.page, world.scaffold.workspaceCwd, 'continuous-conversation-perf')
  1318. const cdp = await world.page.context().newCDPSession(world.page)
  1319. await cdp.send('Performance.enable')
  1320. const conversation = await continueConversation(world, cdp, {
  1321. startingTurns: 0,
  1322. turnCount: SOAK_TURNS,
  1323. turnSpec: soakTurn,
  1324. checkpointInterval: SOAK_CHECKPOINT_INTERVAL,
  1325. })
  1326. expect(conversation.toolTurns).toBe(SOAK_TURNS / SOAK_TOOL_INTERVAL)
  1327. const postSoakUserRender = await measurePostSoakUserRender(world, cdp)
  1328. const { turns, ...retained } = conversation
  1329. console.info(`WEB_PERF_RESULT ${JSON.stringify({
  1330. scenario: 'continuous-100-turn-soak',
  1331. setupMs: rounded(world.setupMs),
  1332. replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
  1333. ...retained,
  1334. windows: summarizeTurnWindows(turns, SOAK_CHECKPOINT_INTERVAL),
  1335. postSoakUserRender,
  1336. }, null, 2)}`)
  1337. expect(world.tripwire.warnings).toEqual([])
  1338. expect(world.tripwire.pageErrors).toEqual([])
  1339. } catch (error) {
  1340. testFailure = error
  1341. throw error
  1342. } finally {
  1343. try {
  1344. await closePerformanceWorld(world)
  1345. } catch (cleanupError) {
  1346. if (testFailure === undefined) throw cleanupError
  1347. throw new AggregateError(
  1348. [testFailure, cleanupError],
  1349. 'continuous conversation performance test and teardown failed',
  1350. )
  1351. }
  1352. }
  1353. })
  1354. })