| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432 |
- // Opt-in browser benchmark for high-cardinality workspace and history
- // rendering. It reports measurements without timing assertions because host
- // speed is not a correctness contract; structural assertions keep the load
- // shape from silently shrinking.
- import { mkdtemp, rm, writeFile } from 'node:fs/promises'
- import { tmpdir } from 'node:os'
- import { join } from 'node:path'
- import { performance } from 'node:perf_hooks'
- import type { Browser, CDPSession, Locator, Page } from 'playwright'
- import { chromium } from 'playwright'
- import { afterAll, beforeAll, describe, expect, it } from 'vitest'
- import type { StreamChunk } from '@deepseek-ai/dsh-llm'
- import {
- CallId,
- createAssistantMessage,
- createToolResultMessage,
- createUserMessage,
- } from '@deepseek-ai/dsh-llm'
- import type { ReplayEntry, ReplayOverrideDoc } from '@deepseek-ai/dsh-llm-replay'
- import type { SessionEvent } from '@deepseek-ai/dsh-session'
- import {
- SESSION_FORMAT_VERSION,
- Session,
- SessionId,
- } from '@deepseek-ai/dsh-session'
- // Carries the session/title event declaration into the fixture builder.
- import type {} from '@deepseek-ai/dsh-session-title'
- import {
- launchWebScaffold,
- seedSession,
- watchConsole,
- webSnapshotMode,
- type WebScaffold,
- } from './scaffold.ts'
- import { connectFreshWorkspace, newEnglishPage } from './support.ts'
- const SIDEBAR_SESSION_COUNT = 1_000
- const LONG_SESSION_ID = 'perf-long-history'
- const LONG_SESSION_TITLE = 'LONG_PERF_SENTINEL 500-turn session'
- const LONG_HISTORY_TURNS = 500
- const TOOL_TURN_INTERVAL = 10
- const TOOLS_PER_TOOL_TURN = 10
- const EXPECTED_TOOL_CALLS = LONG_HISTORY_TURNS / TOOL_TURN_INTERVAL * TOOLS_PER_TOOL_TURN
- const EXPECTED_TRAJECTORY_ROWS = 2_100
- const DEFAULT_HISTORY_TURNS = 24
- const PERF_REPLAY_CONTEXT_WINDOW = 10_000_000
- const STREAM_PACE_MS = 8
- const STREAM_DELTA_COUNT = 120
- const COMPARISON_TURNS = 8
- const COMPARISON_DELTA_COUNT = 24
- const COMPARISON_TOOL_INTERVAL = 3
- const SOAK_TURNS = 100
- const POST_SOAK_RENDER_TURN = SOAK_TURNS + 1
- const SOAK_DELTA_COUNT = 8
- const SOAK_TOOL_INTERVAL = 10
- const SOAK_CHECKPOINT_INTERVAL = 10
- const LONG_CONTINUATION_USER_PREFIX = 'LONG_CONTINUATION_USER'
- const LONG_CONTINUATION_FIRST_PREFIX = 'LONG_CONTINUATION_FIRST'
- const LONG_CONTINUATION_DONE_PREFIX = 'LONG_CONTINUATION_DONE'
- const SOAK_USER_PREFIX = 'SOAK_CONVERSATION_USER'
- const SOAK_FIRST_PREFIX = 'SOAK_CONVERSATION_FIRST'
- const SOAK_DONE_PREFIX = 'SOAK_CONVERSATION_DONE'
- const LIVE_PROMPT_MARKER = 'STREAM_PERF_USER_INPUT'
- const STREAM_FIRST_MARKER = 'STREAM_PERF_FIRST'
- const STREAM_DONE_MARKER = 'STREAM_PERF_DONE'
- const LIVE_PROMPT = [
- LIVE_PROMPT_MARKER,
- 'Analyze the following mixed-language project context and return a concise diagnostic.',
- ...Array.from(
- { length: 48 },
- (_, index) =>
- `Context ${String(index + 1).padStart(2, '0')}: 用户正在检查长会话中的增量渲染性能。`
- + ` Preserve item ${String(index)} and compare ${'payload'.repeat(8)}.`,
- ),
- '```ts',
- ...Array.from(
- { length: 40 },
- (_, index) => `const sample_${String(index)} = ${JSON.stringify(`value-${String(index)}-${'x'.repeat(32)}`)}`,
- ),
- '```',
- ].join('\n')
- const STREAM_DELTAS = Array.from({ length: STREAM_DELTA_COUNT }, (_, index) => {
- if (index === 0) return `${STREAM_FIRST_MARKER} `
- if (index === STREAM_DELTA_COUNT - 1) return `${STREAM_DONE_MARKER}.`
- return `chunk-${String(index).padStart(3, '0')} ${'response'.repeat(3)} `
- })
- interface ChromiumMetrics {
- readonly [name: string]: number
- }
- interface Measurement {
- readonly wallMs: number
- readonly taskMs: number
- readonly scriptMs: number
- readonly layoutMs: number
- readonly recalcStyleMs: number
- readonly devtoolsMs: number
- readonly nodesDelta: number
- readonly listenersDelta: number
- readonly heapDeltaMb: number
- readonly totalNodes: number
- readonly heapMb: number
- }
- interface MutationProbeResult {
- readonly batches: number
- readonly records: number
- }
- interface UserRenderProbeResult {
- readonly trustedClick: boolean
- readonly sendToDomMs: number
- readonly sendToPaintMs: number
- readonly domToPaintMs: number
- readonly mutationBatches: number
- readonly mutationRecords: number
- }
- interface RetainedBrowserState {
- readonly domElements: number
- readonly nodes: number
- readonly listeners: number
- readonly heapMb: number
- }
- interface ContinuedTurnReport {
- readonly ordinal: number
- readonly resultingTurns: number
- readonly kind: 'text' | 'tool'
- readonly promptChars: number
- readonly composerFill: Measurement
- readonly stream: MutationProbeResult & Measurement & {
- readonly paceMs: number
- readonly deltaChunks: number
- readonly persistedChunks: number
- readonly toolCalls: number
- readonly toolResults: number
- readonly clickToUserEchoMs: number
- readonly clickToFirstChunkMs: number
- readonly firstChunkToSettledMs: number
- }
- }
- interface ConversationTurnSpec {
- readonly prompt: string
- readonly deltas: readonly string[]
- readonly userMarker: string
- readonly firstMarker: string
- readonly doneMarker: string
- readonly toolResultMarker?: string
- }
- interface RetainedCheckpoint {
- readonly turns: number
- readonly state: RetainedBrowserState
- }
- interface ConversationReport {
- readonly startingTurns: number
- readonly turnsAdded: number
- readonly toolTurns: number
- readonly retainedBefore: RetainedBrowserState
- readonly retainedAfter: RetainedBrowserState
- readonly retainedDelta: RetainedBrowserState
- readonly checkpoints: readonly RetainedCheckpoint[]
- readonly turns: readonly ContinuedTurnReport[]
- }
- interface PerformanceWorld {
- readonly scaffold: WebScaffold
- readonly page: Page
- readonly tripwire: ReturnType<typeof watchConsole>
- readonly sessionEvents: SessionEvent[]
- readonly setupMs: number
- readonly replayDir?: string
- }
- interface PerformanceWorldOptions {
- readonly browser: Browser
- readonly replay?: ReplayOverrideDoc
- readonly sidebarSessions?: number
- readonly seedLongHistory?: boolean
- }
- function text(value: string): { type: 'text'; text: string }[] {
- return [{ type: 'text', text: value }]
- }
- function appendTitle(session: Session, title: string, messageSeq: number): void {
- session.append('session/title', {
- title,
- messageSeqs: [messageSeq],
- source: { kind: 'fallback' },
- })
- }
- function appendRequestHeader(session: Session, turn: number, step: number): void {
- session.append('request/header', {
- header: {
- config: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
- system: `Synthetic performance system prompt for turn ${String(turn)}, step ${String(step)}.`,
- },
- reason: turn === 1 && step === 1 ? 'initial' : 'change',
- })
- }
- function appendAssistant(
- session: Session,
- turn: number,
- step: number,
- body: string,
- ): void {
- session.append('assistant/message', {
- turn,
- step,
- message: createAssistantMessage({
- content: text(body),
- source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
- }),
- usage: {
- inputTokens: 4_000 + turn * 10,
- outputTokens: 200 + step * 20,
- cacheReadTokens: turn % 2 === 0 ? 2_000 : 0,
- },
- }, { surfaceOp: 'append' })
- }
- function appendToolStep(
- session: Session,
- turn: number,
- step: number,
- toolCount: number,
- ): void {
- const calls = Array.from({ length: toolCount }, (_, index) => {
- const callId = CallId(`perf-call-${String(turn)}-${String(index)}`)
- const args = JSON.stringify({
- turn,
- index,
- payload: 'x'.repeat(120),
- })
- return { callId, index, args }
- })
- session.append('assistant/message', {
- turn,
- step,
- message: createAssistantMessage({
- content: [
- {
- type: 'reasoning',
- text: `Dispatching ${String(toolCount)} synthetic tools for turn ${String(turn)}.`,
- },
- ...calls.map(({ callId, args }) => ({
- type: 'tool-call' as const,
- id: callId,
- name: 'synthetic_tool',
- arguments: args,
- })),
- ],
- source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
- }),
- usage: {
- inputTokens: 6_000 + turn * 10,
- outputTokens: 500,
- cacheReadTokens: 3_000,
- reasoningTokens: 50,
- },
- }, { surfaceOp: 'append' })
- const callEvents = calls.map(({ callId, args }) =>
- session.append('tool/call', {
- turn,
- step,
- callId,
- name: 'synthetic_tool',
- arguments: args,
- }))
- for (const [index, call] of calls.entries()) {
- const source = callEvents[index]
- if (source === undefined) throw new Error(`missing synthetic tool call ${String(index)}`)
- session.append('tool/result', {
- turn,
- step,
- message: createToolResultMessage({
- callId: call.callId,
- content: text(
- `synthetic result turn=${String(turn)} index=${String(call.index)} ${'r'.repeat(400)}`,
- ),
- isError: false,
- }),
- }, { surfaceOp: 'append', sourceEventSeqs: [source.seq] })
- }
- }
- function fencedCode(turn: number): string {
- if (turn % 25 !== 0) return ''
- const lines = Array.from(
- { length: 80 },
- (_, index) => `const value_${String(index)} = ${String(turn + index)}`,
- )
- return `\n\n\`\`\`ts\n${lines.join('\n')}\n\`\`\``
- }
- function fixtureLog(session: Session): string {
- const header = {
- type: 'session',
- version: SESSION_FORMAT_VERSION,
- id: '{{sessionId}}',
- createdAt: Date.now() - 60_000,
- cwd: '{{cwd}}',
- }
- return [
- JSON.stringify(header),
- ...session.events.map(event => JSON.stringify(event)),
- '',
- ].join('\n')
- }
- function smallSidebarFixture(): string {
- const session = Session.create(SessionId('perf-small-template'))
- session.append('turn/start', {
- turn: 1,
- })
- const user = session.append('user/message', createUserMessage({
- content: text('Inspect this compact synthetic session.'),
- source: { kind: 'user' },
- }), { surfaceOp: 'append' })
- appendTitle(session, 'Synthetic sidebar session', user.seq)
- session.append('step/start', { turn: 1, step: 1 })
- appendRequestHeader(session, 1, 1)
- appendToolStep(session, 1, 1, 2)
- session.append('step/end', { turn: 1, step: 1 })
- session.append('step/start', { turn: 1, step: 2 })
- appendRequestHeader(session, 1, 2)
- appendAssistant(session, 1, 2, 'Synthetic sidebar fixture complete.')
- session.append('step/end', { turn: 1, step: 2 })
- session.append('turn/end', { turn: 1, reason: { kind: 'completed' } })
- return fixtureLog(session)
- }
- function longHistoryFixture(): string {
- const session = Session.create(SessionId(LONG_SESSION_ID))
- for (let turn = 1; turn <= LONG_HISTORY_TURNS; turn += 1) {
- session.append('turn/start', {
- turn,
- })
- const user = session.append('user/message', createUserMessage({
- content: text(
- `LONG_PERF_SENTINEL turn ${String(turn)}: analyze payload ${'u'.repeat(200)}`,
- ),
- source: { kind: 'user' },
- }), { surfaceOp: 'append' })
- if (turn === 1) appendTitle(session, LONG_SESSION_TITLE, user.seq)
- session.append('step/start', { turn, step: 1 })
- appendRequestHeader(session, turn, 1)
- if (turn % TOOL_TURN_INTERVAL === 0) {
- appendToolStep(session, turn, 1, TOOLS_PER_TOOL_TURN)
- session.append('step/end', { turn, step: 1 })
- session.append('step/start', { turn, step: 2 })
- appendRequestHeader(session, turn, 2)
- appendAssistant(
- session,
- turn,
- 2,
- `All synthetic tools completed for turn ${String(turn)}. ${'z'.repeat(320)}${fencedCode(turn)}`,
- )
- session.append('step/end', { turn, step: 2 })
- } else {
- appendAssistant(
- session,
- turn,
- 1,
- `Synthetic assistant response for turn ${String(turn)}. ${'a'.repeat(320)}${fencedCode(turn)}`,
- )
- session.append('step/end', { turn, step: 1 })
- }
- session.append('turn/end', { turn, reason: { kind: 'completed' } })
- }
- return fixtureLog(session)
- }
- function textStream(deltas: readonly string[], inputTokens: number): StreamChunk[] {
- const response = deltas.join('')
- return [
- { type: 'block-start', index: 0, blockType: 'text' },
- ...deltas.map(text => ({
- type: 'text-delta' as const,
- index: 0,
- text,
- })),
- {
- type: 'block-end',
- index: 0,
- block: { type: 'text', text: response },
- },
- {
- type: 'usage',
- usage: {
- inputTokens,
- outputTokens: Math.ceil(response.length / 4),
- },
- },
- { type: 'finish', reason: { kind: 'stop' } },
- ]
- }
- function comparisonPrompt(index: number): string {
- if (index === COMPARISON_TURNS) return LIVE_PROMPT
- return (`${LONG_CONTINUATION_USER_PREFIX}_${String(index)} `
- + `继续分析这个长会话的第 ${String(index)} 个增量问题,并保留当前滚动和输入响应。 `
- + 'context '.repeat(80)).trimEnd()
- }
- function comparisonDeltas(index: number): string[] {
- if (index === COMPARISON_TURNS) return STREAM_DELTAS
- return Array.from({ length: COMPARISON_DELTA_COUNT }, (_, chunkIndex) => {
- if (chunkIndex === 0) return `${LONG_CONTINUATION_FIRST_PREFIX}_${String(index)} `
- if (chunkIndex === COMPARISON_DELTA_COUNT - 1) {
- return `${LONG_CONTINUATION_DONE_PREFIX}_${String(index)}.`
- }
- return `turn-${String(index)}-chunk-${String(chunkIndex).padStart(2, '0')} ${'response'.repeat(2)} `
- })
- }
- function comparisonTurn(index: number): ConversationTurnSpec {
- const toolResultMarker = index % COMPARISON_TOOL_INTERVAL === 0
- ? `LONG_CONTINUATION_TOOL_RESULT_${String(index)}`
- : undefined
- return {
- prompt: comparisonPrompt(index),
- deltas: comparisonDeltas(index),
- userMarker: index === COMPARISON_TURNS
- ? LIVE_PROMPT_MARKER
- : `${LONG_CONTINUATION_USER_PREFIX}_${String(index)}`,
- firstMarker: index === COMPARISON_TURNS
- ? STREAM_FIRST_MARKER
- : `${LONG_CONTINUATION_FIRST_PREFIX}_${String(index)}`,
- doneMarker: index === COMPARISON_TURNS
- ? STREAM_DONE_MARKER
- : `${LONG_CONTINUATION_DONE_PREFIX}_${String(index)}`,
- ...toolResultMarker === undefined ? {} : { toolResultMarker },
- }
- }
- function soakTurn(index: number): ConversationTurnSpec {
- const suffix = String(index).padStart(3, '0')
- const toolResultMarker = index % SOAK_TOOL_INTERVAL === 0
- ? `SOAK_CONVERSATION_TOOL_RESULT_${suffix}`
- : undefined
- return {
- prompt: (`${SOAK_USER_PREFIX}_${suffix} `
- + `持续对话第 ${String(index)} 轮,检查增量渲染与保留状态。 `
- + 'context '.repeat(20)).trimEnd(),
- deltas: Array.from({ length: SOAK_DELTA_COUNT }, (_, chunkIndex) => {
- if (chunkIndex === 0) return `${SOAK_FIRST_PREFIX}_${suffix} `
- if (chunkIndex === SOAK_DELTA_COUNT - 1) return `${SOAK_DONE_PREFIX}_${suffix}.`
- return `soak-${suffix}-${String(chunkIndex).padStart(2, '0')} response `
- }),
- userMarker: `${SOAK_USER_PREFIX}_${suffix}`,
- firstMarker: `${SOAK_FIRST_PREFIX}_${suffix}`,
- doneMarker: `${SOAK_DONE_PREFIX}_${suffix}`,
- ...toolResultMarker === undefined ? {} : { toolResultMarker },
- }
- }
- function toolStream(index: number, marker: string): StreamChunk[] {
- const callId = CallId(`performance-tool-${marker.toLowerCase()}-${String(index)}`)
- const args = JSON.stringify({
- command: `printf '${marker}\\n'`,
- description: `Emit performance marker ${String(index)}`,
- })
- return [
- { type: 'block-start', index: 0, blockType: 'tool-call' },
- {
- type: 'tool-call-delta',
- index: 0,
- id: callId,
- name: 'bash',
- argumentsDelta: args,
- },
- {
- type: 'block-end',
- index: 0,
- block: { type: 'tool-call', id: callId, name: 'bash', arguments: args },
- },
- { type: 'usage', usage: { inputTokens: 256, outputTokens: 32 } },
- { type: 'finish', reason: { kind: 'tool-calls' } },
- ]
- }
- function performanceReplayOverride(
- turnCount: number,
- turnSpec: (index: number) => ConversationTurnSpec,
- ): ReplayOverrideDoc {
- const continuationEntries = Array.from(
- { length: turnCount },
- (_, offset): ReplayEntry[] => {
- const index = offset + 1
- const spec = turnSpec(index)
- const finalResponse: ReplayEntry = {
- kind: 'chunks',
- chunks: textStream(spec.deltas, Math.ceil(spec.prompt.length / 4)),
- }
- return spec.toolResultMarker === undefined
- ? [finalResponse]
- : [{ kind: 'chunks', chunks: toolStream(index, spec.toolResultMarker) }, finalResponse]
- },
- ).flat()
- return continuationEntries
- }
- function rounded(value: number): number {
- return Math.round(value * 1_000) / 1_000
- }
- async function chromiumMetrics(cdp: CDPSession): Promise<ChromiumMetrics> {
- const payload = await cdp.send('Performance.getMetrics')
- return Object.fromEntries(payload.metrics.map(metric => [metric.name, metric.value]))
- }
- async function retainedBrowserState(
- cdp: CDPSession,
- page: Page,
- ): Promise<RetainedBrowserState> {
- await cdp.send('HeapProfiler.collectGarbage')
- const metrics = await chromiumMetrics(cdp)
- return {
- domElements: await page.evaluate(() => document.querySelectorAll('*').length),
- nodes: requiredMetric(metrics, 'Nodes'),
- listeners: requiredMetric(metrics, 'JSEventListeners'),
- heapMb: rounded(requiredMetric(metrics, 'JSHeapUsedSize') / 1_048_576),
- }
- }
- function requiredMetric(metrics: ChromiumMetrics, name: string): number {
- const value = metrics[name]
- if (value === undefined) throw new Error(`Chromium performance metric ${name} is unavailable`)
- return value
- }
- function metricDelta(
- before: ChromiumMetrics,
- after: ChromiumMetrics,
- wallMs: number,
- ): Measurement {
- return {
- wallMs: rounded(wallMs),
- taskMs: rounded((requiredMetric(after, 'TaskDuration') - requiredMetric(before, 'TaskDuration')) * 1_000),
- scriptMs: rounded((requiredMetric(after, 'ScriptDuration') - requiredMetric(before, 'ScriptDuration')) * 1_000),
- layoutMs: rounded((requiredMetric(after, 'LayoutDuration') - requiredMetric(before, 'LayoutDuration')) * 1_000),
- recalcStyleMs: rounded(
- (requiredMetric(after, 'RecalcStyleDuration') - requiredMetric(before, 'RecalcStyleDuration')) * 1_000,
- ),
- devtoolsMs: rounded(
- (requiredMetric(after, 'DevToolsCommandDuration') - requiredMetric(before, 'DevToolsCommandDuration')) * 1_000,
- ),
- nodesDelta: requiredMetric(after, 'Nodes') - requiredMetric(before, 'Nodes'),
- listenersDelta: requiredMetric(after, 'JSEventListeners') - requiredMetric(before, 'JSEventListeners'),
- heapDeltaMb: rounded(
- (requiredMetric(after, 'JSHeapUsedSize') - requiredMetric(before, 'JSHeapUsedSize')) / 1_048_576,
- ),
- totalNodes: requiredMetric(after, 'Nodes'),
- heapMb: rounded(requiredMetric(after, 'JSHeapUsedSize') / 1_048_576),
- }
- }
- async function measure<T>(
- cdp: CDPSession,
- action: () => Promise<T>,
- ): Promise<{ measurement: Measurement; value: T }> {
- const before = await chromiumMetrics(cdp)
- const started = performance.now()
- const value = await action()
- const wallMs = performance.now() - started
- const after = await chromiumMetrics(cdp)
- return { measurement: metricDelta(before, after, wallMs), value }
- }
- async function startMutationProbe(page: Page): Promise<void> {
- await page.evaluate(() => {
- const target = document.querySelector('[class*="centerCol"]')
- if (target === null) throw new Error('stream mutation probe target is unavailable')
- const probe = {
- batches: 0,
- records: 0,
- observer: undefined as MutationObserver | undefined,
- }
- const observer = new MutationObserver((records) => {
- probe.batches += 1
- probe.records += records.length
- })
- probe.observer = observer
- observer.observe(target, {
- attributes: true,
- attributeFilter: ['data-streaming'],
- characterData: true,
- childList: true,
- subtree: true,
- })
- Reflect.set(globalThis, '__dshPerfMutationProbe', probe)
- })
- }
- async function stopMutationProbe(page: Page): Promise<MutationProbeResult> {
- return page.evaluate(() => {
- const probe = Reflect.get(globalThis, '__dshPerfMutationProbe') as
- | { batches: number; records: number; observer: MutationObserver }
- | undefined
- if (probe === undefined) throw new Error('stream mutation probe was not started')
- probe.observer.disconnect()
- Reflect.deleteProperty(globalThis, '__dshPerfMutationProbe')
- return { batches: probe.batches, records: probe.records }
- })
- }
- async function startUserRenderProbe(
- page: Page,
- marker: string,
- ): Promise<void> {
- const send = page.getByRole('button', { name: 'Send message', exact: true })
- await send.waitFor({ timeout: 15_000 })
- await expect.poll(() => send.isEnabled(), { timeout: 15_000 }).toBe(true)
- await page.evaluate((expectedMarker) => {
- const target = document.querySelector('[data-conversation-scroll]')
- if (target === null) throw new Error('user render probe target is unavailable')
- const probe: {
- sendAt?: number
- domAt?: number
- paintAt?: number
- trustedClick?: boolean
- batches: number
- records: number
- observer?: MutationObserver
- pollForMarker?: () => void
- } = { batches: 0, records: 0 }
- const transcriptContainsMarker = (): boolean => {
- // The composer projects the draft into a backdrop and hidden sizing
- // mirror; only a matching text node outside that card proves delivery.
- const walker = document.createTreeWalker(target, NodeFilter.SHOW_TEXT)
- for (let node = walker.nextNode(); node !== null; node = walker.nextNode()) {
- if (!node.textContent?.includes(expectedMarker)) continue
- const parent = node.parentElement
- if (parent !== null && parent.closest('[data-composer-card]') === null) return true
- }
- return false
- }
- const markDomVisible = (): void => {
- if (
- probe.sendAt === undefined
- || probe.domAt !== undefined
- || !transcriptContainsMarker()
- ) return
- probe.domAt = globalThis.performance.now()
- // The second callback runs only after a render opportunity for the DOM
- // insertion observed before the first callback.
- requestAnimationFrame(() => {
- requestAnimationFrame(() => {
- probe.paintAt = globalThis.performance.now()
- observer.disconnect()
- })
- })
- }
- const pollForMarker = (): void => {
- markDomVisible()
- if (probe.domAt === undefined) requestAnimationFrame(pollForMarker)
- }
- const observer = new MutationObserver((records) => {
- if (probe.sendAt === undefined) return
- probe.batches += 1
- probe.records += records.length
- markDomVisible()
- })
- probe.observer = observer
- probe.pollForMarker = pollForMarker
- observer.observe(target, {
- attributes: true,
- attributeFilter: ['data-streaming'],
- characterData: true,
- childList: true,
- subtree: true,
- })
- Reflect.set(globalThis, '__dshPerfUserRenderProbe', probe)
- }, marker)
- await send.evaluate((button) => {
- button.addEventListener('click', (event) => {
- const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
- | { sendAt?: number; trustedClick?: boolean; pollForMarker?: () => void }
- | undefined
- if (probe?.pollForMarker === undefined) throw new Error('user render probe was not started')
- probe.trustedClick = event.isTrusted
- probe.sendAt = globalThis.performance.now()
- requestAnimationFrame(probe.pollForMarker)
- }, { capture: true, once: true })
- })
- }
- async function triggerUserRenderProbe(page: Page): Promise<void> {
- const send = page.getByRole('button', { name: 'Send message', exact: true })
- await send.click()
- }
- async function stopUserRenderProbe(
- page: Page,
- marker: string,
- ): Promise<UserRenderProbeResult> {
- try {
- await page.waitForFunction(() => {
- const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
- | { paintAt?: number }
- | undefined
- return probe?.paintAt !== undefined
- }, undefined, { timeout: 15_000 })
- } catch (error) {
- const diagnostic = await page.evaluate((expectedMarker) => {
- const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
- | {
- sendAt?: number
- domAt?: number
- paintAt?: number
- trustedClick?: boolean
- batches: number
- records: number
- }
- | undefined
- const target = document.querySelector('[data-conversation-scroll]')
- let markerOutsideComposer = false
- if (target !== null) {
- const walker = document.createTreeWalker(target, NodeFilter.SHOW_TEXT)
- for (let node = walker.nextNode(); node !== null; node = walker.nextNode()) {
- if (!node.textContent?.includes(expectedMarker)) continue
- const parent = node.parentElement
- if (parent !== null && parent.closest('[data-composer-card]') === null) {
- markerOutsideComposer = true
- break
- }
- }
- }
- return {
- probe,
- markerOutsideComposer,
- markerInComposer: document.querySelector('[data-composer-card]')
- ?.textContent?.includes(expectedMarker) ?? false,
- }
- }, marker)
- throw new Error(`user render probe timed out: ${JSON.stringify(diagnostic)}`, { cause: error })
- }
- return page.evaluate(() => {
- const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as
- | {
- sendAt?: number
- domAt?: number
- paintAt?: number
- trustedClick?: boolean
- batches: number
- records: number
- }
- | undefined
- Reflect.deleteProperty(globalThis, '__dshPerfUserRenderProbe')
- if (
- probe?.trustedClick !== true
- || probe.sendAt === undefined
- || probe.domAt === undefined
- || probe.paintAt === undefined
- ) {
- throw new Error('user render probe did not observe a trusted click, DOM insertion, and paint')
- }
- return {
- trustedClick: probe.trustedClick,
- sendToDomMs: probe.domAt - probe.sendAt,
- sendToPaintMs: probe.paintAt - probe.sendAt,
- domToPaintMs: probe.paintAt - probe.domAt,
- mutationBatches: probe.batches,
- mutationRecords: probe.records,
- }
- })
- }
- async function stableCount(
- locator: Locator,
- accepts: (count: number) => boolean,
- timeoutMs = 60_000,
- ): Promise<number> {
- const deadline = performance.now() + timeoutMs
- let previous = -1
- let stableReads = 0
- while (performance.now() < deadline) {
- const count = await locator.count()
- stableReads = accepts(count) && count === previous ? stableReads + 1 : 0
- if (stableReads >= 4) return count
- previous = count
- await new Promise(resolve => setTimeout(resolve, 50))
- }
- throw new Error(`browser row count did not stabilize; last count ${String(previous)}`)
- }
- async function conversationTurns(page: Page): Promise<number> {
- const stats = page.getByText(/\d+ turns · \d+ steps/, { exact: true }).last()
- await stats.waitFor({ timeout: 15_000 })
- const value = await stats.textContent()
- const match = value?.match(/^(\d+) turns · \d+ steps$/)
- if (match?.[1] === undefined) throw new Error(`unexpected conversation stats ${JSON.stringify(value)}`)
- return Number(match[1])
- }
- function retainedDelta(
- before: RetainedBrowserState,
- after: RetainedBrowserState,
- ): RetainedBrowserState {
- return {
- domElements: after.domElements - before.domElements,
- nodes: after.nodes - before.nodes,
- listeners: after.listeners - before.listeners,
- heapMb: rounded(after.heapMb - before.heapMb),
- }
- }
- async function launchPerformanceWorld(
- options: PerformanceWorldOptions,
- ): Promise<PerformanceWorld> {
- const setupStarted = performance.now()
- let replayDir: string | undefined
- let scaffold: WebScaffold | undefined
- let page: Page | undefined
- try {
- if (options.replay === undefined) {
- scaffold = await launchWebScaffold()
- } else {
- replayDir = await mkdtemp(join(tmpdir(), 'dsh-web-perf-replay-'))
- const replayOverride = join(replayDir, 'replay.override.json')
- await writeFile(replayOverride, JSON.stringify(options.replay))
- scaffold = await launchWebScaffold({
- // The override-only fixture supplies one positional script for this
- // world's single live session.
- replayFixture: join(replayDir, 'override-only.jsonl'),
- replayOverride,
- paceMs: STREAM_PACE_MS,
- replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
- })
- }
- const sessionEvents: SessionEvent[] = []
- scaffold.ctx.on('session/event', (_session, event: SessionEvent) => {
- sessionEvents.push(event)
- })
- if ((options.sidebarSessions ?? 0) > 0) {
- const small = smallSidebarFixture()
- for (let index = 0; index < (options.sidebarSessions ?? 0); index += 1) {
- await seedSession(scaffold, small, `perf-sidebar-${String(index).padStart(4, '0')}`)
- }
- }
- if (options.seedLongHistory === true) {
- await seedSession(scaffold, longHistoryFixture(), LONG_SESSION_ID)
- }
- const setupMs = performance.now() - setupStarted
- page = await newEnglishPage(options.browser)
- return {
- scaffold,
- page,
- tripwire: watchConsole(page),
- sessionEvents,
- setupMs,
- ...replayDir === undefined ? {} : { replayDir },
- }
- } catch (error) {
- const failures: unknown[] = [error]
- if (page !== undefined) {
- try {
- await page.close()
- } catch (cleanupError) {
- failures.push(cleanupError)
- }
- }
- if (scaffold !== undefined) {
- try {
- await scaffold.close()
- } catch (cleanupError) {
- failures.push(cleanupError)
- }
- }
- if (replayDir !== undefined) {
- try {
- await rm(replayDir, { recursive: true, force: true })
- } catch (cleanupError) {
- failures.push(cleanupError)
- }
- }
- if (failures.length === 1) throw error
- throw new AggregateError(failures, 'web performance setup and cleanup failed')
- }
- }
- async function closePerformanceWorld(world: PerformanceWorld): Promise<void> {
- const failures: unknown[] = []
- await world.page.close().catch((error: unknown) => failures.push(error))
- await world.scaffold.close().catch((error: unknown) => failures.push(error))
- if (world.replayDir !== undefined) {
- await rm(world.replayDir, { recursive: true, force: true })
- .catch((error: unknown) => failures.push(error))
- }
- if (failures.length === 1) throw failures[0]
- if (failures.length > 1) throw new AggregateError(failures, 'web performance teardown failed')
- }
- async function openPerformancePage(
- world: PerformanceWorld,
- expectedSessions: number,
- ): Promise<Locator> {
- await world.page.goto(world.scaffold.baseUrl, { waitUntil: 'load' })
- await world.page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
- const group = world.page.getByRole('treeitem').first()
- await expect.poll(() => group.textContent(), { timeout: 30_000 })
- .toContain(`${String(expectedSessions)} ${expectedSessions === 1 ? 'session' : 'sessions'}`)
- return group
- }
- async function openLongHistory(page: Page): Promise<number> {
- await page.getByRole('textbox', { name: 'Search name, keywords...', exact: true })
- .fill('LONG_PERF_SENTINEL')
- const results = page.getByRole('tree', { name: 'Search results' }).getByRole('treeitem')
- await expect.poll(() => results.count(), { timeout: 60_000 }).toBe(1)
- await results.first().click()
- await page.getByRole('tab', { name: 'Chat', exact: true }).waitFor({ timeout: 30_000 })
- return conversationTurns(page)
- }
- async function continueConversation(
- world: PerformanceWorld,
- cdp: CDPSession,
- options: {
- readonly startingTurns: number
- readonly turnCount: number
- readonly turnSpec: (index: number) => ConversationTurnSpec
- readonly expectedSessionId?: SessionId
- readonly checkpointInterval?: number
- },
- ): Promise<ConversationReport> {
- const composer = world.page.locator('textarea:enabled').last()
- await composer.waitFor({ timeout: 15_000 })
- const retainedBefore = await retainedBrowserState(cdp, world.page)
- const checkpoints: RetainedCheckpoint[] = [{ turns: options.startingTurns, state: retainedBefore }]
- const turns: ContinuedTurnReport[] = []
- let liveSessionId = options.expectedSessionId
- for (let index = 1; index <= options.turnCount; index += 1) {
- const spec = options.turnSpec(index)
- const composerFill = await measure(cdp, async () => {
- await composer.fill(spec.prompt)
- await expect.poll(() => composer.inputValue()).toBe(spec.prompt)
- return (await composer.inputValue()).length
- })
- expect(composerFill.value).toBe(spec.prompt.length)
- const eventStart = world.sessionEvents.length
- await startMutationProbe(world.page)
- const streamBefore = await chromiumMetrics(cdp)
- const streamStarted = performance.now()
- const settled = world.scaffold.whenTurnSettled(60_000)
- await world.page.getByRole('button', { name: 'Send message', exact: true }).click()
- await world.page.getByText(spec.userMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
- const clickToUserEchoMs = performance.now() - streamStarted
- await world.page.getByText(spec.firstMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
- const clickToFirstChunkMs = performance.now() - streamStarted
- const settledSessionId = await settled
- if (liveSessionId === undefined) {
- liveSessionId = settledSessionId
- } else {
- expect(settledSessionId).toBe(liveSessionId)
- }
- await expect.poll(
- () => world.page.locator('[data-streaming="true"]').count(),
- { timeout: 15_000 },
- ).toBe(0)
- await world.page.getByText(spec.doneMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
- const clickToSettledMs = performance.now() - streamStarted
- const streamAfter = await chromiumMetrics(cdp)
- const mutations = await stopMutationProbe(world.page)
- const turnEvents = world.sessionEvents.slice(eventStart)
- const chunks = turnEvents.filter(event => event.type === 'assistant/chunk')
- const toolCalls = turnEvents.filter(event => event.type === 'tool/call')
- const toolResults = turnEvents.filter(event => event.type === 'tool/result')
- const toolTurn = spec.toolResultMarker !== undefined
- expect(chunks).toHaveLength(spec.deltas.length + (toolTurn ? 9 : 4))
- expect(toolCalls).toHaveLength(toolTurn ? 1 : 0)
- expect(toolResults).toHaveLength(toolTurn ? 1 : 0)
- if (spec.toolResultMarker !== undefined) {
- expect(toolCalls[0]?.data.name).toBe('bash')
- const toolResult = toolResults[0]
- if (toolResult?.type !== 'tool/result') {
- throw new Error(`continued turn ${String(index)} did not log its tool result`)
- }
- const resultBlock = toolResult.data.message.content.find(block => block.type === 'tool-result')
- expect(resultBlock?.isError).toBe(false)
- expect(resultBlock?.content
- .filter(block => block.type === 'text')
- .map(block => block.text)
- .join('')).toContain(spec.toolResultMarker)
- }
- const user = turnEvents.find(
- event => event.type === 'user/message' && event.data.source.kind === 'user',
- )
- if (user?.type !== 'user/message') {
- throw new Error(`continued turn ${String(index)} did not log its user message`)
- }
- expect(user.data.content
- .filter(block => block.type === 'text')
- .map(block => block.text)
- .join('')).toBe(spec.prompt)
- const resultingTurns = await conversationTurns(world.page)
- expect(resultingTurns).toBe(options.startingTurns + index)
- turns.push({
- ordinal: index,
- resultingTurns,
- kind: toolTurn ? 'tool' : 'text',
- promptChars: spec.prompt.length,
- composerFill: composerFill.measurement,
- stream: {
- paceMs: STREAM_PACE_MS,
- deltaChunks: spec.deltas.length,
- persistedChunks: chunks.length,
- toolCalls: toolCalls.length,
- toolResults: toolResults.length,
- clickToUserEchoMs: rounded(clickToUserEchoMs),
- clickToFirstChunkMs: rounded(clickToFirstChunkMs),
- firstChunkToSettledMs: rounded(clickToSettledMs - clickToFirstChunkMs),
- ...mutations,
- ...metricDelta(streamBefore, streamAfter, clickToSettledMs),
- },
- })
- if (options.checkpointInterval !== undefined && index % options.checkpointInterval === 0) {
- checkpoints.push({
- turns: options.startingTurns + index,
- state: await retainedBrowserState(cdp, world.page),
- })
- }
- }
- const lastCheckpoint = checkpoints.at(-1)
- const retainedAfter = lastCheckpoint?.turns === options.startingTurns + options.turnCount
- ? lastCheckpoint.state
- : await retainedBrowserState(cdp, world.page)
- if (lastCheckpoint?.turns !== options.startingTurns + options.turnCount) {
- checkpoints.push({ turns: options.startingTurns + options.turnCount, state: retainedAfter })
- }
- return {
- startingTurns: options.startingTurns,
- turnsAdded: options.turnCount,
- toolTurns: turns.filter(turn => turn.kind === 'tool').length,
- retainedBefore,
- retainedAfter,
- retainedDelta: retainedDelta(retainedBefore, retainedAfter),
- checkpoints,
- turns,
- }
- }
- async function measurePostSoakUserRender(
- world: PerformanceWorld,
- cdp: CDPSession,
- ): Promise<object> {
- const spec = soakTurn(POST_SOAK_RENDER_TURN)
- if (spec.toolResultMarker !== undefined) {
- throw new Error('post-soak render probe must remain a text-only turn')
- }
- const composer = world.page.locator('textarea:enabled').last()
- const composerFill = await measure(cdp, async () => {
- await composer.fill(spec.prompt)
- await expect.poll(() => composer.inputValue()).toBe(spec.prompt)
- return (await composer.inputValue()).length
- })
- expect(composerFill.value).toBe(spec.prompt.length)
- const eventStart = world.sessionEvents.length
- await startUserRenderProbe(world.page, spec.userMarker)
- const browserBefore = await chromiumMetrics(cdp)
- const fullTurnStarted = performance.now()
- const settled = world.scaffold.whenTurnSettled(60_000)
- await triggerUserRenderProbe(world.page)
- const browserTiming = await stopUserRenderProbe(world.page, spec.userMarker)
- const browserAfter = await chromiumMetrics(cdp)
- const browserAfterPaintSample = metricDelta(
- browserBefore,
- browserAfter,
- performance.now() - fullTurnStarted,
- )
- await world.page.getByText(spec.firstMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
- const settledSessionId = await settled
- await expect.poll(
- () => world.page.locator('[data-streaming="true"]').count(),
- { timeout: 15_000 },
- ).toBe(0)
- await world.page.getByText(spec.doneMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
- const fullTurnMs = performance.now() - fullTurnStarted
- const turnEvents = world.sessionEvents.slice(eventStart)
- const chunks = turnEvents.filter(event => event.type === 'assistant/chunk')
- const user = turnEvents.find(
- event => event.type === 'user/message' && event.data.source.kind === 'user',
- )
- if (user?.type !== 'user/message') throw new Error('post-soak render probe did not log its user message')
- expect(user.data.content
- .filter(block => block.type === 'text')
- .map(block => block.text)
- .join('')).toBe(spec.prompt)
- expect(chunks).toHaveLength(spec.deltas.length + 4)
- expect(turnEvents.filter(event => event.type === 'tool/call')).toHaveLength(0)
- expect(turnEvents.filter(event => event.type === 'tool/result')).toHaveLength(0)
- expect(world.scaffold.ctx.agents.get(settledSessionId)).toBeDefined()
- expect(await conversationTurns(world.page)).toBe(POST_SOAK_RENDER_TURN)
- return {
- ordinal: POST_SOAK_RENDER_TURN,
- resultingTurns: POST_SOAK_RENDER_TURN,
- promptChars: spec.prompt.length,
- composerFill: composerFill.measurement,
- trustedClick: browserTiming.trustedClick,
- sendToDomMs: rounded(browserTiming.sendToDomMs),
- sendToPaintMs: rounded(browserTiming.sendToPaintMs),
- domToPaintMs: rounded(browserTiming.domToPaintMs),
- mutationBatchesThroughPaint: browserTiming.mutationBatches,
- mutationRecordsThroughPaint: browserTiming.mutationRecords,
- browserAfterPaintSample,
- fullTurnMs: rounded(fullTurnMs),
- persistedChunks: chunks.length,
- }
- }
- function average(values: readonly number[]): number {
- return rounded(values.reduce((sum, value) => sum + value, 0) / values.length)
- }
- function p95(values: readonly number[]): number {
- const sorted = [...values].sort((left, right) => left - right)
- return sorted[Math.max(0, Math.ceil(sorted.length * 0.95) - 1)] ?? 0
- }
- function summarizeTurnWindows(
- turns: readonly ContinuedTurnReport[],
- windowSize: number,
- ): object[] {
- const windows: object[] = []
- for (let start = 0; start < turns.length; start += windowSize) {
- const window = turns.slice(start, start + windowSize)
- const streamWall = window.map(turn => turn.stream.wallMs)
- const userEcho = window.map(turn => turn.stream.clickToUserEchoMs)
- const firstChunk = window.map(turn => turn.stream.clickToFirstChunkMs)
- windows.push({
- turns: `${String(start + 1)}-${String(start + window.length)}`,
- toolTurns: window.filter(turn => turn.kind === 'tool').length,
- average: {
- composerFillWallMs: average(window.map(turn => turn.composerFill.wallMs)),
- streamWallMs: average(streamWall),
- clickToUserEchoMs: average(userEcho),
- clickToFirstChunkMs: average(firstChunk),
- taskMs: average(window.map(turn => turn.stream.taskMs)),
- scriptMs: average(window.map(turn => turn.stream.scriptMs)),
- recalcStyleMs: average(window.map(turn => turn.stream.recalcStyleMs)),
- mutationBatches: average(window.map(turn => turn.stream.batches)),
- },
- p95: {
- streamWallMs: rounded(p95(streamWall)),
- clickToUserEchoMs: rounded(p95(userEcho)),
- clickToFirstChunkMs: rounded(p95(firstChunk)),
- },
- })
- }
- return windows
- }
- describe('manual web performance: complex workspace and history', () => {
- let browser: Browser
- beforeAll(async () => {
- if (webSnapshotMode() === 'record') {
- throw new Error('manual web performance runs only with deterministic replay')
- }
- browser = await chromium.launch()
- })
- afterAll(async () => {
- await browser?.close()
- })
- it('reports workspace, history, and trajectory cardinality costs', async () => {
- const world = await launchPerformanceWorld({
- browser,
- sidebarSessions: SIDEBAR_SESSION_COUNT,
- seedLongHistory: true,
- })
- try {
- const bootStarted = performance.now()
- const group = await openPerformancePage(world, SIDEBAR_SESSION_COUNT + 1)
- const bootReadyMs = performance.now() - bootStarted
- const page = world.page
- const cdp = await page.context().newCDPSession(page)
- await cdp.send('Performance.enable')
- const firstContentfulPaintMs = await page.evaluate(
- () => globalThis.performance.getEntriesByName('first-contentful-paint')[0]?.startTime,
- )
- const sidebar = await measure(cdp, async () => {
- await group.click()
- return stableCount(
- page.getByRole('treeitem'),
- count => count === SIDEBAR_SESSION_COUNT + 2,
- )
- })
- expect(sidebar.value).toBe(SIDEBAR_SESSION_COUNT + 2)
- await group.click()
- await expect.poll(() => page.getByRole('treeitem').count()).toBe(1)
- const contentSearch = await measure(cdp, async () => {
- await page.getByRole('textbox', { name: 'Search name, keywords...', exact: true })
- .fill('LONG_PERF_SENTINEL')
- const results = page.getByRole('tree', { name: 'Search results' }).getByRole('treeitem')
- await expect.poll(() => results.count(), { timeout: 60_000 }).toBe(1)
- await results.first().waitFor({ timeout: 60_000 })
- return results.first()
- })
- const opened = await measure(cdp, async () => {
- await contentSearch.value.click()
- await page.getByRole('tab', { name: 'Trajectory', exact: true }).waitFor({ timeout: 30_000 })
- return conversationTurns(page)
- })
- expect(opened.value).toBe(DEFAULT_HISTORY_TURNS)
- const trajectoryRows = page.getByRole('row')
- const coldTrajectory = await measure(cdp, async () => {
- await page.getByRole('tab', { name: 'Trajectory', exact: true }).click()
- return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS)
- })
- expect(coldTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS)
- const collapseTurns = await measure(cdp, async () => {
- await page.getByRole('button', { name: 'Collapse turns', exact: true }).click()
- return stableCount(trajectoryRows, count => count > 0 && count < EXPECTED_TRAJECTORY_ROWS)
- })
- expect(collapseTurns.value).toBeLessThan(EXPECTED_TRAJECTORY_ROWS)
- const trajectorySearch = await measure(cdp, async () => {
- await page.getByRole('searchbox', { name: 'Search trajectory', exact: true }).fill('turn 499')
- return stableCount(trajectoryRows, count => count > 0 && count < 20)
- })
- expect(trajectorySearch.value).toBeLessThan(20)
- await page.getByRole('tab', { name: 'Chat', exact: true }).click()
- const historyPages: { turns: number; measurement: Measurement }[] = []
- let turns = await conversationTurns(page)
- while (turns < LONG_HISTORY_TURNS) {
- const previousTurns = turns
- const older = await measure(cdp, async () => {
- await page.getByRole('button', { name: 'Load earlier', exact: true }).click()
- await expect.poll(() => conversationTurns(page), { timeout: 30_000 })
- .toBeGreaterThan(previousTurns)
- return conversationTurns(page)
- })
- turns = older.value
- historyPages.push({ turns, measurement: older.measurement })
- }
- const warmTrajectory = await measure(cdp, async () => {
- await page.getByRole('tab', { name: 'Trajectory', exact: true }).click()
- return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS)
- })
- expect(warmTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS)
- const warmConversation = await measure(cdp, async () => {
- await page.getByRole('tab', { name: 'Chat', exact: true }).click()
- return conversationTurns(page)
- })
- expect(warmConversation.value).toBe(LONG_HISTORY_TURNS)
- console.info(`WEB_PERF_RESULT ${JSON.stringify({
- scenario: 'workspace-history-trajectory',
- fixture: {
- sidebarSessions: SIDEBAR_SESSION_COUNT,
- totalSessions: SIDEBAR_SESSION_COUNT + 1,
- longHistoryTurns: LONG_HISTORY_TURNS,
- toolCalls: EXPECTED_TOOL_CALLS,
- trajectoryRows: EXPECTED_TRAJECTORY_ROWS,
- },
- setupMs: rounded(world.setupMs),
- boot: {
- readyMs: rounded(bootReadyMs),
- firstContentfulPaintMs: firstContentfulPaintMs === undefined
- ? null
- : rounded(firstContentfulPaintMs),
- },
- sidebarExpand: sidebar.measurement,
- contentSearch: contentSearch.measurement,
- openLongHistory: { initialTurns: opened.value, ...opened.measurement },
- coldTrajectory: { rows: coldTrajectory.value, ...coldTrajectory.measurement },
- collapseTurns: { rows: collapseTurns.value, ...collapseTurns.measurement },
- trajectorySearch: { rows: trajectorySearch.value, ...trajectorySearch.measurement },
- historyPages,
- warmTrajectory: { rows: warmTrajectory.value, ...warmTrajectory.measurement },
- warmConversation: { turns: warmConversation.value, ...warmConversation.measurement },
- }, null, 2)}`)
- expect(world.tripwire.warnings).toEqual([])
- expect(world.tripwire.pageErrors).toEqual([])
- } finally {
- await closePerformanceWorld(world)
- }
- })
- it('reports default 24-turn history plus eight continued turns', async () => {
- const world = await launchPerformanceWorld({
- browser,
- replay: performanceReplayOverride(COMPARISON_TURNS, comparisonTurn),
- seedLongHistory: true,
- })
- try {
- await openPerformancePage(world, 1)
- const cdp = await world.page.context().newCDPSession(world.page)
- await cdp.send('Performance.enable')
- const opened = await measure(cdp, () => openLongHistory(world.page))
- expect(opened.value).toBe(DEFAULT_HISTORY_TURNS)
- const conversation = await continueConversation(world, cdp, {
- startingTurns: DEFAULT_HISTORY_TURNS,
- turnCount: COMPARISON_TURNS,
- turnSpec: comparisonTurn,
- expectedSessionId: SessionId(LONG_SESSION_ID),
- })
- expect(conversation.toolTurns).toBe(2)
- console.info(`WEB_PERF_RESULT ${JSON.stringify({
- scenario: 'default-resume-24-plus-8',
- setupMs: rounded(world.setupMs),
- replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
- openLongHistory: { turns: opened.value, ...opened.measurement },
- conversation,
- }, null, 2)}`)
- expect(world.tripwire.warnings).toEqual([])
- expect(world.tripwire.pageErrors).toEqual([])
- } finally {
- await closePerformanceWorld(world)
- }
- })
- it('reports fully expanded 500-turn history plus eight continued turns', async () => {
- const world = await launchPerformanceWorld({
- browser,
- replay: performanceReplayOverride(COMPARISON_TURNS, comparisonTurn),
- seedLongHistory: true,
- })
- try {
- await openPerformancePage(world, 1)
- const cdp = await world.page.context().newCDPSession(world.page)
- await cdp.send('Performance.enable')
- expect(await openLongHistory(world.page)).toBe(DEFAULT_HISTORY_TURNS)
- const historyPages: { turns: number; measurement: Measurement }[] = []
- let turns = DEFAULT_HISTORY_TURNS
- while (turns < LONG_HISTORY_TURNS) {
- const previousTurns = turns
- const older = await measure(cdp, async () => {
- await world.page.getByRole('button', { name: 'Load earlier', exact: true }).click()
- await expect.poll(() => conversationTurns(world.page), { timeout: 30_000 })
- .toBeGreaterThan(previousTurns)
- return conversationTurns(world.page)
- })
- turns = older.value
- historyPages.push({ turns, measurement: older.measurement })
- }
- expect(turns).toBe(LONG_HISTORY_TURNS)
- const conversation = await continueConversation(world, cdp, {
- startingTurns: LONG_HISTORY_TURNS,
- turnCount: COMPARISON_TURNS,
- turnSpec: comparisonTurn,
- expectedSessionId: SessionId(LONG_SESSION_ID),
- })
- expect(conversation.toolTurns).toBe(2)
- console.info(`WEB_PERF_RESULT ${JSON.stringify({
- scenario: 'expanded-history-500-plus-8',
- setupMs: rounded(world.setupMs),
- replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
- historyPages,
- conversation,
- }, null, 2)}`)
- expect(world.tripwire.warnings).toEqual([])
- expect(world.tripwire.pageErrors).toEqual([])
- } finally {
- await closePerformanceWorld(world)
- }
- })
- it('reports one hundred generated turns and the next user-message paint', async () => {
- const world = await launchPerformanceWorld({
- browser,
- replay: performanceReplayOverride(POST_SOAK_RENDER_TURN, soakTurn),
- })
- let testFailure: unknown
- try {
- await world.page.goto(world.scaffold.baseUrl, { waitUntil: 'load' })
- await world.page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
- await connectFreshWorkspace(world.page, world.scaffold.workspaceCwd, 'continuous-conversation-perf')
- const cdp = await world.page.context().newCDPSession(world.page)
- await cdp.send('Performance.enable')
- const conversation = await continueConversation(world, cdp, {
- startingTurns: 0,
- turnCount: SOAK_TURNS,
- turnSpec: soakTurn,
- checkpointInterval: SOAK_CHECKPOINT_INTERVAL,
- })
- expect(conversation.toolTurns).toBe(SOAK_TURNS / SOAK_TOOL_INTERVAL)
- const postSoakUserRender = await measurePostSoakUserRender(world, cdp)
- const { turns, ...retained } = conversation
- console.info(`WEB_PERF_RESULT ${JSON.stringify({
- scenario: 'continuous-100-turn-soak',
- setupMs: rounded(world.setupMs),
- replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
- ...retained,
- windows: summarizeTurnWindows(turns, SOAK_CHECKPOINT_INTERVAL),
- postSoakUserRender,
- }, null, 2)}`)
- expect(world.tripwire.warnings).toEqual([])
- expect(world.tripwire.pageErrors).toEqual([])
- } catch (error) {
- testFailure = error
- throw error
- } finally {
- try {
- await closePerformanceWorld(world)
- } catch (cleanupError) {
- if (testFailure === undefined) throw cleanupError
- throw new AggregateError(
- [testFailure, cleanupError],
- 'continuous conversation performance test and teardown failed',
- )
- }
- }
- })
- })
|