|
|
@@ -16,7 +16,7 @@ import {
|
|
|
createToolResultMessage,
|
|
|
createUserMessage,
|
|
|
} from '@deepseek-ai/dsh-llm'
|
|
|
-import type { ReplayOverrideDoc } from '@deepseek-ai/dsh-llm-replay'
|
|
|
+import type { ReplayEntry, ReplayOverrideDoc } from '@deepseek-ai/dsh-llm-replay'
|
|
|
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
|
|
import {
|
|
|
SESSION_FORMAT_VERSION,
|
|
|
@@ -42,8 +42,23 @@ const TOOL_TURN_INTERVAL = 10
|
|
|
const TOOLS_PER_TOOL_TURN = 10
|
|
|
const EXPECTED_TOOL_CALLS = LONG_HISTORY_TURNS / TOOL_TURN_INTERVAL * TOOLS_PER_TOOL_TURN
|
|
|
const EXPECTED_TRAJECTORY_ROWS = 2_100
|
|
|
+const DEFAULT_HISTORY_TURNS = 24
|
|
|
+const PERF_REPLAY_CONTEXT_WINDOW = 10_000_000
|
|
|
const STREAM_PACE_MS = 8
|
|
|
const STREAM_DELTA_COUNT = 120
|
|
|
+const COMPARISON_TURNS = 8
|
|
|
+const COMPARISON_DELTA_COUNT = 24
|
|
|
+const COMPARISON_TOOL_INTERVAL = 3
|
|
|
+const SOAK_TURNS = 100
|
|
|
+const SOAK_DELTA_COUNT = 8
|
|
|
+const SOAK_TOOL_INTERVAL = 10
|
|
|
+const SOAK_CHECKPOINT_INTERVAL = 10
|
|
|
+const LONG_CONTINUATION_USER_PREFIX = 'LONG_CONTINUATION_USER'
|
|
|
+const LONG_CONTINUATION_FIRST_PREFIX = 'LONG_CONTINUATION_FIRST'
|
|
|
+const LONG_CONTINUATION_DONE_PREFIX = 'LONG_CONTINUATION_DONE'
|
|
|
+const SOAK_USER_PREFIX = 'SOAK_CONVERSATION_USER'
|
|
|
+const SOAK_FIRST_PREFIX = 'SOAK_CONVERSATION_FIRST'
|
|
|
+const SOAK_DONE_PREFIX = 'SOAK_CONVERSATION_DONE'
|
|
|
const LIVE_PROMPT_MARKER = 'STREAM_PERF_USER_INPUT'
|
|
|
const STREAM_FIRST_MARKER = 'STREAM_PERF_FIRST'
|
|
|
const STREAM_DONE_MARKER = 'STREAM_PERF_DONE'
|
|
|
@@ -68,7 +83,6 @@ const STREAM_DELTAS = Array.from({ length: STREAM_DELTA_COUNT }, (_, index) => {
|
|
|
if (index === STREAM_DELTA_COUNT - 1) return `${STREAM_DONE_MARKER}.`
|
|
|
return `chunk-${String(index).padStart(3, '0')} ${'response'.repeat(3)} `
|
|
|
})
|
|
|
-const STREAM_RESPONSE = STREAM_DELTAS.join('')
|
|
|
|
|
|
interface ChromiumMetrics {
|
|
|
readonly [name: string]: number
|
|
|
@@ -93,6 +107,72 @@ interface MutationProbeResult {
|
|
|
readonly records: number
|
|
|
}
|
|
|
|
|
|
+interface RetainedBrowserState {
|
|
|
+ readonly domElements: number
|
|
|
+ readonly nodes: number
|
|
|
+ readonly listeners: number
|
|
|
+ readonly heapMb: number
|
|
|
+}
|
|
|
+
|
|
|
+interface ContinuedTurnReport {
|
|
|
+ readonly ordinal: number
|
|
|
+ readonly resultingTurns: number
|
|
|
+ readonly kind: 'text' | 'tool'
|
|
|
+ readonly promptChars: number
|
|
|
+ readonly composerFill: Measurement
|
|
|
+ readonly stream: MutationProbeResult & Measurement & {
|
|
|
+ readonly paceMs: number
|
|
|
+ readonly deltaChunks: number
|
|
|
+ readonly persistedChunks: number
|
|
|
+ readonly toolCalls: number
|
|
|
+ readonly toolResults: number
|
|
|
+ readonly clickToUserEchoMs: number
|
|
|
+ readonly clickToFirstChunkMs: number
|
|
|
+ readonly firstChunkToSettledMs: number
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+interface ConversationTurnSpec {
|
|
|
+ readonly prompt: string
|
|
|
+ readonly deltas: readonly string[]
|
|
|
+ readonly userMarker: string
|
|
|
+ readonly firstMarker: string
|
|
|
+ readonly doneMarker: string
|
|
|
+ readonly toolResultMarker?: string
|
|
|
+}
|
|
|
+
|
|
|
+interface RetainedCheckpoint {
|
|
|
+ readonly turns: number
|
|
|
+ readonly state: RetainedBrowserState
|
|
|
+}
|
|
|
+
|
|
|
+interface ConversationReport {
|
|
|
+ readonly startingTurns: number
|
|
|
+ readonly turnsAdded: number
|
|
|
+ readonly toolTurns: number
|
|
|
+ readonly retainedBefore: RetainedBrowserState
|
|
|
+ readonly retainedAfter: RetainedBrowserState
|
|
|
+ readonly retainedDelta: RetainedBrowserState
|
|
|
+ readonly checkpoints: readonly RetainedCheckpoint[]
|
|
|
+ readonly turns: readonly ContinuedTurnReport[]
|
|
|
+}
|
|
|
+
|
|
|
+interface PerformanceWorld {
|
|
|
+ readonly scaffold: WebScaffold
|
|
|
+ readonly page: Page
|
|
|
+ readonly tripwire: ReturnType<typeof watchConsole>
|
|
|
+ readonly sessionEvents: SessionEvent[]
|
|
|
+ readonly setupMs: number
|
|
|
+ readonly replayDir?: string
|
|
|
+}
|
|
|
+
|
|
|
+interface PerformanceWorldOptions {
|
|
|
+ readonly browser: Browser
|
|
|
+ readonly replay?: ReplayOverrideDoc
|
|
|
+ readonly sidebarSessions?: number
|
|
|
+ readonly seedLongHistory?: boolean
|
|
|
+}
|
|
|
+
|
|
|
function text(value: string): { type: 'text'; text: string }[] {
|
|
|
return [{ type: 'text', text: value }]
|
|
|
}
|
|
|
@@ -108,7 +188,7 @@ function appendTitle(session: Session, title: string, messageSeq: number): void
|
|
|
function appendRequestHeader(session: Session, turn: number, step: number): void {
|
|
|
session.append('request/header', {
|
|
|
header: {
|
|
|
- config: { provider: 'synthetic-perf', model: 'synthetic-perf' },
|
|
|
+ config: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
|
|
|
system: `Synthetic performance system prompt for turn ${String(turn)}, step ${String(step)}.`,
|
|
|
},
|
|
|
reason: turn === 1 && step === 1 ? 'initial' : 'change',
|
|
|
@@ -126,7 +206,7 @@ function appendAssistant(
|
|
|
step,
|
|
|
message: createAssistantMessage({
|
|
|
content: text(body),
|
|
|
- source: { provider: 'synthetic-perf', model: 'synthetic-perf' },
|
|
|
+ source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
|
|
|
}),
|
|
|
usage: {
|
|
|
inputTokens: 4_000 + turn * 10,
|
|
|
@@ -168,7 +248,7 @@ function appendToolStep(
|
|
|
arguments: args,
|
|
|
})),
|
|
|
],
|
|
|
- source: { provider: 'synthetic-perf', model: 'synthetic-perf' },
|
|
|
+ source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
|
|
|
}),
|
|
|
usage: {
|
|
|
inputTokens: 6_000 + turn * 10,
|
|
|
@@ -294,10 +374,11 @@ function longHistoryFixture(): string {
|
|
|
return fixtureLog(session)
|
|
|
}
|
|
|
|
|
|
-function streamingReplayOverride(): ReplayOverrideDoc {
|
|
|
- const chunks: StreamChunk[] = [
|
|
|
+function textStream(deltas: readonly string[], inputTokens: number): StreamChunk[] {
|
|
|
+ const response = deltas.join('')
|
|
|
+ return [
|
|
|
{ type: 'block-start', index: 0, blockType: 'text' },
|
|
|
- ...STREAM_DELTAS.map(text => ({
|
|
|
+ ...deltas.map(text => ({
|
|
|
type: 'text-delta' as const,
|
|
|
index: 0,
|
|
|
text,
|
|
|
@@ -305,18 +386,122 @@ function streamingReplayOverride(): ReplayOverrideDoc {
|
|
|
{
|
|
|
type: 'block-end',
|
|
|
index: 0,
|
|
|
- block: { type: 'text', text: STREAM_RESPONSE },
|
|
|
+ block: { type: 'text', text: response },
|
|
|
},
|
|
|
{
|
|
|
type: 'usage',
|
|
|
usage: {
|
|
|
- inputTokens: Math.ceil(LIVE_PROMPT.length / 4),
|
|
|
- outputTokens: Math.ceil(STREAM_RESPONSE.length / 4),
|
|
|
+ inputTokens,
|
|
|
+ outputTokens: Math.ceil(response.length / 4),
|
|
|
},
|
|
|
},
|
|
|
{ type: 'finish', reason: { kind: 'stop' } },
|
|
|
]
|
|
|
- return [{ kind: 'chunks', chunks }]
|
|
|
+}
|
|
|
+
|
|
|
+function comparisonPrompt(index: number): string {
|
|
|
+ if (index === COMPARISON_TURNS) return LIVE_PROMPT
|
|
|
+ return (`${LONG_CONTINUATION_USER_PREFIX}_${String(index)} `
|
|
|
+ + `继续分析这个长会话的第 ${String(index)} 个增量问题，并保留当前滚动和输入响应。 `
|
|
|
+ + 'context '.repeat(80)).trimEnd()
|
|
|
+}
|
|
|
+
|
|
|
+function comparisonDeltas(index: number): string[] {
|
|
|
+ if (index === COMPARISON_TURNS) return STREAM_DELTAS
|
|
|
+ return Array.from({ length: COMPARISON_DELTA_COUNT }, (_, chunkIndex) => {
|
|
|
+ if (chunkIndex === 0) return `${LONG_CONTINUATION_FIRST_PREFIX}_${String(index)} `
|
|
|
+ if (chunkIndex === COMPARISON_DELTA_COUNT - 1) {
|
|
|
+ return `${LONG_CONTINUATION_DONE_PREFIX}_${String(index)}.`
|
|
|
+ }
|
|
|
+ return `turn-${String(index)}-chunk-${String(chunkIndex).padStart(2, '0')} ${'response'.repeat(2)} `
|
|
|
+ })
|
|
|
+}
|
|
|
+
|
|
|
+function comparisonTurn(index: number): ConversationTurnSpec {
|
|
|
+ const toolResultMarker = index % COMPARISON_TOOL_INTERVAL === 0
|
|
|
+ ? `LONG_CONTINUATION_TOOL_RESULT_${String(index)}`
|
|
|
+ : undefined
|
|
|
+ return {
|
|
|
+ prompt: comparisonPrompt(index),
|
|
|
+ deltas: comparisonDeltas(index),
|
|
|
+ userMarker: index === COMPARISON_TURNS
|
|
|
+ ? LIVE_PROMPT_MARKER
|
|
|
+ : `${LONG_CONTINUATION_USER_PREFIX}_${String(index)}`,
|
|
|
+ firstMarker: index === COMPARISON_TURNS
|
|
|
+ ? STREAM_FIRST_MARKER
|
|
|
+ : `${LONG_CONTINUATION_FIRST_PREFIX}_${String(index)}`,
|
|
|
+ doneMarker: index === COMPARISON_TURNS
|
|
|
+ ? STREAM_DONE_MARKER
|
|
|
+ : `${LONG_CONTINUATION_DONE_PREFIX}_${String(index)}`,
|
|
|
+ ...toolResultMarker === undefined ? {} : { toolResultMarker },
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+function soakTurn(index: number): ConversationTurnSpec {
|
|
|
+ const suffix = String(index).padStart(3, '0')
|
|
|
+ const toolResultMarker = index % SOAK_TOOL_INTERVAL === 0
|
|
|
+ ? `SOAK_CONVERSATION_TOOL_RESULT_${suffix}`
|
|
|
+ : undefined
|
|
|
+ return {
|
|
|
+ prompt: (`${SOAK_USER_PREFIX}_${suffix} `
|
|
|
+ + `持续对话第 ${String(index)} 轮，检查增量渲染与保留状态。 `
|
|
|
+ + 'context '.repeat(20)).trimEnd(),
|
|
|
+ deltas: Array.from({ length: SOAK_DELTA_COUNT }, (_, chunkIndex) => {
|
|
|
+ if (chunkIndex === 0) return `${SOAK_FIRST_PREFIX}_${suffix} `
|
|
|
+ if (chunkIndex === SOAK_DELTA_COUNT - 1) return `${SOAK_DONE_PREFIX}_${suffix}.`
|
|
|
+ return `soak-${suffix}-${String(chunkIndex).padStart(2, '0')} response `
|
|
|
+ }),
|
|
|
+ userMarker: `${SOAK_USER_PREFIX}_${suffix}`,
|
|
|
+ firstMarker: `${SOAK_FIRST_PREFIX}_${suffix}`,
|
|
|
+ doneMarker: `${SOAK_DONE_PREFIX}_${suffix}`,
|
|
|
+ ...toolResultMarker === undefined ? {} : { toolResultMarker },
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+function toolStream(index: number, marker: string): StreamChunk[] {
|
|
|
+ const callId = CallId(`performance-tool-${marker.toLowerCase()}-${String(index)}`)
|
|
|
+ const args = JSON.stringify({
|
|
|
+ command: `printf '${marker}\\n'`,
|
|
|
+ description: `Emit performance marker ${String(index)}`,
|
|
|
+ })
|
|
|
+ return [
|
|
|
+ { type: 'block-start', index: 0, blockType: 'tool-call' },
|
|
|
+ {
|
|
|
+ type: 'tool-call-delta',
|
|
|
+ index: 0,
|
|
|
+ id: callId,
|
|
|
+ name: 'bash',
|
|
|
+ argumentsDelta: args,
|
|
|
+ },
|
|
|
+ {
|
|
|
+ type: 'block-end',
|
|
|
+ index: 0,
|
|
|
+ block: { type: 'tool-call', id: callId, name: 'bash', arguments: args },
|
|
|
+ },
|
|
|
+ { type: 'usage', usage: { inputTokens: 256, outputTokens: 32 } },
|
|
|
+ { type: 'finish', reason: { kind: 'tool-calls' } },
|
|
|
+ ]
|
|
|
+}
|
|
|
+
|
|
|
+function performanceReplayOverride(
|
|
|
+ turnCount: number,
|
|
|
+ turnSpec: (index: number) => ConversationTurnSpec,
|
|
|
+): ReplayOverrideDoc {
|
|
|
+ const continuationEntries = Array.from(
|
|
|
+ { length: turnCount },
|
|
|
+ (_, offset): ReplayEntry[] => {
|
|
|
+ const index = offset + 1
|
|
|
+ const spec = turnSpec(index)
|
|
|
+ const finalResponse: ReplayEntry = {
|
|
|
+ kind: 'chunks',
|
|
|
+ chunks: textStream(spec.deltas, Math.ceil(spec.prompt.length / 4)),
|
|
|
+ }
|
|
|
+ return spec.toolResultMarker === undefined
|
|
|
+ ? [finalResponse]
|
|
|
+ : [{ kind: 'chunks', chunks: toolStream(index, spec.toolResultMarker) }, finalResponse]
|
|
|
+ },
|
|
|
+ ).flat()
|
|
|
+ return continuationEntries
|
|
|
}
|
|
|
|
|
|
function rounded(value: number): number {
|
|
|
@@ -328,6 +513,20 @@ async function chromiumMetrics(cdp: CDPSession): Promise<ChromiumMetrics> {
|
|
|
return Object.fromEntries(payload.metrics.map(metric => [metric.name, metric.value]))
|
|
|
}
|
|
|
|
|
|
+async function retainedBrowserState(
|
|
|
+ cdp: CDPSession,
|
|
|
+ page: Page,
|
|
|
+): Promise<RetainedBrowserState> {
|
|
|
+ await cdp.send('HeapProfiler.collectGarbage')
|
|
|
+ const metrics = await chromiumMetrics(cdp)
|
|
|
+ return {
|
|
|
+ domElements: await page.evaluate(() => document.querySelectorAll('*').length),
|
|
|
+ nodes: requiredMetric(metrics, 'Nodes'),
|
|
|
+ listeners: requiredMetric(metrics, 'JSEventListeners'),
|
|
|
+ heapMb: rounded(requiredMetric(metrics, 'JSHeapUsedSize') / 1_048_576),
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
function requiredMetric(metrics: ChromiumMetrics, name: string): number {
|
|
|
const value = metrics[name]
|
|
|
if (value === undefined) throw new Error(`Chromium performance metric ${name} is unavailable`)
|
|
|
@@ -436,256 +635,543 @@ async function conversationTurns(page: Page): Promise<number> {
|
|
|
return Number(match[1])
|
|
|
}
|
|
|
|
|
|
-describe('manual web performance: complex workspace and history', () => {
|
|
|
- let scaffold: WebScaffold
|
|
|
- let browser: Browser
|
|
|
- let page: Page
|
|
|
- let setupMs = 0
|
|
|
- let tripwire: ReturnType<typeof watchConsole>
|
|
|
- let replayDir: string | undefined
|
|
|
- const sessionEvents: SessionEvent[] = []
|
|
|
+function retainedDelta(
|
|
|
+ before: RetainedBrowserState,
|
|
|
+ after: RetainedBrowserState,
|
|
|
+): RetainedBrowserState {
|
|
|
+ return {
|
|
|
+ domElements: after.domElements - before.domElements,
|
|
|
+ nodes: after.nodes - before.nodes,
|
|
|
+ listeners: after.listeners - before.listeners,
|
|
|
+ heapMb: rounded(after.heapMb - before.heapMb),
|
|
|
+ }
|
|
|
+}
|
|
|
|
|
|
- beforeAll(async () => {
|
|
|
- if (webSnapshotMode() === 'record') {
|
|
|
- throw new Error('manual web performance runs only with deterministic replay')
|
|
|
+async function launchPerformanceWorld(
|
|
|
+ options: PerformanceWorldOptions,
|
|
|
+): Promise<PerformanceWorld> {
|
|
|
+ const setupStarted = performance.now()
|
|
|
+ let replayDir: string | undefined
|
|
|
+ let scaffold: WebScaffold | undefined
|
|
|
+ let page: Page | undefined
|
|
|
+ try {
|
|
|
+ if (options.replay === undefined) {
|
|
|
+ scaffold = await launchWebScaffold()
|
|
|
+ } else {
|
|
|
+ replayDir = await mkdtemp(join(tmpdir(), 'dsh-web-perf-replay-'))
|
|
|
+ const replayOverride = join(replayDir, 'replay.override.json')
|
|
|
+ await writeFile(replayOverride, JSON.stringify(options.replay))
|
|
|
+ scaffold = await launchWebScaffold({
|
|
|
+ // The override-only fixture supplies one positional script for this
|
|
|
+ // world's single live session.
|
|
|
+ replayFixture: join(replayDir, 'override-only.jsonl'),
|
|
|
+ replayOverride,
|
|
|
+ paceMs: STREAM_PACE_MS,
|
|
|
+ replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
|
|
|
+ })
|
|
|
}
|
|
|
- const setupStarted = performance.now()
|
|
|
- replayDir = await mkdtemp(join(tmpdir(), 'dsh-web-perf-replay-'))
|
|
|
- const replayOverride = join(replayDir, 'replay.override.json')
|
|
|
- await writeFile(replayOverride, JSON.stringify(streamingReplayOverride()))
|
|
|
- scaffold = await launchWebScaffold({
|
|
|
- // A whole-script override needs no committed JSONL, but replayFixture
|
|
|
- // selects the replay adapter and supplies its diagnostic identity.
|
|
|
- replayFixture: join(replayDir, 'override-only.jsonl'),
|
|
|
- replayOverride,
|
|
|
- paceMs: STREAM_PACE_MS,
|
|
|
- })
|
|
|
+
|
|
|
+ const sessionEvents: SessionEvent[] = []
|
|
|
scaffold.ctx.on('session/event', (_session, event: SessionEvent) => {
|
|
|
sessionEvents.push(event)
|
|
|
})
|
|
|
- const small = smallSidebarFixture()
|
|
|
- for (let index = 0; index < SIDEBAR_SESSION_COUNT; index += 1) {
|
|
|
- await seedSession(scaffold, small, `perf-sidebar-${String(index).padStart(4, '0')}`)
|
|
|
+ if ((options.sidebarSessions ?? 0) > 0) {
|
|
|
+ const small = smallSidebarFixture()
|
|
|
+ for (let index = 0; index < (options.sidebarSessions ?? 0); index += 1) {
|
|
|
+ await seedSession(scaffold, small, `perf-sidebar-${String(index).padStart(4, '0')}`)
|
|
|
+ }
|
|
|
}
|
|
|
- await seedSession(scaffold, longHistoryFixture(), LONG_SESSION_ID)
|
|
|
- setupMs = performance.now() - setupStarted
|
|
|
-
|
|
|
- browser = await chromium.launch()
|
|
|
- page = await newEnglishPage(browser)
|
|
|
- tripwire = watchConsole(page)
|
|
|
- })
|
|
|
-
|
|
|
- afterAll(async () => {
|
|
|
- const failures: unknown[] = []
|
|
|
- await browser?.close().catch((error: unknown) => failures.push(error))
|
|
|
- await scaffold?.close().catch((error: unknown) => failures.push(error))
|
|
|
- if (replayDir !== undefined) {
|
|
|
- await rm(replayDir, { recursive: true, force: true })
|
|
|
- .catch((error: unknown) => failures.push(error))
|
|
|
+ if (options.seedLongHistory === true) {
|
|
|
+ await seedSession(scaffold, longHistoryFixture(), LONG_SESSION_ID)
|
|
|
}
|
|
|
- if (failures.length === 1) throw failures[0]
|
|
|
- if (failures.length > 1) {
|
|
|
- throw new AggregateError(failures, 'web performance teardown failed')
|
|
|
+ const setupMs = performance.now() - setupStarted
|
|
|
+ page = await newEnglishPage(options.browser)
|
|
|
+ return {
|
|
|
+ scaffold,
|
|
|
+ page,
|
|
|
+ tripwire: watchConsole(page),
|
|
|
+ sessionEvents,
|
|
|
+ setupMs,
|
|
|
+ ...replayDir === undefined ? {} : { replayDir },
|
|
|
}
|
|
|
- })
|
|
|
-
|
|
|
- it('reports sidebar, paging, and trajectory rendering costs', async () => {
|
|
|
- const bootStarted = performance.now()
|
|
|
- await page.goto(scaffold.baseUrl, { waitUntil: 'load' })
|
|
|
- await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
|
|
|
- const group = page.getByRole('treeitem').first()
|
|
|
- await expect.poll(() => group.textContent(), { timeout: 30_000 })
|
|
|
- .toContain(`${String(SIDEBAR_SESSION_COUNT + 1)} sessions`)
|
|
|
- const bootReadyMs = performance.now() - bootStarted
|
|
|
-
|
|
|
- const cdp = await page.context().newCDPSession(page)
|
|
|
- await cdp.send('Performance.enable')
|
|
|
- const firstContentfulPaintMs = await page.evaluate(
|
|
|
- () => globalThis.performance.getEntriesByName('first-contentful-paint')[0]?.startTime,
|
|
|
- )
|
|
|
-
|
|
|
- const sidebar = await measure(cdp, async () => {
|
|
|
- await group.click()
|
|
|
- return stableCount(
|
|
|
- page.getByRole('treeitem'),
|
|
|
- count => count === SIDEBAR_SESSION_COUNT + 2,
|
|
|
- )
|
|
|
- })
|
|
|
- expect(sidebar.value).toBe(SIDEBAR_SESSION_COUNT + 2)
|
|
|
-
|
|
|
- await group.click()
|
|
|
- await expect.poll(() => page.getByRole('treeitem').count()).toBe(1)
|
|
|
-
|
|
|
- const contentSearch = await measure(cdp, async () => {
|
|
|
- await page.getByRole('textbox', { name: 'Search name, keywords...', exact: true })
|
|
|
- .fill('LONG_PERF_SENTINEL')
|
|
|
- const results = page.getByRole('tree', { name: 'Search results' })
|
|
|
- .getByRole('treeitem')
|
|
|
- await expect.poll(() => results.count(), { timeout: 60_000 }).toBe(1)
|
|
|
- const result = results.first()
|
|
|
- await result.waitFor({ timeout: 60_000 })
|
|
|
- return result
|
|
|
- })
|
|
|
-
|
|
|
- const openLongHistory = await measure(cdp, async () => {
|
|
|
- await contentSearch.value.click()
|
|
|
- await page.getByRole('tab', { name: 'Trajectory', exact: true }).waitFor({ timeout: 30_000 })
|
|
|
- return conversationTurns(page)
|
|
|
- })
|
|
|
- expect(openLongHistory.value).toBeGreaterThan(0)
|
|
|
-
|
|
|
- const trajectoryRows = page.getByRole('row')
|
|
|
- const coldTrajectory = await measure(cdp, async () => {
|
|
|
- await page.getByRole('tab', { name: 'Trajectory', exact: true }).click()
|
|
|
- return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS)
|
|
|
- })
|
|
|
- expect(coldTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS)
|
|
|
+ } catch (error) {
|
|
|
+ const failures: unknown[] = [error]
|
|
|
+ if (page !== undefined) {
|
|
|
+ try {
|
|
|
+ await page.close()
|
|
|
+ } catch (cleanupError) {
|
|
|
+ failures.push(cleanupError)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (scaffold !== undefined) {
|
|
|
+ try {
|
|
|
+ await scaffold.close()
|
|
|
+ } catch (cleanupError) {
|
|
|
+ failures.push(cleanupError)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (replayDir !== undefined) {
|
|
|
+ try {
|
|
|
+ await rm(replayDir, { recursive: true, force: true })
|
|
|
+ } catch (cleanupError) {
|
|
|
+ failures.push(cleanupError)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (failures.length === 1) throw error
|
|
|
+ throw new AggregateError(failures, 'web performance setup and cleanup failed')
|
|
|
+ }
|
|
|
+}
|
|
|
|
|
|
- const collapseTurns = await measure(cdp, async () => {
|
|
|
- await page.getByRole('button', { name: 'Collapse turns', exact: true }).click()
|
|
|
- return stableCount(trajectoryRows, count => count > 0 && count < EXPECTED_TRAJECTORY_ROWS)
|
|
|
- })
|
|
|
- expect(collapseTurns.value).toBeLessThan(EXPECTED_TRAJECTORY_ROWS)
|
|
|
+async function closePerformanceWorld(world: PerformanceWorld): Promise<void> {
|
|
|
+ const failures: unknown[] = []
|
|
|
+ await world.page.close().catch((error: unknown) => failures.push(error))
|
|
|
+ await world.scaffold.close().catch((error: unknown) => failures.push(error))
|
|
|
+ if (world.replayDir !== undefined) {
|
|
|
+ await rm(world.replayDir, { recursive: true, force: true })
|
|
|
+ .catch((error: unknown) => failures.push(error))
|
|
|
+ }
|
|
|
+ if (failures.length === 1) throw failures[0]
|
|
|
+ if (failures.length > 1) throw new AggregateError(failures, 'web performance teardown failed')
|
|
|
+}
|
|
|
|
|
|
- const trajectorySearch = await measure(cdp, async () => {
|
|
|
- await page.getByRole('searchbox', { name: 'Search trajectory', exact: true }).fill('turn 499')
|
|
|
- return stableCount(trajectoryRows, count => count > 0 && count < 20)
|
|
|
- })
|
|
|
- expect(trajectorySearch.value).toBeLessThan(20)
|
|
|
-
|
|
|
- await page.getByRole('tab', { name: 'Chat', exact: true }).click()
|
|
|
- const historyPages: { turns: number; measurement: Measurement }[] = []
|
|
|
- let turns = await conversationTurns(page)
|
|
|
- for (let pageIndex = 0; pageIndex < 5; pageIndex += 1) {
|
|
|
- const previousTurns = turns
|
|
|
- const older = await measure(cdp, async () => {
|
|
|
- await page.getByRole('button', { name: 'Load earlier', exact: true }).click()
|
|
|
- await expect.poll(() => conversationTurns(page), { timeout: 30_000 }).toBeGreaterThan(previousTurns)
|
|
|
- return conversationTurns(page)
|
|
|
- })
|
|
|
- turns = older.value
|
|
|
- historyPages.push({ turns, measurement: older.measurement })
|
|
|
- }
|
|
|
+async function openPerformancePage(
|
|
|
+ world: PerformanceWorld,
|
|
|
+ expectedSessions: number,
|
|
|
+): Promise<Locator> {
|
|
|
+ await world.page.goto(world.scaffold.baseUrl, { waitUntil: 'load' })
|
|
|
+ await world.page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
|
|
|
+ const group = world.page.getByRole('treeitem').first()
|
|
|
+ await expect.poll(() => group.textContent(), { timeout: 30_000 })
|
|
|
+ .toContain(`${String(expectedSessions)} ${expectedSessions === 1 ? 'session' : 'sessions'}`)
|
|
|
+ return group
|
|
|
+}
|
|
|
|
|
|
- const warmTrajectory = await measure(cdp, async () => {
|
|
|
- await page.getByRole('tab', { name: 'Trajectory', exact: true }).click()
|
|
|
- return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS)
|
|
|
- })
|
|
|
- expect(warmTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS)
|
|
|
+async function openLongHistory(page: Page): Promise<number> {
|
|
|
+ await page.getByRole('textbox', { name: 'Search name, keywords...', exact: true })
|
|
|
+ .fill('LONG_PERF_SENTINEL')
|
|
|
+ const results = page.getByRole('tree', { name: 'Search results' }).getByRole('treeitem')
|
|
|
+ await expect.poll(() => results.count(), { timeout: 60_000 }).toBe(1)
|
|
|
+ await results.first().click()
|
|
|
+ await page.getByRole('tab', { name: 'Chat', exact: true }).waitFor({ timeout: 30_000 })
|
|
|
+ return conversationTurns(page)
|
|
|
+}
|
|
|
|
|
|
- const newSession = await measure(cdp, async () => {
|
|
|
- await page.getByRole('button', { name: 'New session', exact: true }).last().click()
|
|
|
- await connectFreshWorkspace(page, 'stream-perf')
|
|
|
- const input = page.locator(
|
|
|
- 'textarea:enabled[placeholder="Describe what you want to build"]',
|
|
|
- )
|
|
|
- await input.waitFor({ timeout: 15_000 })
|
|
|
- return input
|
|
|
- })
|
|
|
+async function continueConversation(
|
|
|
+ world: PerformanceWorld,
|
|
|
+ cdp: CDPSession,
|
|
|
+ options: {
|
|
|
+ readonly startingTurns: number
|
|
|
+ readonly turnCount: number
|
|
|
+ readonly turnSpec: (index: number) => ConversationTurnSpec
|
|
|
+ readonly expectedSessionId?: SessionId
|
|
|
+ readonly checkpointInterval?: number
|
|
|
+ },
|
|
|
+): Promise<ConversationReport> {
|
|
|
+ const composer = world.page.locator('textarea:enabled').last()
|
|
|
+ await composer.waitFor({ timeout: 15_000 })
|
|
|
+ const retainedBefore = await retainedBrowserState(cdp, world.page)
|
|
|
+ const checkpoints: RetainedCheckpoint[] = [{ turns: options.startingTurns, state: retainedBefore }]
|
|
|
+ const turns: ContinuedTurnReport[] = []
|
|
|
+ let liveSessionId = options.expectedSessionId
|
|
|
|
|
|
- const composerInput = await measure(cdp, async () => {
|
|
|
- await newSession.value.fill(LIVE_PROMPT)
|
|
|
- await expect.poll(() => newSession.value.inputValue()).toBe(LIVE_PROMPT)
|
|
|
- return (await newSession.value.inputValue()).length
|
|
|
+ for (let index = 1; index <= options.turnCount; index += 1) {
|
|
|
+ const spec = options.turnSpec(index)
|
|
|
+ const composerFill = await measure(cdp, async () => {
|
|
|
+ await composer.fill(spec.prompt)
|
|
|
+ await expect.poll(() => composer.inputValue()).toBe(spec.prompt)
|
|
|
+ return (await composer.inputValue()).length
|
|
|
})
|
|
|
- expect(composerInput.value).toBe(LIVE_PROMPT.length)
|
|
|
+ expect(composerFill.value).toBe(spec.prompt.length)
|
|
|
|
|
|
- await startMutationProbe(page)
|
|
|
+ const eventStart = world.sessionEvents.length
|
|
|
+ await startMutationProbe(world.page)
|
|
|
const streamBefore = await chromiumMetrics(cdp)
|
|
|
const streamStarted = performance.now()
|
|
|
- const settled = scaffold.whenTurnSettled(60_000)
|
|
|
- await page.getByRole('button', { name: 'Send message', exact: true }).click()
|
|
|
- await page.getByText(LIVE_PROMPT_MARKER, { exact: false }).last()
|
|
|
- .waitFor({ timeout: 15_000 })
|
|
|
+ const settled = world.scaffold.whenTurnSettled(60_000)
|
|
|
+ await world.page.getByRole('button', { name: 'Send message', exact: true }).click()
|
|
|
+ await world.page.getByText(spec.userMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
|
|
|
const clickToUserEchoMs = performance.now() - streamStarted
|
|
|
- await page.getByText(STREAM_FIRST_MARKER, { exact: false }).last()
|
|
|
- .waitFor({ timeout: 15_000 })
|
|
|
+ await world.page.getByText(spec.firstMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
|
|
|
const clickToFirstChunkMs = performance.now() - streamStarted
|
|
|
- await settled
|
|
|
+ const settledSessionId = await settled
|
|
|
+ if (liveSessionId === undefined) {
|
|
|
+ liveSessionId = settledSessionId
|
|
|
+ } else {
|
|
|
+ expect(settledSessionId).toBe(liveSessionId)
|
|
|
+ }
|
|
|
await expect.poll(
|
|
|
- () => page.locator('[data-streaming="true"]').count(),
|
|
|
+ () => world.page.locator('[data-streaming="true"]').count(),
|
|
|
{ timeout: 15_000 },
|
|
|
).toBe(0)
|
|
|
- await page.getByText(STREAM_DONE_MARKER, { exact: false }).last()
|
|
|
- .waitFor({ timeout: 15_000 })
|
|
|
+ await world.page.getByText(spec.doneMarker, { exact: false }).last().waitFor({ timeout: 15_000 })
|
|
|
const clickToSettledMs = performance.now() - streamStarted
|
|
|
const streamAfter = await chromiumMetrics(cdp)
|
|
|
- const streamMutations = await stopMutationProbe(page)
|
|
|
- const streamMeasurement = metricDelta(
|
|
|
- streamBefore,
|
|
|
- streamAfter,
|
|
|
- clickToSettledMs,
|
|
|
- )
|
|
|
-
|
|
|
- const assistantChunks = sessionEvents.filter(
|
|
|
- event => event.type === 'assistant/chunk',
|
|
|
+ const mutations = await stopMutationProbe(world.page)
|
|
|
+ const turnEvents = world.sessionEvents.slice(eventStart)
|
|
|
+ const chunks = turnEvents.filter(event => event.type === 'assistant/chunk')
|
|
|
+ const toolCalls = turnEvents.filter(event => event.type === 'tool/call')
|
|
|
+ const toolResults = turnEvents.filter(event => event.type === 'tool/result')
|
|
|
+ const toolTurn = spec.toolResultMarker !== undefined
|
|
|
+ expect(chunks).toHaveLength(spec.deltas.length + (toolTurn ? 9 : 4))
|
|
|
+ expect(toolCalls).toHaveLength(toolTurn ? 1 : 0)
|
|
|
+ expect(toolResults).toHaveLength(toolTurn ? 1 : 0)
|
|
|
+ if (spec.toolResultMarker !== undefined) {
|
|
|
+ expect(toolCalls[0]?.data.name).toBe('bash')
|
|
|
+ const toolResult = toolResults[0]
|
|
|
+ if (toolResult?.type !== 'tool/result') {
|
|
|
+ throw new Error(`continued turn ${String(index)} did not log its tool result`)
|
|
|
+ }
|
|
|
+ const resultBlock = toolResult.data.message.content.find(block => block.type === 'tool-result')
|
|
|
+ expect(resultBlock?.isError).toBe(false)
|
|
|
+ expect(resultBlock?.content
|
|
|
+ .filter(block => block.type === 'text')
|
|
|
+ .map(block => block.text)
|
|
|
+ .join('')).toContain(spec.toolResultMarker)
|
|
|
+ }
|
|
|
+ const user = turnEvents.find(
|
|
|
+ event => event.type === 'user/message' && event.data.source.kind === 'user',
|
|
|
)
|
|
|
- expect(assistantChunks).toHaveLength(STREAM_DELTA_COUNT + 4)
|
|
|
- const liveUser = sessionEvents.filter(event => event.type === 'user/message').at(-1)
|
|
|
- if (liveUser?.type !== 'user/message') throw new Error('live performance prompt was not logged')
|
|
|
- const liveUserText = liveUser.data.content
|
|
|
+ if (user?.type !== 'user/message') {
|
|
|
+ throw new Error(`continued turn ${String(index)} did not log its user message`)
|
|
|
+ }
|
|
|
+ expect(user.data.content
|
|
|
.filter(block => block.type === 'text')
|
|
|
.map(block => block.text)
|
|
|
- .join('')
|
|
|
- expect(liveUserText).toBe(LIVE_PROMPT)
|
|
|
-
|
|
|
- const report = {
|
|
|
- fixture: {
|
|
|
- sidebarSessions: SIDEBAR_SESSION_COUNT,
|
|
|
- totalSessions: SIDEBAR_SESSION_COUNT + 1,
|
|
|
- longHistoryTurns: LONG_HISTORY_TURNS,
|
|
|
- toolCalls: EXPECTED_TOOL_CALLS,
|
|
|
- trajectoryRows: EXPECTED_TRAJECTORY_ROWS,
|
|
|
+ .join('')).toBe(spec.prompt)
|
|
|
+ const resultingTurns = await conversationTurns(world.page)
|
|
|
+ expect(resultingTurns).toBe(options.startingTurns + index)
|
|
|
+ turns.push({
|
|
|
+ ordinal: index,
|
|
|
+ resultingTurns,
|
|
|
+ kind: toolTurn ? 'tool' : 'text',
|
|
|
+ promptChars: spec.prompt.length,
|
|
|
+ composerFill: composerFill.measurement,
|
|
|
+ stream: {
|
|
|
+ paceMs: STREAM_PACE_MS,
|
|
|
+ deltaChunks: spec.deltas.length,
|
|
|
+ persistedChunks: chunks.length,
|
|
|
+ toolCalls: toolCalls.length,
|
|
|
+ toolResults: toolResults.length,
|
|
|
+ clickToUserEchoMs: rounded(clickToUserEchoMs),
|
|
|
+ clickToFirstChunkMs: rounded(clickToFirstChunkMs),
|
|
|
+ firstChunkToSettledMs: rounded(clickToSettledMs - clickToFirstChunkMs),
|
|
|
+ ...mutations,
|
|
|
+ ...metricDelta(streamBefore, streamAfter, clickToSettledMs),
|
|
|
},
|
|
|
- setupMs: rounded(setupMs),
|
|
|
- boot: {
|
|
|
- readyMs: rounded(bootReadyMs),
|
|
|
- firstContentfulPaintMs: firstContentfulPaintMs === undefined
|
|
|
- ? null
|
|
|
- : rounded(firstContentfulPaintMs),
|
|
|
- },
|
|
|
- sidebarExpand: sidebar.measurement,
|
|
|
- contentSearch: contentSearch.measurement,
|
|
|
- openLongHistory: {
|
|
|
- initialTurns: openLongHistory.value,
|
|
|
- ...openLongHistory.measurement,
|
|
|
- },
|
|
|
- coldTrajectory: {
|
|
|
- rows: coldTrajectory.value,
|
|
|
- ...coldTrajectory.measurement,
|
|
|
- },
|
|
|
- collapseTurns: {
|
|
|
- rows: collapseTurns.value,
|
|
|
- ...collapseTurns.measurement,
|
|
|
- },
|
|
|
- trajectorySearch: {
|
|
|
- rows: trajectorySearch.value,
|
|
|
- ...trajectorySearch.measurement,
|
|
|
+ })
|
|
|
+
|
|
|
+ if (options.checkpointInterval !== undefined && index % options.checkpointInterval === 0) {
|
|
|
+ checkpoints.push({
|
|
|
+ turns: options.startingTurns + index,
|
|
|
+ state: await retainedBrowserState(cdp, world.page),
|
|
|
+ })
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ const lastCheckpoint = checkpoints.at(-1)
|
|
|
+ const retainedAfter = lastCheckpoint?.turns === options.startingTurns + options.turnCount
|
|
|
+ ? lastCheckpoint.state
|
|
|
+ : await retainedBrowserState(cdp, world.page)
|
|
|
+ if (lastCheckpoint?.turns !== options.startingTurns + options.turnCount) {
|
|
|
+ checkpoints.push({ turns: options.startingTurns + options.turnCount, state: retainedAfter })
|
|
|
+ }
|
|
|
+ return {
|
|
|
+ startingTurns: options.startingTurns,
|
|
|
+ turnsAdded: options.turnCount,
|
|
|
+ toolTurns: turns.filter(turn => turn.kind === 'tool').length,
|
|
|
+ retainedBefore,
|
|
|
+ retainedAfter,
|
|
|
+ retainedDelta: retainedDelta(retainedBefore, retainedAfter),
|
|
|
+ checkpoints,
|
|
|
+ turns,
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+function average(values: readonly number[]): number {
|
|
|
+ return rounded(values.reduce((sum, value) => sum + value, 0) / values.length)
|
|
|
+}
|
|
|
+
|
|
|
+function p95(values: readonly number[]): number {
|
|
|
+ const sorted = [...values].sort((left, right) => left - right)
|
|
|
+ return sorted[Math.max(0, Math.ceil(sorted.length * 0.95) - 1)] ?? 0
|
|
|
+}
|
|
|
+
|
|
|
+function summarizeTurnWindows(
|
|
|
+ turns: readonly ContinuedTurnReport[],
|
|
|
+ windowSize: number,
|
|
|
+): object[] {
|
|
|
+ const windows: object[] = []
|
|
|
+ for (let start = 0; start < turns.length; start += windowSize) {
|
|
|
+ const window = turns.slice(start, start + windowSize)
|
|
|
+ const streamWall = window.map(turn => turn.stream.wallMs)
|
|
|
+ const userEcho = window.map(turn => turn.stream.clickToUserEchoMs)
|
|
|
+ const firstChunk = window.map(turn => turn.stream.clickToFirstChunkMs)
|
|
|
+ windows.push({
|
|
|
+ turns: `${String(start + 1)}-${String(start + window.length)}`,
|
|
|
+ toolTurns: window.filter(turn => turn.kind === 'tool').length,
|
|
|
+ average: {
|
|
|
+ composerFillWallMs: average(window.map(turn => turn.composerFill.wallMs)),
|
|
|
+ streamWallMs: average(streamWall),
|
|
|
+ clickToUserEchoMs: average(userEcho),
|
|
|
+ clickToFirstChunkMs: average(firstChunk),
|
|
|
+ taskMs: average(window.map(turn => turn.stream.taskMs)),
|
|
|
+ scriptMs: average(window.map(turn => turn.stream.scriptMs)),
|
|
|
+ recalcStyleMs: average(window.map(turn => turn.stream.recalcStyleMs)),
|
|
|
+ mutationBatches: average(window.map(turn => turn.stream.batches)),
|
|
|
},
|
|
|
- historyPages,
|
|
|
- warmTrajectory: {
|
|
|
- rows: warmTrajectory.value,
|
|
|
- ...warmTrajectory.measurement,
|
|
|
+ p95: {
|
|
|
+ streamWallMs: rounded(p95(streamWall)),
|
|
|
+ clickToUserEchoMs: rounded(p95(userEcho)),
|
|
|
+ clickToFirstChunkMs: rounded(p95(firstChunk)),
|
|
|
},
|
|
|
- liveInput: {
|
|
|
- promptChars: LIVE_PROMPT.length,
|
|
|
- newSession: newSession.measurement,
|
|
|
- composerFill: composerInput.measurement,
|
|
|
- stream: {
|
|
|
- paceMs: STREAM_PACE_MS,
|
|
|
- deltaChunks: STREAM_DELTA_COUNT,
|
|
|
- persistedChunks: assistantChunks.length,
|
|
|
- clickToUserEchoMs: rounded(clickToUserEchoMs),
|
|
|
- clickToFirstChunkMs: rounded(clickToFirstChunkMs),
|
|
|
- firstChunkToSettledMs: rounded(clickToSettledMs - clickToFirstChunkMs),
|
|
|
- mutationBatches: streamMutations.batches,
|
|
|
- mutationRecords: streamMutations.records,
|
|
|
- ...streamMeasurement,
|
|
|
+ })
|
|
|
+ }
|
|
|
+ return windows
|
|
|
+}
|
|
|
+
|
|
|
+describe('manual web performance: complex workspace and history', () => {
|
|
|
+ let browser: Browser
|
|
|
+
|
|
|
+ beforeAll(async () => {
|
|
|
+ if (webSnapshotMode() === 'record') {
|
|
|
+ throw new Error('manual web performance runs only with deterministic replay')
|
|
|
+ }
|
|
|
+ browser = await chromium.launch()
|
|
|
+ })
|
|
|
+
|
|
|
+ afterAll(async () => {
|
|
|
+ await browser?.close()
|
|
|
+ })
|
|
|
+
|
|
|
+ it('reports workspace, history, and trajectory cardinality costs', async () => {
|
|
|
+ const world = await launchPerformanceWorld({
|
|
|
+ browser,
|
|
|
+ sidebarSessions: SIDEBAR_SESSION_COUNT,
|
|
|
+ seedLongHistory: true,
|
|
|
+ })
|
|
|
+ try {
|
|
|
+ const bootStarted = performance.now()
|
|
|
+ const group = await openPerformancePage(world, SIDEBAR_SESSION_COUNT + 1)
|
|
|
+ const bootReadyMs = performance.now() - bootStarted
|
|
|
+ const page = world.page
|
|
|
+ const cdp = await page.context().newCDPSession(page)
|
|
|
+ await cdp.send('Performance.enable')
|
|
|
+ const firstContentfulPaintMs = await page.evaluate(
|
|
|
+ () => globalThis.performance.getEntriesByName('first-contentful-paint')[0]?.startTime,
|
|
|
+ )
|
|
|
+
|
|
|
+ const sidebar = await measure(cdp, async () => {
|
|
|
+ await group.click()
|
|
|
+ return stableCount(
|
|
|
+ page.getByRole('treeitem'),
|
|
|
+ count => count === SIDEBAR_SESSION_COUNT + 2,
|
|
|
+ )
|
|
|
+ })
|
|
|
+ expect(sidebar.value).toBe(SIDEBAR_SESSION_COUNT + 2)
|
|
|
+ await group.click()
|
|
|
+ await expect.poll(() => page.getByRole('treeitem').count()).toBe(1)
|
|
|
+
|
|
|
+ const contentSearch = await measure(cdp, async () => {
|
|
|
+ await page.getByRole('textbox', { name: 'Search name, keywords...', exact: true })
|
|
|
+ .fill('LONG_PERF_SENTINEL')
|
|
|
+ const results = page.getByRole('tree', { name: 'Search results' }).getByRole('treeitem')
|
|
|
+ await expect.poll(() => results.count(), { timeout: 60_000 }).toBe(1)
|
|
|
+ await results.first().waitFor({ timeout: 60_000 })
|
|
|
+ return results.first()
|
|
|
+ })
|
|
|
+ const opened = await measure(cdp, async () => {
|
|
|
+ await contentSearch.value.click()
|
|
|
+ await page.getByRole('tab', { name: 'Trajectory', exact: true }).waitFor({ timeout: 30_000 })
|
|
|
+ return conversationTurns(page)
|
|
|
+ })
|
|
|
+ expect(opened.value).toBe(DEFAULT_HISTORY_TURNS)
|
|
|
+
|
|
|
+ const trajectoryRows = page.getByRole('row')
|
|
|
+ const coldTrajectory = await measure(cdp, async () => {
|
|
|
+ await page.getByRole('tab', { name: 'Trajectory', exact: true }).click()
|
|
|
+ return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS)
|
|
|
+ })
|
|
|
+ expect(coldTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS)
|
|
|
+
|
|
|
+ const collapseTurns = await measure(cdp, async () => {
|
|
|
+ await page.getByRole('button', { name: 'Collapse turns', exact: true }).click()
|
|
|
+ return stableCount(trajectoryRows, count => count > 0 && count < EXPECTED_TRAJECTORY_ROWS)
|
|
|
+ })
|
|
|
+ expect(collapseTurns.value).toBeLessThan(EXPECTED_TRAJECTORY_ROWS)
|
|
|
+ const trajectorySearch = await measure(cdp, async () => {
|
|
|
+ await page.getByRole('searchbox', { name: 'Search trajectory', exact: true }).fill('turn 499')
|
|
|
+ return stableCount(trajectoryRows, count => count > 0 && count < 20)
|
|
|
+ })
|
|
|
+ expect(trajectorySearch.value).toBeLessThan(20)
|
|
|
+
|
|
|
+ await page.getByRole('tab', { name: 'Chat', exact: true }).click()
|
|
|
+ const historyPages: { turns: number; measurement: Measurement }[] = []
|
|
|
+ let turns = await conversationTurns(page)
|
|
|
+ while (turns < LONG_HISTORY_TURNS) {
|
|
|
+ const previousTurns = turns
|
|
|
+ const older = await measure(cdp, async () => {
|
|
|
+ await page.getByRole('button', { name: 'Load earlier', exact: true }).click()
|
|
|
+ await expect.poll(() => conversationTurns(page), { timeout: 30_000 })
|
|
|
+ .toBeGreaterThan(previousTurns)
|
|
|
+ return conversationTurns(page)
|
|
|
+ })
|
|
|
+ turns = older.value
|
|
|
+ historyPages.push({ turns, measurement: older.measurement })
|
|
|
+ }
|
|
|
+
|
|
|
+ const warmTrajectory = await measure(cdp, async () => {
|
|
|
+ await page.getByRole('tab', { name: 'Trajectory', exact: true }).click()
|
|
|
+ return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS)
|
|
|
+ })
|
|
|
+ expect(warmTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS)
|
|
|
+ const warmConversation = await measure(cdp, async () => {
|
|
|
+ await page.getByRole('tab', { name: 'Chat', exact: true }).click()
|
|
|
+ return conversationTurns(page)
|
|
|
+ })
|
|
|
+ expect(warmConversation.value).toBe(LONG_HISTORY_TURNS)
|
|
|
+
|
|
|
+ console.info(`WEB_PERF_RESULT ${JSON.stringify({
|
|
|
+ scenario: 'workspace-history-trajectory',
|
|
|
+ fixture: {
|
|
|
+ sidebarSessions: SIDEBAR_SESSION_COUNT,
|
|
|
+ totalSessions: SIDEBAR_SESSION_COUNT + 1,
|
|
|
+ longHistoryTurns: LONG_HISTORY_TURNS,
|
|
|
+ toolCalls: EXPECTED_TOOL_CALLS,
|
|
|
+ trajectoryRows: EXPECTED_TRAJECTORY_ROWS,
|
|
|
},
|
|
|
- },
|
|
|
+ setupMs: rounded(world.setupMs),
|
|
|
+ boot: {
|
|
|
+ readyMs: rounded(bootReadyMs),
|
|
|
+ firstContentfulPaintMs: firstContentfulPaintMs === undefined
|
|
|
+ ? null
|
|
|
+ : rounded(firstContentfulPaintMs),
|
|
|
+ },
|
|
|
+ sidebarExpand: sidebar.measurement,
|
|
|
+ contentSearch: contentSearch.measurement,
|
|
|
+ openLongHistory: { initialTurns: opened.value, ...opened.measurement },
|
|
|
+ coldTrajectory: { rows: coldTrajectory.value, ...coldTrajectory.measurement },
|
|
|
+ collapseTurns: { rows: collapseTurns.value, ...collapseTurns.measurement },
|
|
|
+ trajectorySearch: { rows: trajectorySearch.value, ...trajectorySearch.measurement },
|
|
|
+ historyPages,
|
|
|
+ warmTrajectory: { rows: warmTrajectory.value, ...warmTrajectory.measurement },
|
|
|
+ warmConversation: { turns: warmConversation.value, ...warmConversation.measurement },
|
|
|
+ }, null, 2)}`)
|
|
|
+ expect(world.tripwire.warnings).toEqual([])
|
|
|
+ expect(world.tripwire.pageErrors).toEqual([])
|
|
|
+ } finally {
|
|
|
+ await closePerformanceWorld(world)
|
|
|
+ }
|
|
|
+ })
|
|
|
+
|
|
|
+ it('reports default 24-turn history plus eight continued turns', async () => {
|
|
|
+ const world = await launchPerformanceWorld({
|
|
|
+ browser,
|
|
|
+ replay: performanceReplayOverride(COMPARISON_TURNS, comparisonTurn),
|
|
|
+ seedLongHistory: true,
|
|
|
+ })
|
|
|
+ try {
|
|
|
+ await openPerformancePage(world, 1)
|
|
|
+ const cdp = await world.page.context().newCDPSession(world.page)
|
|
|
+ await cdp.send('Performance.enable')
|
|
|
+ const opened = await measure(cdp, () => openLongHistory(world.page))
|
|
|
+ expect(opened.value).toBe(DEFAULT_HISTORY_TURNS)
|
|
|
+ const conversation = await continueConversation(world, cdp, {
|
|
|
+ startingTurns: DEFAULT_HISTORY_TURNS,
|
|
|
+ turnCount: COMPARISON_TURNS,
|
|
|
+ turnSpec: comparisonTurn,
|
|
|
+ expectedSessionId: SessionId(LONG_SESSION_ID),
|
|
|
+ })
|
|
|
+ expect(conversation.toolTurns).toBe(2)
|
|
|
+ console.info(`WEB_PERF_RESULT ${JSON.stringify({
|
|
|
+ scenario: 'default-resume-24-plus-8',
|
|
|
+ setupMs: rounded(world.setupMs),
|
|
|
+ replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
|
|
|
+ openLongHistory: { turns: opened.value, ...opened.measurement },
|
|
|
+ conversation,
|
|
|
+ }, null, 2)}`)
|
|
|
+ expect(world.tripwire.warnings).toEqual([])
|
|
|
+ expect(world.tripwire.pageErrors).toEqual([])
|
|
|
+ } finally {
|
|
|
+ await closePerformanceWorld(world)
|
|
|
+ }
|
|
|
+ })
|
|
|
+
|
|
|
+ it('reports fully expanded 500-turn history plus eight continued turns', async () => {
|
|
|
+ const world = await launchPerformanceWorld({
|
|
|
+ browser,
|
|
|
+ replay: performanceReplayOverride(COMPARISON_TURNS, comparisonTurn),
|
|
|
+ seedLongHistory: true,
|
|
|
+ })
|
|
|
+ try {
|
|
|
+ await openPerformancePage(world, 1)
|
|
|
+ const cdp = await world.page.context().newCDPSession(world.page)
|
|
|
+ await cdp.send('Performance.enable')
|
|
|
+ expect(await openLongHistory(world.page)).toBe(DEFAULT_HISTORY_TURNS)
|
|
|
+ const historyPages: { turns: number; measurement: Measurement }[] = []
|
|
|
+ let turns = DEFAULT_HISTORY_TURNS
|
|
|
+ while (turns < LONG_HISTORY_TURNS) {
|
|
|
+ const previousTurns = turns
|
|
|
+ const older = await measure(cdp, async () => {
|
|
|
+ await world.page.getByRole('button', { name: 'Load earlier', exact: true }).click()
|
|
|
+ await expect.poll(() => conversationTurns(world.page), { timeout: 30_000 })
|
|
|
+ .toBeGreaterThan(previousTurns)
|
|
|
+ return conversationTurns(world.page)
|
|
|
+ })
|
|
|
+ turns = older.value
|
|
|
+ historyPages.push({ turns, measurement: older.measurement })
|
|
|
+ }
|
|
|
+ expect(turns).toBe(LONG_HISTORY_TURNS)
|
|
|
+ const conversation = await continueConversation(world, cdp, {
|
|
|
+ startingTurns: LONG_HISTORY_TURNS,
|
|
|
+ turnCount: COMPARISON_TURNS,
|
|
|
+ turnSpec: comparisonTurn,
|
|
|
+ expectedSessionId: SessionId(LONG_SESSION_ID),
|
|
|
+ })
|
|
|
+ expect(conversation.toolTurns).toBe(2)
|
|
|
+ console.info(`WEB_PERF_RESULT ${JSON.stringify({
|
|
|
+ scenario: 'expanded-history-500-plus-8',
|
|
|
+ setupMs: rounded(world.setupMs),
|
|
|
+ replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
|
|
|
+ historyPages,
|
|
|
+ conversation,
|
|
|
+ }, null, 2)}`)
|
|
|
+ expect(world.tripwire.warnings).toEqual([])
|
|
|
+ expect(world.tripwire.pageErrors).toEqual([])
|
|
|
+ } finally {
|
|
|
+ await closePerformanceWorld(world)
|
|
|
+ }
|
|
|
+ })
|
|
|
+
|
|
|
+ it('reports one hundred continuously generated turns with retained checkpoints', async () => {
|
|
|
+ const world = await launchPerformanceWorld({
|
|
|
+ browser,
|
|
|
+ replay: performanceReplayOverride(SOAK_TURNS, soakTurn),
|
|
|
+ })
|
|
|
+ try {
|
|
|
+ await world.page.goto(world.scaffold.baseUrl, { waitUntil: 'load' })
|
|
|
+ await world.page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
|
|
|
+ await connectFreshWorkspace(world.page, 'continuous-conversation-perf')
|
|
|
+ const cdp = await world.page.context().newCDPSession(world.page)
|
|
|
+ await cdp.send('Performance.enable')
|
|
|
+ const conversation = await continueConversation(world, cdp, {
|
|
|
+ startingTurns: 0,
|
|
|
+ turnCount: SOAK_TURNS,
|
|
|
+ turnSpec: soakTurn,
|
|
|
+ checkpointInterval: SOAK_CHECKPOINT_INTERVAL,
|
|
|
+ })
|
|
|
+ expect(conversation.toolTurns).toBe(SOAK_TURNS / SOAK_TOOL_INTERVAL)
|
|
|
+ const { turns, ...retained } = conversation
|
|
|
+ console.info(`WEB_PERF_RESULT ${JSON.stringify({
|
|
|
+ scenario: 'continuous-100-turn-soak',
|
|
|
+ setupMs: rounded(world.setupMs),
|
|
|
+ replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW,
|
|
|
+ ...retained,
|
|
|
+ windows: summarizeTurnWindows(turns, SOAK_CHECKPOINT_INTERVAL),
|
|
|
+ }, null, 2)}`)
|
|
|
+ expect(world.tripwire.warnings).toEqual([])
|
|
|
+ expect(world.tripwire.pageErrors).toEqual([])
|
|
|
+ } finally {
|
|
|
+ await closePerformanceWorld(world)
|
|
|
}
|
|
|
- console.info(`WEB_PERF_RESULT ${JSON.stringify(report, null, 2)}`)
|
|
|
- expect(tripwire.warnings).toEqual([])
|
|
|
- expect(tripwire.pageErrors).toEqual([])
|
|
|
})
|
|
|
})
|
kingwl