|
|
@@ -3,16 +3,18 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
|
|
import { tmpdir } from 'node:os'
|
|
|
import { join } from 'node:path'
|
|
|
import { performance } from 'node:perf_hooks'
|
|
|
-import { chromium, type Page, type CDPSession } from 'playwright'
|
|
|
+import { chromium, type Page, type CDPSession, type Locator } from 'playwright'
|
|
|
import { expect, it } from 'vitest'
|
|
|
import { launchWebScaffold, seedSession, watchConsole, webSnapshotMode } from '../../apps/web/tests/scaffold.ts'
|
|
|
import { newEnglishPage } from '../../apps/web/tests/support.ts'
|
|
|
-import { ciTimeBudget } from '../support/calibration.ts'
|
|
|
+import { ciTimeBudget, PERFORMANCE_BUDGET_HEADROOM } from '../support/calibration.ts'
|
|
|
import { HISTORY_TURNS, SESSION_ID, FIRST, DONE, DELTAS, PACE_MS, syntheticHistory, syntheticReply } from './synthetic-history.ts'
|
|
|
|
|
|
const SAMPLES = 3
|
|
|
const TAIL = '[data-chat-flow-key^="9:turn-tail"]'
|
|
|
const REFERENCE = { open: 200, page: 260, trajectory: 160, first: 1100, streamTask: 1800, input: 500, streamWall: 1000 }
|
|
|
+const EXPECTED_OPEN_CI_MS = 700
|
|
|
+const OPEN_BUDGET_MS = Math.ceil(EXPECTED_OPEN_CI_MS * PERFORMANCE_BUDGET_HEADROOM)
|
|
|
const REPLAY_DURATION_MS = (DELTAS + 4) * PACE_MS
|
|
|
|
|
|
async function painted(page: Page): Promise<void> {
|
|
|
@@ -38,6 +40,36 @@ function median(values: number[]): number {
|
|
|
return values.toSorted((a, b) => a - b)[Math.floor(values.length / 2)]!
|
|
|
}
|
|
|
|
|
|
+function expectEndpointWithinBudget(value: number, budget: number): void {
|
|
|
+ expect(value).toBeLessThanOrEqual(budget)
|
|
|
+}
|
|
|
+
|
|
|
+function expectInputOverlap(value: boolean): void {
|
|
|
+ expect(value).toBe(true)
|
|
|
+}
|
|
|
+
|
|
|
+async function watchInputOverlap(composer: Locator): Promise<void> {
|
|
|
+ await composer.evaluate((element, markers) => {
|
|
|
+ element.removeAttribute('data-benchmark-input-witness')
|
|
|
+ element.removeAttribute('data-benchmark-input-overlap')
|
|
|
+ element.addEventListener('input', (event) => {
|
|
|
+ const transcript = Array.from(document.querySelectorAll('[data-chat-flow-kind="assistant-step"]')).at(-1)?.textContent ?? ''
|
|
|
+ element.setAttribute('data-benchmark-input-overlap', String(event.isTrusted && transcript.includes(markers.first) && !transcript.includes(markers.done)))
|
|
|
+ element.setAttribute('data-benchmark-input-witness', JSON.stringify({ trusted: event.isTrusted, first: transcript.includes(markers.first), done: transcript.includes(markers.done) }))
|
|
|
+ }, { once: true })
|
|
|
+ }, { first: FIRST, done: DONE })
|
|
|
+}
|
|
|
+
|
|
|
+it('accepts recorded hosted open samples and rejects slower endpoints', () => {
|
|
|
+ for (const value of [681.276514, 541.051233]) {
|
|
|
+ expect(() => expectEndpointWithinBudget(value, ciTimeBudget(REFERENCE.open))).toThrow()
|
|
|
+ expectEndpointWithinBudget(value, OPEN_BUDGET_MS)
|
|
|
+ }
|
|
|
+ expect(OPEN_BUDGET_MS).toBe(875)
|
|
|
+ expect(() => expectEndpointWithinBudget(OPEN_BUDGET_MS + 1, OPEN_BUDGET_MS)).toThrow()
|
|
|
+ expect(() => expectEndpointWithinBudget(2000, OPEN_BUDGET_MS)).toThrow()
|
|
|
+})
|
|
|
+
|
|
|
it('opens, pages, navigates and streams into a 240-turn browser history', async () => {
|
|
|
if (webSnapshotMode() !== 'replay') throw new Error('browser benchmarks require keyless replay mode')
|
|
|
const samples: { open: number; page: number; trajectory: number; first: number; streamTask: number; streamWall: number; input: number; inputOverlapped: boolean; heapMb: number; nodes: number }[] = []
|
|
|
@@ -97,18 +129,12 @@ it('opens, pages, navigates and streams into a 240-turn browser history', async
|
|
|
() => ({ ok: true as const }),
|
|
|
(error: unknown) => ({ ok: false as const, error }),
|
|
|
)
|
|
|
+ await watchInputOverlap(composer)
|
|
|
const started = performance.now()
|
|
|
await page.locator('[data-composer-seat]').getByRole('button', { name: 'Send message', exact: true }).click()
|
|
|
const reply = page.locator('[data-chat-flow-kind="assistant-step"]').last()
|
|
|
await reply.getByText(FIRST, { exact: false }).last().waitFor()
|
|
|
- await painted(page)
|
|
|
const first = performance.now() - started
|
|
|
- await composer.evaluate((element, markers) => {
|
|
|
- element.addEventListener('input', (event) => {
|
|
|
- const transcript = Array.from(document.querySelectorAll('[data-chat-flow-kind="assistant-step"]')).at(-1)?.textContent ?? ''
|
|
|
- element.setAttribute('data-benchmark-input-overlap', String(event.isTrusted && transcript.includes(markers.first) && !transcript.includes(markers.done)))
|
|
|
- }, { once: true })
|
|
|
- }, { first: FIRST, done: DONE })
|
|
|
// Observe the actual trusted input event, not state before asynchronous click/typing.
|
|
|
const input = await measure(page, async () => {
|
|
|
await composer.click()
|
|
|
@@ -116,7 +142,8 @@ it('opens, pages, navigates and streams into a 240-turn browser history', async
|
|
|
await expect.poll(() => composer.textContent()).toBe('next synthetic question')
|
|
|
})
|
|
|
const inputOverlapped = await composer.getAttribute('data-benchmark-input-overlap') === 'true'
|
|
|
- expect(inputOverlapped).toBe(true)
|
|
|
+ console.log(JSON.stringify({ benchmark: 'long-session-browser/input', sample, first, input, witness: await composer.getAttribute('data-benchmark-input-witness') }))
|
|
|
+ expectInputOverlap(inputOverlapped)
|
|
|
await reply.getByText(DONE, { exact: false }).last().waitFor()
|
|
|
const settlement = await settled
|
|
|
if (!settlement.ok) throw settlement.error
|
|
|
@@ -130,6 +157,12 @@ it('opens, pages, navigates and streams into a 240-turn browser history', async
|
|
|
if (heap === undefined) throw new Error('Chromium heap metric missing')
|
|
|
samples.push({ open, page: Math.max(...pages), trajectory, first, streamTask, streamWall, input, inputOverlapped, heapMb: heap.value / 1048576, nodes: await page.locator('*').count() })
|
|
|
console.log(JSON.stringify({ benchmark: 'long-session-browser/sample', sample, initialTurns, pages, ...samples.at(-1) }))
|
|
|
+ await watchInputOverlap(composer)
|
|
|
+ await composer.click()
|
|
|
+ await page.keyboard.type('!')
|
|
|
+ const lateInputOverlapped = await composer.getAttribute('data-benchmark-input-overlap') === 'true'
|
|
|
+ expect(await composer.getAttribute('data-benchmark-input-witness')).toBe(JSON.stringify({ trusted: true, first: true, done: true }))
|
|
|
+ expect(() => expectInputOverlap(lateInputOverlapped)).toThrow()
|
|
|
expect(consoleWatch.pageErrors).toEqual([])
|
|
|
expect(consoleWatch.warnings).toEqual([])
|
|
|
} catch (error) { failures.push(error) } finally {
|
|
|
@@ -144,7 +177,7 @@ it('opens, pages, navigates and streams into a 240-turn browser history', async
|
|
|
if (failures.length > 0) throw new AggregateError(failures, 'browser benchmark failed')
|
|
|
}
|
|
|
const aggregate = Object.fromEntries(Object.keys(REFERENCE).map(key => [key, median(samples.map(sample => sample[key as keyof typeof REFERENCE]))]))
|
|
|
- const budgets = Object.fromEntries(Object.entries(REFERENCE).map(([key, value]) => [key, ciTimeBudget(value) + (key === 'streamWall' ? REPLAY_DURATION_MS : 0)]))
|
|
|
- console.log(JSON.stringify({ benchmark: 'long-session-browser/median', turns: HISTORY_TURNS, deltas: DELTAS, paceMs: PACE_MS, samples, aggregate, referenceMs: REFERENCE, budgets }))
|
|
|
- for (const [key, value] of Object.entries(aggregate)) expect.soft(value, key).toBeLessThanOrEqual(budgets[key]!)
|
|
|
+ const budgets = Object.fromEntries(Object.entries(REFERENCE).map(([key, value]) => [key, key === 'open' ? OPEN_BUDGET_MS : ciTimeBudget(value) + (key === 'streamWall' ? REPLAY_DURATION_MS : 0)]))
|
|
|
+ console.log(JSON.stringify({ benchmark: 'long-session-browser/median', turns: HISTORY_TURNS, deltas: DELTAS, paceMs: PACE_MS, samples, aggregate, referenceMs: REFERENCE, expectedOpenCiMs: EXPECTED_OPEN_CI_MS, budgets }))
|
|
|
+ for (const [key, value] of Object.entries(aggregate)) expectEndpointWithinBudget(value, budgets[key]!)
|
|
|
})
|
Tianyi Cui