Browse Source

test(web): make real title smoke deterministic

Tianyi Cui 3 months ago
parent
commit
2a62da8ff0
1 changed files with 62 additions and 9 deletions
  1. 62 9
      apps/web/tests/smoke-real.e2e.ts

+ 62 - 9
apps/web/tests/smoke-real.e2e.ts

@@ -77,6 +77,55 @@ async function rpc<T>(baseUrl: string, method: string, payload: unknown): Promis
   return body.result.value
 }
 
+interface HistoryPage {
+  events: { event: { type: string; data: unknown } }[]
+  hasMore: boolean
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null
+}
+
+function providerTitle(page: HistoryPage): string | undefined {
+  for (let index = page.events.length - 1; index >= 0; index--) {
+    const event = page.events[index]!.event
+    if (event.type !== 'session/title' || !isRecord(event.data)) continue
+    const source = event.data.source
+    if (typeof event.data.title === 'string' && isRecord(source) && source.kind === 'provider') {
+      return event.data.title
+    }
+  }
+  return undefined
+}
+
+function hasAssistantMarker(page: HistoryPage, marker: string): boolean {
+  return page.events.some(({ event }) => {
+    if (event.type !== 'assistant/message' || !isRecord(event.data) || !Array.isArray(event.data.content)) return false
+    return event.data.content.some(block =>
+      isRecord(block) && block.type === 'text' && typeof block.text === 'string' && block.text.includes(marker))
+  })
+}
+
+async function history(baseUrl: string, sessionId: string): Promise<HistoryPage> {
+  return rpc<HistoryPage>(baseUrl, 'session.history', { sessionId, maxMessages: 10 })
+}
+
+async function waitForProviderTitle(baseUrl: string, sessionId: string): Promise<string> {
+  let observed: string | undefined
+  await expect.poll(async () => {
+    observed = providerTitle(await history(baseUrl, sessionId))
+    return observed
+  }, { timeout: 90_000 }).toEqual(expect.any(String))
+  if (observed === undefined) throw new Error('provider-backed session title was not observed')
+  return observed
+}
+
+async function waitForAssistantMarker(baseUrl: string, sessionId: string, marker: string): Promise<void> {
+  await expect.poll(async () => hasAssistantMarker(await history(baseUrl, sessionId), marker), {
+    timeout: 120_000,
+  }).toBe(true)
+}
+
 /** W5 screenshot: evidence for the figma comparison, not a failure artifact. */
 async function screen(page: Page, name: string): Promise<void> {
   await page.screenshot({ path: join(REPO_ROOT, '.artifacts', `w5-${name}.png`) })
@@ -282,7 +331,6 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || notReady.length > 0)('web smoke
     await input.waitFor({ timeout: 10_000 })
     await screen(page, '02-empty-state')
     const prompt = `Please answer this request carefully: explain event sourcing in two sentences, ending with exactly ${ROUND_DONE_MARKER}.`
-    const fallbackTitle = 'Please answer this request carefully:'
     await input.fill(prompt)
     await input.press('Enter')
     // startSession chain: session mounts, composer moves to the bottom.
@@ -296,14 +344,18 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || notReady.length > 0)('web smoke
       undefined,
       { timeout: 15_000 },
     )
+    await expect.poll(async () => (await rpc<{ items: { sessionId: string }[] }>(baseUrl, 'session.list', {})).items.length, {
+      timeout: 15_000,
+    }).toBe(1)
+    const sessions = await rpc<{ items: { sessionId: string }[] }>(baseUrl, 'session.list', {})
+    const sessionId = sessions.items[0]?.sessionId
+    if (sessionId === undefined) throw new Error('created Web session was not listed')
+    const durableTitle = await waitForProviderTitle(baseUrl, sessionId)
     await page.waitForFunction(
-      expected => document.title !== `${expected} — DeepSeek Harness`
-        && document.title.endsWith(' — DeepSeek Harness'),
-      fallbackTitle,
-      { timeout: 90_000 },
+      expected => document.title === `${expected} — DeepSeek Harness`,
+      durableTitle,
+      { timeout: 15_000 },
     )
-    const durableTitle = (await page.title()).replace(/ — DeepSeek Harness$/, '')
-    expect(durableTitle).not.toBe(fallbackTitle)
     const sessionTree = page.getByRole('tree', { name: 'Sessions' })
     const projectRow = sessionTree.getByRole('treeitem').first()
     if (await projectRow.getAttribute('aria-expanded') === 'false') await projectRow.click()
@@ -311,7 +363,8 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || notReady.length > 0)('web smoke
       sessionTree.getByText(durableTitle, { exact: true }).waitFor({ timeout: 10_000 }),
       page.getByRole('navigation').getByText(durableTitle, { exact: true }).waitFor({ timeout: 10_000 }),
     ])
-    await page.waitForFunction(marker => document.body.innerText.includes(marker), ROUND_DONE_MARKER, { timeout: 120_000 })
+    await waitForAssistantMarker(baseUrl, sessionId, ROUND_DONE_MARKER)
+    await page.locator('p').filter({ hasText: ROUND_DONE_MARKER }).waitFor({ timeout: 10_000 })
     await screen(page, '04-round-complete')
   }, 150_000)
 
@@ -386,7 +439,7 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || notReady.length > 0)('web smoke
     onTestFailed(() => saveFailureShot(page, 'w5-reload'))
     await page.reload({ waitUntil: 'load' })
     await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
-    await page.waitForFunction(marker => document.body.innerText.includes(marker), ROUND_DONE_MARKER, { timeout: 30_000 })
+    await page.locator('p').filter({ hasText: ROUND_DONE_MARKER }).waitFor({ timeout: 30_000 })
     await screen(page, '12-reload-recovery')
   })