deepseek-harness/apps/web/tests/live-interactions.e2e.ts

// Web e2e scenarios: live-turn interactions — cancellation, error surfacing,
// and transient-retry recovery, all through the real composition and wire.
// The model seam is dsh-llm-replay with override sidecars: `hang` (+ a
// readyFile marker) makes mid-stream cancel deterministic by construction,
// `throw` entries express provider failures by stable code, and `{ patches }`
// augmentation injects a transient throw before the recorded success so
// llm-retry's recovery is proven end-to-end in the browser. Sidecar CONTENT
// is authored here (single-sourced against the fixture via deriveReplayScript
// — no committed copy of recorded chunks); the file is a per-run artifact in
// the temp workspace. One recorded base fixture serves all three scenarios.
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { existsSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { fileURLToPath } from 'node:url'
import { join } from 'node:path'
import type { Browser, Page } from 'playwright'
import { chromium } from 'playwright'
import { afterEach, describe, expect, it, onTestFailed } from 'vitest'
import { deriveReplayScript, parseSessionLog } from '@deepseek-ai/dsh-llm-replay'
import type { ReplayOverrideDoc } from '@deepseek-ai/dsh-llm-replay'
import type { SessionEvent } from '@deepseek-ai/dsh-session'
import {
  assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts,
  launchWebScaffold, recordFixture, watchConsole, webSnapshotMode, type WebScaffold,
} from './scaffold.ts'
import { connectFreshWorkspace, newEnglishPage, saveFailureShot } from './support.ts'

const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/live-interactions', import.meta.url))
const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl')
// One golden pins the stable mid-turn loading state; the other three capture
// what the user is left looking at after cancel, after a non-retryable failure,
// and after retry recovery.
const CANCEL_EXPECTED = join(SNAPSHOT_DIR, 'cancel.expected.md')
const LOADING_EXPECTED = join(SNAPSHOT_DIR, 'loading.expected.md')
const ERROR_EXPECTED = join(SNAPSHOT_DIR, 'error-auth.expected.md')
const RETRY_EXPECTED = join(SNAPSHOT_DIR, 'retry.expected.md')
const MODE = webSnapshotMode()
const AUTH_PROVIDER_MESSAGE = 'Authentication Fails, Your api key: sk-preview-secret is invalid'

// The recorded base: one text-only turn whose derived script the sidecars
// patch. Kept deliberately tool-free so the derived script is exactly one
// model call.
const PROMPT = 'Reply with a one-sentence description of event sourcing, then stop.'

/** turn/end reasons observed, in order. */
function turnEndReasons(events: SessionEvent[]): string[] {
  return events
    .filter(e => e.type === 'turn/end')
    .map(e => (e as SessionEvent & { data: { reason: { kind: string } } }).data.reason.kind)
}

describe('web e2e: live-turn interactions (cancel / error / retry)', () => {
  let scaffold: WebScaffold | undefined
  let browser: Browser | undefined
  let page: Page
  let tripwire: ReturnType<typeof watchConsole>
  let sessionEvents: SessionEvent[]
  let sidecarDir: string | undefined

  afterEach(async () => {
    // scaffold.close() failures MUST fail the scenario: assertConsumed() is
    // the fixture-drift tripwire and cleanup problems are real defects. Run
    // every teardown step regardless, then rethrow what failed.
    const failures: unknown[] = []
    await browser?.close().catch((error: unknown) => failures.push(error))
    browser = undefined
    const closing = scaffold
    scaffold = undefined
    await closing?.close().catch((error: unknown) => failures.push(error))
    if (sidecarDir !== undefined) await rm(sidecarDir, { recursive: true, force: true }).catch((error: unknown) => failures.push(error))
    sidecarDir = undefined
    if (failures.length === 1) throw failures[0]
    if (failures.length > 1) throw new AggregateError(failures, 'live-interactions teardown failed')
  })

  /** Boot scaffold + page with an optional override doc materialized per run. */
  async function launch(buildOverride?: (sidecarHome: string) => ReplayOverrideDoc): Promise<void> {
    sessionEvents = []
    let overridePath: string | undefined
    if (buildOverride !== undefined) {
      // The sidecar CONTENT is authored in this spec; the file is a per-run
      // artifact minted in a spec-owned temp dir. It must exist BEFORE the
      // scaffold boots — installLlmReplay resolves the script at install.
      sidecarDir = await mkdtemp(join(tmpdir(), 'dsh-web-e2e-sidecar-'))
      overridePath = join(sidecarDir, 'replay.override.json')
      await writeFile(overridePath, JSON.stringify(buildOverride(sidecarDir)))
    }
    scaffold = await launchWebScaffold({
      replayFixture: FIXTURE,
      ...(overridePath === undefined ? {} : { replayOverride: overridePath }),
    })
    scaffold.ctx.on('session/event', (_session, event: SessionEvent) => { sessionEvents.push(event) })
    browser = await chromium.launch()
    page = await newEnglishPage(browser)
    tripwire = watchConsole(page)
    await page.goto(scaffold.baseUrl, { waitUntil: 'load' })
    await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
    // Fresh world: connect a Workspace so the composer scenarios start live.
    await connectFreshWorkspace(page, scaffold.workspaceCwd)
  }

  /**
   * Type the recorded prompt and send, with the settled barrier pre-armed.
   * Returned WRAPPED ({ settled }) — a bare returned promise would be
   * flattened by the caller's await, blocking on turn/end before the caller
   * can act mid-turn (the cancel scenario's whole point).
   */
  async function sendPrompt(timeoutMs?: number): Promise<{ settled: ReturnType<WebScaffold['whenTurnSettled']> }> {
    const input = page.locator('textarea').first()
    await input.waitFor({ timeout: 10_000 })
    const settled = scaffold!.whenTurnSettled(timeoutMs)
    await input.fill(PROMPT)
    await input.press('Enter')
    return { settled }
  }

  it.skipIf(MODE !== 'record')('records the base fixture live through the composer', async () => {
    await launch()
    onTestFailed(() => saveFailureShot(page, 'web-e2e-interactions-record'))
    const { settled } = await sendPrompt(180_000)
    const sessionId = await settled
    await recordFixture(scaffold!, sessionId, FIXTURE)
  }, 200_000)

  it.skipIf(MODE === 'record')('cancels a hung stream deterministically via the readyFile marker', async () => {
    expect(fixtureUserPrompts(await readFile(FIXTURE, 'utf8'))).toEqual([PROMPT])
    let marker = ''
    await launch((sidecarHome) => {
      marker = join(sidecarHome, '.hang-ready')
      return { patches: [{ at: 0, entry: { kind: 'hang', readyFile: marker } }] }
    })
    onTestFailed(() => saveFailureShot(page, 'web-e2e-cancel'))
    const { settled } = await sendPrompt()
    // The marker IS the synchronization: the stream is provably parked in the
    // hang (prefix chunks delivered to the loop) before the stop click.
    await expect.poll(() => existsSync(marker), { timeout: 15_000 }).toBe(true)
    await expect.poll(
      () => page.getByRole('status').filter({ hasText: 'Deep diving...' }).isVisible(),
      { timeout: 10_000 },
    ).toBe(true)
    const loadingSnapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd)
    await compareOrRefreshGolden(LOADING_EXPECTED, loadingSnapshot, MODE)
    await page.getByRole('button', { name: 'Stop generating' }).click()
    await settled
    expect(turnEndReasons(sessionEvents).at(-1)).toBe('aborted')
    // Composer recovered; no streaming node lingers. The host settled first
    // (awaited above), but the abort frame reaches the browser over SSE — the
    // frozen-partial swap is eventually consistent, so poll rather than count.
    await expect.poll(() => page.locator('textarea').first().isEnabled(), { timeout: 10_000 }).toBe(true)
    await expect.poll(() => page.locator('[data-streaming="true"]').count(), { timeout: 10_000 }).toBe(0)
    // Golden of the aborted end-state: the prompt bubble plus the frozen
    // partial ('partial' is the hang entry's replayed prefix) and no more.
    const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd)
    await compareOrRefreshGolden(CANCEL_EXPECTED, snapshot, MODE)
    expect(tripwire.pageErrors).toEqual([])
    expect(tripwire.warnings).toEqual([])
  }, 120_000)

  it.skipIf(MODE === 'record')('surfaces a non-retryable AUTH failure without retrying', async () => {
    await launch(() => ({
      patches: [{ at: 0, entry: { kind: 'throw', chunks: [], message: AUTH_PROVIDER_MESSAGE, code: 'AUTH' } }],
    }))
    onTestFailed(() => saveFailureShot(page, 'web-e2e-error-auth'))
    const { settled } = await sendPrompt()
    await settled
    expect(turnEndReasons(sessionEvents).at(-1)).toBe('error')
    // AUTH is outside llm-retry's retryable set: no retry record.
    expect(sessionEvents.filter(e => e.type === 'llm/retry').length).toBe(0)
    await expect.poll(() => page.locator('textarea').first().isEnabled(), { timeout: 10_000 }).toBe(true)
    expect(await page.locator('[data-streaming="true"]').count()).toBe(0)
    const errorStatus = page.getByRole('status').filter({ hasText: 'This turn failed' })
    await errorStatus.waitFor({ timeout: 10_000 })
    expect(await errorStatus.textContent()).toContain('API key is invalid')
    expect(await errorStatus.textContent()).toContain('AUTH')
    expect(await page.locator('body').textContent()).not.toContain('sk-preview-secret')
    const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd)
    await compareOrRefreshGolden(ERROR_EXPECTED, snapshot, MODE)
    await page.getByRole('tab', { name: 'Trajectory' }).click()
    const requestMarker = page.locator('tr[data-request-only="true"]').last()
      .getByRole('button', { name: /Request #/ })
    await requestMarker.click()
    await page.getByText('API key is invalid', { exact: true }).waitFor({ timeout: 10_000 })
    expect(await page.locator('body').textContent()).not.toContain('sk-preview-secret')
    expect(tripwire.pageErrors).toEqual([])
    expect(tripwire.warnings).toEqual([])
  }, 120_000)

  it.skipIf(MODE === 'record')('keeps a terminal request marker inside the trajectory table', async () => {
    await launch(() => ({
      patches: [{ at: 0, entry: { kind: 'throw', chunks: [], message: AUTH_PROVIDER_MESSAGE, code: 'AUTH' } }],
    }))
    const { settled } = await sendPrompt()
    await settled
    await page.getByRole('tab', { name: 'Trajectory' }).click()
    // The boundary marker row itself is a 0-height hairline except at the
    // table tail; the marker button is absolutely positioned and stays
    // visible, so wait on it directly.
    const tailRequest = page.locator('tr[data-request-only="true"]').last()
    const requestMarker = tailRequest.getByRole('button', { name: /Request #/ })
    await requestMarker.waitFor({ timeout: 10_000 })

    const markerWithinTable = await requestMarker.evaluate((element) => {
      const marker = element.getBoundingClientRect()
      const table = element.closest('table')?.getBoundingClientRect()
      if (table === undefined) throw new Error('request marker has no table')
      return marker.bottom <= table.bottom
    })

    expect(markerWithinTable).toBe(true)
    expect(tripwire.pageErrors).toEqual([])
    expect(tripwire.warnings).toEqual([])
  }, 120_000)

  it.skipIf(MODE === 'record')('recovers a transient SERVER failure through llm-retry and completes', async () => {
    const derived = deriveReplayScript(parseSessionLog(await readFile(FIXTURE, 'utf8')))
    expect(derived).toHaveLength(1)
    await launch(() => ({
      patches: [
        { at: 0, entry: { kind: 'throw', chunks: [], message: 'upstream 503', code: 'SERVER' } },
        // Append the fixture's own success as the retry attempt — single-
        // sourced from the recording, never copied into a committed sidecar.
        { at: 1, entry: derived[0]! },
      ],
    }))
    onTestFailed(() => saveFailureShot(page, 'web-e2e-retry'))
    // llm-retry backs off ~500ms before the second attempt.
    const { settled } = await sendPrompt(60_000)
    await settled
    expect(turnEndReasons(sessionEvents).at(-1)).toBe('completed')
    // The durable retry record proves the second attempt (request/header logs
    // only on change, so attempt count is invisible there).
    expect(sessionEvents.filter(e => e.type === 'llm/retry').length).toBeGreaterThanOrEqual(1)
    await expect.poll(() => page.getByText('event sourcing', { exact: false }).count(), { timeout: 10_000 }).toBeGreaterThan(0)
    // Golden of the recovered end-state: the discarded partial stays absent,
    // while the settled retry row remains as durable recovery context.
    const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd)
    await compareOrRefreshGolden(RETRY_EXPECTED, snapshot, MODE)
    expect(tripwire.pageErrors).toEqual([])
    expect(tripwire.warnings).toEqual([])
  }, 120_000)

  it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => {
    await assertFixtureInventory(SNAPSHOT_DIR, [
      'session.jsonl', 'cancel.expected.md', 'loading.expected.md', 'error-auth.expected.md', 'retry.expected.md',
    ])
  })
})