smoke-real.e2e.ts 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657
  1. // W5 real-host smoke: spawn `dsh web` with a real key, walk the full W5 flow
  2. // list in a real chromium, screenshot every screen into .artifacts/ for the
  3. // figma comparison pass. Self-skips without DEEPSEEK_API_KEY (repo e2e
  4. // convention); vitest.web.config.ts loads the repo-root .env before this file
  5. // runs (the CLI only auto-loads .env from its cwd — a temp dir here, so
  6. // sessions never land in the repo's .sessions).
  7. //
  8. // Selector convention: CSS Modules hash as [hash]_[local], so class-substring
  9. // selectors are unreliable — anchor on data-* attributes (data-variant /
  10. // data-sample) or visible text. The one [class*=] use below
  11. // (frame/handle) rides local names that survive hashing as suffixes; prefer
  12. // data-* for anything new.
  13. //
  14. // Flow order matters: chat rounds first (5 depends on 3's session), geometry
  15. // and theme after, reload recovery last. Tests run sequentially in-file.
  16. import type { ChildProcess } from 'node:child_process'
  17. import { spawn } from 'node:child_process'
  18. import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
  19. import { createServer } from 'node:http'
  20. import { createRequire } from 'node:module'
  21. import { tmpdir } from 'node:os'
  22. import { join } from 'node:path'
  23. import { fileURLToPath, pathToFileURL } from 'node:url'
  24. import type { Browser, Page } from 'playwright'
  25. import { chromium } from 'playwright'
  26. import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
  27. import { REPO_ROOT, connectFreshWorkspace, newEnglishPage, probeFreePort, requireDist, saveFailureShot } from './support.ts'
  28. const DEVELOPMENT_PROMPT = fileURLToPath(new URL('./snapshots/web-runtime-context/development-prompt.expected.md', import.meta.url))
  29. function waitForReadyLine(child: ChildProcess): Promise<string> {
  30. return new Promise((resolveReady, reject) => {
  31. let out = ''
  32. const timer = setTimeout(() => { reject(new Error(`dsh web not ready in 90s; output:\n${out}`)) }, 90_000)
  33. const onData = (chunk: Buffer): void => {
  34. out += chunk.toString()
  35. const match = /dsh web: (http:\/\/[^\s]+)/.exec(out)
  36. if (match?.[1] !== undefined) {
  37. clearTimeout(timer)
  38. resolveReady(match[1])
  39. }
  40. }
  41. child.stdout?.on('data', onData)
  42. child.stderr?.on('data', onData)
  43. child.once('exit', (code) => {
  44. clearTimeout(timer)
  45. reject(new Error(`dsh web exited early (code ${code}); output:\n${out}`))
  46. })
  47. })
  48. }
  49. async function rpc<T>(baseUrl: string, method: string, payload: unknown): Promise<T> {
  50. const response = await fetch(`${baseUrl}/api/${method}`, {
  51. method: 'POST',
  52. headers: { 'content-type': 'application/json' },
  53. body: JSON.stringify({
  54. type: 'client-request',
  55. rpcId: `smoke-${method}`,
  56. method,
  57. payload,
  58. }),
  59. })
  60. if (!response.ok) throw new Error(`${method} failed over HTTP ${response.status}: ${await response.text()}`)
  61. const body = await response.json() as {
  62. result: { ok: true; value: T } | { ok: false; error: { code: string; message: string } }
  63. }
  64. if (!body.result.ok) throw new Error(`${method} failed: ${body.result.error.code}: ${body.result.error.message}`)
  65. return body.result.value
  66. }
  67. interface HistoryPage {
  68. events: { event: { type: string; data: unknown } }[]
  69. hasMore: boolean
  70. }
  71. function isRecord(value: unknown): value is Record<string, unknown> {
  72. return typeof value === 'object' && value !== null
  73. }
  74. function providerTitle(page: HistoryPage): string | undefined {
  75. for (let index = page.events.length - 1; index >= 0; index--) {
  76. const event = page.events[index]!.event
  77. if (event.type !== 'session/title' || !isRecord(event.data)) continue
  78. const source = event.data.source
  79. if (typeof event.data.title === 'string' && isRecord(source) && source.kind === 'provider') {
  80. return event.data.title
  81. }
  82. }
  83. return undefined
  84. }
  85. function hasAssistantMarker(page: HistoryPage, marker: string): boolean {
  86. return page.events.some(({ event }) => {
  87. if (event.type !== 'assistant/message' || !isRecord(event.data) || !isRecord(event.data.message)) return false
  88. const content = event.data.message.content
  89. if (!Array.isArray(content)) return false
  90. return content.some(block =>
  91. isRecord(block) && block.type === 'text' && typeof block.text === 'string' && block.text.includes(marker))
  92. })
  93. }
  94. async function history(baseUrl: string, sessionId: string): Promise<HistoryPage> {
  95. return rpc<HistoryPage>(baseUrl, 'session.history', { sessionId, maxMessages: 10 })
  96. }
  97. async function waitForProviderTitle(baseUrl: string, sessionId: string): Promise<string> {
  98. let observed: string | undefined
  99. await expect.poll(async () => {
  100. observed = providerTitle(await history(baseUrl, sessionId))
  101. return observed
  102. }, { timeout: 90_000 }).toEqual(expect.any(String))
  103. if (observed === undefined) throw new Error('provider-backed session title was not observed')
  104. return observed
  105. }
  106. async function waitForAssistantMarker(baseUrl: string, sessionId: string, marker: string): Promise<void> {
  107. await expect.poll(async () => hasAssistantMarker(await history(baseUrl, sessionId), marker), {
  108. timeout: 120_000,
  109. }).toBe(true)
  110. }
  111. /** W5 screenshot: evidence for the figma comparison, not a failure artifact. */
  112. async function screen(page: Page, name: string): Promise<void> {
  113. await page.screenshot({ path: join(REPO_ROOT, '.artifacts', `w5-${name}.png`) })
  114. }
  115. /** First column track (px string) of the frame grid. */
  116. async function firstTrack(page: Page): Promise<string> {
  117. return (await page.locator('[class*="frame"]').evaluate(
  118. el => getComputedStyle(el).gridTemplateColumns)).split(' ')[0]!
  119. }
  120. /** Last column track (details) as a number of pixels. */
  121. async function detailsTrack(page: Page): Promise<number> {
  122. const cols = await page.locator('[class*="frame"]').evaluate(
  123. el => getComputedStyle(el).gridTemplateColumns)
  124. return Number(cols.split(' ').pop()!.replace('px', ''))
  125. }
  126. // Readiness gate: `dsh web` serves all ten production manifest plugins; until every UI
  127. // plugin's client bundle exists and exports apply, the loader fail-louds and
  128. // the frame never appears.
  129. const UI_PLUGIN_DIRS = [
  130. 'connection', 'runtime', 'ui-theme', 'locale', 'ui-layout', 'ui-sidebar',
  131. 'ui-settings', 'ui-settings-general', 'ui-models', 'ui-conversation',
  132. 'ui-model', 'ui-question', 'ui-trajectory',
  133. ]
  134. const ROUND_DONE_MARKER = 'WEB_ROUND_DONE'
  135. const notReady = UI_PLUGIN_DIRS.filter((dir) => {
  136. const bundle = join(REPO_ROOT, 'packages/client', dir, 'lib/client.js')
  137. return !existsSync(bundle) || !readFileSync(bundle, 'utf8').includes('exports.apply')
  138. })
  139. if (notReady.length > 0) console.warn(`[smoke-real] skipped — client bundles not ready: ${notReady.join(', ')}`)
  140. describe('dsh web keyless CLI smoke', () => {
  141. it('listens on 127.0.0.1 by default', async () => {
  142. requireDist()
  143. const sessionsDir = mkdtempSync(join(tmpdir(), 'dsh-web-keyless-'))
  144. const tsxLoader = pathToFileURL(createRequire(join(REPO_ROOT, 'package.json')).resolve('tsx')).href
  145. const child = spawn(
  146. process.execPath,
  147. ['--import', tsxLoader, join(REPO_ROOT, 'apps/cli/src/bin.ts'), 'web', '--port', '0'],
  148. {
  149. cwd: sessionsDir,
  150. env: {
  151. ...process.env,
  152. DEEPSEEK_API_KEY: 'keyless-web-no-call',
  153. DSH_HOME: join(sessionsDir, '.dsh'),
  154. DSH_AGENTS_HOME: join(sessionsDir, '.agents'),
  155. TSX_TSCONFIG_PATH: join(REPO_ROOT, 'tsconfig.json'),
  156. },
  157. stdio: ['ignore', 'pipe', 'pipe'],
  158. },
  159. )
  160. try {
  161. const readyUrl = await waitForReadyLine(child)
  162. expect(readyUrl).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/)
  163. expect((await fetch(readyUrl)).status).toBe(200)
  164. } finally {
  165. const closed = child.exitCode === null
  166. ? new Promise<void>((resolveClose) => { child.once('close', () => { resolveClose() }) })
  167. : Promise.resolve()
  168. if (child.exitCode === null) child.kill('SIGTERM')
  169. await closed
  170. rmSync(sessionsDir, { recursive: true, force: true })
  171. }
  172. })
  173. it('routes --dev runtime context and workspace instructions through the real CLI request', async () => {
  174. requireDist()
  175. const workspace = mkdtempSync(join(tmpdir(), 'dsh-web-workspace-'))
  176. mkdirSync(join(workspace, '.git'))
  177. writeFileSync(join(workspace, 'AGENTS.md'), 'web-workspace-context-probe\n')
  178. interface NativeProviderRequest {
  179. messages?: { role?: string; content?: string }[]
  180. tools?: { function?: { name?: string } }[]
  181. }
  182. let resolveProviderRequests!: (requests: NativeProviderRequest[]) => void
  183. const requests: NativeProviderRequest[] = []
  184. const providerRequests = new Promise<NativeProviderRequest[]>((resolve) => {
  185. resolveProviderRequests = resolve
  186. })
  187. const provider = createServer((request, response) => {
  188. let body = ''
  189. request.setEncoding('utf8')
  190. request.on('data', (chunk: string) => { body += chunk })
  191. request.on('end', () => {
  192. const parsed = JSON.parse(body) as NativeProviderRequest
  193. if ((parsed.tools?.length ?? 0) > 0) requests.push(parsed)
  194. if (requests.length === 1) resolveProviderRequests(requests)
  195. response.writeHead(200, { 'content-type': 'text/event-stream' })
  196. response.end([
  197. 'data: {"choices":[{"delta":{"role":"assistant","content":null,"reasoning_content":""}}]}',
  198. 'data: {"choices":[{"delta":{"content":"done"}}]}',
  199. 'data: {"choices":[{"delta":{"content":""},"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":1}}',
  200. 'data: [DONE]',
  201. '',
  202. ].join('\n\n'))
  203. })
  204. })
  205. await new Promise<void>(resolve => provider.listen(0, '127.0.0.1', resolve))
  206. const address = provider.address()
  207. if (address === null || typeof address === 'string') throw new Error('mock provider did not bind a TCP port')
  208. const tsxLoader = pathToFileURL(createRequire(join(REPO_ROOT, 'package.json')).resolve('tsx')).href
  209. const child = spawn(
  210. process.execPath,
  211. ['--import', tsxLoader, join(REPO_ROOT, 'apps/cli/src/bin.ts'), 'web', '--port', '0', '--dev'],
  212. {
  213. cwd: workspace,
  214. env: {
  215. ...process.env,
  216. DEEPSEEK_API_KEY: 'keyless-web-workspace',
  217. DEEPSEEK_BASE_URL: `http://127.0.0.1:${address.port}`,
  218. DSH_HOME: join(workspace, '.dsh'),
  219. DSH_AGENTS_HOME: join(workspace, '.agents'),
  220. TSX_TSCONFIG_PATH: join(REPO_ROOT, 'tsconfig.json'),
  221. },
  222. stdio: ['ignore', 'pipe', 'pipe'],
  223. },
  224. )
  225. try {
  226. const baseUrl = await waitForReadyLine(child)
  227. const created = await rpc<{ sessionId: string }>(baseUrl, 'session.create', {})
  228. await rpc<{ accepted: true }>(baseUrl, 'session.prompt', {
  229. sessionId: created.sessionId,
  230. mode: 'queue',
  231. content: [{ type: 'text', text: 'go' }],
  232. })
  233. const capturedRequests = await Promise.race([
  234. providerRequests,
  235. new Promise<never>((_resolve, reject) => {
  236. setTimeout(() => { reject(new Error('provider request not received in 10s')) }, 10_000).unref()
  237. }),
  238. ])
  239. const captured = capturedRequests[0]
  240. if (captured === undefined) {
  241. throw new Error('provider did not receive the workspace projection request')
  242. }
  243. const workspaceMessage = captured.messages?.find(message =>
  244. message.role === 'user' && message.content?.includes('web-workspace-context-probe'))
  245. const systemMessage = captured.messages?.find(message => message.role === 'system')
  246. const expectedWebSection = readFileSync(DEVELOPMENT_PROMPT, 'utf8').trimEnd()
  247. .replace('{{webUrl}}', baseUrl)
  248. expect(systemMessage?.content).toContain(expectedWebSection)
  249. expect(workspaceMessage).toMatchInlineSnapshot(`
  250. {
  251. "content": "<system-reminder>
  252. The following workspace instructions may be relevant to your work. Use them as guidance when applicable. More specific instructions take precedence over broader ones. They do not override system, developer, or direct user instructions.
  253. Instructions from: AGENTS.md
  254. web-workspace-context-probe
  255. </system-reminder>",
  256. "role": "user",
  257. }
  258. `)
  259. expect(captured.tools?.map(tool => tool.function?.name)
  260. .filter(name => name === 'web_search' || name === 'web_fetch'))
  261. .toMatchInlineSnapshot(`
  262. [
  263. "web_search",
  264. ]
  265. `)
  266. } finally {
  267. const closed = child.exitCode === null
  268. ? new Promise<void>((resolveClose) => { child.once('close', () => { resolveClose() }) })
  269. : Promise.resolve()
  270. if (child.exitCode === null) child.kill('SIGTERM')
  271. await closed
  272. await new Promise<void>(resolveClose => provider.close(() => { resolveClose() }))
  273. rmSync(workspace, { recursive: true, force: true })
  274. }
  275. })
  276. it('retries a partial transport failure through the shipped Web composition', async () => {
  277. requireDist()
  278. const workspace = mkdtempSync(join(tmpdir(), 'dsh-web-retry-'))
  279. const promptMarker = 'WEB_RETRY_REQUEST'
  280. const recoveredMarker = 'WEB_RETRY_RECOVERED'
  281. let mainAttempts = 0
  282. const provider = createServer((request, response) => {
  283. let body = ''
  284. request.setEncoding('utf8')
  285. request.on('data', (chunk: string) => { body += chunk })
  286. request.on('end', () => {
  287. const parsed = JSON.parse(body) as { max_tokens?: number; messages?: unknown[] }
  288. const titleRequest = parsed.max_tokens === 64
  289. const mainRequest = !titleRequest && body.includes(promptMarker)
  290. response.writeHead(200, { 'content-type': 'text/event-stream' })
  291. if (!mainRequest) {
  292. response.end([
  293. 'data: {"choices":[{"delta":{"content":"Web retry title"}}]}',
  294. 'data: {"choices":[{"delta":{"content":""},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}',
  295. 'data: [DONE]',
  296. '',
  297. ].join('\n\n'))
  298. return
  299. }
  300. mainAttempts++
  301. if (mainAttempts === 1) {
  302. response.write('data: {"choices":[{"delta":{"content":"WEB_RETRY_DISCARDED"}}]}\n\n')
  303. setTimeout(() => { response.destroy() }, 20)
  304. return
  305. }
  306. response.end([
  307. `data: {"choices":[{"delta":{"content":"${recoveredMarker}"}}]}`,
  308. 'data: {"choices":[{"delta":{"content":""},"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":1}}',
  309. 'data: [DONE]',
  310. '',
  311. ].join('\n\n'))
  312. })
  313. })
  314. await new Promise<void>(resolve => provider.listen(0, '127.0.0.1', resolve))
  315. const address = provider.address()
  316. if (address === null || typeof address === 'string') throw new Error('mock provider did not bind a TCP port')
  317. const tsxLoader = pathToFileURL(createRequire(join(REPO_ROOT, 'package.json')).resolve('tsx')).href
  318. const child = spawn(
  319. process.execPath,
  320. ['--import', tsxLoader, join(REPO_ROOT, 'apps/cli/src/bin.ts'), 'web', '--port', '0'],
  321. {
  322. cwd: workspace,
  323. env: {
  324. ...process.env,
  325. DEEPSEEK_API_KEY: 'keyless-web-retry',
  326. DEEPSEEK_BASE_URL: `http://127.0.0.1:${address.port}`,
  327. DSH_HOME: join(workspace, '.dsh'),
  328. TSX_TSCONFIG_PATH: join(REPO_ROOT, 'tsconfig.json'),
  329. },
  330. stdio: ['ignore', 'pipe', 'pipe'],
  331. },
  332. )
  333. try {
  334. const baseUrl = await waitForReadyLine(child)
  335. const created = await rpc<{ sessionId: string }>(baseUrl, 'session.create', {})
  336. await rpc<{ accepted: true }>(baseUrl, 'session.prompt', {
  337. sessionId: created.sessionId,
  338. mode: 'queue',
  339. content: [{ type: 'text', text: promptMarker }],
  340. })
  341. let page: HistoryPage | undefined
  342. await expect.poll(async () => {
  343. page = await history(baseUrl, created.sessionId)
  344. return hasAssistantMarker(page, recoveredMarker)
  345. }, { timeout: 20_000 }).toBe(true)
  346. if (page === undefined) throw new Error('retry history was not observed')
  347. const retry = page.events.find(({ event }) => event.type === 'llm/retry')?.event
  348. expect(mainAttempts).toBe(2)
  349. expect(retry?.data).toMatchObject({
  350. turn: 1,
  351. step: 1,
  352. retry: 1,
  353. maxRetries: 2,
  354. failure: { code: 'TRANSPORT' },
  355. })
  356. expect(JSON.stringify(page.events)).toContain('WEB_RETRY_DISCARDED')
  357. } finally {
  358. const closed = child.exitCode === null
  359. ? new Promise<void>((resolveClose) => { child.once('close', () => { resolveClose() }) })
  360. : Promise.resolve()
  361. if (child.exitCode === null) child.kill('SIGTERM')
  362. await closed
  363. await new Promise<void>(resolveClose => provider.close(() => { resolveClose() }))
  364. rmSync(workspace, { recursive: true, force: true })
  365. }
  366. }, 30_000)
  367. it('DSH_TOOLS_MODE=code collapses the provider wire tools to run_code with the SDK prompt section', async () => {
  368. requireDist()
  369. const workspace = mkdtempSync(join(tmpdir(), 'dsh-web-code-mode-'))
  370. interface CodeModeProviderRequest {
  371. messages?: { role?: string; content?: string }[]
  372. tools?: { function?: { name?: string } }[]
  373. }
  374. let resolveProviderRequest!: (request: CodeModeProviderRequest) => void
  375. const providerRequest = new Promise<CodeModeProviderRequest>((resolve) => {
  376. resolveProviderRequest = resolve
  377. })
  378. const provider = createServer((request, response) => {
  379. let body = ''
  380. request.setEncoding('utf8')
  381. request.on('data', (chunk: string) => { body += chunk })
  382. request.on('end', () => {
  383. resolveProviderRequest(JSON.parse(body) as CodeModeProviderRequest)
  384. response.writeHead(200, { 'content-type': 'text/event-stream' })
  385. response.end([
  386. 'data: {"choices":[{"delta":{"role":"assistant","content":null,"reasoning_content":""}}]}',
  387. 'data: {"choices":[{"delta":{"content":"done"}}]}',
  388. 'data: {"choices":[{"delta":{"content":""},"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":1}}',
  389. 'data: [DONE]',
  390. '',
  391. ].join('\n\n'))
  392. })
  393. })
  394. await new Promise<void>(resolve => provider.listen(0, '127.0.0.1', resolve))
  395. const address = provider.address()
  396. if (address === null || typeof address === 'string') throw new Error('mock provider did not bind a TCP port')
  397. const tsxLoader = pathToFileURL(createRequire(join(REPO_ROOT, 'package.json')).resolve('tsx')).href
  398. const child = spawn(
  399. process.execPath,
  400. ['--import', tsxLoader, join(REPO_ROOT, 'apps/cli/src/bin.ts'), 'web', '--port', '0'],
  401. {
  402. cwd: workspace,
  403. env: {
  404. ...process.env,
  405. DEEPSEEK_API_KEY: 'keyless-web-code-mode',
  406. DEEPSEEK_BASE_URL: `http://127.0.0.1:${address.port}`,
  407. DSH_TOOLS_MODE: 'code',
  408. DSH_HOME: join(workspace, '.dsh'),
  409. DSH_AGENTS_HOME: join(workspace, '.agents'),
  410. TSX_TSCONFIG_PATH: join(REPO_ROOT, 'tsconfig.json'),
  411. },
  412. stdio: ['ignore', 'pipe', 'pipe'],
  413. },
  414. )
  415. try {
  416. const baseUrl = await waitForReadyLine(child)
  417. const created = await rpc<{ sessionId: string }>(baseUrl, 'session.create', {})
  418. await rpc<{ accepted: true }>(baseUrl, 'session.prompt', {
  419. sessionId: created.sessionId,
  420. mode: 'queue',
  421. content: [{ type: 'text', text: 'go' }],
  422. })
  423. const captured = await Promise.race([
  424. providerRequest,
  425. new Promise<never>((_resolve, reject) => {
  426. setTimeout(() => { reject(new Error('provider request not received in 10s')) }, 10_000).unref()
  427. }),
  428. ])
  429. expect(captured.tools?.map(tool => tool.function?.name)).toEqual(['run_code'])
  430. const system = captured.messages?.find(message => message.role === 'system')
  431. expect(system?.content).toContain('## Writing code for run_code')
  432. expect(system?.content).toContain('declare const tools')
  433. } finally {
  434. const closed = child.exitCode === null
  435. ? new Promise<void>((resolveClose) => { child.once('close', () => { resolveClose() }) })
  436. : Promise.resolve()
  437. if (child.exitCode === null) child.kill('SIGTERM')
  438. await closed
  439. await new Promise<void>(resolveClose => provider.close(() => { resolveClose() }))
  440. rmSync(workspace, { recursive: true, force: true })
  441. }
  442. })
  443. })
  444. describe.skipIf(!process.env.DEEPSEEK_API_KEY || notReady.length > 0)('web smoke (real host, real key, W5)', () => {
  445. let child: ChildProcess
  446. let sessionsDir: string
  447. let baseUrl: string
  448. let browser: Browser
  449. let page: Page
  450. const pageErrors: string[] = []
  451. beforeAll(async () => {
  452. requireDist()
  453. sessionsDir = mkdtempSync(join(tmpdir(), 'dsh-web-w5-'))
  454. const port = await probeFreePort()
  455. // tsx boot mirrors demo:web — lib/ may be unbuilt in this worktree. Isolate
  456. // the host-level Harness and shared-agent homes inside the temp world; tsx
  457. // also needs the repo's loader and tsconfig paths pointed at explicitly.
  458. const tsxLoader = pathToFileURL(createRequire(join(REPO_ROOT, 'package.json')).resolve('tsx')).href
  459. child = spawn(
  460. process.execPath,
  461. [
  462. '--import', tsxLoader, join(REPO_ROOT, 'apps/cli/src/bin.ts'), 'web', '--port', String(port),
  463. // Pin the in-browser picker: the shipped `-auto` row would resolve to
  464. // the native OS chooser on this bind, and no page can drive that.
  465. '--config', fileURLToPath(new URL('./pin-browse-picker.overlay.yml', import.meta.url)),
  466. ],
  467. {
  468. cwd: sessionsDir,
  469. env: {
  470. ...process.env,
  471. DSH_HOME: join(sessionsDir, '.dsh'),
  472. DSH_AGENTS_HOME: join(sessionsDir, '.agents'),
  473. TSX_TSCONFIG_PATH: join(REPO_ROOT, 'tsconfig.json'),
  474. },
  475. stdio: ['ignore', 'pipe', 'pipe'],
  476. },
  477. )
  478. baseUrl = (await waitForReadyLine(child)).replace('0.0.0.0', '127.0.0.1')
  479. browser = await chromium.launch()
  480. page = await newEnglishPage(browser)
  481. page.on('pageerror', e => pageErrors.push(String(e)))
  482. await page.goto(baseUrl, { waitUntil: 'load' })
  483. }, 120_000)
  484. afterAll(async () => {
  485. await browser?.close()
  486. if (child !== undefined && child.exitCode === null) {
  487. const gone = new Promise<void>(resolveExit => child.once('exit', () => { resolveExit() }))
  488. child.kill('SIGTERM')
  489. await Promise.race([gone, new Promise(r => setTimeout(r, 10_000).unref())])
  490. if (child.exitCode === null) child.kill('SIGKILL')
  491. }
  492. if (sessionsDir !== undefined) rmSync(sessionsDir, { recursive: true, force: true })
  493. })
  494. it('1 cold start: loading page settles into the three-column frame', async () => {
  495. onTestFailed(() => saveFailureShot(page, 'w5-cold-start'))
  496. await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
  497. expect(await page.locator('text=Failed to load plugins').count()).toBe(0)
  498. const template = await page.locator('[class*="frame"]').evaluate(el => getComputedStyle(el).gridTemplateColumns)
  499. expect(template.split(' ').length).toBe(3)
  500. await screen(page, '01-cold-start')
  501. })
  502. it('2+3 empty-state first send completes a real model round', async () => {
  503. onTestFailed(() => saveFailureShot(page, 'w5-first-round'))
  504. // This scenario spawns its own server against a fresh $DSH_HOME, so the
  505. // first-run welcome notice is unacknowledged and its overlay owns pointer
  506. // events (the shared scaffold acknowledges it before boot instead). The
  507. // notice is anchored structurally, not by its copy: this spec sits in the
  508. // client TypeScript program, which does not reference the package that
  509. // owns the strings.
  510. const welcome = page.locator('[class*="onboardingOverlay"]')
  511. await welcome.waitFor({ timeout: 15_000 })
  512. await welcome.getByRole('button').click()
  513. await welcome.waitFor({ state: 'detached', timeout: 15_000 })
  514. // Fresh world: connect a Workspace so the composer starts live.
  515. await connectFreshWorkspace(page, sessionsDir)
  516. const input = page.locator('textarea').first()
  517. await input.waitFor({ timeout: 10_000 })
  518. await screen(page, '02-empty-state')
  519. const prompt = `Please answer this request carefully: explain event sourcing in two sentences, ending with exactly ${ROUND_DONE_MARKER}.`
  520. await input.fill(prompt)
  521. await input.press('Enter')
  522. // The first send must keep the session tree mounted; a near-empty body
  523. // reveals a duplicate runtime bundle with incompatible scope tags.
  524. await page.waitForFunction(() => document.body.innerText.length > 50, undefined, { timeout: 15_000 })
  525. expect(pageErrors).toEqual([])
  526. await page.waitForFunction(
  527. () => document.title !== 'DeepSeek Harness' && document.title.endsWith(' — DeepSeek Harness'),
  528. undefined,
  529. { timeout: 15_000 },
  530. )
  531. await expect.poll(async () => (await rpc<{ items: { sessionId: string }[] }>(baseUrl, 'session.list', {})).items.length, {
  532. timeout: 15_000,
  533. }).toBe(1)
  534. const sessions = await rpc<{ items: { sessionId: string }[] }>(baseUrl, 'session.list', {})
  535. const sessionId = sessions.items[0]?.sessionId
  536. if (sessionId === undefined) throw new Error('created Web session was not listed')
  537. const durableTitle = await waitForProviderTitle(baseUrl, sessionId)
  538. await page.waitForFunction(
  539. expected => document.title === `${expected} — DeepSeek Harness`,
  540. durableTitle,
  541. { timeout: 15_000 },
  542. )
  543. const sessionTree = page.getByRole('tree', { name: 'Sessions' })
  544. const projectRow = sessionTree.getByRole('treeitem').first()
  545. if (await projectRow.getAttribute('aria-expanded') === 'false') await projectRow.click()
  546. await Promise.all([
  547. sessionTree.getByText(durableTitle, { exact: true }).waitFor({ timeout: 10_000 }),
  548. page.getByRole('navigation').getByText(durableTitle, { exact: true }).waitFor({ timeout: 10_000 }),
  549. ])
  550. await waitForAssistantMarker(baseUrl, sessionId, ROUND_DONE_MARKER)
  551. await page.locator('p').filter({ hasText: ROUND_DONE_MARKER }).waitFor({ timeout: 10_000 })
  552. await screen(page, '04-round-complete')
  553. }, 150_000)
  554. it('view tabs: Chat and Trajectory switch', async () => {
  555. onTestFailed(() => saveFailureShot(page, 'w5-tabs'))
  556. await page.locator('button', { hasText: /Trajectory/i }).first().click()
  557. await screen(page, '05-trajectory-tab')
  558. await page.getByLabel('Trajectory timeline').waitFor()
  559. await expect.poll(() => page.getByRole('tab', { name: 'Waterfall' }).count()).toBe(0)
  560. await page.locator('button', { hasText: /^Chat$/i }).first().click()
  561. await screen(page, '07-back-to-chat')
  562. })
  563. it('5 bash differential rendering: tool row click leaves the default details column closed', async () => {
  564. onTestFailed(() => saveFailureShot(page, 'w5-tool-details'))
  565. const input = page.locator('textarea').first()
  566. await input.fill('请用 bash 工具运行命令 echo w5marker 然后告诉我结果')
  567. await input.press('Enter')
  568. // Wait for the tool ROW, not response text (the reply echoes any marker).
  569. // Bash renders through the third-party sample registration. Match that
  570. // exact row: other clickable variants (for example Think disclosure)
  571. // may precede the tool call in document order.
  572. const toolRow = page.locator('[data-sample="bash"]')
  573. await toolRow.waitFor({ timeout: 120_000 })
  574. await screen(page, '08-bash-round')
  575. expect(await detailsTrack(page)).toBe(0)
  576. await toolRow.click()
  577. // Tool rows no longer drive layout.openDetails; the default column stays closed.
  578. expect(await detailsTrack(page)).toBe(0)
  579. await screen(page, '09-details-closed')
  580. }, 150_000)
  581. it('6 sidebar drag widens the column and resets across reload', async () => {
  582. onTestFailed(() => saveFailureShot(page, 'w5-drag'))
  583. const before = await firstTrack(page)
  584. const handle = page.locator('[class*="handle"]').first()
  585. const box = await handle.boundingBox()
  586. expect(box).not.toBeNull()
  587. await page.mouse.move(box!.x + box!.width / 2, box!.y + 300)
  588. await page.mouse.down()
  589. await page.mouse.move(box!.x + 70, box!.y + 300, { steps: 6 })
  590. await page.mouse.up()
  591. const after = await firstTrack(page)
  592. expect(after).not.toBe(before)
  593. await screen(page, '10-sidebar-dragged')
  594. await page.reload({ waitUntil: 'load' })
  595. await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
  596. expect(await firstTrack(page)).toBe(before)
  597. })
  598. it('7 dark mode: the body attribute cascades the token sheets', async () => {
  599. onTestFailed(() => saveFailureShot(page, 'w5-dark'))
  600. // theme.apply === toggling this attribute (v3 §8); no switcher UI owns it
  601. // in P-I, so the acceptance drives the documented mechanism directly.
  602. const dark = await page.evaluate(() => {
  603. document.body.setAttribute('data-ds-dark-theme', '')
  604. return getComputedStyle(document.body).backgroundColor
  605. })
  606. await screen(page, '11-dark-mode')
  607. const light = await page.evaluate(() => {
  608. document.body.removeAttribute('data-ds-dark-theme')
  609. return getComputedStyle(document.body).backgroundColor
  610. })
  611. expect(dark).not.toBe(light)
  612. })
  613. it('8 reload recovery: history replays after a fresh boot', async () => {
  614. onTestFailed(() => saveFailureShot(page, 'w5-reload'))
  615. await page.reload({ waitUntil: 'load' })
  616. await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
  617. await page.locator('p').filter({ hasText: ROUND_DONE_MARKER }).waitFor({ timeout: 30_000 })
  618. await screen(page, '12-reload-recovery')
  619. })
  620. it('stayed clean: no page errors across every flow', () => {
  621. expect(pageErrors).toEqual([])
  622. })
  623. })