ci-workflow.spec.ts 56 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057
  1. import { readFileSync } from 'node:fs'
  2. import { resolve } from 'node:path'
  3. import * as yaml from 'js-yaml'
  4. import { describe, expect, it } from 'vitest'
  5. const root = resolve(import.meta.dirname, '..')
  6. const runnerPrivatePnpmDestination = /^\$\{\{ runner\.temp \}\}\/setup-pnpm-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}$/
  7. const nativeWindowsPnpmDestination = '${{ runner.temp }}/setup-pnpm-js-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}'
  8. describe('CI workflow', () => {
  9. it.each(['ci.yml', 'ci-master.yml', 'e2e.yml', 'release.yml', 'release-vendor.yml'])(
  10. '%s cancels superseded validation runs without crossing workflow or ref boundaries', (name) => {
  11. const workflow = loadWorkflow('.github/workflows/' + name)
  12. expect(workflow.concurrency).toEqual({
  13. group: '${{ github.workflow }}-${{ github.ref }}',
  14. 'cancel-in-progress': true,
  15. })
  16. },
  17. )
  18. it('cancels reusable CI builds without cancelling release-owned builds', () => {
  19. const workflow = loadWorkflow('.github/workflows/build-exe-for-python-sdk.yml')
  20. expect(workflow.concurrency).toEqual({
  21. group: 'build-single-exe-${{ github.workflow }}-${{ github.ref }}',
  22. 'cancel-in-progress': '${{ !inputs.release }}',
  23. })
  24. })
  25. it('does not cancel protected publication or deployment transactions', () => {
  26. for (const name of ['release-publish.yml', 'release-vendor-publish.yml']) {
  27. const publish = workflowJob(loadWorkflow('.github/workflows/' + name), 'publish')
  28. expect(publish.concurrency).toMatchObject({ 'cancel-in-progress': false })
  29. }
  30. for (const name of ['python-release.yml', 'node-addon-system-release.yml', 'docs-pages.yml']) {
  31. expect(loadWorkflow('.github/workflows/' + name).concurrency).toMatchObject({ 'cancel-in-progress': false })
  32. }
  33. })
  34. it('skips coverage-history uploads on cancellation but retains Wine cleanup', () => {
  35. const coverage = workflowJob(loadWorkflow('.github/workflows/ci.yml'), 'windows-coverage')
  36. const wine = workflowJob(loadWorkflow('.github/workflows/ci-master.yml'), 'windows')
  37. expect(coverage.steps).toContainEqual(expect.objectContaining({
  38. name: 'Save coverage duration history', if: '${{ !cancelled() }}',
  39. }))
  40. expect(wine.steps).toContainEqual(expect.objectContaining({ name: 'Shut down wineserver', if: 'always()' }))
  41. })
  42. it('isolates every pnpm action setup destination per runner', () => {
  43. const files = ['.github/workflows/ci.yml', '.github/workflows/ci-master.yml']
  44. const setups: Array<{ jobName: string; step: unknown }> = []
  45. for (const file of files) {
  46. const workflow: unknown = yaml.load(readFileSync(resolve(root, file), 'utf8'))
  47. if (!isRecord(workflow) || !isRecord(workflow.jobs)) throw new TypeError(`${file} must define jobs`)
  48. for (const [jobName, job] of Object.entries(workflow.jobs)) {
  49. if (!isRecord(job) || !Array.isArray(job.steps)) continue
  50. for (const step of job.steps) {
  51. if (!isRecord(step) || typeof step.uses !== 'string' || !step.uses.startsWith('pnpm/action-setup@')) continue
  52. setups.push({ jobName, step })
  53. }
  54. }
  55. }
  56. expect(setups.length).toBeGreaterThan(0)
  57. for (const { jobName, step } of setups) {
  58. const stepDest = (step as { with?: { dest?: unknown } }).with?.dest
  59. if (jobName.startsWith('windows-')) {
  60. expect(stepDest, `${jobName} must use the native Windows pnpm destination`).toBe(nativeWindowsPnpmDestination)
  61. expect(step).not.toMatchObject({ with: { standalone: true } })
  62. } else {
  63. expect(typeof stepDest, `${jobName} must use a runner-and-run-private pnpm destination`).toBe('string')
  64. expect(stepDest as string).toMatch(runnerPrivatePnpmDestination)
  65. }
  66. }
  67. })
  68. it.each(['node-24', 'node-24-coverage', 'node-24-consumers'])(
  69. '%s keeps tool and fixture temporary files under runner cleanup',
  70. (jobName) => {
  71. const job = workflowJob(loadWorkflow('.github/workflows/ci.yml'), jobName)
  72. if (!Array.isArray(job.steps)) throw new TypeError(`${jobName} must define steps`)
  73. expect(job.steps[0]).toEqual({
  74. name: 'Use runner-owned temporary storage',
  75. run: [
  76. 'echo "TMPDIR=${{ runner.temp }}" >> "$GITHUB_ENV"',
  77. ...(jobName === 'node-24-consumers'
  78. ? ['echo "PLAYWRIGHT_BROWSERS_PATH=${RUNNER_TEMP%/*}/ms-playwright" >> "$GITHUB_ENV"']
  79. : []),
  80. '',
  81. ].join('\n'),
  82. })
  83. if (jobName === 'node-24-consumers') {
  84. const browserCache: unknown = job.steps.find(step => isRecord(step) && isRecord(step.with)
  85. && step.with.path === '${{ env.PLAYWRIGHT_BROWSERS_PATH }}')
  86. expect(browserCache).toMatchObject({ uses: 'actions/cache/restore@v4' })
  87. }
  88. const store: unknown = job.steps.find(step => isRecord(step) && step.name === 'Configure pnpm store path')
  89. expect(store).toMatchObject({
  90. run: [
  91. 'store_root="$HOME/.local/share/pnpm/store"',
  92. 'echo "PNPM_CONFIG_STORE_DIR=$store_root" >> "$GITHUB_ENV"',
  93. 'store_path=$(PNPM_CONFIG_STORE_DIR="$store_root" pnpm store path --silent)',
  94. 'echo "path=$store_path" >> "$GITHUB_OUTPUT"',
  95. '',
  96. ].join('\n'),
  97. })
  98. for (const step of job.steps) {
  99. if (isRecord(step) && isRecord(step.env)) {
  100. expect(step.env.TMPDIR).toBeUndefined()
  101. expect(step.env.npm_config_cache).toBeUndefined()
  102. }
  103. }
  104. },
  105. )
  106. it('isolates the python SDK exe pnpm setup destination per job', () => {
  107. const workflow: unknown = yaml.load(readFileSync(resolve(root, '.github/workflows/build-exe-for-python-sdk.yml'), 'utf8'))
  108. if (!isRecord(workflow) || !isRecord(workflow.jobs)) throw new TypeError('build-exe-for-python-sdk.yml must define jobs')
  109. const setups: Array<{ step: unknown }> = []
  110. for (const job of Object.values(workflow.jobs)) {
  111. if (!isRecord(job) || !Array.isArray(job.steps)) continue
  112. for (const step of job.steps) {
  113. if (!isRecord(step) || typeof step.uses !== 'string' || !step.uses.startsWith('pnpm/action-setup@')) continue
  114. setups.push({ step })
  115. }
  116. }
  117. expect(setups.length).toBeGreaterThan(0)
  118. for (const { step } of setups) {
  119. expect(step).toMatchObject({
  120. with: { dest: nativeWindowsPnpmDestination },
  121. })
  122. }
  123. })
  124. it('keeps split native Windows PR jobs with failover, plus a master-only standby', () => {
  125. const workflow = loadWorkflow('.github/workflows/ci.yml')
  126. const masterWorkflow = loadWorkflow('.github/workflows/ci-master.yml')
  127. if (!isRecord(workflow.jobs)
  128. || !isRecord(workflow.jobs['windows-build'])
  129. || !isRecord(workflow.jobs['windows-coverage'])
  130. || !isRecord(workflow.jobs['windows-native-tests'])
  131. || !isRecord(workflow.jobs['windows-observational'])
  132. || !isRecord(workflow.jobs['node-24'])
  133. || !isRecord(workflow.jobs['node-24-coverage'])
  134. || !isRecord(workflow.jobs['node-24-bench'])
  135. || !isRecord(workflow.jobs['node-24-consumers'])
  136. || !isRecord(workflow.jobs['node-compat'])
  137. || !isRecord(workflow.jobs['all-checks-passed'])
  138. || !isRecord(masterWorkflow.jobs)
  139. || !isRecord(masterWorkflow.jobs['serial-windows'])) {
  140. throw new TypeError('CI workflow must define windows-build, windows-coverage, windows-native-tests, windows-observational, node-24, node-24-coverage, node-24-bench, node-24-consumers, node-compat, and all-checks-passed; ci-master must define serial-windows')
  141. }
  142. const windowsBuild = workflow.jobs['windows-build']
  143. const windowsCoverage = workflow.jobs['windows-coverage']
  144. const windowsNativeTests = workflow.jobs['windows-native-tests']
  145. const windowsObservational = workflow.jobs['windows-observational']
  146. const serialWindows = masterWorkflow.jobs['serial-windows']
  147. const node24 = workflow.jobs['node-24']
  148. const node24Coverage = workflow.jobs['node-24-coverage']
  149. const node24Bench = workflow.jobs['node-24-bench']
  150. const node24Consumers = workflow.jobs['node-24-consumers']
  151. const nodeCompat = workflow.jobs['node-compat']
  152. const aggregate = workflow.jobs['all-checks-passed']
  153. if (!Array.isArray(aggregate.needs)) {
  154. throw new TypeError('CI aggregate must define needs')
  155. }
  156. // The split native jobs all resolve their pool through the Windows switch.
  157. for (const [jobName, job] of [['windows-build', windowsBuild], ['windows-coverage', windowsCoverage], ['windows-native-tests', windowsNativeTests], ['windows-observational', windowsObservational]] as const) {
  158. expect(typeof job['runs-on']).toBe('string')
  159. expect(job['runs-on'], `${jobName} runs-on must use the Windows failover switch`).toContain('DSH_CI_FAILOVER_WINDOWS')
  160. expect(job['runs-on'], `${jobName} runs-on must not use the Linux failover switch`).not.toContain('DSH_CI_FAILOVER_LINUX')
  161. expect(job['runs-on']).toContain('self-hosted')
  162. expect(job['runs-on']).toContain('dsh-win-ci')
  163. expect(job['runs-on']).toContain('dsh-windows-2025-16core')
  164. expect(job.if).toBe("github.event_name == 'pull_request'")
  165. }
  166. // windows-build runs the blocking build/site pair.
  167. expect(windowsBuild.name).toBe('windows node 24 / build')
  168. const buildSteps = windowsBuild.steps as unknown[]
  169. const buildCommands = buildSteps.filter((step): step is Record<string, unknown> & { run: string } => (
  170. isRecord(step) && typeof step.run === 'string'
  171. ))
  172. expect(buildCommands.map(step => step.run)).toContain('pnpm run check:ci:windows-blocking')
  173. // The four native Windows installs branch on the workspace filesystem:
  174. // clone (ReFS block clone) only on ReFS, plain install elsewhere. This
  175. // keeps the TS6231 store-path leak (see the Windows ReFS store note) out
  176. // of the self-hosted pool without forcing clone onto hosted NTFS, which
  177. // rejects copy-on-write. The branch must stay, or a hosted fallback would
  178. // fail installs with ERR_PNPM_LINKING_FAILED.
  179. for (const [jobName, job] of [['windows-build', windowsBuild], ['windows-coverage', windowsCoverage], ['windows-native-tests', windowsNativeTests], ['windows-observational', windowsObservational]] as const) {
  180. const steps = job.steps as unknown[]
  181. const install = steps.find((step): step is Record<string, unknown> & { run: string } => (
  182. isRecord(step) && step.name === 'Install (immutable)' && typeof step.run === 'string'
  183. ))
  184. expect(install, `${jobName} must define the filesystem-branched install`).toBeDefined()
  185. expect(install!.run).toContain("$fs -eq 'ReFS'")
  186. expect(install!.run).toContain('--package-import-method=clone')
  187. expect(install!.run).toContain('corepack pnpm install')
  188. // The else branch must keep the plain hosted install as a distinct line
  189. // (not the corepack clone line, which contains the same substring);
  190. // dropping it or making both branches clone would force clone onto
  191. // NTFS, which rejects copy-on-write (ERR_PNPM_LINKING_FAILED). The
  192. // YAML folded block keeps the first statement on line 1 and folds the
  193. // rest with leading two-space indents.
  194. const installLines = install!.run.split('\n').map(line => line.trim())
  195. expect(installLines).toContain('} else {')
  196. expect(installLines.some(line => line === 'pnpm install --frozen-lockfile'), `${jobName} else branch must keep the plain hosted install`).toBe(true)
  197. // The ReFS branch must not use the interpolated empty-flag form, which
  198. // passes a stray "" positional argument to pnpm.
  199. expect(install!.run).not.toContain('$cloneFlag')
  200. }
  201. // windows-coverage uses the lower 4-partition profile.
  202. expect(windowsCoverage.name).toBe('windows node 24 / coverage')
  203. expect(windowsCoverage.env).toMatchObject({ DSH_COVERAGE_PARTITIONS: '4' })
  204. const coverageSteps = windowsCoverage.steps as unknown[]
  205. const coverageCommands = coverageSteps.filter((step): step is Record<string, unknown> & { run: string } => (
  206. isRecord(step) && typeof step.run === 'string'
  207. ))
  208. expect(coverageCommands.map(step => step.run)).toContain('pnpm run check:ci:coverage')
  209. // Windows coverage runs zero-build like the Linux lane: workspace imports
  210. // resolve to src through the tsconfig paths map, and the lib-consuming
  211. // suites (webworker-packer image-loadable, webworker-runtime
  212. // transform-corpus, client ui-trajectory client-bundle) self-skip on
  213. // unbuilt checkouts. The regex catches a regression spelled as
  214. // 'corepack pnpm run build' or folded into a multi-line run block, which
  215. // an exact string match would miss.
  216. expect(coverageCommands.every(step => !/\bpnpm\s+run\s+build(?:\s|$)/.test(step.run))).toBe(true)
  217. // windows-native-tests runs the Windows-specific specs.
  218. expect(windowsNativeTests.name).toBe('windows node 24 / native tests')
  219. const nativeTestSteps = windowsNativeTests.steps as unknown[]
  220. const nativeTestCommands = nativeTestSteps.filter((step): step is Record<string, unknown> & { run: string } => (
  221. isRecord(step) && typeof step.run === 'string'
  222. ))
  223. const nativeTestCommand = nativeTestCommands.map(step => step.run).join('\n')
  224. expect(nativeTestCommand).toContain('--no-file-parallelism')
  225. expect(nativeTestCommand).toContain('--testTimeout 90000')
  226. expect(nativeTestCommand).toContain('tool-pwsh/tests/loader.spec.ts')
  227. expect(nativeTestCommand).toContain('workflow-worker-thread.spec.ts')
  228. // windows-observational is non-blocking.
  229. expect(windowsObservational.name).toBe('windows node 24 / observational')
  230. expect(windowsObservational['continue-on-error']).toBe(true)
  231. // serial-windows: master-only standby, self-hosted, non-blocking, lives in ci-master.
  232. expect(serialWindows.if).toBe("github.event_name == 'push' && github.ref == 'refs/heads/master'")
  233. expect(serialWindows['runs-on']).toEqual(['self-hosted', 'dsh-win-ci', 'windows'])
  234. expect(serialWindows.name).toBe('serial / windows (self-hosted standby)')
  235. // Its store must share the ReFS workspace volume for clone; the install
  236. // must carry the same filesystem branch as the PR jobs.
  237. const serialSteps = serialWindows.steps as unknown[]
  238. const serialStore = serialSteps.find((step): step is Record<string, unknown> & { run: string } => (
  239. isRecord(step) && step.name === 'Configure persistent pnpm store' && typeof step.run === 'string'
  240. ))
  241. expect(serialStore).toBeDefined()
  242. expect(serialStore!.run).toContain('F:\\.pnpm-store')
  243. const serialInstall = serialSteps.find((step): step is Record<string, unknown> & { run: string } => (
  244. isRecord(step) && step.name === 'Install (immutable)' && typeof step.run === 'string'
  245. ))
  246. expect(serialInstall).toBeDefined()
  247. expect(serialInstall!.run).toContain("$fs -eq 'ReFS'")
  248. expect(serialInstall!.run).toContain('--package-import-method=clone')
  249. expect(serialInstall!.run).toContain('corepack pnpm install')
  250. // Distinct else-branch line, as for the PR jobs: the corepack clone line
  251. // contains the plain-install substring too.
  252. expect(serialInstall!.run.split('\n').map(line => line.trim())).toContain('} else {')
  253. expect(serialInstall!.run.split('\n').map(line => line.trim())).toContain('pnpm install --frozen-lockfile')
  254. expect(serialInstall!.run).not.toContain('$cloneFlag')
  255. // The unsharded reference runs the whole coverage inventory at the same
  256. // per-test budget the PR coverage lane grants; the default 5000ms times
  257. // out load-sensitive store scans (e.g. gen-third-party-notices).
  258. const serialGate = serialSteps.find((step): step is Record<string, unknown> & { env?: Record<string, unknown> } => (
  259. isRecord(step) && step.name === 'Run complete unsharded Windows gate inventory serially'
  260. ))
  261. expect(serialGate).toBeDefined()
  262. expect(serialGate!.env).toMatchObject({ DSH_COVERAGE_TEST_TIMEOUT_MS: '90000' })
  263. // windows-coverage is temporarily non-blocking while Windows ACP
  264. // half-close tests are stabilized; observational stays out too.
  265. expect(aggregate.needs).not.toContain('windows')
  266. expect(aggregate.needs).toContain('windows-build')
  267. // The benchmark lane is a required verdict input and runs alone so its
  268. // wall-clock budgets never share a runner with a concurrent aggregate.
  269. expect(aggregate.needs).toContain('node-24-bench')
  270. expect(node24Bench.name).toBe('node 24 / benchmarks')
  271. expect(node24Bench.env).toBeUndefined()
  272. expect(node24Bench.steps).toContainEqual({
  273. name: 'Install benchmark browser and hosted dependencies',
  274. run: 'pnpm --filter @deepseek-ai/dsh-benchmarks exec playwright install --with-deps chromium',
  275. })
  276. expect(JSON.stringify(node24Bench.steps)).not.toContain('DSH_CI_FAILOVER_LINUX')
  277. expect(node24Bench.steps).toContainEqual({
  278. name: 'Run performance benchmarks',
  279. env: { DSH_GATE_VERBOSE: '1' },
  280. run: 'pnpm run check:ci:bench',
  281. })
  282. expect(aggregate.needs).not.toContain('windows-coverage')
  283. expect(aggregate.needs).toContain('windows-native-tests')
  284. expect(aggregate.needs).not.toContain('windows-observational')
  285. expect(aggregate.needs).not.toContain('serial-windows')
  286. // Linux failover is a separate switch: the three enterprise Linux workers
  287. // and the verdict job resolve their pool through DSH_CI_FAILOVER_LINUX,
  288. // never the Windows switch.
  289. for (const [jobName, job] of [['node-24', node24], ['node-24-coverage', node24Coverage], ['node-24-consumers', node24Consumers]] as const) {
  290. expect(typeof job['runs-on']).toBe('string')
  291. expect(job['runs-on'], `${jobName} runs-on must use the Linux failover switch`).toContain('DSH_CI_FAILOVER_LINUX')
  292. expect(job['runs-on'], `${jobName} runs-on must not use the Windows failover switch`).not.toContain('DSH_CI_FAILOVER_WINDOWS')
  293. expect(job['runs-on']).toContain('vm-backup')
  294. }
  295. expect(aggregate['runs-on']).toContain('DSH_CI_FAILOVER_LINUX')
  296. expect(aggregate['runs-on']).not.toContain('DSH_CI_FAILOVER_WINDOWS')
  297. expect(aggregate['runs-on']).toContain('vm-backup')
  298. // The run-gates aggregate lanes stop at the first blocking gate failure so
  299. // a red aggregate does not keep burning runner time on the remaining
  300. // gates. Removing the flag silently reverts to running every independent
  301. // gate to completion.
  302. for (const [jobName, job] of [['node-24', node24], ['node-24-coverage', node24Coverage], ['node-24-consumers', node24Consumers], ['node-compat', nodeCompat]] as const) {
  303. expect(job.env, `${jobName} must enable fail-fast`).toMatchObject({ DSH_GATE_FAIL_FAST: '1' })
  304. }
  305. // The native Windows lanes with run-gates aggregates fail fast for the
  306. // same reason: a failing gate aborts the sibling gate instead of waiting
  307. // out the multi-minute instrumented coverage run.
  308. expect(windowsBuild.env, 'windows-build must enable fail-fast').toMatchObject({ DSH_GATE_FAIL_FAST: '1' })
  309. expect(windowsCoverage.env, 'windows-coverage must enable fail-fast').toMatchObject({ DSH_GATE_FAIL_FAST: '1' })
  310. // The observational lane stays complete: it is continue-on-error by design
  311. // and exists to collect as much Windows-native evidence per run as
  312. // possible, so the first failure must not truncate the rest.
  313. expect(windowsObservational.env).toBeDefined()
  314. expect(windowsObservational.env).not.toMatchObject({ DSH_GATE_FAIL_FAST: '1' })
  315. })
  316. it('runs required benchmarks on standard hosted Linux independently of failover', () => {
  317. const workflow = loadWorkflow('.github/workflows/ci.yml')
  318. const benchmark = workflowJob(workflow, 'node-24-bench')
  319. const aggregate = workflowJob(workflow, 'all-checks-passed')
  320. expect(benchmark['runs-on']).toBe('ubuntu-24.04')
  321. expect(benchmark.if).toBe("github.event_name == 'pull_request'")
  322. expect(benchmark.needs).toBeUndefined()
  323. expect(benchmark['continue-on-error']).toBeUndefined()
  324. expect(benchmark.env).toBeUndefined()
  325. expect(aggregate.needs).toContain('node-24-bench')
  326. })
  327. it('always restores the hosted benchmark pnpm cache', () => {
  328. const benchmark = workflowJob(loadWorkflow('.github/workflows/ci.yml'), 'node-24-bench')
  329. if (!Array.isArray(benchmark.steps)) throw new TypeError('benchmark job must define steps')
  330. const caches = benchmark.steps.filter(step => isRecord(step) && step.uses === 'actions/cache/restore@v4')
  331. expect(caches).toHaveLength(1)
  332. expect(caches[0]).not.toHaveProperty('if')
  333. expect(caches[0]).toMatchObject({
  334. with: {
  335. path: '${{ steps.pnpm-store.outputs.path }}',
  336. key: "${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }}",
  337. },
  338. })
  339. })
  340. it('bounds the complete benchmark job to fifteen minutes', () => {
  341. const benchmark = workflowJob(loadWorkflow('.github/workflows/ci.yml'), 'node-24-bench')
  342. expect(benchmark['timeout-minutes']).toBe(15)
  343. expect(benchmark.steps).toContainEqual({
  344. name: 'Run performance benchmarks',
  345. env: { DSH_GATE_VERBOSE: '1' },
  346. run: 'pnpm run check:ci:bench',
  347. })
  348. })
  349. it('gives the Wine Host TypeScript compile the repository heap budget', () => {
  350. const wineGates = readFileSync(resolve(root, 'scripts/wine-windows-gates.sh'), 'utf8')
  351. expect(wineGates).toContain(
  352. 'wine_node "$scratch/logs/host-tsc.log" --max-old-space-size=4096 "$tsc_js" -b tsconfig.host.json --pretty false',
  353. )
  354. })
  355. it('cancels superseded master runs without changing the post-merge job inventory', () => {
  356. const workflow = loadWorkflow('.github/workflows/ci-master.yml')
  357. const prWorkflow = loadWorkflow('.github/workflows/ci.yml')
  358. if (!isRecord(workflow.jobs) || !isRecord(workflow.concurrency)) {
  359. throw new TypeError('ci-master workflow must define jobs and a workflow-level concurrency block')
  360. }
  361. if (!isRecord(prWorkflow.jobs)) {
  362. throw new TypeError('ci workflow must define jobs')
  363. }
  364. expect(workflow.concurrency).toEqual({
  365. group: '${{ github.workflow }}-${{ github.ref }}',
  366. 'cancel-in-progress': true,
  367. })
  368. expect(prWorkflow.concurrency).toEqual(workflow.concurrency)
  369. // The exact event sets are what keep master-only jobs out of the PR check
  370. // panel: ci-master triggers only on push(master) + workflow_dispatch and
  371. // never on pull_request; ci.yml is exactly pull_request-only. Assert the
  372. // full sets so losing the wrong event, or gaining an extra one, fails.
  373. if (!isRecord(workflow.on) || !isRecord(prWorkflow.on)) {
  374. throw new TypeError('both CI workflows must define on')
  375. }
  376. expect(Object.keys(workflow.on).sort()).toEqual(['push', 'workflow_dispatch'])
  377. expect(Object.keys(prWorkflow.on)).toEqual(['pull_request'])
  378. // Drills share the parent run’s supersession policy.
  379. for (const name of ['serial-linux-selfhosted', 'serial-windows']) {
  380. const job = workflow.jobs[name]
  381. if (!isRecord(job)) throw new TypeError(`${name} must be defined`)
  382. expect(job.concurrency).toBeUndefined()
  383. // Standby drills remain post-merge work, but share run cancellation.
  384. expect(job.if).toBe("github.event_name == 'push' && github.ref == 'refs/heads/master'")
  385. }
  386. // Pin the post-merge runtime, Wine, and standby inventory.
  387. const NOT_PUSH_REACHABLE = new Set([
  388. "github.event_name == 'workflow_dispatch' && inputs.suite == 'larger-runner-benchmark'",
  389. "github.event_name == 'workflow_dispatch' && inputs.suite == 'consolidated-runner-benchmark'",
  390. ])
  391. const pushReachable = Object.entries(workflow.jobs)
  392. .filter(([, job]) => {
  393. if (!isRecord(job)) return false
  394. if (job.if === undefined) return true // unconditional: runs on every event
  395. if (job.if === false) return false // `if: false` parses as a boolean
  396. if (typeof job.if !== 'string') return true // unrecognized shape: surface it
  397. return !NOT_PUSH_REACHABLE.has(job.if.trim())
  398. })
  399. .map(([name]) => name)
  400. .sort()
  401. expect(pushReachable).toEqual(['python-runtime', 'serial-linux-selfhosted', 'serial-windows', 'windows'])
  402. // Manual benchmarks retain their bounded fan-out.
  403. for (const name of ['larger-runner-benchmark', 'consolidated-runner-benchmark']) {
  404. const job = workflow.jobs[name]
  405. if (!isRecord(job) || !isRecord(job.strategy)) {
  406. throw new TypeError(`${name} must define a matrix strategy`)
  407. }
  408. expect(job.strategy['max-parallel']).toBe(12)
  409. expect(job['timeout-minutes']).toBe(15)
  410. }
  411. })
  412. it('redirects the Node compile cache to the data-volume runner temp before the first pnpm call', () => {
  413. const prWorkflow = loadWorkflow('.github/workflows/ci.yml')
  414. const masterWorkflow = loadWorkflow('.github/workflows/ci-master.yml')
  415. const redirectLanes = [
  416. [prWorkflow, 'node-24'],
  417. [prWorkflow, 'node-24-coverage'],
  418. [prWorkflow, 'node-24-consumers'],
  419. [masterWorkflow, 'serial-linux-selfhosted'],
  420. ] as const
  421. for (const [workflow, jobKey] of redirectLanes) {
  422. const job = workflowJob(workflow, jobKey)
  423. if (!Array.isArray(job.steps)) throw new TypeError(`${jobKey} must define steps`)
  424. const redirectStepIndex = job.steps.findIndex((step): step is Record<string, unknown> & { run: string } => (
  425. isRecord(step) && typeof step.run === 'string'
  426. && step.run.includes('NODE_COMPILE_CACHE=${{ runner.temp }}/node-compile-cache')
  427. && step.run.includes('"$GITHUB_ENV"')
  428. ))
  429. // Removing this injection would send every pnpm call in the lane (setup,
  430. // store-path probe, install, and the gate) back to the root partition's
  431. // /tmp; rationale in
  432. // .agents/notes/implemented/process/2026-08-28-ci-node-compile-cache-data-disk.md.
  433. expect(redirectStepIndex, `${jobKey} must inject NODE_COMPILE_CACHE into GITHUB_ENV`).toBeGreaterThan(-1)
  434. const pnpmSetupIndex = job.steps.findIndex((step): step is Record<string, unknown> & { uses: string } => (
  435. isRecord(step) && typeof step.uses === 'string' && step.uses.includes('pnpm/action-setup')
  436. ))
  437. expect(pnpmSetupIndex, `${jobKey} must run pnpm/action-setup`).toBeGreaterThan(-1)
  438. expect(redirectStepIndex, `${jobKey} must redirect before pnpm/action-setup runs pnpm`).toBeLessThan(pnpmSetupIndex)
  439. }
  440. })
  441. it('keeps supported LSP source under native Windows coverage', () => {
  442. const config = readFileSync(resolve(root, 'vitest.config.ts'), 'utf8')
  443. expect(config).not.toContain('packages/lsp/lsp-stdio/src/connection.ts')
  444. expect(config).not.toContain('packages/lsp/lsp-stdio/src/index.ts')
  445. expect(config).not.toContain('packages/lsp/lsp-stdio/src/instance.ts')
  446. })
  447. it('requires release-shaped Python runtime validation on Linux and Windows x64', () => {
  448. const workflow = loadWorkflow('.github/workflows/ci.yml')
  449. const pythonRuntime = workflowJob(workflow, 'python-runtime')
  450. const aggregate = workflowJob(workflow, 'all-checks-passed')
  451. if (!Array.isArray(aggregate.needs)) {
  452. throw new TypeError('CI aggregate must define required job dependencies')
  453. }
  454. expect(pythonRuntime).toMatchObject({
  455. if: "github.event_name == 'pull_request'",
  456. name: 'python runtime / release-shaped matrix',
  457. uses: './.github/workflows/build-exe-for-python-sdk.yml',
  458. with: {
  459. targets: 'node24-linux-x64,node24-win-x64',
  460. ci: true,
  461. },
  462. secrets: {
  463. DEEPSEEK_API_KEY_EXTERNAL: '${{ secrets.DEEPSEEK_API_KEY_EXTERNAL }}',
  464. },
  465. })
  466. expect(aggregate.needs).toContain('python-runtime')
  467. })
  468. it('keeps every Vitest project process-isolated on native Windows', () => {
  469. const config = readFileSync(resolve(root, 'vitest.config.ts'), 'utf8')
  470. expect(config).not.toContain("pool: process.platform === 'win32' ? 'threads' : 'forks'")
  471. expect(config.match(/pool: 'forks'/g)).toHaveLength(2)
  472. })
  473. })
  474. describe('DeepSeek e2e workflow', () => {
  475. it('prepares bubblewrap from the pinned payload without a package transaction', () => {
  476. const workflow = loadWorkflow('.github/workflows/e2e.yml')
  477. const e2e = workflowJob(workflow, 'e2e')
  478. if (!Array.isArray(e2e.steps)) throw new TypeError('DeepSeek e2e workflow must define steps')
  479. const steps = e2e.steps.filter(isRecord)
  480. expect(steps.find(step => step.name === 'Prepare bubblewrap (unrestrict userns)')).toMatchObject({
  481. run: 'bash scripts/prepare-ci-bubblewrap.sh',
  482. })
  483. expect(JSON.stringify(steps)).not.toContain('apt-get')
  484. })
  485. it('bounds profile subprocess fan-out to the tested e2e default', () => {
  486. const workflow = loadWorkflow('.github/workflows/e2e.yml')
  487. const e2e = workflowJob(workflow, 'e2e')
  488. if (!Array.isArray(e2e.steps)) throw new TypeError('DeepSeek e2e workflow must define steps')
  489. const step = e2e.steps.filter(isRecord).find(candidate => candidate.name === 'E2E tests (real DeepSeek API)')
  490. expect(step).toMatchObject({ env: { DSH_E2E_MAX_WORKERS: 4 } })
  491. })
  492. })
  493. describe('E2B e2e workflow', () => {
  494. it('is manual-only and fails loud before running the focused live suite', () => {
  495. const workflow = loadWorkflow('.github/workflows/e2b-e2e.yml')
  496. expect(workflow.on).toEqual({ workflow_dispatch: null })
  497. if (!isRecord(workflow.jobs) || !isRecord(workflow.jobs.e2b) || !Array.isArray(workflow.jobs.e2b.steps)) {
  498. throw new TypeError('E2B e2e workflow must define the e2b job steps')
  499. }
  500. const steps = workflow.jobs.e2b.steps.filter(isRecord)
  501. const preflight = steps.find(step => step.name === 'Preflight (require E2B API key)')
  502. const e2b = steps.find(step => step.name === 'E2B tests (live sandbox)')
  503. expect(preflight).toMatchObject({
  504. env: { E2B_API_KEY: '${{ secrets.E2B_API_KEY_EXTERNAL }}' },
  505. })
  506. expect(preflight?.run).toContain('E2B_API_KEY_EXTERNAL repository secret')
  507. expect(e2b).toMatchObject({
  508. env: {
  509. E2B_API_KEY: '${{ secrets.E2B_API_KEY_EXTERNAL }}',
  510. DSH_E2E_MAX_WORKERS: '1',
  511. DSH_EXAMPLE_MODE: 'lib',
  512. },
  513. })
  514. expect(e2b?.run).toContain('packages/e2b/e2b/tests/composition.e2e.ts')
  515. })
  516. })
  517. describe('Python release workflows', () => {
  518. it('keeps complete wheel validation separate from protected public publication', () => {
  519. const workflow = loadWorkflow('.github/workflows/python-release.yml')
  520. const dispatch = workflowEvent(workflow, 'workflow_dispatch')
  521. const build = workflowJob(workflow, 'build')
  522. const pythonCompat = workflowJob(workflow, 'python-compat')
  523. const validate = workflowJob(workflow, 'validate')
  524. const publishRuntime = workflowJob(workflow, 'publish-runtime')
  525. const publishSdk = workflowJob(workflow, 'publish-sdk')
  526. if (!isRecord(dispatch.inputs)
  527. || !isRecord(dispatch.inputs.publish)
  528. || !Array.isArray(pythonCompat.steps)
  529. || !Array.isArray(validate.steps)
  530. || !Array.isArray(publishRuntime.steps)
  531. || !Array.isArray(publishSdk.steps)) {
  532. throw new TypeError('Python release workflow must define publish input and release steps')
  533. }
  534. expect(dispatch.inputs.publish).toMatchObject({ type: 'boolean', default: false })
  535. if (!isRecord(workflow.on)) throw new TypeError('python-release workflow must define on')
  536. expect(Object.keys(workflow.on)).toEqual(['workflow_dispatch'])
  537. expect(build).toMatchObject({
  538. uses: './.github/workflows/build-exe-for-python-sdk.yml',
  539. with: {
  540. targets: 'node24-linux-x64,node24-linux-arm64,node24-macos-arm64,node24-macos-x64,node24-win-x64',
  541. release: true,
  542. },
  543. })
  544. expect(pythonCompat.strategy).toMatchObject({ matrix: { python: ['3.10', '3.14'] } })
  545. const pythonCompatSteps = JSON.stringify(pythonCompat.steps)
  546. expect(pythonCompatSteps).toContain('dist/deepseek_harness_sdk-$VERSION-py3-none-any.whl')
  547. expect(pythonCompatSteps).toContain('dist/deepseek_harness_runtime_bin-$VERSION-py3-none-manylinux_2_28_x86_64.whl')
  548. expect(pythonCompatSteps).not.toContain('--find-links')
  549. const validateSteps = JSON.stringify(validate.steps)
  550. const authorize = validate.steps.filter(isRecord).find(step => step.name === 'Authorize publication request')
  551. if (!isRecord(authorize) || typeof authorize.run !== 'string') {
  552. throw new TypeError('Python release validation must authorize publication requests')
  553. }
  554. expect(validateSteps).toContain('PUBLIC_PYPI_RELEASE_ENABLED')
  555. expect(authorize).toMatchObject({
  556. env: {
  557. PYPI_PUBLISHER_REPOSITORY: '${{ vars.PYPI_PUBLISHER_REPOSITORY }}',
  558. REPOSITORY: '${{ github.repository }}',
  559. },
  560. })
  561. expect(authorize.run).toContain('[ "$REPOSITORY" = "$PYPI_PUBLISHER_REPOSITORY" ]')
  562. expect(validateSteps).toContain('100000000')
  563. expect(publishRuntime).toMatchObject({
  564. if: "github.event_name == 'workflow_dispatch' && inputs.publish",
  565. needs: 'validate',
  566. environment: 'pypi-runtime',
  567. permissions: { contents: 'read', 'id-token': 'write' },
  568. })
  569. expect(publishSdk).toMatchObject({
  570. if: "github.event_name == 'workflow_dispatch' && inputs.publish",
  571. needs: ['validate', 'publish-runtime'],
  572. environment: 'pypi',
  573. permissions: { contents: 'read', 'id-token': 'write' },
  574. })
  575. const runtimeSteps = publishRuntime.steps.filter(isRecord)
  576. const sdkSteps = publishSdk.steps.filter(isRecord)
  577. const runtimePublish = runtimeSteps.find(step => step.name === 'Publish runtime wheels')
  578. const sdkPublish = sdkSteps.find(step => step.name === 'Publish SDK wheel')
  579. const runtimeHashes = runtimeSteps.find(step => step.name === 'Verify release artifact hashes')
  580. const sdkHashes = sdkSteps.find(step => step.name === 'Verify release artifact hashes')
  581. expect([...runtimeSteps, ...sdkSteps].some(
  582. step => typeof step.uses === 'string' && step.uses.startsWith('actions/checkout@'),
  583. )).toBe(false)
  584. expect([...runtimeSteps, ...sdkSteps].filter(
  585. step => step.uses === 'pypa/gh-action-pypi-publish@release/v1',
  586. )).toHaveLength(2)
  587. expect(runtimePublish).toMatchObject({
  588. with: { 'packages-dir': 'dist/runtime/', attestations: false },
  589. })
  590. expect(sdkPublish).toMatchObject({
  591. with: { 'packages-dir': 'dist/sdk/', attestations: false },
  592. })
  593. expect(runtimeHashes).toMatchObject({ run: 'cd dist && sha256sum -c SHA256SUMS' })
  594. expect(sdkHashes).toMatchObject({ run: 'cd dist && sha256sum -c SHA256SUMS' })
  595. })
  596. it('exposes the native wheel builder to the release caller with normalized versions', () => {
  597. const workflow = loadWorkflow('.github/workflows/build-exe-for-python-sdk.yml')
  598. expect(Object.keys(workflow.on as Record<string, unknown>).sort()).toEqual(['workflow_call', 'workflow_dispatch'])
  599. const call = workflowEvent(workflow, 'workflow_call')
  600. const plan = workflowJob(workflow, 'plan')
  601. const build = workflowJob(workflow, 'build')
  602. if (!isRecord(call.inputs) || !isRecord(call.secrets) || !Array.isArray(plan.steps) || !Array.isArray(build.steps)) {
  603. throw new TypeError('Python wheel builder must define workflow_call inputs and plan steps')
  604. }
  605. const buildSteps: unknown[] = build.steps
  606. const manylinuxAddon = buildSteps.find(step => isRecord(step) && step.name === 'Rebuild Linux node-pty against manylinux 2.28')
  607. const macosCheck = buildSteps.find(step => isRecord(step) && step.name === 'Check macOS payload architecture and deployment target')
  608. const manylinuxSmoke = buildSteps.find(step => isRecord(step) && step.name === 'Run wheel in a manylinux 2.28 container')
  609. const cleanVenvPosix = buildSteps.find(step => isRecord(step) && step.name === 'Install local SDK and runtime wheels into a clean venv (POSIX)')
  610. const cleanVenvWindows = buildSteps.find(step => isRecord(step) && step.name === 'Install local SDK and runtime wheels into a clean venv (Windows)')
  611. const installedKeylessPosix = buildSteps.find(step => isRecord(step) && step.name === 'Run installed-wheel keyless black-box tests (POSIX)')
  612. const installedKeylessWindows = buildSteps.find(step => isRecord(step) && step.name === 'Run installed-wheel keyless black-box tests (Windows)')
  613. const realApiPreflightPosix = buildSteps.find(step => isRecord(step) && step.name === 'Preflight installed-wheel real API test (POSIX)')
  614. const realApiPreflightWindows = buildSteps.find(step => isRecord(step) && step.name === 'Preflight installed-wheel real API test (Windows)')
  615. const installedRealApiPosix = buildSteps.find(step => isRecord(step) && step.name === 'Run installed-wheel real API black-box test (POSIX)')
  616. const installedRealApiWindows = buildSteps.find(step => isRecord(step) && step.name === 'Run installed-wheel real API black-box test (Windows)')
  617. if (!isRecord(macosCheck) || typeof macosCheck.run !== 'string'
  618. || !isRecord(cleanVenvPosix) || !isRecord(cleanVenvWindows)
  619. || !isRecord(installedKeylessPosix) || !isRecord(installedKeylessWindows)
  620. || !isRecord(realApiPreflightPosix) || !isRecord(realApiPreflightWindows)
  621. || !isRecord(installedRealApiPosix) || !isRecord(installedRealApiWindows)) {
  622. throw new TypeError('Python wheel builder must define native POSIX and Windows installed-wheel steps')
  623. }
  624. expect(call.inputs).toHaveProperty('targets')
  625. expect(call.inputs).toMatchObject({
  626. ci: { type: 'boolean', default: false },
  627. release: { type: 'boolean', default: false },
  628. })
  629. expect(call.secrets).toMatchObject({
  630. DEEPSEEK_API_KEY_EXTERNAL: { required: false },
  631. })
  632. expect(workflow.concurrency).toMatchObject({
  633. group: 'build-single-exe-${{ github.workflow }}-${{ github.ref }}',
  634. })
  635. expect(build.defaults).toBeUndefined()
  636. expect(plan.if).toContain('inputs.ci')
  637. expect(plan.if).toContain('inputs.release')
  638. expect(JSON.stringify(plan.steps)).toContain('pep440_version')
  639. const workflowJson = JSON.stringify(workflow)
  640. expect(workflowJson).toContain('macosx_14_0_arm64')
  641. expect(workflowJson).toContain('macosx_14_0_x86_64')
  642. expect(workflowJson).toContain('node24-macos-x64')
  643. expect(workflowJson).toContain('macos-15-intel')
  644. expect(workflowJson).toContain('win_amd64')
  645. expect(workflowJson).toContain('node24-win-x64')
  646. expect(workflowJson).toContain('windows-2025')
  647. expect(workflowJson).toContain('dist-python/$SDK_WHEEL')
  648. expect(workflowJson).toContain('dist-python/$RUNTIME_WHEEL')
  649. expect(workflowJson).toContain('/work/dist-python/$SDK_WHEEL')
  650. expect(workflowJson).toContain('/work/dist-python/$RUNTIME_WHEEL')
  651. expect(workflowJson).not.toContain('--find-links dist-python')
  652. expect(workflowJson).not.toContain('--find-links /work/dist-python')
  653. expect(workflowJson).not.toContain('cygpath')
  654. expect(manylinuxAddon).toMatchObject({ if: "runner.os == 'Linux'" })
  655. expect(JSON.stringify(manylinuxAddon)).toContain('manylinux_2_28_x86_64')
  656. expect(JSON.stringify(manylinuxAddon)).toContain('manylinux_2_28_aarch64')
  657. expect(JSON.stringify(manylinuxAddon)).toContain('npm_config_build_from_source=true pnpm run install')
  658. expect(JSON.stringify(manylinuxAddon)).toContain('pnpm_setup_root')
  659. expect(JSON.stringify(manylinuxAddon)).toContain('$pnpm_setup_root:$pnpm_setup_root:ro')
  660. expect(JSON.stringify(manylinuxAddon)).toContain('node-pty-glibc-versions.txt')
  661. expect(JSON.stringify(manylinuxAddon)).toContain('le 2.28')
  662. expect(macosCheck).toMatchObject({ if: "runner.os == 'macOS'" })
  663. expect(macosCheck.run).toContain('scripts/check-macos-deployment-target.py')
  664. expect(macosCheck.run).toContain('lipo "$payload" -verify_arch')
  665. expect(macosCheck.run).toContain('$EXE-rg')
  666. expect(macosCheck.run).toContain('$EXE-spawn-helper')
  667. expect(JSON.stringify(installedKeylessPosix)).toContain('--scenario all')
  668. expect(JSON.stringify(installedKeylessPosix)).toContain('env -u PYTHONPATH')
  669. expect(JSON.stringify(installedKeylessWindows)).toContain('--scenario all --installed-wheel')
  670. expect(installedKeylessWindows).toMatchObject({ if: "runner.os == 'Windows'", shell: 'pwsh' })
  671. expect(cleanVenvWindows).toMatchObject({ if: "runner.os == 'Windows'", shell: 'pwsh' })
  672. expect(JSON.stringify(cleanVenvWindows)).toContain('Scripts\\\\python.exe')
  673. expect(realApiPreflightPosix).toMatchObject({
  674. env: { DEEPSEEK_API_KEY: '${{ secrets.DEEPSEEK_API_KEY_EXTERNAL }}' },
  675. })
  676. expect(String(realApiPreflightPosix.if)).toContain('inputs.ci')
  677. expect(String(realApiPreflightPosix.if)).toContain('head.repo.fork')
  678. expect(String(realApiPreflightPosix.if)).toContain('dependabot[bot]')
  679. expect(realApiPreflightWindows).toMatchObject({ shell: 'pwsh' })
  680. expect(installedRealApiPosix).toMatchObject({
  681. env: {
  682. DEEPSEEK_API_KEY: '${{ secrets.DEEPSEEK_API_KEY_EXTERNAL }}',
  683. DEEPSEEK_BASE_URL: 'https://api.deepseek.com',
  684. },
  685. })
  686. expect(JSON.stringify(installedRealApiPosix)).toContain('--scenario sdk-live')
  687. expect(JSON.stringify(installedRealApiPosix)).toContain('-u DSH_RUNTIME_MODE')
  688. expect(installedRealApiWindows).toMatchObject({ shell: 'pwsh' })
  689. expect(JSON.stringify(installedRealApiWindows)).toContain('--scenario sdk-live --installed-wheel')
  690. expect(manylinuxSmoke).toMatchObject({ if: "runner.os == 'Linux'" })
  691. expect(JSON.stringify(manylinuxSmoke)).toContain('-e DSH_TELEMETRY_DISABLED')
  692. })
  693. it('uses the shared macOS deployment-target check in GitLab', () => {
  694. const workflow = loadWorkflow('.gitlab-ci.yml')
  695. const runtimeWheel = workflow['.runtime-wheel']
  696. if (!isRecord(runtimeWheel) || !Array.isArray(runtimeWheel.script)) {
  697. throw new TypeError('GitLab CI must define the runtime wheel script')
  698. }
  699. const runtimeScript: unknown[] = runtimeWheel.script
  700. const macosCheck = runtimeScript.find(
  701. step => typeof step === 'string' && step.includes('${PLATFORM#macos-}'),
  702. )
  703. if (typeof macosCheck !== 'string') {
  704. throw new TypeError('GitLab CI must check the macOS deployment target')
  705. }
  706. expect(macosCheck).toContain('scripts/check-macos-deployment-target.py')
  707. expect(macosCheck).toContain('lipo "$payload" -verify_arch')
  708. expect(macosCheck).toContain('"$EXE" "$EXE-rg" "$EXE-spawn-helper"')
  709. })
  710. it('builds the macOS x64 wheel on the matching GitLab runner', () => {
  711. const workflow = loadWorkflow('.gitlab-ci.yml')
  712. const macosX64 = workflow['runtime-macos-x64']
  713. const publish = workflow['publish-python']
  714. if (!isRecord(macosX64) || !isRecord(publish) || !Array.isArray(publish.needs)) {
  715. throw new TypeError('GitLab CI must define the macOS x64 runtime and publication jobs')
  716. }
  717. expect(macosX64.tags).toEqual(['macos-x64'])
  718. expect(macosX64.variables).toMatchObject({ PKG_TARGET: 'node24-macos-x64', PLATFORM: 'macos-x64' })
  719. expect(publish.needs).toContainEqual({ job: 'runtime-macos-x64', artifacts: true })
  720. expect(JSON.stringify(publish.script)).toContain('macosx_14_0_x86_64.whl')
  721. })
  722. it('builds and black-box tests the Windows x64 wheel in GitLab', () => {
  723. const workflow = loadWorkflow('.gitlab-ci.yml')
  724. const windows = workflow['runtime-windows-x64']
  725. const publish = workflow['publish-python']
  726. if (!isRecord(windows) || !Array.isArray(windows.before_script) || !Array.isArray(windows.script)
  727. || !isRecord(publish) || !Array.isArray(publish.needs)) {
  728. throw new TypeError('GitLab CI must define the Windows runtime and aggregate publication jobs')
  729. }
  730. expect(windows.tags).toEqual(['windows-x64'])
  731. expect(windows.variables).toMatchObject({ PKG_TARGET: 'node24-win-x64', PLATFORM: 'win-x64' })
  732. expect(JSON.stringify(windows.before_script)).toContain('.ci-python\\\\Scripts')
  733. expect(JSON.stringify(windows.before_script)).toContain('[IO.Path]::PathSeparator')
  734. expect(JSON.stringify(windows.script)).toContain('win_amd64.whl')
  735. expect(JSON.stringify(windows.script)).toContain('--scenario all --installed-wheel')
  736. expect(publish.needs).toContainEqual({ job: 'runtime-windows-x64', artifacts: true })
  737. })
  738. })
  739. describe('Weighted approval workflow', () => {
  740. it('publishes from the trusted default branch after pull request and review updates', () => {
  741. const publisher = loadWorkflow('.github/workflows/weighted-approval.yml')
  742. const reviewEvent = loadWorkflow('.github/workflows/weighted-approval-review-event.yml')
  743. const pullRequest = workflowEvent(publisher, 'pull_request_target')
  744. const workflowRun = workflowEvent(publisher, 'workflow_run')
  745. const review = workflowEvent(reviewEvent, 'pull_request_review')
  746. const job = workflowJob(publisher, 'publish-status')
  747. const recordJob = workflowJob(reviewEvent, 'record-review-event')
  748. if (!isRecord(publisher.on)) throw new TypeError('weighted-approval workflow must define events')
  749. if (!isRecord(reviewEvent.on)) throw new TypeError('weighted-approval review event workflow must define events')
  750. if (!Array.isArray(job.steps)) throw new TypeError('weighted-approval job must define steps')
  751. if (!Array.isArray(recordJob.steps)) throw new TypeError('weighted-approval review event job must define steps')
  752. const steps = job.steps.filter(isRecord)
  753. const checkout = steps.find(step => step.name === 'Check out trusted approval policy')
  754. const publish = steps.find(step => step.name === 'Publish weighted approval status')
  755. const recordSteps = recordJob.steps.filter(isRecord)
  756. const record = recordSteps.find(step => step.name === 'Record review event')
  757. expect(publisher.name).toBe('weighted-approval')
  758. expect(Object.keys(publisher.on)).toEqual(['pull_request_target', 'workflow_run'])
  759. expect(pullRequest.types).toEqual(['opened', 'synchronize', 'reopened', 'ready_for_review', 'converted_to_draft'])
  760. expect(workflowRun).toEqual({ workflows: ['weighted-approval-review-event'], types: ['completed'] })
  761. expect(reviewEvent.name).toBe('weighted-approval-review-event')
  762. expect(reviewEvent['run-name']).toBe('weighted-approval-review-event:${{ github.event.pull_request.number }}')
  763. expect(Object.keys(reviewEvent.on)).toEqual(['pull_request_review'])
  764. expect(review.types).toEqual(['submitted', 'edited', 'dismissed'])
  765. expect(reviewEvent.permissions).toEqual({})
  766. expect(publisher.permissions).toEqual({
  767. contents: 'read',
  768. 'pull-requests': 'read',
  769. statuses: 'write',
  770. })
  771. expect(publisher.concurrency).toEqual({
  772. group: 'weighted-approval-${{ github.event.pull_request.number || github.event.workflow_run.head_sha }}',
  773. 'cancel-in-progress': false,
  774. })
  775. expect(job).toMatchObject({
  776. if: "github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'",
  777. name: 'weighted approval publisher',
  778. 'runs-on': 'ubuntu-latest',
  779. 'timeout-minutes': 5,
  780. })
  781. expect(checkout).toMatchObject({
  782. uses: 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1',
  783. with: {
  784. ref: '${{ github.event.repository.default_branch }}',
  785. 'persist-credentials': false,
  786. },
  787. })
  788. expect(publish).toMatchObject({
  789. env: {
  790. GITHUB_TOKEN: '${{ github.token }}',
  791. GITHUB_RUN_URL: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}',
  792. },
  793. run: 'node .github/review-ownership/check-approval.mjs',
  794. })
  795. expect(recordJob).toMatchObject({
  796. name: 'record weighted approval review event',
  797. 'runs-on': 'ubuntu-latest',
  798. 'timeout-minutes': 2,
  799. })
  800. expect(record).toBeDefined()
  801. expect(record?.run).toBe("echo 'Recorded a weighted approval review event.'")
  802. expect(recordSteps).toHaveLength(1)
  803. expect(JSON.stringify(publisher)).not.toContain('github.event.pull_request.head')
  804. expect(JSON.stringify(publisher)).not.toContain('secrets.')
  805. expect(JSON.stringify(reviewEvent)).not.toContain('github.token')
  806. expect(JSON.stringify(reviewEvent)).not.toContain('secrets.')
  807. })
  808. })
  809. describe('Issue lifecycle workflow', () => {
  810. it('runs the lifecycle job on every PR/review event but gates token and board steps', () => {
  811. const lifecycle = loadWorkflow('.github/workflows/issue-lifecycle.yml')
  812. const policy = loadWorkflow('.github/workflows/issue-policy.yml')
  813. const lifecycleJob = workflowJob(lifecycle, 'lifecycle')
  814. if (!Array.isArray(lifecycleJob.steps)) throw new TypeError('Issue lifecycle job must define steps')
  815. // The job has no job-level `if`, so it is listed on every pull_request /
  816. // pull_request_review event and reports success instead of a gray skip. The
  817. // write-capable steps are gated at step level so approved/commented reviews
  818. // never mint a Project/Issue App token nor touch the board.
  819. expect(lifecycle.on).toHaveProperty('pull_request')
  820. expect(lifecycle.on).toHaveProperty('pull_request_review')
  821. expect(lifecycleJob.if).toBeUndefined()
  822. // Keep the subscription-type gates: issue-lifecycle does not re-subscribe
  823. // ready_for_review (issue-policy owns that) and only reacts to submitted
  824. // review events.
  825. const lifecyclePullRequest = workflowEvent(lifecycle, 'pull_request')
  826. const lifecycleReview = workflowEvent(lifecycle, 'pull_request_review')
  827. expect(lifecyclePullRequest.types).toContain('opened')
  828. expect(lifecyclePullRequest.types).not.toContain('ready_for_review')
  829. expect(lifecyclePullRequest.types).toContain('review_requested')
  830. expect(lifecycleReview.types).toEqual(['submitted'])
  831. const gated = "${{ github.event_name != 'pull_request_review' || github.event.review.state == 'changes_requested' }}"
  832. const steps = lifecycleJob.steps.filter(isRecord)
  833. const tokenStep = steps.find(s => s.name === 'Create project token')
  834. const handleStep = steps.find(s => s.name === 'Handle repository event')
  835. expect(tokenStep).toMatchObject({ if: gated })
  836. expect(handleStep).toMatchObject({ if: gated })
  837. // issue-policy owns PR validation; it is read-only and a real gate.
  838. const policyPullRequest = workflowEvent(policy, 'pull_request')
  839. expect(policyPullRequest.types).toContain('ready_for_review')
  840. })
  841. it('uses a read-only Project token only for human pull request policy metadata', () => {
  842. const policy = loadWorkflow('.github/workflows/issue-policy.yml')
  843. const policyJob = workflowJob(policy, 'policy')
  844. if (!Array.isArray(policyJob.steps)) throw new TypeError('Issue policy job must define steps')
  845. const steps = policyJob.steps.filter(isRecord)
  846. const tokenStep = steps.find(step => step.name === 'Create Project read token')
  847. const validateStep = steps.find(step => step.name === 'Validate pull request')
  848. const humanPullRequest =
  849. "${{ github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.type != 'App' }}"
  850. expect(tokenStep).toMatchObject({
  851. id: 'app-token',
  852. if: humanPullRequest,
  853. uses: 'actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1',
  854. with: {
  855. 'client-id': '${{ vars.DSH_ISSUE_APP_CLIENT_ID }}',
  856. 'private-key': '${{ secrets.DSH_ISSUE_APP_PRIVATE_KEY }}',
  857. owner: 'deepseek-harness',
  858. repositories: 'deepseek-harness',
  859. 'permission-issues': 'read',
  860. 'permission-organization-projects': 'read',
  861. },
  862. })
  863. expect(validateStep).toMatchObject({
  864. if: humanPullRequest,
  865. env: {
  866. GITHUB_TOKEN: '${{ github.token }}',
  867. PROJECT_TOKEN: '${{ steps.app-token.outputs.token }}',
  868. },
  869. })
  870. })
  871. })
  872. describe('npm release workflows', () => {
  873. it('keeps publication dispatch-only and pack in the PR workflow', () => {
  874. // pack stays in the PR/master release workflows so a PR proves the set packs.
  875. for (const file of ['release.yml', 'release-vendor.yml']) {
  876. const workflow = loadWorkflow(`.github/workflows/${file}`)
  877. if (!isRecord(workflow.jobs)) throw new TypeError(`${file} must define jobs`)
  878. expect(Object.keys(workflow.jobs).sort()).toEqual(file === 'release.yml' ? ['dependencies', 'pack'] : ['pack'])
  879. }
  880. // publication is workflow_dispatch-only (never a PR check) and keeps the
  881. // npm-publish environment plus the shared dist-tag group.
  882. for (const file of ['release-publish.yml', 'release-vendor-publish.yml']) {
  883. const workflow = loadWorkflow(`.github/workflows/${file}`)
  884. if (!isRecord(workflow.on) || !isRecord(workflow.jobs)) throw new TypeError(`${file} must define on and jobs`)
  885. expect(Object.keys(workflow.on)).toEqual(['workflow_dispatch'])
  886. const publish = workflow.jobs.publish
  887. if (!isRecord(publish)) throw new TypeError(`${file} must define a publish job`)
  888. expect(publish.environment).toBe('npm-publish')
  889. expect(publish.concurrency).toMatchObject({ group: 'Release-publish' })
  890. }
  891. })
  892. it('runs dependency policy and npm layout checks in the DSH release workflow', () => {
  893. const workflow = loadWorkflow('.github/workflows/release.yml')
  894. const dependencies = workflowJob(workflow, 'dependencies')
  895. if (!isRecord(workflow.on) || !Array.isArray(dependencies.steps)) {
  896. throw new TypeError('DSH release workflow must define triggers and dependency steps')
  897. }
  898. const commands = dependencies.steps.flatMap(step =>
  899. isRecord(step) && typeof step.run === 'string' ? [step.run] : [])
  900. expect(Object.keys(workflow.on).sort()).toEqual(['pull_request', 'push', 'workflow_dispatch'])
  901. expect(commands).toContain('pnpm run verify-package-dependencies')
  902. expect(commands).toContain('pnpm run verify-npm-install-layout')
  903. })
  904. })
  905. describe('Documentation site publication', () => {
  906. it('keeps Pages deployment dispatch-only from a dsh-v* tag', () => {
  907. const workflow = loadWorkflow('.github/workflows/docs-pages.yml')
  908. const build = workflowJob(workflow, 'build')
  909. const deploy = workflowJob(workflow, 'deploy')
  910. if (!isRecord(workflow.on) || !isRecord(workflow.env) || !Array.isArray(build.steps)) {
  911. throw new TypeError('Documentation deployment must define on, env, and build steps')
  912. }
  913. // The site presents a released snapshot: a merge must never publish it, and
  914. // publication must never appear as a PR check.
  915. expect(Object.keys(workflow.on)).toEqual(['workflow_dispatch'])
  916. // RELEASE_PUBLISH makes release:verify reject every ref that is not a dsh-v*
  917. // tag naming this tree's version, so the site and the npm sequence share one
  918. // definition of a released version.
  919. const steps = build.steps.filter(isRecord)
  920. const verify = steps.find(step => step.name === 'Verify release version')
  921. const checkout = steps.find(
  922. step => typeof step.uses === 'string' && step.uses.startsWith('actions/checkout@'),
  923. )
  924. expect(verify).toMatchObject({
  925. env: { RELEASE_PUBLISH: 'true' },
  926. run: 'pnpm run release:verify --family dsh',
  927. })
  928. // Complete history: the release scripts read tags.
  929. expect(checkout).toMatchObject({ with: { 'fetch-depth': 0 } })
  930. // Projected source links stay on the public repository's master. That
  931. // repository advances only to each release commit, so its master never
  932. // carries unreleased work, while it retains only the most recent tags:
  933. // following the dispatched tag would leave every source link on a deploy
  934. // from an older tag unresolvable.
  935. expect(workflow.env.DOCS_REPOSITORY_REF).toBe('master')
  936. // The environment owns the deployment tag policy and the required reviewers.
  937. expect(deploy.environment).toMatchObject({ name: 'github-pages' })
  938. })
  939. })
  940. describe('Git hooks', () => {
  941. it('leaves frozen Agent Note sidecars to the archive verifier', () => {
  942. const lefthook = loadWorkflow('lefthook.yml')
  943. for (const hookName of ['pre-commit', 'pre-merge-commit']) {
  944. const hook = lefthook[hookName]
  945. if (!isRecord(hook) || !Array.isArray(hook.jobs)) {
  946. throw new TypeError(`lefthook must define ${hookName} jobs`)
  947. }
  948. const pairing: unknown = hook.jobs.find(
  949. (job: unknown) => isRecord(job) && job.name === 'translation pairing (staged records)',
  950. )
  951. expect(pairing).toMatchObject({ exclude: ['.agents/notes/archived/**'] })
  952. }
  953. })
  954. })
  955. function loadWorkflow(path: string): Record<string, unknown> {
  956. const workflow: unknown = yaml.load(readFileSync(resolve(root, path), 'utf8'))
  957. if (!isRecord(workflow)) throw new TypeError(`${path} must define a workflow`)
  958. return workflow
  959. }
  960. function workflowEvent(workflow: Record<string, unknown>, event: string): Record<string, unknown> {
  961. if (!isRecord(workflow.on) || !isRecord(workflow.on[event])) {
  962. throw new TypeError(`workflow must define the ${event} event`)
  963. }
  964. return workflow.on[event]
  965. }
  966. function workflowJob(workflow: Record<string, unknown>, job: string): Record<string, unknown> {
  967. if (!isRecord(workflow.jobs) || !isRecord(workflow.jobs[job])) {
  968. throw new TypeError(`workflow must define the ${job} job`)
  969. }
  970. return workflow.jobs[job]
  971. }
  972. function isRecord(value: unknown): value is Record<string, unknown> {
  973. return typeof value === 'object' && value !== null && !Array.isArray(value)
  974. }