run-gates.ts 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973
  1. /**
  2. * Run local and CI quality gates with bounded in-process scheduling.
  3. *
  4. * Package scripts own public aggregate names; this runner owns their validated
  5. * dependency graphs, scheduler environment, and process diagnostics.
  6. * @see ../.agents/notes/implemented/process/2026-07-06-parallel-pre-push-gates.md
  7. */
  8. import { spawn } from 'node:child_process'
  9. import { availableParallelism } from 'node:os'
  10. import { resolve } from 'node:path'
  11. import { performance } from 'node:perf_hooks'
  12. import { CLIENT_BUILD_PROFILE_SELECTOR } from './client-build-environment.ts'
  13. import { COVERAGE_EXEMPT_ENV, coverageExemptHeavySuites } from './coverage-exempt.ts'
  14. import {
  15. COVERAGE_PARTITIONS_ENV,
  16. COVERAGE_TEST_TIMEOUT_ENV,
  17. coverageTestTimeoutArgs,
  18. parseCoveragePartitionCount,
  19. } from './coverage-partitions.ts'
  20. import { pnpmInvocation } from './pnpm-invocation.ts'
  21. /** A named aggregate exposed by the gate runner. */
  22. export type Mode =
  23. | 'ci-primary'
  24. | 'ci-linux-primary'
  25. | 'ci-static'
  26. | 'ci-lint-contracts-ready'
  27. | 'ci-coverage'
  28. | 'ci-snapshot'
  29. | 'ci-artifacts'
  30. | 'ci-consumers'
  31. | 'ci-windows-blocking'
  32. | 'ci-windows-complete'
  33. | 'ci-windows-observational'
  34. | 'node-compat'
  35. | 'check-all'
  36. | 'hygiene'
  37. | 'doc-sync'
  38. type GateResultStatus = 'passed' | 'failed' | 'skipped'
  39. type GateState = 'pending' | 'running' | GateResultStatus
  40. /** A command and its dependency metadata inside one aggregate. */
  41. export interface Gate {
  42. id: string
  43. label: string
  44. displayCommand: string
  45. command: string
  46. args: string[]
  47. needs?: string[]
  48. /** Gate ids that must settle, regardless of outcome, before this gate starts. */
  49. after?: string[]
  50. env?: Record<string, string | undefined>
  51. /** Keep a failure visible without failing the aggregate. */
  52. allowFailure?: boolean
  53. /** Write child output as it arrives instead of buffering it until completion. */
  54. streamOutput?: boolean
  55. }
  56. /** The observed outcome of one gate process. */
  57. export interface GateResult {
  58. gate: Gate
  59. status: GateResultStatus
  60. durationMs: number
  61. output: GateOutputChunk[]
  62. exitCode: number | null
  63. signalCode: NodeJS.Signals | null
  64. error?: string
  65. }
  66. interface GateOutputChunk {
  67. stream: 'stdout' | 'stderr'
  68. text: string
  69. }
  70. interface RunningGate {
  71. gate: Gate
  72. promise: Promise<GateResult>
  73. }
  74. interface ConcurrencyDefault {
  75. workers: number
  76. source: string
  77. }
  78. type GateExecutor = (gate: Gate) => Promise<GateResult>
  79. type ResultObserver = (result: GateResult) => void
  80. const root = resolve(import.meta.dirname, '..')
  81. if (import.meta.main) {
  82. process.exitCode = await main(process.argv.slice(2))
  83. }
  84. async function main(args: string[]): Promise<number> {
  85. const mode = parseMode(args[0])
  86. const gates = gatesForMode(mode)
  87. const concurrencyDefault = defaultConcurrency(mode, gates.length)
  88. const concurrencyOverride = process.env.DSH_GATE_CONCURRENCY
  89. const maxConcurrency = concurrencyFromEnv('DSH_GATE_CONCURRENCY', concurrencyDefault.workers)
  90. const concurrencySource = concurrencyOverride === undefined || concurrencyOverride === ''
  91. ? concurrencyDefault.source
  92. : '$DSH_GATE_CONCURRENCY'
  93. const startedAt = performance.now()
  94. console.log(`run-gates: ${mode} running ${gates.length} gate(s) with ${maxConcurrency} worker(s) from ${concurrencySource}.`)
  95. const results = await runGates(gates, maxConcurrency, runGate, printResult)
  96. printSummary(results, performance.now() - startedAt)
  97. return results.some(result => result.gate.allowFailure !== true && (result.status === 'failed' || result.status === 'skipped'))
  98. ? 1
  99. : 0
  100. }
  101. function parseMode(raw: string | undefined): Mode {
  102. switch (raw) {
  103. case 'ci-primary':
  104. case 'ci-linux-primary':
  105. case 'ci-static':
  106. case 'ci-lint-contracts-ready':
  107. case 'ci-coverage':
  108. case 'ci-snapshot':
  109. case 'ci-artifacts':
  110. case 'ci-consumers':
  111. case 'ci-windows-blocking':
  112. case 'ci-windows-complete':
  113. case 'ci-windows-observational':
  114. case 'node-compat':
  115. case 'check-all':
  116. case 'hygiene':
  117. case 'doc-sync':
  118. return raw
  119. default:
  120. throw new Error(
  121. `run-gates: expected mode ci-primary | ci-linux-primary | ci-static | ci-lint-contracts-ready | ci-coverage | ci-snapshot | ci-artifacts | ci-consumers | ci-windows-blocking | ci-windows-complete | ci-windows-observational | node-compat | check-all | hygiene | doc-sync, got ${JSON.stringify(raw)}.`,
  122. )
  123. }
  124. }
  125. /**
  126. * Resolve the default worker count for one aggregate.
  127. * @param selectedMode - aggregate whose resource posture applies.
  128. * @param total - number of gates in the aggregate.
  129. * @param available - host CPU availability for ordinary modes.
  130. * @returns the default worker count and its diagnostic source.
  131. */
  132. export function defaultConcurrency(
  133. selectedMode: Mode,
  134. total: number,
  135. available = availableParallelism(),
  136. ): ConcurrencyDefault {
  137. if (selectedMode === 'ci-consumers') return { workers: total, source: 'ci-consumers gate count' }
  138. // Local modes cap workers: several doc gates each build a full ts.Program,
  139. // so an uncapped default on a large host trades wall clock for memory blowups.
  140. const localCap = selectedMode === 'check-all' || selectedMode === 'hygiene' || selectedMode === 'doc-sync'
  141. const modeLimit = localCap ? Math.min(4, available) : available
  142. return {
  143. workers: Math.min(total, modeLimit),
  144. source: localCap
  145. ? `${available} available CPU(s), ${selectedMode} cap 4`
  146. : `${available} available CPU(s)`,
  147. }
  148. }
  149. function concurrencyFromEnv(name: string, fallback: number): number {
  150. const raw = process.env[name]
  151. if (raw === undefined || raw === '') return fallback
  152. const parsed = Number.parseInt(raw, 10)
  153. if (!Number.isSafeInteger(parsed) || parsed < 1) {
  154. throw new Error(`run-gates: ${name} must be a positive integer, got ${JSON.stringify(raw)}.`)
  155. }
  156. return parsed
  157. }
  158. function pnpmScript(id: string, script: string, options: Partial<Gate> = {}): Gate {
  159. return {
  160. id,
  161. label: options.label ?? script,
  162. displayCommand: `pnpm run ${script}`,
  163. ...pnpmInvocation(['run', script]),
  164. ...options,
  165. }
  166. }
  167. /** Build official client artifacts inside a CI aggregate without changing sibling gate environments. */
  168. function ciBuildGate(id = 'build', options: Partial<Gate> = {}): Gate {
  169. return pnpmScript(id, 'build', {
  170. ...options,
  171. env: { ...options.env, [CLIENT_BUILD_PROFILE_SELECTOR]: 'official' },
  172. })
  173. }
  174. function pnpmExec(id: string, args: string[], options: Partial<Gate> = {}): Gate {
  175. return {
  176. id,
  177. label: options.label ?? `pnpm exec ${args.join(' ')}`,
  178. displayCommand: `pnpm exec ${args.join(' ')}`,
  179. ...pnpmInvocation(['exec', ...args]),
  180. ...options,
  181. }
  182. }
  183. /**
  184. * Construct the complete gate list for a named aggregate.
  185. * @param selected - aggregate mode to construct.
  186. * @returns the aggregate's gate graph.
  187. */
  188. export function gatesForMode(selected: Mode): Gate[] {
  189. switch (selected) {
  190. case 'ci-primary':
  191. return ciPrimaryGates()
  192. case 'ci-linux-primary':
  193. return [...ciPrimaryGates(), webSnapshotGate(['built-package-invariants'])]
  194. case 'ci-static':
  195. return ciStaticGates({ ownsBuild: false })
  196. case 'ci-lint-contracts-ready':
  197. return [
  198. lintGate(),
  199. pnpmScript('duplication', 'duplication'),
  200. ]
  201. case 'ci-coverage':
  202. return coverageGates()
  203. case 'ci-snapshot':
  204. return [ciBuildGate(), snapshotGate()]
  205. case 'ci-artifacts':
  206. return ciArtifactGates()
  207. case 'ci-consumers':
  208. return ciConsumerGates()
  209. case 'ci-windows-blocking':
  210. return ciWindowsBlockingGates()
  211. case 'ci-windows-complete':
  212. return ciWindowsCompleteGates()
  213. case 'ci-windows-observational':
  214. return ciWindowsObservationalGates()
  215. case 'node-compat':
  216. return nodeCompatGates()
  217. case 'check-all':
  218. return [
  219. pnpmScript('runtime-closure', 'verify-runtime-closure', { label: 'runtime closure' }),
  220. pnpmScript('cordis-config', 'verify-cordis-config', { label: 'Cordis config' }),
  221. pnpmScript('client-domain-graph', 'verify-client-domain-graph', { label: 'client domain graph' }),
  222. pnpmScript('test', 'test'),
  223. pnpmScript('issue-management', 'test:issue-management', { label: 'Issue management policy' }),
  224. pnpmScript('duplication', 'duplication'),
  225. snapshotGate(),
  226. pnpmScript('build', 'build'),
  227. pnpmScript('build:web', 'build:web'),
  228. ...hygieneLeafGates({ artifactNeeds: ['build'] }),
  229. ...docSyncLeafGates({
  230. docTypecheckNeeds: ['build'],
  231. docTypecheckEnv: { DSH_DOC_TYPECHECK_USE_BUILD_OUTPUT: '1' },
  232. docTypecheckScript: 'doc-typecheck:contracts-ready',
  233. }),
  234. pnpmScript('module-graph', 'verify-module-graph', { label: 'module graph' }),
  235. ]
  236. case 'hygiene':
  237. return [
  238. ...hygieneLeafGates(),
  239. pnpmScript('cordis-config', 'verify-cordis-config', { label: 'Cordis config' }),
  240. pnpmScript('runtime-closure', 'verify-runtime-closure', { label: 'runtime closure' }),
  241. pnpmScript('vendored-links', 'verify-vendored-links', { label: 'vendored links' }),
  242. ]
  243. case 'doc-sync':
  244. return docSyncLeafGates()
  245. }
  246. }
  247. function ciSharedStaticGates(): Gate[] {
  248. return [
  249. pnpmScript('runtime-closure', 'verify-runtime-closure', { label: 'runtime closure' }),
  250. pnpmScript('application-entrypoints', 'verify-application-entrypoints', { label: 'application entrypoints' }),
  251. pnpmScript('constraints', 'constraints'),
  252. pnpmScript('dsh-package-licenses', 'verify-dsh-package-licenses', { label: 'DSH package licenses' }),
  253. pnpmScript('package-invariants', 'verify-package-invariants', { label: 'package invariants' }),
  254. pnpmScript('cordis-config', 'verify-cordis-config', { label: 'Cordis config' }),
  255. pnpmScript('optional-dependency-imports', 'verify-optional-dependency-imports', {
  256. label: 'optional dependency imports',
  257. }),
  258. pnpmScript('client-packages', 'verify-client-packages', { label: 'client packages' }),
  259. pnpmScript('issue-management', 'test:issue-management', { label: 'Issue management policy' }),
  260. ]
  261. }
  262. function ciPrimaryGates(): Gate[] {
  263. return [
  264. ...ciSharedStaticGates(),
  265. typertContractsGate(),
  266. pnpmScript('typecheck', 'typecheck:contracts-ready', { needs: ['typert-contracts'] }),
  267. lintGate({ needs: ['typert-contracts'] }),
  268. pnpmScript('duplication', 'duplication'),
  269. ...coverageGates(),
  270. ...nodeCompatSmokeGates(),
  271. snapshotGate(),
  272. ...docSyncLeafGates({
  273. docTypecheckNeeds: ['typert-contracts'],
  274. docTypecheckScript: 'doc-typecheck:contracts-ready',
  275. }),
  276. pnpmScript('module-graph', 'verify-module-graph', { label: 'module graph' }),
  277. pnpmScript('knip', 'knip'),
  278. // The prepared typecheck and build both drive Client tsc, while build also
  279. // repeats the Host contract pass. Wait for all three consumers so build
  280. // neither races tsbuildinfo nor replaces declarations while they are read.
  281. ciBuildGate('build', { needs: ['typecheck', 'lint', 'doc-typecheck'] }),
  282. pnpmScript('publint', 'publint', { needs: ['build'] }),
  283. pnpmScript('node-next-types', 'verify-node-next-types', {
  284. label: 'node-next types',
  285. needs: ['build'],
  286. }),
  287. builtPackageInvariantsGate(['build']),
  288. builtBinSmokeGate(),
  289. ]
  290. }
  291. function nodeCompatGates(): Gate[] {
  292. const typecheck = flagEnabled('DSH_NODE_COMPAT_SKIP_TYPECHECK')
  293. ? []
  294. : [pnpmScript('typecheck', 'typecheck')]
  295. if (runningNodeMajor() !== 22) {
  296. return [...typecheck, ...nodeCompatSmokeGates()]
  297. }
  298. return [
  299. ...typecheck,
  300. pnpmScript('build', 'build', {
  301. ...typecheck.length === 0 ? {} : { needs: ['typecheck'] },
  302. }),
  303. pnpmScript('build:web', 'build:web', {
  304. label: 'Web frontend build',
  305. needs: ['build'],
  306. }),
  307. ...nodeCompatSmokeGates({ cliSmoke: true }),
  308. ]
  309. }
  310. function nodeCompatSmokeGates(options: { cliSmoke?: boolean } = {}): Gate[] {
  311. const gates: Gate[] = [
  312. pnpmExec('source-worker-smoke', [
  313. 'vitest',
  314. 'run',
  315. 'packages/workflow/workflow-worker-thread/tests/source-worker.compat.spec.ts',
  316. ], { label: 'source worker smoke' }),
  317. pnpmExec('jsonl-zstd-smoke', [
  318. 'vitest',
  319. 'run',
  320. 'packages/session/session-persistence-jsonl/tests/zstd.compat.spec.ts',
  321. ], { label: 'JSONL Zstandard smoke' }),
  322. pnpmExec('dsh-source-launch-smoke', [
  323. 'vitest',
  324. 'run',
  325. 'apps/cli/tests/source-launch.compat.spec.ts',
  326. ], { label: 'dsh source-launch smoke' }),
  327. pnpmExec('vitest-jsdom-smoke', [
  328. 'vitest',
  329. 'run',
  330. 'scripts/vitest-environment.compat.spec.ts',
  331. ], { label: 'Vitest jsdom smoke' }),
  332. ]
  333. if (options.cliSmoke) {
  334. gates.push(
  335. pnpmExec('cli-lazy-search-startup-smoke', [
  336. 'vitest',
  337. 'run',
  338. 'apps/cli/tests/lazy-search-startup.compat.spec.ts',
  339. ], {
  340. label: 'CLI lazy-search startup smoke',
  341. env: { DSH_REQUIRE_BUILT_CLI_SMOKE: '1' },
  342. needs: ['build:web'],
  343. }),
  344. )
  345. }
  346. return gates
  347. }
  348. /** Active Node major used to select version-specific compatibility checks. */
  349. function runningNodeMajor(): number {
  350. const major = Number.parseInt(process.versions.node.split('.')[0] ?? '', 10)
  351. if (!Number.isSafeInteger(major)) {
  352. throw new Error(`run-gates: cannot parse Node version ${JSON.stringify(process.versions.node)}.`)
  353. }
  354. return major
  355. }
  356. function ciStaticGates(options: { ownsBuild: boolean }): Gate[] {
  357. return [
  358. ...ciSharedStaticGates(),
  359. ...options.ownsBuild ? [ciBuildGate()] : [],
  360. ...docSyncLeafGates({
  361. includeDocTypecheck: options.ownsBuild,
  362. ...options.ownsBuild
  363. ? {
  364. docTypecheckNeeds: ['build'],
  365. docTypecheckEnv: { DSH_DOC_TYPECHECK_USE_BUILD_OUTPUT: '1' },
  366. docTypecheckScript: 'doc-typecheck:contracts-ready',
  367. }
  368. : {},
  369. docsBuildScript: 'docs:build:mpa',
  370. }),
  371. pnpmScript('module-graph', 'verify-module-graph', { label: 'module graph' }),
  372. pnpmScript('knip', 'knip'),
  373. ]
  374. }
  375. function ciArtifactGates(): Gate[] {
  376. return [
  377. ciBuildGate(),
  378. pnpmScript('publint', 'publint', { needs: ['build'] }),
  379. pnpmScript('node-next-types', 'verify-node-next-types', {
  380. label: 'node-next types',
  381. needs: ['build'],
  382. }),
  383. builtPackageInvariantsGate(['build']),
  384. builtBinSmokeGate(),
  385. ]
  386. }
  387. function ciConsumerGates(): Gate[] {
  388. const builtTree = ['build']
  389. const validatedBuild = ['built-package-invariants']
  390. return [
  391. ciBuildGate(),
  392. pnpmScript('node-compat', 'check:node-compat', {
  393. label: 'Node compatibility',
  394. env: { [CLIENT_BUILD_PROFILE_SELECTOR]: 'official' },
  395. }),
  396. pnpmScript('publint', 'publint', { needs: builtTree }),
  397. builtPackageInvariantsGate(builtTree),
  398. pnpmScript('lint-and-duplication', 'check:ci:lint:contracts-ready', {
  399. label: 'lint and duplication',
  400. needs: validatedBuild,
  401. }),
  402. snapshotGate(validatedBuild),
  403. webSnapshotGate(validatedBuild),
  404. pnpmScript('doc-typecheck', 'doc-typecheck:contracts-ready', {
  405. needs: validatedBuild,
  406. env: { DSH_DOC_TYPECHECK_USE_BUILD_OUTPUT: '1' },
  407. }),
  408. pnpmScript('node-next-types', 'verify-node-next-types', {
  409. label: 'node-next types',
  410. needs: validatedBuild,
  411. }),
  412. builtBinSmokeGate(validatedBuild),
  413. ]
  414. }
  415. function webSnapshotGate(needs: string[]): Gate {
  416. const workerRaw = process.env.DSH_WEB_SNAPSHOT_WORKERS
  417. if (workerRaw !== undefined && workerRaw !== '') {
  418. const workers = Number.parseInt(workerRaw, 10)
  419. if (!Number.isSafeInteger(workers) || workers < 2 || String(workers) !== workerRaw) {
  420. throw new Error(`run-gates: DSH_WEB_SNAPSHOT_WORKERS must be an integer greater than 1, got ${JSON.stringify(workerRaw)}.`)
  421. }
  422. return pnpmScript('web-snapshot', 'test:web:ci', {
  423. label: 'web browser snapshot',
  424. displayCommand: `DSH_SNAPSHOT=replay DSH_WEB_SNAPSHOT_WORKERS=${workers} pnpm run test:web:ci`,
  425. env: { DSH_SNAPSHOT: 'replay' },
  426. needs,
  427. streamOutput: true,
  428. })
  429. }
  430. return pnpmScript('web-snapshot', 'test:web:built', {
  431. label: 'web browser snapshot',
  432. displayCommand: 'DSH_SNAPSHOT=replay pnpm run test:web:built',
  433. env: { DSH_SNAPSHOT: 'replay' },
  434. needs,
  435. })
  436. }
  437. function ciWindowsBlockingGates(): Gate[] {
  438. return [
  439. ciBuildGate('windows-build', { label: 'build' }),
  440. pnpmScript('windows-site', 'docs:build', { label: 'production site' }),
  441. ]
  442. }
  443. function ciWindowsCompleteGates(): Gate[] {
  444. const coverage = coverageGates().map(gate => gate.id === 'coverage-exempt-heavy'
  445. ? {
  446. ...gate,
  447. needs: [...new Set(['build', ...(gate.needs ?? [])])],
  448. after: [...new Set(['coverage', ...(gate.after ?? [])])],
  449. }
  450. : gate)
  451. const observational = ciWindowsObservationalGates()
  452. // The required production site replaces the observational MPA build; both
  453. // VitePress modes write the same output directory and cannot overlap.
  454. .filter(gate => gate.id !== 'build' && gate.id !== 'docs-site-build')
  455. .map(gate => ({
  456. ...gate,
  457. allowFailure: true,
  458. after: [...new Set(['coverage', ...(gate.after ?? [])])],
  459. }))
  460. return [
  461. ciBuildGate(),
  462. pnpmScript('windows-site', 'docs:build', { label: 'production site' }),
  463. ...coverage,
  464. ...observational,
  465. ]
  466. }
  467. function ciWindowsObservationalGates(): Gate[] {
  468. return [
  469. ...ciStaticGates({ ownsBuild: true }),
  470. // Linux owns required lint and snapshots; Windows omits those duplicates.
  471. pnpmScript('duplication', 'duplication'),
  472. pnpmScript('publint', 'publint', { needs: ['build'] }),
  473. pnpmScript('node-next-types', 'verify-node-next-types', {
  474. label: 'node-next types',
  475. needs: ['build'],
  476. }),
  477. builtPackageInvariantsGate(['build']),
  478. builtBinSmokeGate(),
  479. ]
  480. }
  481. function typertContractsGate(): Gate {
  482. return pnpmScript('typert-contracts', 'build:lib:host', { label: 'Typert contracts' })
  483. }
  484. function lintGate(options: { needs?: string[] } = {}): Gate {
  485. const raw = process.env.DSH_OXLINT_THREADS
  486. const script = 'lint:contracts-ready'
  487. return pnpmScript('lint', script, {
  488. ...raw === undefined || raw === ''
  489. ? {}
  490. : { displayCommand: `DSH_OXLINT_THREADS=${raw} pnpm run ${script}` },
  491. ...options.needs === undefined ? {} : { needs: options.needs },
  492. })
  493. }
  494. // The heavy suites run uninstrumented beside the thresholded gate: their
  495. // compiler- and subprocess-bound fixtures pay a multiple of their runtime
  496. // under v8 instrumentation while contributing nothing the thresholds need
  497. // (membership rules in scripts/coverage-exempt.ts).
  498. //
  499. // DSH_COVERAGE_MAX_WORKERS is the ordinary lane's worker budget, so the two
  500. // parallel gates split it instead of each claiming it whole. When
  501. // DSH_COVERAGE_PARTITIONS is set, its single-worker processes replace the
  502. // instrumented share while this budget still sizes the exempt gate. The exempt
  503. // gate's wall clock is dominated by its longest single file, so it takes the
  504. // small share. A budget of 1 gives each gate 1 worker; lanes that need a strict
  505. // total of one (the serial reference jobs) also set DSH_GATE_CONCURRENCY=1,
  506. // which keeps the gates from overlapping at all.
  507. // DSH_COVERAGE_TEST_TIMEOUT_MS raises Vitest's per-test and expect.poll
  508. // defaults together for instrumented lanes whose scheduling overhead exceeds
  509. // those defaults. Explicit fixture timeouts remain authoritative.
  510. function coverageWorkerArgs(): { instrumented: string[]; exempt: string[] } {
  511. const [flag] = positiveIntArg('DSH_COVERAGE_MAX_WORKERS', '--maxWorkers')
  512. if (flag === undefined) return { instrumented: [], exempt: [] }
  513. const total = Number.parseInt(flag.split('=')[1] ?? '', 10)
  514. const exempt = Math.max(1, Math.floor(total / 3))
  515. const instrumented = Math.max(1, total - exempt)
  516. return {
  517. instrumented: [`--maxWorkers=${String(instrumented)}`],
  518. exempt: [`--maxWorkers=${String(exempt)}`],
  519. }
  520. }
  521. function coverageGates(): Gate[] {
  522. const workers = coverageWorkerArgs()
  523. const timeouts = coverageTestTimeoutArgs(process.env[COVERAGE_TEST_TIMEOUT_ENV])
  524. const partitions = parseCoveragePartitionCount(process.env[COVERAGE_PARTITIONS_ENV])
  525. const instrumented = partitions === undefined
  526. ? pnpmExec('coverage', [
  527. 'vitest',
  528. 'run',
  529. '--coverage',
  530. ...workers.instrumented,
  531. ...timeouts,
  532. ], {
  533. label: 'test:coverage',
  534. env: { [COVERAGE_EXEMPT_ENV]: '1' },
  535. })
  536. : pnpmScript('coverage', 'test:coverage:partitioned', {
  537. label: 'test:coverage',
  538. displayCommand: `${COVERAGE_PARTITIONS_ENV}=${partitions} pnpm run test:coverage:partitioned`,
  539. env: { [COVERAGE_EXEMPT_ENV]: '1' },
  540. streamOutput: true,
  541. })
  542. return [
  543. instrumented,
  544. pnpmExec('coverage-exempt-heavy', [
  545. 'vitest',
  546. 'run',
  547. ...coverageExemptHeavySuites.map(suite => suite.filter),
  548. ...workers.exempt,
  549. ...timeouts,
  550. ], {
  551. label: 'test:coverage-exempt-heavy',
  552. }),
  553. ]
  554. }
  555. // Example and package snapshots boot their bins in `lib` mode (built artifacts under plain Node,
  556. // plugins via real exports); script snapshots execute their real source entry path.
  557. // Callers wait either on `build` or on a validation gate that transitively owns that build.
  558. function snapshotGate(needs: string[] = ['build']): Gate {
  559. return pnpmScript('snapshot', 'test:snapshot', {
  560. env: { DSH_EXAMPLE_MODE: 'lib' },
  561. needs,
  562. })
  563. }
  564. function builtPackageInvariantsGate(needs?: string[]): Gate {
  565. return pnpmScript('built-package-invariants', 'verify-built-package-invariants', {
  566. label: 'built package invariants',
  567. ...needs === undefined ? {} : { needs },
  568. })
  569. }
  570. function positiveIntArg(envName: string, flag: string): string[] {
  571. const raw = process.env[envName]
  572. if (raw === undefined || raw === '') return []
  573. const parsed = Number.parseInt(raw, 10)
  574. if (!Number.isSafeInteger(parsed) || parsed < 1 || String(parsed) !== raw) {
  575. throw new Error(`run-gates: ${envName} must be a positive integer, got ${JSON.stringify(raw)}.`)
  576. }
  577. return [`${flag}=${raw}`]
  578. }
  579. function flagEnabled(envName: string): boolean {
  580. const raw = process.env[envName]
  581. if (raw === undefined || raw === '') return false
  582. if (raw !== '1') throw new Error(`run-gates: ${envName} must be 1 when set, got ${JSON.stringify(raw)}.`)
  583. return true
  584. }
  585. function hygieneLeafGates(options: { artifactNeeds?: string[] } = {}): Gate[] {
  586. const artifactOptions = options.artifactNeeds === undefined ? {} : { needs: options.artifactNeeds }
  587. return [
  588. pnpmScript('rescope-vendor', 'rescope-vendor:check', { label: 'vendor rescope' }),
  589. pnpmScript('knip', 'knip'),
  590. pnpmScript('publint', 'publint', artifactOptions),
  591. pnpmScript('constraints', 'constraints'),
  592. pnpmScript('application-entrypoints', 'verify-application-entrypoints', { label: 'application entrypoints' }),
  593. pnpmScript('dsh-package-licenses', 'verify-dsh-package-licenses', { label: 'DSH package licenses' }),
  594. pnpmScript('package-invariants', 'verify-package-invariants', { label: 'package invariants' }),
  595. builtPackageInvariantsGate(options.artifactNeeds),
  596. pnpmScript('node-next-types', 'verify-node-next-types', {
  597. label: 'node-next types',
  598. ...artifactOptions,
  599. }),
  600. pnpmScript('optional-dependency-imports', 'verify-optional-dependency-imports', {
  601. label: 'optional dependency imports',
  602. }),
  603. pnpmScript('client-packages', 'verify-client-packages', { label: 'client packages' }),
  604. ]
  605. }
  606. function docSyncLeafGates(options: {
  607. includeDocTypecheck?: boolean
  608. docTypecheckNeeds?: string[]
  609. docTypecheckEnv?: Record<string, string | undefined>
  610. docTypecheckScript?: 'doc-typecheck' | 'doc-typecheck:contracts-ready'
  611. docsBuildScript?: 'docs:build' | 'docs:build:mpa'
  612. } = {}): Gate[] {
  613. const docTypecheckOptions: Partial<Gate> = {}
  614. if (options.docTypecheckNeeds !== undefined) docTypecheckOptions.needs = options.docTypecheckNeeds
  615. if (options.docTypecheckEnv !== undefined) docTypecheckOptions.env = options.docTypecheckEnv
  616. return [
  617. // Stable FIFO starts the longest leaves first; only docs-site-build writes website/.generated.
  618. ...options.includeDocTypecheck === false
  619. ? []
  620. : [pnpmScript('doc-typecheck', options.docTypecheckScript ?? 'doc-typecheck', docTypecheckOptions)],
  621. pnpmScript('docs-site-build', options.docsBuildScript ?? 'docs:build', { label: 'documentation build' }),
  622. pnpmScript('doc-graphs', 'verify-doc-graphs', { label: 'doc graphs' }),
  623. pnpmScript('markdown-links', 'verify-md-links', { label: 'markdown links' }),
  624. pnpmScript('type-equivalence', 'verify-type-equiv', { label: 'type equivalence' }),
  625. pnpmScript('cordis-catalog', 'verify-cordis-catalog', { label: 'cordis catalog' }),
  626. pnpmScript('mermaid', 'verify-mermaid'),
  627. pnpmScript('scoped-events', 'verify-scoped-events', { label: 'scoped events' }),
  628. pnpmScript('translation-pairing', 'verify-translation-pairing', { label: 'translation pairing' }),
  629. pnpmScript('markdown-wrap', 'verify-md-wrap', { label: 'markdown wrap' }),
  630. pnpmScript('client-catalog', 'verify-client-catalog', { label: 'client catalog' }),
  631. pnpmScript('export-jsdoc', 'verify-export-jsdoc', { label: 'export jsdoc' }),
  632. pnpmScript('tool-catalog', 'verify-tool-catalog', { label: 'tool catalog' }),
  633. pnpmScript('config-catalog', 'verify-config-catalog', { label: 'config catalog' }),
  634. pnpmScript('persistence-catalog', 'verify-persistence-catalog', { label: 'persistence catalog' }),
  635. pnpmScript('public-repository-links', 'verify-public-repository-links', { label: 'public repository links' }),
  636. pnpmScript('doc-refs', 'verify-doc-refs', { label: 'doc refs' }),
  637. pnpmScript('subsystem-pages', 'verify-subsystem-pages', { label: 'subsystem pages' }),
  638. pnpmScript('package-paths', 'verify-package-paths', { label: 'package paths' }),
  639. pnpmScript('config-source-ownership', 'verify-config-source-ownership', { label: 'config source ownership' }),
  640. pnpmScript('package-readme-model-experience', 'verify-package-readme-model-experience', { label: 'package README model experience' }),
  641. pnpmScript('agent-note-classification', 'verify-agent-note-classification', { label: 'agent note classification' }),
  642. pnpmScript('agent-note-format', 'verify-agent-note-format', { label: 'agent note format' }),
  643. pnpmScript('archived-agent-notes', 'verify-archived-agent-notes', { label: 'archived agent notes' }),
  644. pnpmScript('skill-invocation-metadata', 'verify-skill-invocation-metadata', { label: 'skill invocation metadata' }),
  645. pnpmScript('translation-prompt', 'verify-translation-prompt', { label: 'translation prompt' }),
  646. pnpmScript('doc-budgets', 'verify-doc-budgets', { label: 'doc budgets' }),
  647. pnpmExec('docs-site-projection', ['vitest', 'run', 'scripts/project-doc-site.spec.ts', 'scripts/verify-doc-site-fragments.spec.ts'], {
  648. label: 'documentation site checks',
  649. }),
  650. pnpmScript('package-readme-limitations', 'verify-package-readme-limitations', { label: 'package README limitations' }),
  651. ]
  652. }
  653. function builtBinSmokeGate(needs: string[] = ['build']): Gate {
  654. return pnpmExec('built-bin-smoke', [
  655. 'vitest',
  656. 'run',
  657. '--config',
  658. 'vitest.e2e.config.ts',
  659. 'examples/headless-agent/tests/keyless-smoke.e2e.ts',
  660. 'apps/cli/tests/built-bin.e2e.ts',
  661. 'packages/host/directory-picker-native/tests/built-worker.e2e.ts',
  662. 'packages/sdk/server/tests/built-scope-carrier.e2e.ts',
  663. 'packages/subagent/subagent-codex/tests/loader-composition.e2e.ts',
  664. 'packages/subagent/subagent-claude-code/tests/loader-composition.e2e.ts',
  665. 'packages/api/remotes/tests/built-lib.e2e.ts',
  666. // Built execution consumers: the only automated proof that package-name
  667. // imports reach their lib/ entrypoints under plain Node. The e2e lane runs
  668. // unbuilt, so these files self-skip there.
  669. 'packages/workflow/workflow-worker-thread/tests/built-worker.e2e.ts',
  670. 'packages/code-runtime/code-runtime-worker-thread/tests/built-lib.e2e.ts',
  671. 'packages/lsp/lsp-stdio/tests/built-lib.e2e.ts',
  672. ], {
  673. label: 'built-bin smoke',
  674. needs,
  675. env: { DSH_EXAMPLE_MODE: 'lib' },
  676. })
  677. }
  678. /**
  679. * Reject a gate list whose graph cannot be executed unambiguously.
  680. * @param gates - complete aggregate to validate.
  681. */
  682. function validateGateGraph(gates: readonly Gate[]): void {
  683. if (gates.length === 0) throw new Error('run-gates: gate graph has no gates.')
  684. const ids = new Set<string>()
  685. for (const gate of gates) {
  686. if (ids.has(gate.id)) throw new Error(`run-gates: duplicate gate id ${JSON.stringify(gate.id)}.`)
  687. ids.add(gate.id)
  688. }
  689. for (const gate of gates) {
  690. for (const dependency of gate.needs ?? []) {
  691. if (!ids.has(dependency)) {
  692. throw new Error(`run-gates: gate ${JSON.stringify(gate.id)} depends on unknown gate ${JSON.stringify(dependency)}.`)
  693. }
  694. }
  695. for (const predecessor of gate.after ?? []) {
  696. if (!ids.has(predecessor)) {
  697. throw new Error(`run-gates: gate ${JSON.stringify(gate.id)} waits for unknown gate ${JSON.stringify(predecessor)}.`)
  698. }
  699. }
  700. }
  701. const cycle = findDependencyCycle(gates)
  702. if (cycle !== undefined) throw new Error(`run-gates: dependency cycle: ${cycle.join(' -> ')}.`)
  703. }
  704. function findDependencyCycle(gates: readonly Gate[]): string[] | undefined {
  705. const byId = new Map(gates.map(gate => [gate.id, gate]))
  706. const complete = new Set<string>()
  707. const active = new Map<string, number>()
  708. const path: string[] = []
  709. const visit = (id: string): string[] | undefined => {
  710. if (complete.has(id)) return undefined
  711. const cycleStart = active.get(id)
  712. if (cycleStart !== undefined) return [...path.slice(cycleStart), id]
  713. const gate = byId.get(id)
  714. if (gate === undefined) return undefined
  715. active.set(id, path.length)
  716. path.push(id)
  717. for (const predecessor of [...(gate.needs ?? []), ...(gate.after ?? [])]) {
  718. const cycle = visit(predecessor)
  719. if (cycle !== undefined) return cycle
  720. }
  721. path.pop()
  722. active.delete(id)
  723. complete.add(id)
  724. return undefined
  725. }
  726. for (const gate of gates) {
  727. const cycle = visit(gate.id)
  728. if (cycle !== undefined) return cycle
  729. }
  730. return undefined
  731. }
  732. /**
  733. * Validate and run one aggregate before the injected executor can start a child.
  734. * @param gates - complete aggregate to execute.
  735. * @param maxActive - maximum concurrent child count.
  736. * @param execute - child-process executor.
  737. * @param observe - result observer invoked when each gate settles.
  738. * @returns results in aggregate order.
  739. */
  740. export async function runGates(
  741. gates: Gate[],
  742. maxActive: number,
  743. execute: GateExecutor,
  744. observe: ResultObserver = () => {},
  745. ): Promise<GateResult[]> {
  746. validateGateGraph(gates)
  747. if (!Number.isSafeInteger(maxActive) || maxActive < 1) {
  748. throw new Error(`run-gates: max concurrency must be a positive integer, got ${JSON.stringify(maxActive)}.`)
  749. }
  750. const states = new Map<string, GateState>(gates.map(gate => [gate.id, 'pending']))
  751. const results = new Map<string, GateResult>()
  752. const running: RunningGate[] = []
  753. for (;;) {
  754. let madeProgress = false
  755. while (running.length < maxActive) {
  756. const ready = gates.find(gate => states.get(gate.id) === 'pending' && predecessorsReady(gate, states))
  757. if (ready === undefined) break
  758. states.set(ready.id, 'running')
  759. running.push({ gate: ready, promise: execute(ready) })
  760. console.log(`run-gates: start ${ready.label}`)
  761. madeProgress = true
  762. }
  763. if (running.length === 0) {
  764. const pending = gates.filter(gate => states.get(gate.id) === 'pending')
  765. if (pending.length === 0) break
  766. const gate = pending.find(item => (item.needs ?? []).some(id => gateFailed(states.get(id))))
  767. if (gate === undefined) throw new Error('run-gates: validated graph stalled without a failed dependency.')
  768. const failedDeps = (gate.needs ?? []).filter(id => gateFailed(states.get(id)))
  769. const result: GateResult = {
  770. gate,
  771. status: 'skipped',
  772. durationMs: 0,
  773. output: [],
  774. exitCode: null,
  775. signalCode: null,
  776. error: `dependency failed or skipped: ${failedDeps.join(', ')}`,
  777. }
  778. states.set(gate.id, 'skipped')
  779. results.set(gate.id, result)
  780. observe(result)
  781. continue
  782. }
  783. if (!madeProgress) {
  784. const settled = await Promise.race(running.map(async item => ({ item, result: await item.promise })))
  785. running.splice(running.indexOf(settled.item), 1)
  786. states.set(settled.item.gate.id, settled.result.status)
  787. results.set(settled.item.gate.id, settled.result)
  788. observe(settled.result)
  789. }
  790. }
  791. return gates.map((gate) => {
  792. const result = results.get(gate.id)
  793. if (result === undefined) throw new Error(`run-gates: missing result for ${gate.id}.`)
  794. return result
  795. })
  796. }
  797. function predecessorsReady(gate: Gate, states: Map<string, GateState>): boolean {
  798. return (gate.needs ?? []).every(id => states.get(id) === 'passed')
  799. && (gate.after ?? []).every(id => gateSettled(states.get(id)))
  800. }
  801. function gateSettled(state: GateState | undefined): boolean {
  802. return state === 'passed' || state === 'failed' || state === 'skipped'
  803. }
  804. function gateFailed(state: GateState | undefined): boolean {
  805. return state === 'failed' || state === 'skipped'
  806. }
  807. /**
  808. * Execute one gate through the real shell-free child-process boundary.
  809. * @param gate - command and scheduler environment to execute.
  810. * @returns the complete process outcome.
  811. */
  812. export async function runGate(gate: Gate): Promise<GateResult> {
  813. const started = performance.now()
  814. const output: GateOutputChunk[] = []
  815. let spawnError: string | undefined
  816. const outcome = await new Promise<{
  817. exitCode: number | null
  818. signalCode: NodeJS.Signals | null
  819. }>((resolveExit) => {
  820. const child = spawn(gate.command, gate.args, {
  821. cwd: root,
  822. env: { ...process.env, ...gate.env },
  823. stdio: ['pipe', 'pipe', 'pipe'],
  824. })
  825. child.stdout.setEncoding('utf8')
  826. child.stderr.setEncoding('utf8')
  827. child.stdout.on('data', (chunk: string) => {
  828. if (gate.streamOutput === true) process.stdout.write(chunk)
  829. else output.push({ stream: 'stdout', text: chunk })
  830. })
  831. child.stderr.on('data', (chunk: string) => {
  832. if (gate.streamOutput === true) process.stderr.write(chunk)
  833. else output.push({ stream: 'stderr', text: chunk })
  834. })
  835. child.on('error', (error) => {
  836. spawnError = `failed to start command: ${error.message}`
  837. resolveExit({ exitCode: null, signalCode: null })
  838. })
  839. child.on('close', (exitCode, signalCode) => {
  840. resolveExit({ exitCode, signalCode })
  841. })
  842. child.stdin.end()
  843. })
  844. const { exitCode, signalCode } = outcome
  845. const status: GateResultStatus = exitCode === 0 && signalCode === null && spawnError === undefined ? 'passed' : 'failed'
  846. const result: GateResult = {
  847. gate,
  848. status,
  849. durationMs: performance.now() - started,
  850. output,
  851. exitCode,
  852. signalCode,
  853. }
  854. if (spawnError !== undefined) result.error = spawnError
  855. return result
  856. }
  857. /**
  858. * Format every independently observed failure fact for the aggregate summary.
  859. * @param result - unsuccessful gate result.
  860. * @returns error, exit, and signal facts without allowing one to hide another.
  861. */
  862. export function formatGateResultReason(result: GateResult): string {
  863. const facts: string[] = []
  864. if (result.error !== undefined) facts.push(result.error)
  865. if (result.exitCode !== null) facts.push(`exit ${result.exitCode}`)
  866. if (result.signalCode !== null) facts.push(`signal ${result.signalCode}`)
  867. return facts.length === 0 ? 'no exit code or signal' : facts.join(', ')
  868. }
  869. function printResult(result: GateResult): void {
  870. const verbose = process.env.DSH_GATE_VERBOSE === '1'
  871. const seconds = (result.durationMs / 1000).toFixed(2)
  872. if (result.status === 'passed' && !verbose) {
  873. console.log(`run-gates: PASS ${result.gate.label} (${seconds}s)`)
  874. return
  875. }
  876. const heading = `${result.status.toUpperCase()} ${result.gate.label} (${seconds}s)`
  877. const writeHeading = result.status === 'passed' ? console.log : console.error
  878. writeHeading(`\n== ${heading} ==`)
  879. if (result.status !== 'passed') {
  880. console.error(`command: ${result.gate.displayCommand}`)
  881. console.error(`outcome: ${formatGateResultReason(result)}`)
  882. }
  883. if (result.gate.streamOutput !== true) printOutput(result.output)
  884. }
  885. function printSummary(results: GateResult[], durationMs: number): void {
  886. const passed = results.filter(result => result.status === 'passed').length
  887. const failed = results.filter(result => result.status === 'failed').length
  888. const skipped = results.filter(result => result.status === 'skipped').length
  889. const seconds = (durationMs / 1000).toFixed(2)
  890. console.log(`\nrun-gates: ${passed} passed, ${failed} failed, ${skipped} skipped in ${seconds}s.`)
  891. const unsuccessful = results.filter(result => result.status === 'failed' || result.status === 'skipped')
  892. if (unsuccessful.length === 0) return
  893. console.error('run-gates: unsuccessful gates:')
  894. for (const result of unsuccessful) {
  895. const duration = (result.durationMs / 1000).toFixed(2)
  896. const reason = formatGateResultReason(result)
  897. const disposition = result.gate.allowFailure === true ? 'NON-BLOCKING ' : ''
  898. console.error(` - ${disposition}${result.status.toUpperCase()} ${result.gate.label} (${duration}s, ${reason})`)
  899. console.error(` ${result.gate.displayCommand}`)
  900. }
  901. }
  902. function printOutput(output: GateOutputChunk[]): void {
  903. for (const chunk of output) {
  904. if (chunk.stream === 'stdout') process.stdout.write(chunk.text)
  905. else process.stderr.write(chunk.text)
  906. }
  907. }