Przeglądaj źródła

ci: split primary cold-path budget

Tianyi Cui 2 miesięcy temu
rodzic
commit
9d81078f2f
3 zmienionych plików z 91 dodań i 30 usunięć
  1. 48 13
      .github/workflows/ci.yml
  2. 1 0
      package.json
  3. 42 17
      scripts/run-gates.ts

+ 48 - 13
.github/workflows/ci.yml

@@ -319,19 +319,16 @@ jobs:
         run: ${{ matrix.command }}
 
   # One large runner pays hosted setup once, then the repository scheduler
-  # overlaps the unsharded primary inventory except the production site build.
+  # overlaps the unsharded core inventory. CPU and site chains use coarse lanes.
   node-24:
     if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners'
     runs-on: dsh-ubuntu-24-04-96core
     name: node 24 / core
     env:
-      # Thirty-two Vitest forks intermittently crash Node's CJS lexer on this image.
-      DSH_COVERAGE_MAX_WORKERS: '16'
       DSH_ESLINT_CACHE: '1'
       DSH_ESLINT_CONCURRENCY: '32'
       DSH_GATE_CONCURRENCY: '32'
       DSH_PUBLINT_CONCURRENCY: '32'
-      DSH_SNAPSHOT_MAX_CONCURRENCY: '32'
     steps:
       - uses: actions/checkout@v6
 
@@ -383,8 +380,8 @@ jobs:
       - name: Run unsharded primary Node core CI concurrently
         run: pnpm run check:ci:large-runner
 
-  # Keep only the longest independent Linux gate on a second coarse-grained
-  # runner so cold install variance does not push the primary box over a minute.
+  # Keep the longest independent Linux gate on its own coarse-grained runner so
+  # cold install variance does not push either primary aggregate over a minute.
   node-24-site:
     if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners'
     runs-on: dsh-ubuntu-24-04-16core
@@ -418,24 +415,39 @@ jobs:
   node-compat:
     if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners'
     # Distinct larger-runner pools avoid both standard-runner setup outliers and
-    # the delayed second allocation observed when two jobs shared one pool.
+    # the delayed second allocation observed when two jobs shared one pool. The
+    # primary Node row also owns the CPU/dependency-critical primary gate chain.
     runs-on: ${{ matrix.runner }}
-    name: node ${{ matrix.node }}
+    name: ${{ matrix.name }}
     env:
-      DSH_GATE_CONCURRENCY: '2'
+      DSH_COVERAGE_MAX_WORKERS: ${{ matrix.coverage_workers }}
+      DSH_GATE_CONCURRENCY: ${{ matrix.gate_concurrency }}
       DSH_NODE_COMPAT_SKIP_TYPECHECK: ${{ matrix.skip_typecheck }}
+      DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ matrix.snapshot_workers }}
     strategy:
       fail-fast: false
       matrix:
         include:
           - node: '22.19'
+            name: node 22.19
             runner: dsh-ubuntu-24-04-4core
+            command: pnpm run check:node-compat
+            gate_concurrency: '2'
             skip_typecheck: '1'
           - node: 24
-            runner: dsh-ubuntu-24-04-8core
+            name: node 24 / cpu
+            runner: dsh-ubuntu-24-04-64core
+            command: pnpm run check:ci:primary-cpu
+            coverage_workers: '16'
+            gate_concurrency: '6'
+            primary_cpu: true
+            snapshot_workers: '16'
             skip_typecheck: '1'
           - node: 26
+            name: node 26
             runner: dsh-ubuntu-24-04-32core
+            command: pnpm run check:node-compat
+            gate_concurrency: '2'
             skip_typecheck: '1'
     steps:
       - uses: actions/checkout@v6
@@ -458,14 +470,37 @@ jobs:
             ${{ runner.os }}-node-${{ matrix.node }}-pnpm-
 
       - name: Install (immutable)
+        if: matrix.primary_cpu != true
         run: pnpm install --frozen-lockfile
 
-      - name: Run compatibility gates
-        run: pnpm run check:node-compat
+      - name: Install and prepare bubblewrap
+        if: matrix.primary_cpu == true
+        run: |
+          pnpm install --frozen-lockfile &
+          install_pid=$!
+          (
+            if ! sudo apt-get install -yq --no-install-recommends bubblewrap; then
+              echo "initial bubblewrap install failed; refreshing APT indexes and retrying"
+              sudo apt-get update -q
+              sudo apt-get install -yq --no-install-recommends bubblewrap
+            fi
+            sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \
+              || echo "apparmor userns knob absent — the functional probe decides"
+          ) &
+          sandbox_pid=$!
+          install_status=0
+          wait "$install_pid" || install_status=$?
+          sandbox_status=0
+          wait "$sandbox_pid" || sandbox_status=$?
+          if (( install_status != 0 )); then exit "$install_status"; fi
+          exit "$sandbox_status"
+
+      - name: Run assigned gates
+        run: ${{ matrix.command }}
 
   python-sdk:
     if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners'
-    runs-on: dsh-ubuntu-24-04-64core
+    runs-on: dsh-ubuntu-24-04-8core
     name: python 3.10 / keyless SDK
     steps:
       - uses: actions/checkout@v6

+ 1 - 0
package.json

@@ -26,6 +26,7 @@
     "test:snapshot:record": "DSH_SNAPSHOT=record vitest run --config vitest.snapshot.config.ts --update",
     "test:snapshot:refresh": "DSH_SNAPSHOT=refresh vitest run --config vitest.snapshot.config.ts",
     "check:ci": "tsx scripts/run-gates.ts ci-primary",
+    "check:ci:primary-cpu": "tsx scripts/run-gates.ts ci-primary-cpu",
     "check:ci:large-runner": "tsx scripts/run-gates.ts ci-primary-large-runner",
     "check:ci:static": "tsx scripts/run-gates.ts ci-static",
     "check:ci:lint": "tsx scripts/run-gates.ts ci-lint",

+ 42 - 17
scripts/run-gates.ts

@@ -15,6 +15,7 @@ import { selectStaticGates } from './static-shards.ts'
 
 type Mode =
   | 'ci-primary'
+  | 'ci-primary-cpu'
   | 'ci-primary-large-runner'
   | 'ci-static'
   | 'ci-lint'
@@ -92,6 +93,7 @@ if (results.some(result => result.gate.allowFailure !== true && (result.status =
 function parseMode(raw: string | undefined): Mode {
   switch (raw) {
     case 'ci-primary':
+    case 'ci-primary-cpu':
     case 'ci-primary-large-runner':
     case 'ci-static':
     case 'ci-lint':
@@ -107,7 +109,7 @@ function parseMode(raw: string | undefined): Mode {
       return raw
     default:
       throw new Error(
-        `run-gates: expected mode ci-primary | ci-primary-large-runner | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-complete | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`,
+        `run-gates: expected mode ci-primary | ci-primary-cpu | ci-primary-large-runner | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-complete | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`,
       )
   }
 }
@@ -173,6 +175,8 @@ function gatesForMode(selected: Mode): Gate[] {
   switch (selected) {
     case 'ci-primary':
       return ciPrimaryGates()
+    case 'ci-primary-cpu':
+      return ciPrimaryCpuGates()
     case 'ci-primary-large-runner':
       return ciPrimaryLargeRunnerGates()
     case 'ci-static':
@@ -199,19 +203,7 @@ function gatesForMode(selected: Mode): Gate[] {
     case 'ci-windows-observational':
       return ciWindowsObservationalGates()
     case 'node-compat':
-      return [
-        ...flagEnabled('DSH_NODE_COMPAT_SKIP_TYPECHECK') ? [] : [pnpmScript('typecheck', 'typecheck')],
-        pnpmExec('source-worker-smoke', [
-          'vitest',
-          'run',
-          'packages/workflow/workflow-workerthread/tests/source-worker.compat.spec.ts',
-        ], { label: 'source worker smoke' }),
-        pnpmExec('jsonl-zstd-smoke', [
-          'vitest',
-          'run',
-          'packages/session-persistence/session-persistence-jsonl/tests/zstd.compat.spec.ts',
-        ], { label: 'JSONL Zstandard smoke' }),
-      ]
+      return nodeCompatGates()
     case 'pre-push':
       return [
         pnpmScript('runtime-closure', 'verify-runtime-closure', { label: 'runtime closure' }),
@@ -258,10 +250,10 @@ function ciPrimaryGates(): Gate[] {
 }
 
 function ciPrimaryLargeRunnerGates(): Gate[] {
-  // Typecheck does not consume build output, so a large runner can start both
-  // together while snapshot and artifact consumers still wait for the build.
+  // The CPU lane owns typecheck, coverage, and the build-to-snapshot chain.
+  // This core lane starts its own build eagerly for the remaining artifact consumers.
   return ciPrimaryGates()
-    .filter(gate => gate.id !== 'docs-site-build')
+    .filter(gate => !['coverage', 'docs-site-build', 'snapshot', 'typecheck'].includes(gate.id))
     .map((gate) => {
       if (gate.id !== 'build') return gate
       const eagerBuild = { ...gate }
@@ -270,6 +262,39 @@ function ciPrimaryLargeRunnerGates(): Gate[] {
     })
 }
 
+function ciPrimaryCpuGates(): Gate[] {
+  // Build and snapshot stay together so the dependent replay consumes this lane's output.
+  return [
+    pnpmScript('typecheck', 'typecheck'),
+    coverageGate(),
+    pnpmScript('build', 'build'),
+    snapshotGate(),
+    ...nodeCompatSmokeGates(),
+  ]
+}
+
+function nodeCompatGates(): Gate[] {
+  return [
+    ...flagEnabled('DSH_NODE_COMPAT_SKIP_TYPECHECK') ? [] : [pnpmScript('typecheck', 'typecheck')],
+    ...nodeCompatSmokeGates(),
+  ]
+}
+
+function nodeCompatSmokeGates(): Gate[] {
+  return [
+    pnpmExec('source-worker-smoke', [
+      'vitest',
+      'run',
+      'packages/workflow/workflow-workerthread/tests/source-worker.compat.spec.ts',
+    ], { label: 'source worker smoke' }),
+    pnpmExec('jsonl-zstd-smoke', [
+      'vitest',
+      'run',
+      'packages/session-persistence/session-persistence-jsonl/tests/zstd.compat.spec.ts',
+    ], { label: 'JSONL Zstandard smoke' }),
+  ]
+}
+
 function ciStaticGates(): Gate[] {
   const gates = [
     pnpmScript('runtime-closure', 'verify-runtime-closure', { label: 'runtime closure' }),