hai/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-05-17 09:41:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

test: rewrite 5 plan-mode E2E tests on the real-PTY harness

Browse Source 
Replaces SDK-based assertions with runPlanSkillObservation contract. Each
test launches real claude --permission-mode plan, invokes the skill, and
asserts the outcome reaches 'asked' or 'plan_ready' within a 300s budget
(no silent Write/Edit, no crash, no timeout).

Affected:
- test/skill-e2e-plan-ceo-plan-mode.test.ts
- test/skill-e2e-plan-eng-plan-mode.test.ts
- test/skill-e2e-plan-design-plan-mode.test.ts
- test/skill-e2e-plan-devex-plan-mode.test.ts
- test/skill-e2e-plan-mode-no-op.test.ts (inPlanMode: false; tests the
  preamble plan-mode-info no-op path)

test/e2e-harness-audit.test.ts — recognize runPlanSkillObservation as a
valid coverage path alongside the legacy canUseTool / runPlanModeSkillTest.

test/helpers/touchfiles.ts — point the 5 plan-mode test selections and
the e2e-harness-audit selection at test/helpers/claude-pty-runner.ts
instead of the deleted plan-mode-helpers.ts.

Proof: bun test EVALS=1 EVALS_TIER=gate on these 5 files runs sequentially
in 790s and passes 5/5. Same tests were 0/5 on origin/main, on v1.0.0.0,
and on this branch with the SDK harness.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-04-25 21:23:54 -07:00
parent 1b1fd30ec7
commit 38f31e3b1d
7 changed files with 140 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
test/skill-e2e-plan-design-plan-mode.test.ts
View File

@@ -1,31 +1,36 @@
/**
* plan-design-review plan-mode smoke test (gate tier, paid).
* plan-design-review plan-mode smoke (gate, paid, real-PTY).
*
* See test/skill-e2e-plan-ceo-plan-mode.test.ts for the shared assertion
* contract. Exercises the same assertions against /plan-design-review.
* contract. Exercises the same contract against /plan-design-review.
*
* Note: on no-UI-scope branches plan-design-review legitimately short-
* circuits to plan_ready without firing AskUserQuestion. Both 'asked' and
* 'plan_ready' are valid pass outcomes.
*/
import { describe, test, expect } from 'bun:test';
import {
runPlanModeSkillTest,
assertNotHandshakeShape,
} from './helpers/plan-mode-helpers';
import { runPlanSkillObservation } from './helpers/claude-pty-runner';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate';
const describeE2E = shouldRun ? describe : describe.skip;
describeE2E('plan-design-review plan-mode smoke (gate)', () => {
test('goes straight to first design question, no handshake, no silent writes', async () => {
const result = await runPlanModeSkillTest({
test('reaches a terminal outcome (asked or plan_ready) without silent writes', async () => {
const obs = await runPlanSkillObservation({
skillName: 'plan-design-review',
// First question for design review varies; pick any reasonable match.
// The substring match falls back to the first option if no match.
firstAnswerSubstring: '7',
inPlanMode: true,
timeoutMs: 300_000,
});
expect(result.askUserQuestions.length).toBeGreaterThanOrEqual(1);
assertNotHandshakeShape(result.askUserQuestions[0]!);
expect(result.writeOrEditBeforeAsk).toBe(false);
expect(result.exitPlanModeBeforeAsk).toBe(false);
}, 120_000);
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
throw new Error(
`plan-design-review plan-mode smoke FAILED: outcome=${obs.outcome}\n` +
`summary: ${obs.summary}\n` +
`elapsed: ${obs.elapsedMs}ms\n` +
`--- evidence (last 2KB visible) ---\n${obs.evidence}`,
);
}
expect(['asked', 'plan_ready']).toContain(obs.outcome);
}, 360_000);
});