hai/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-05-21 03:40:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(eval): parametric overlay-efficacy harness (runner + fixtures)

Browse Source 
`test/helpers/agent-sdk-runner.ts` wraps @anthropic-ai/claude-agent-sdk
with explicit `AgentSdkResult` types, process-level API concurrency
semaphore, and 3-shape 429 retry (thrown error, result-message error,
mid-stream SDKRateLimitEvent). Pins the local claude binary via
`pathToClaudeCodeExecutable`.

`test/fixtures/overlay-nudges.ts` holds the typed registry. Two
fixtures for the first measurement: `opus-4-7-fanout-toy` (3-file read)
and `opus-4-7-fanout-realistic` (mixed-tool audit). Strict validator
rejects duplicate ids, non-integer trials, unsafe overlay paths, non-safe
id chars, and missing overlay files at module load.

Adding a future overlay nudge eval = one fixture entry.
This commit is contained in:
Garry Tan
2026-04-23 09:13:58 -07:00
parent 66887b2f05
commit 6b85262422
2 changed files with 644 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

179
test/fixtures/overlay-nudges.ts vendored Normal file
View File

@@ -0,0 +1,179 @@
/**
* Overlay-efficacy fixture registry.
*
* Each fixture defines a reproducible A/B test for one behavioral nudge
* embedded in a model-overlays/*.md file. The harness at
* test/skill-e2e-overlay-harness.test.ts iterates this registry and runs
* `fixture.trials` A/B trials per fixture, asserting `fixture.pass(arms)`.
*
* Adding a new overlay eval = one entry in this list. The harness handles
* arm wiring, concurrency, artifact storage, rate-limit retries, and the
* cross-harness diagnostic.
*/
import * as fs from 'fs';
import * as path from 'path';
import {
firstTurnParallelism,
type AgentSdkResult,
} from '../helpers/agent-sdk-runner';
const REPO_ROOT = path.resolve(__dirname, '..', '..');
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface OverlayFixture {
/** Unique, lowercase/digits/dash only. Used in artifact paths. */
id: string;
/** Path to the overlay file, relative to repo root. */
overlayPath: string;
/** API model ID, not the overlay family name. */
model: string;
/** Integer >= 3. Trials per arm. */
trials: number;
/** Max concurrent queries for this fixture's arms. Default 3. */
concurrency?: number;
/** Populate the workspace dir before each trial. */
setupWorkspace: (dir: string) => void;
/** The prompt the model receives. Non-empty. */
userPrompt: string;
/** Compute the per-trial metric from the typed SDK result. */
metric: (r: AgentSdkResult) => number;
/** Acceptance predicate across all arms' per-trial metrics. */
pass: (arms: { overlay: number[]; off: number[] }) => boolean;
}
// ---------------------------------------------------------------------------
// Validation
// ---------------------------------------------------------------------------
export function validateFixtures(fixtures: OverlayFixture[]): void {
const ids = new Set<string>();
for (const f of fixtures) {
if (!f.id || !/^[a-z0-9-]+$/.test(f.id)) {
throw new Error(
`fixture id must be non-empty, lowercase/digits/dash only: ${JSON.stringify(f.id)}`,
);
}
if (ids.has(f.id)) {
throw new Error(`duplicate fixture id: ${f.id}`);
}
ids.add(f.id);
if (!Number.isInteger(f.trials) || f.trials < 3) {
throw new Error(`${f.id}: trials must be an integer >= 3 (got ${f.trials})`);
}
if (
f.concurrency !== undefined &&
(!Number.isInteger(f.concurrency) || f.concurrency < 1)
) {
throw new Error(
`${f.id}: concurrency must be an integer >= 1 (got ${f.concurrency})`,
);
}
if (!f.model) throw new Error(`${f.id}: model must be non-empty`);
if (!f.userPrompt) throw new Error(`${f.id}: userPrompt must be non-empty`);
if (path.isAbsolute(f.overlayPath) || f.overlayPath.includes('..')) {
throw new Error(
`${f.id}: overlayPath must be relative and must not contain '..' (got ${f.overlayPath})`,
);
}
const fullPath = path.resolve(REPO_ROOT, f.overlayPath);
if (!fs.existsSync(fullPath)) {
throw new Error(`${f.id}: overlay file not found at ${f.overlayPath}`);
}
for (const fn of ['setupWorkspace', 'metric', 'pass'] as const) {
if (typeof f[fn] !== 'function') {
throw new Error(`${f.id}: ${fn} must be a function`);
}
}
}
}
// ---------------------------------------------------------------------------
// Metric + predicate helpers
// ---------------------------------------------------------------------------
function mean(xs: number[]): number {
if (xs.length === 0) return 0;
return xs.reduce((a, b) => a + b, 0) / xs.length;
}
/**
* Standard fanout predicate: overlay mean beats off mean by at least 0.5
* parallel tool_use blocks in first turn, AND at least 3 of the overlay
* trials emit >= 2 parallel tool_use blocks.
*
* The combined rule catches both "overlay nudges every trial slightly"
* (mean) and "overlay sometimes triggers real fanout" (floor). A single
* 0.5 lift with every trial still emitting 1 call would be suspicious;
* this predicate rejects it.
*/
export function fanoutPass(arms: { overlay: number[]; off: number[] }): boolean {
const lift = mean(arms.overlay) - mean(arms.off);
const floorHits = arms.overlay.filter((n) => n >= 2).length;
return lift >= 0.5 && floorHits >= 3;
}
// ---------------------------------------------------------------------------
// Fixtures
// ---------------------------------------------------------------------------
export const OVERLAY_FIXTURES: OverlayFixture[] = [
{
id: 'opus-4-7-fanout-toy',
overlayPath: 'model-overlays/opus-4-7.md',
model: 'claude-opus-4-7',
trials: 10,
concurrency: 3,
setupWorkspace: (dir) => {
fs.writeFileSync(path.join(dir, 'alpha.txt'), 'Alpha file: used in module A.\n');
fs.writeFileSync(path.join(dir, 'beta.txt'), 'Beta file: used in module B.\n');
fs.writeFileSync(path.join(dir, 'gamma.txt'), 'Gamma file: used in module C.\n');
},
userPrompt:
'Read alpha.txt, beta.txt, and gamma.txt and summarize each in one line.',
metric: (r) => firstTurnParallelism(r.assistantTurns[0]),
pass: fanoutPass,
},
{
id: 'opus-4-7-fanout-realistic',
overlayPath: 'model-overlays/opus-4-7.md',
model: 'claude-opus-4-7',
trials: 10,
concurrency: 3,
setupWorkspace: (dir) => {
fs.writeFileSync(
path.join(dir, 'app.ts'),
"import { config } from './config';\nimport { util } from './src/util';\n\nexport function main() { return config.name + ':' + util(); }\n",
);
fs.writeFileSync(
path.join(dir, 'config.ts'),
"export const config = { name: 'demo', version: 1 };\n",
);
fs.writeFileSync(
path.join(dir, 'README.md'),
'# demo project\n\nA small demo. Entry: `app.ts`. Config: `config.ts`.\n',
);
fs.mkdirSync(path.join(dir, 'src'), { recursive: true });
fs.writeFileSync(
path.join(dir, 'src', 'util.ts'),
"export function util() { return 'util-result'; }\n",
);
},
userPrompt:
'Audit this project: read app.ts, config.ts, and README.md, and glob for ' +
'every .ts file under src/. Summarize what you find in 3 bullet points.',
metric: (r) => firstTurnParallelism(r.assistantTurns[0]),
pass: fanoutPass,
},
];
// Validate at module load so a broken fixture fails fast at test startup,
// not mid-run after burning API dollars.
validateFixtures(OVERLAY_FIXTURES);

465
test/helpers/agent-sdk-runner.ts Normal file
View File

@@ -0,0 +1,465 @@
/**
* Claude Agent SDK wrapper for the overlay-efficacy harness.
*
* This sits alongside session-runner.ts (which drives `claude -p` as a
* subprocess) but runs the model via the published @anthropic-ai/claude-agent-sdk
* instead. The SDK exposes the same harness primitives Claude Code itself uses,
* so overlay-driven behavior change is measured against a closer approximation
* of real Claude Code than the `claude -p` subprocess path provides.
*
* Explicit design rules (from plan review):
* - Use SDK-exported SDKMessage types. No `| unknown` union collapse.
* - Permission surface is explicit: bypassPermissions + settingSources:[] +
* disallowedTools inverse. Without these, the SDK inherits user settings,
* project .claude/, and local hooks, and arms are no longer comparable.
* - Binary pinning via pathToClaudeCodeExecutable. Resolve with `which claude`
* at setup time; the SDK would otherwise use its bundled binary.
* - 3-shape rate-limit detection: thrown error, result-message error subtype,
* mid-stream SDKRateLimitEvent. All three recover on retry.
* - On retry, caller resets workspace via a setupWorkspace callback so
* partial Bash side-effects don't contaminate the next attempt.
* - Process-level semaphore caps concurrent queries across all callers in
* the same bun-test process. Composes with bun's own --concurrent flag.
*/
import {
query,
type SDKMessage,
type SDKAssistantMessage,
type SDKResultMessage,
type SDKSystemMessage,
type PermissionMode,
type SettingSource,
type Options,
} from '@anthropic-ai/claude-agent-sdk';
import * as fs from 'fs';
import * as path from 'path';
import { execSync } from 'child_process';
import type { SkillTestResult } from './session-runner';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface AgentSdkResult {
/** Full raw event stream for forensic recovery. */
events: SDKMessage[];
/** Assistant-typed subset, in order. */
assistantTurns: SDKAssistantMessage[];
/** Flat tool-call list, in order of emission. */
toolCalls: Array<{ tool: string; input: unknown; output: string }>;
/** Concatenated assistant text, newline-joined. */
output: string;
/** 'success' | 'error_during_execution' | 'error_max_turns' | ... */
exitReason: string;
turnsUsed: number;
durationMs: number;
firstResponseMs: number;
maxInterTurnMs: number;
costUsd: number;
model: string;
sdkVersion: string;
/** claude_code_version from the SDK's system/init event (authoritative). */
sdkClaudeCodeVersion: string;
/** Path to the claude binary we pinned. */
resolvedBinaryPath: string;
/** browse-error pattern scan for SkillTestResult parity. Always empty here. */
browseErrors: string[];
}
/** Signature matching `query()` from the SDK. DI hook for unit tests. */
export type QueryProvider = typeof query;
/** Subset of SDK Options['systemPrompt'] we support. */
export type SystemPromptOption =
| string
| { type: 'preset'; preset: 'claude_code'; append?: string; excludeDynamicSections?: boolean };
export interface RunAgentSdkOptions {
/**
* System prompt surface.
* - bare string "" -> omit entirely (SDK default: no system prompt)
* - bare string "...text..." -> REPLACE default with given text (use sparingly)
* - { type:'preset', preset:'claude_code' } -> use Claude Code default
* - { type:'preset', preset:'claude_code', append: "..." } -> default + append
*
* For overlay-efficacy measurement, the preset+append pattern is the right
* one: it measures "does adding overlay text to the REAL Claude Code system
* prompt change behavior" rather than "does the overlay alone (stripped of
* base scaffolding) change behavior".
*/
systemPrompt: SystemPromptOption;
userPrompt: string;
workingDirectory: string;
model?: string;
maxTurns?: number;
allowedTools?: string[];
disallowedTools?: string[];
permissionMode?: PermissionMode;
settingSources?: SettingSource[];
env?: Record<string, string>;
pathToClaudeCodeExecutable?: string;
testName?: string;
runId?: string;
fixtureId?: string;
queryProvider?: QueryProvider;
/** Max 429 retries per call. Default 3. */
maxRetries?: number;
/**
* Caller provides this when retry should reset the workspace. The harness
* invokes it with a fresh dir after a rate-limit failure. When omitted,
* retries reuse the original workingDirectory (fine for read-only tests).
*/
onRetry?: (freshDir: string) => void;
}
export class RateLimitExhaustedError extends Error {
readonly attempts: number;
constructor(attempts: number, cause?: unknown) {
super(`rate limit exhausted after ${attempts} attempts`);
this.name = 'RateLimitExhaustedError';
this.attempts = attempts;
if (cause !== undefined) (this as { cause?: unknown }).cause = cause;
}
}
// ---------------------------------------------------------------------------
// Process-level semaphore for API concurrency
// ---------------------------------------------------------------------------
/**
* Bounded token bucket. Shared across all runAgentSdkTest calls in this
* process so that bun's --concurrent flag does not compound with in-test
* concurrency to blow past Anthropic's rate limits.
*
* Default capacity 3. Override via GSTACK_SDK_MAX_CONCURRENCY env var.
*/
class Semaphore {
private available: number;
private readonly queue: Array<() => void> = [];
constructor(capacity: number) {
this.available = capacity;
}
async acquire(): Promise<void> {
if (this.available > 0) {
this.available--;
return;
}
await new Promise<void>((resolve) => this.queue.push(resolve));
}
release(): void {
const next = this.queue.shift();
if (next) {
next();
} else {
this.available++;
}
}
/** For tests. Returns tokens currently in-flight. */
inFlight(): number {
// Not introspectable from outside without tracking; approximate.
return this.queue.length;
}
}
const DEFAULT_SDK_CONCURRENCY = Number(process.env.GSTACK_SDK_MAX_CONCURRENCY ?? 3);
let _apiSemaphore: Semaphore | null = null;
function getApiSemaphore(): Semaphore {
if (!_apiSemaphore) _apiSemaphore = new Semaphore(DEFAULT_SDK_CONCURRENCY);
return _apiSemaphore;
}
/** Test-only. Resets the process-level semaphore. */
export function __resetSemaphoreForTests(capacity: number): void {
_apiSemaphore = new Semaphore(capacity);
}
// ---------------------------------------------------------------------------
// Rate-limit detection
// ---------------------------------------------------------------------------
/** True if `err` looks like a rate-limit thrown from the SDK. */
export function isRateLimitThrown(err: unknown): boolean {
if (!err || typeof err !== 'object') return false;
const msg = (err as { message?: string }).message ?? '';
const name = (err as { name?: string }).name ?? '';
const status = (err as { status?: number }).status;
return (
status === 429 ||
/rate.?limit|429|too many requests/i.test(msg) ||
/RateLimit/i.test(name)
);
}
/** True if a SDKResultMessage is a rate-limit-shaped error. */
export function isRateLimitResult(msg: SDKMessage): boolean {
if (msg.type !== 'result') return false;
const r = msg as SDKResultMessage;
if (r.subtype === 'success') return false;
// subtype === 'error_during_execution' | 'error_max_turns' | 'error_max_budget_usd' | ...
if (r.subtype !== 'error_during_execution') return false;
const errs = (r as { errors?: string[] }).errors ?? [];
return errs.some((e) => /rate.?limit|429|too many requests/i.test(e));
}
/** True if mid-stream SDKRateLimitEvent indicates a blocking rate-limit. */
export function isRateLimitEvent(msg: SDKMessage): boolean {
if (msg.type !== 'rate_limit_event') return false;
const info = (msg as { rate_limit_info?: { status?: string } }).rate_limit_info;
return info?.status === 'rejected';
}
// ---------------------------------------------------------------------------
// Version resolution (cached)
// ---------------------------------------------------------------------------
let _sdkVersionCache: string | null = null;
function resolveSdkVersion(): string {
if (_sdkVersionCache) return _sdkVersionCache;
try {
const pkgPath = require.resolve('@anthropic-ai/claude-agent-sdk/package.json');
const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf-8')) as { version?: string };
_sdkVersionCache = pkg.version ?? 'unknown';
} catch {
_sdkVersionCache = 'unknown';
}
return _sdkVersionCache;
}
export function resolveClaudeBinary(): string | null {
try {
return execSync('which claude', { encoding: 'utf-8' }).trim() || null;
} catch {
return null;
}
}
// ---------------------------------------------------------------------------
// Main runner
// ---------------------------------------------------------------------------
/**
* Execute a single SDK query with retries. Returns a typed result.
*
* The retry loop treats 429 as recoverable and any other error as fatal.
* Exponential backoff: 1s, 2s, 4s. After maxRetries failures, throws
* RateLimitExhaustedError so the caller can decide what to do with the run.
*/
export async function runAgentSdkTest(
opts: RunAgentSdkOptions,
): Promise<AgentSdkResult> {
const sem = getApiSemaphore();
const maxRetries = opts.maxRetries ?? 3;
const queryImpl: QueryProvider = opts.queryProvider ?? query;
const model = opts.model ?? 'claude-opus-4-7';
let attempt = 0;
let lastErr: unknown = null;
while (attempt <= maxRetries) {
await sem.acquire();
const startMs = Date.now();
try {
const sdkOpts: Options = {
model,
cwd: opts.workingDirectory,
maxTurns: opts.maxTurns ?? 5,
tools: opts.allowedTools ?? ['Read', 'Glob', 'Grep', 'Bash'],
disallowedTools: opts.disallowedTools,
allowedTools: opts.allowedTools ?? ['Read', 'Glob', 'Grep', 'Bash'],
permissionMode: opts.permissionMode ?? 'bypassPermissions',
allowDangerouslySkipPermissions:
(opts.permissionMode ?? 'bypassPermissions') === 'bypassPermissions',
settingSources: opts.settingSources ?? [],
env: opts.env,
pathToClaudeCodeExecutable: opts.pathToClaudeCodeExecutable,
};
// Empty bare string means "omit entirely" (SDK runs with no override).
// Any object or non-empty string is passed through.
if (typeof opts.systemPrompt === 'object' || opts.systemPrompt !== '') {
sdkOpts.systemPrompt = opts.systemPrompt;
}
const events: SDKMessage[] = [];
const assistantTurns: SDKAssistantMessage[] = [];
const toolCalls: Array<{ tool: string; input: unknown; output: string }> = [];
const assistantTextParts: string[] = [];
let firstResponseMs = 0;
let lastEventMs = startMs;
let maxInterTurnMs = 0;
let systemInitVersion = 'unknown';
let rateLimited: unknown = null;
let terminalResult: SDKResultMessage | null = null;
const q = queryImpl({
prompt: opts.userPrompt,
options: sdkOpts,
});
for await (const ev of q) {
const now = Date.now();
if (firstResponseMs === 0) firstResponseMs = now - startMs;
const interTurn = now - lastEventMs;
if (interTurn > maxInterTurnMs) maxInterTurnMs = interTurn;
lastEventMs = now;
events.push(ev);
if (ev.type === 'system' && (ev as SDKSystemMessage).subtype === 'init') {
systemInitVersion =
(ev as SDKSystemMessage).claude_code_version ?? 'unknown';
} else if (ev.type === 'assistant') {
const am = ev as SDKAssistantMessage;
assistantTurns.push(am);
const content = am.message?.content;
if (Array.isArray(content)) {
for (const block of content as Array<
| { type: 'text'; text?: string }
| { type: 'tool_use'; name?: string; input?: unknown }
| { type: string }
>) {
if (block.type === 'text') {
const t = (block as { text?: string }).text;
if (t) assistantTextParts.push(t);
} else if (block.type === 'tool_use') {
const tb = block as { name?: string; input?: unknown };
toolCalls.push({
tool: tb.name ?? 'unknown',
input: tb.input ?? {},
output: '',
});
}
}
}
} else if (isRateLimitEvent(ev)) {
rateLimited = new Error(
`mid-stream rate limit: ${JSON.stringify(
(ev as { rate_limit_info?: unknown }).rate_limit_info,
)}`,
);
} else if (ev.type === 'result') {
terminalResult = ev as SDKResultMessage;
if (isRateLimitResult(ev)) {
rateLimited = new Error(
`result-message rate limit: ${((ev as { errors?: string[] }).errors ?? []).join('; ')}`,
);
}
}
}
if (rateLimited) {
throw rateLimited;
}
if (!terminalResult) {
throw new Error('query stream ended without a result event');
}
const durationMs = Date.now() - startMs;
const costUsd =
(terminalResult as { total_cost_usd?: number }).total_cost_usd ?? 0;
const turnsUsed =
(terminalResult as { num_turns?: number }).num_turns ??
assistantTurns.length;
const exitReason =
(terminalResult as { subtype?: string }).subtype ?? 'unknown';
return {
events,
assistantTurns,
toolCalls,
output: assistantTextParts.join('\n'),
exitReason,
turnsUsed,
durationMs,
firstResponseMs,
maxInterTurnMs,
costUsd,
model,
sdkVersion: resolveSdkVersion(),
sdkClaudeCodeVersion: systemInitVersion,
resolvedBinaryPath: opts.pathToClaudeCodeExecutable ?? 'sdk-default',
browseErrors: [],
};
} catch (err) {
lastErr = err;
const isRetryable = isRateLimitThrown(err);
if (!isRetryable || attempt >= maxRetries) {
if (isRetryable) {
throw new RateLimitExhaustedError(attempt + 1, err);
}
throw err;
}
attempt++;
// backoff: 1s, 2s, 4s
await new Promise((r) => setTimeout(r, 1000 * Math.pow(2, attempt - 1)));
// Let caller reset workspace since prior attempt may have partially
// mutated files via Bash.
if (opts.onRetry) {
opts.onRetry(opts.workingDirectory);
}
} finally {
sem.release();
}
}
throw new RateLimitExhaustedError(attempt + 1, lastErr);
}
// ---------------------------------------------------------------------------
// Legacy shape mapper
// ---------------------------------------------------------------------------
/**
* Adapt AgentSdkResult to the legacy SkillTestResult shape so helpers that
* expect the old `claude -p` output (extractToolSummary, etc) work unchanged.
*/
export function toSkillTestResult(r: AgentSdkResult): SkillTestResult {
// Cost estimate: use SDK's authoritative cost; back-compute chars.
// session-runner.ts:30 requires inputChars/outputChars/estimatedTokens.
// These are rough; real consumers of CostEstimate use cost + turns.
const outputChars = r.output.length;
const inputChars = 0; // unknown from SDK path; not used for pass/fail
const estimatedTokens = Math.round((inputChars + outputChars) / 4);
// Build a flat transcript list mimicking the NDJSON shape:
// parseNDJSON emits [{ type: 'assistant', message: {...} }, ...].
// Use the SDK's assistantTurns directly since their shape matches.
const transcript: unknown[] = r.events.slice();
return {
toolCalls: r.toolCalls,
browseErrors: r.browseErrors,
exitReason: r.exitReason,
duration: r.durationMs,
output: r.output,
costEstimate: {
inputChars,
outputChars,
estimatedTokens,
estimatedCost: r.costUsd,
turnsUsed: r.turnsUsed,
},
transcript,
model: r.model,
firstResponseMs: r.firstResponseMs,
maxInterTurnMs: r.maxInterTurnMs,
};
}
// ---------------------------------------------------------------------------
// Metric helpers (re-exported for fixtures)
// ---------------------------------------------------------------------------
/**
* Count `tool_use` blocks in the first assistant turn of an SDK result.
* Returns 0 if there is no first turn or no content array.
*
* This is the core "fanout" metric. A turn with N tool_use blocks = N
* parallel tool invocations.
*/
export function firstTurnParallelism(firstTurn: SDKAssistantMessage | undefined): number {
if (!firstTurn) return 0;
const content = firstTurn.message?.content;
if (!Array.isArray(content)) return 0;
return (content as Array<{ type: string }>).filter((b) => b.type === 'tool_use').length;
}