Переглянути джерело

test(acp): require a real sandbox denial

Tianyi Cui 2 місяців тому
батько
коміт
be62eecc60

+ 1 - 1
.agents/notes/implemented/feature/2026-07-06-sandbox.md

@@ -117,7 +117,7 @@ fs/web/todo execute in-process, so their sandbox semantics are policy at their s
 
 - **Unit:** pin platform selection and profiles, fail-closed runner classification, per-call mode/root resolution, per-process facts, escalation validation and outcomes, permission preset folding and write-through, narrator coalescing, ACP advertisement and validation, and turn-enclosed config writes.
 - **Keyless real-runner:** exercise bwrap, Landlock, and Seatbelt against real filesystem effects at provider and bash-consumer layers; one real Cordis context concurrently drives two project sessions through shipped bash and fs tools, proving own-root success and sibling-root denial. Packed-install coverage proves the registry launcher remains executable. The real ACP composition pins permission switching and rejects unknown presets. CI rejects a silent all-skip.
-- **With-key:** drive a real model, runner, bridge answerer, and disk effect through granted and rejected escalation; unavailable credentials or runners self-skip.
+- **With-key:** start the real ACP composition in read-only mode, let a model-driven bash write hit the runner's denial marker, then drive the bridge answerer and disk effect through granted and rejected workspace-write retries; unavailable credentials or runners self-skip.
 - **Snapshot:** pin the permission config-option wire, preset and knob events, prompt deltas and notices, and both scripted approval branches. A real ACP example scenario places its session under the user home while the deployment fallback points at `/tmp`, then pins a successful workspace-write mutation; this distinguishes session-root resolution from the process fallback without depending on runner-specific denial text. Other snapshots start unconfined so unrelated fixtures remain platform-independent, and policy scenarios switch explicitly.
 
 ## Deferred phases

+ 40 - 18
examples/acp-agent/tests/escalation.e2e.ts

@@ -25,13 +25,11 @@ import { cleanupAcpExampleTest } from './cleanup.ts'
  * model nor a sandbox runner is ever exercised.
  *
  * With-key escalation flow (self-skips without DEEPSEEK_API_KEY or a usable
- * platform runner): a scripted ACP client plays the human. The prompt asserts
- * a prior denial (the organic denial→marker path lives on the sandbox e2e
- * legs and unit tiers), the real model escalates with `sandbox_permissions` +
- * `justification`, the bridge prompts THIS client over
- * `session/request_permission`, the client answers `allow-once`, and the
- * retried write must land ON DISK (world-verified) — under the granted mode,
- * a temp-dir session cwd is writable either way.
+ * platform runner): a scripted ACP client plays the human. The subprocess
+ * starts read-only, its first real bash write is denied, the model retries with
+ * `sandbox_permissions` + `justification`, and the bridge prompts THIS client
+ * over `session/request_permission`. An approved workspace-write retry must
+ * then land ON DISK (world-verified).
  */
 
 const AGENT: AgentUnderTest = {
@@ -58,14 +56,21 @@ interface Spawned extends LaunchedAcpTestAgent {
   permissionRequests: RequestPermissionRequest[]
 }
 
-/** Boot the example as an ACP subprocess; the scripted client answers every permission prompt with `answer`. */
-function launchExampleAcpAgent(cwd: string, answer: 'allow-once' | 'reject-once'): Spawned {
+/** Boot the example with an optional sandbox override; the scripted client answers every permission prompt with `answer`. */
+function launchExampleAcpAgent(
+  cwd: string,
+  answer: 'allow-once' | 'reject-once',
+  sandboxMode?: 'read-only' | 'workspace-write' | 'danger-full-access',
+): Spawned {
   const permissionRequests: RequestPermissionRequest[] = []
   const launched = launchAcpTestAgent({
     agent: AGENT,
     cwd,
     // A dummy key lets the adapter boot keylessly; live tests carry the real key.
-    env: { DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY ?? 'sk-dummy-for-boot' },
+    env: {
+      DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY ?? 'sk-dummy-for-boot',
+      DSH_PERMISSION_MODE: sandboxMode,
+    },
     requestPermission(params) {
       permissionRequests.push(params)
       const option = params.options.find(o => o.optionId === answer)
@@ -78,6 +83,17 @@ function launchExampleAcpAgent(cwd: string, answer: 'allow-once' | 'reject-once'
   return Object.assign(launched, { permissionRequests })
 }
 
+function escalationPrompt(path: string, content: string): string {
+  return `Create ${path} containing exactly ${JSON.stringify(content)} using bash, not filesystem tools. `
+    + 'First try the command without sandbox_permissions. If the sandbox denies it, retry that exact command once '
+    + 'with sandbox_permissions set to workspace-write and a one-sentence justification.'
+}
+
+function includesReadOnlyDenial(updates: LaunchedAcpTestAgent['updates']): boolean {
+  return updates.some(update => update.sessionUpdate === 'tool_call_update'
+    && JSON.stringify(update.content).includes('[sandbox: file access denied under read-only mode]'))
+}
+
 let spawned: Spawned | undefined
 let workdir: string | undefined
 
@@ -137,17 +153,20 @@ describe('default sandbox composition keyless smoke (real cordis.yml via the Loa
 describe.skipIf(!process.env.DEEPSEEK_API_KEY || !hasRunner)('default sandbox composition e2e: the live approval loop', () => {
   it('denial → model escalation → editor prompt → allow-once → the retried write lands on disk', async () => {
     workdir = await mkdtemp(join(tmpdir(), 'sandbox-acp-e2e-'))
-    spawned = launchExampleAcpAgent(workdir, 'allow-once')
-    const { client, permissionRequests } = spawned
+    spawned = launchExampleAcpAgent(workdir, 'allow-once', 'read-only')
+    const { client, permissionRequests, updates } = spawned
 
     await client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })
     const { sessionId } = await client.newSession({ cwd: workdir, mcpServers: [] })
     const res = await client.prompt({
       sessionId,
-      prompt: [{ type: 'text', text: `The sandbox already denied writing ${workdir}/escalated.txt. Create it now containing exactly "ACP_ESCALATION_OK": `
-        + 'one single bash call with sandbox_permissions set to danger-full-access and a one-sentence justification, then stop.' }],
+      prompt: [{
+        type: 'text',
+        text: `${escalationPrompt(join(workdir, 'escalated.txt'), 'ACP_ESCALATION_OK')} Then stop.`,
+      }],
     })
     expect(['end_turn', 'max_tokens']).toContain(res.stopReason)
+    expect(includesReadOnlyDenial(updates)).toBe(true)
 
     // The WORLD: the approved escalated retry landed the write.
     const proof = await readFile(join(workdir, 'escalated.txt'), 'utf8')
@@ -166,17 +185,20 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || !hasRunner)('default sandbox co
 
   it('a rejected escalation stays denied: no write lands, the turn still ends', async () => {
     workdir = await mkdtemp(join(tmpdir(), 'sandbox-acp-e2e-'))
-    spawned = launchExampleAcpAgent(workdir, 'reject-once')
-    const { client, permissionRequests } = spawned
+    spawned = launchExampleAcpAgent(workdir, 'reject-once', 'read-only')
+    const { client, permissionRequests, updates } = spawned
 
     await client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })
     const { sessionId } = await client.newSession({ cwd: workdir, mcpServers: [] })
     const res = await client.prompt({
       sessionId,
-      prompt: [{ type: 'text', text: `The sandbox already denied writing ${workdir}/refused.txt. Create it now containing "NO": `
-        + 'one single bash call with sandbox_permissions set to danger-full-access and a one-sentence justification. If that is rejected, stop and say so.' }],
+      prompt: [{
+        type: 'text',
+        text: `${escalationPrompt(join(workdir, 'refused.txt'), 'NO')} If approval is rejected, stop and say so.`,
+      }],
     })
     expect(['end_turn', 'max_tokens']).toContain(res.stopReason)
+    expect(includesReadOnlyDenial(updates)).toBe(true)
 
     // The WORLD: rejected means the file never appeared.
     await expect(readFile(join(workdir, 'refused.txt'), 'utf8')).rejects.toThrow()