Просмотр исходного кода

fix(agent-loop): contain finalizer append-listener throws (review #32 round 2)

The prior fix handled a throwing session/event listener on the turn/start
append, but the SAME push-before-notify hazard remained on the three
FINALIZER appends. Session.append pushes the event before notifying, so a
throwing listener on a finalizer event left the event logged but aborted
the rest of finalization — stranding the turn open.

- failTurn(): set `reason` BEFORE appending the `error` event, and contain
  a throwing session/event listener on it (the event is already logged
  either way). Otherwise reason stayed unset, agent/error was skipped, and
  the caller's closeTurn(false) never ran → open turn.
- closeStep(): the try/catch wrapped only the agent/step-end EMIT, not the
  step/end APPEND. A throwing session/event listener on step/end escaped —
  fatal when closeStep runs from the outer catch during finalization
  (turn/start + step/end but no turn/end). Now both the append and the emit
  are contained and surface as a turn error via failTurn.
- closeTurn(): contain a throwing session/event listener on the turn/end
  append (it would propagate to the runLoop backstop from closeTurn(false),
  or skip the turn-end emit from closeTurn(true)). turn/end is logged
  either way, so the turn stays balanced.

Regressions: a throwing session/event listener on the error event, on
step/end during finalization (driven by a throwing agent/step-start), and
on turn/end — each leaves a balanced turn and the loop survives.
Tianyi Cui 3 месяцев назад
Родитель
Сommit
3e1ca8a425
2 измененных файлов с 145 добавлено и 8 удалено
  1. 41 8
      packages/agent-loop/src/loop.ts
  2. 104 0
      packages/agent-loop/tests/review-fixes.spec.ts

+ 41 - 8
packages/agent-loop/src/loop.ts

@@ -181,16 +181,27 @@ async function runTurn(ctx: Context, agent: LoopAgent, handle: LoopHandle, turn:
   const closeStep = (): void => {
     if (!stepOpen) return
     stepOpen = false
-    session.append('step/end', { turn, step })
+    // Session.append pushes step/end BEFORE notifying session/event listeners,
+    // so a throwing listener leaves step/end in the log (balance holds) but
+    // would otherwise abort finalization. Contain it and surface it as a turn
+    // error below — the same outcome as a throwing agent/step-end listener.
+    let failure: unknown
+    try {
+      session.append('step/end', { turn, step })
+    } catch (error: unknown) {
+      failure = error
+    }
     try {
       ctx.emit('agent/step-end', agent, turn, step)
     } catch (error: unknown) {
-      // step/end is already recorded so balance holds; surface the throwing
-      // listener as a turn error via failTurn (idempotent). This prevents a
-      // throwing step-end listener from producing a silent "completed" turn
-      // when the step itself succeeded (the normal-path closeStep call).
-      failTurn(toError(error))
+      failure ??= error
     }
+    // A throwing step/end session-event listener OR a throwing agent/step-end
+    // listener surfaces as a turn error via failTurn (idempotent). This prevents
+    // a throwing listener from producing a silent "completed" turn when the step
+    // itself succeeded, AND keeps finalization going when closeStep runs from
+    // the outer catch.
+    if (failure !== undefined) failTurn(toError(failure))
   }
 
   // Record a step/turn failure exactly once: append the single `error` event
@@ -208,8 +219,18 @@ async function runTurn(ctx: Context, agent: LoopAgent, handle: LoopHandle, turn:
     // backend treats it as a crash tail and drops it on resume (ADR 0017). In
     // that case report via agent/error + the logger only; the turn is balanced.
     if (!turnEnded) {
-      session.append('error', { turn, step, ...errorData(err) })
+      // Set `reason` BEFORE the append: Session.append pushes the error event
+      // before notifying session/event listeners, so a throwing listener would
+      // otherwise leave `reason` unset (and closeTurn would record the wrong
+      // reason / the outer catch would skip closeTurn). The append is contained
+      // — the error event is already in the log either way; a throwing listener
+      // must not abort finalization.
       reason = { kind: 'error', ...errorData(err) }
+      try {
+        session.append('error', { turn, step, ...errorData(err) })
+      } catch (appendError: unknown) {
+        ctx.logger.warn(`agent "${agent.id}": session/event listener threw on the error event at turn ${turn}: ${toError(appendError).message}`)
+      }
     } else {
       ctx.logger.warn(`agent "${agent.id}": agent/turn-end listener threw after turn ${turn} closed: ${err.message}`)
     }
@@ -229,7 +250,19 @@ async function runTurn(ctx: Context, agent: LoopAgent, handle: LoopHandle, turn:
   const closeTurn = (emit: boolean): void => {
     if (turnEnded) return
     turnEnded = true
-    session.append('turn/end', { turn, reason })
+    // Session.append pushes turn/end BEFORE notifying session/event listeners,
+    // so a throwing listener leaves turn/end in the log (the turn is balanced)
+    // but would otherwise escape — from the outer catch's closeTurn(false) it
+    // would propagate to the runLoop backstop, and from the normal-path
+    // closeTurn(true) it would skip the agent/turn-end emit. Contain it: the
+    // boundary is durable either way, and finalization must not abort on a bad
+    // listener. (On the normal path the outer catch also re-runs closeTurn,
+    // which is an idempotent no-op once turnEnded is set.)
+    try {
+      session.append('turn/end', { turn, reason })
+    } catch (error: unknown) {
+      ctx.logger.warn(`agent "${agent.id}": session/event listener threw on turn/end at turn ${turn}: ${toError(error).message}`)
+    }
     if (emit) ctx.emit('agent/turn-end', agent, turn, reason)
   }
 

+ 104 - 0
packages/agent-loop/tests/review-fixes.spec.ts

@@ -969,6 +969,110 @@ describe('P1-5: a started turn (and any open step) is always closed on a boundar
     await waitForIdle(ctx, agent)
     expect(boundaryCounts(agent).turnEnd).toBe(2)
   })
+
+  it('a throwing session/event listener on the error event still closes the turn (finalizer containment)', async () => {
+    // failTurn appends the `error` event; Session.append pushes it BEFORE
+    // notifying session/event listeners, so a throwing listener leaves `error`
+    // in the log but must NOT abort finalization — `reason` is set before the
+    // append and the throw is contained, so closeTurn(false) still runs and
+    // turn/end is appended (the turn is balanced, not left open).
+    // Plain harness (no invariants oracle): the throwing listener is itself a
+    // session/event subscriber. A finish-error drives the boundary-error path.
+    const errorStream: StreamChunk[] = [{ type: 'finish', reason: { kind: 'error', message: 'provider down' } }]
+    const adapter = new MockAdapter([errorStream, textResponse('turn 2 ok')])
+    const ctx = await harness(adapter)
+    const agent = ctx.agentLoop.create('a-errthrow', { model: 'mock' })
+
+    let threw = false
+    ctx.on('session/event', (_s, event) => {
+      if (!threw && event.type === 'error') { threw = true; throw new Error('boom error-event listener') }
+    })
+
+    send(agent, 'go')
+    await waitForIdle(ctx, agent)
+
+    const e = [...agent.session.events]
+    // The error event is in the log (pushed before the listener threw)…
+    expect(e.some(x => x.type === 'error')).toBe(true)
+    // …and the turn was still closed with the error reason (finalization did not
+    // abort): the last event is turn/end carrying the error reason.
+    const last = e.at(-1)
+    expect(last?.type).toBe('turn/end')
+    expect(last?.type === 'turn/end' && last.data.reason).toMatchObject({ kind: 'error', message: 'provider down' })
+
+    // loop survives: a second turn runs normally.
+    send(agent, 'again')
+    await waitForIdle(ctx, agent)
+    expect(adapter.requests).toHaveLength(2)
+  })
+
+  it('a throwing session/event listener on step/end during finalization still appends turn/end', async () => {
+    // A throwing agent/step-start listener drives the outer catch, which calls
+    // closeStep() during finalization. closeStep appends step/end; a
+    // session/event listener throwing on THAT must not abort the catch before
+    // closeTurn(false) — step/end is already logged (balance holds) and the
+    // throw is contained + surfaced via failTurn, so turn/end is still appended.
+    const adapter = new MockAdapter([textResponse('never reached')])
+    const ctx = await harness(adapter)
+    const agent = ctx.agentLoop.create('a-stependthrow', { model: 'mock' })
+
+    // Open a step, then make the agent/step-start emit throw (boundary throw →
+    // outer catch → closeStep during finalization).
+    ctx.on('agent/step-start', () => { throw new Error('boom step-start') })
+    let threw = false
+    ctx.on('session/event', (_s, event) => {
+      if (!threw && event.type === 'step/end') { threw = true; throw new Error('boom step/end listener') }
+    })
+    const errors: Error[] = []
+    ctx.on('agent/error', (_a, _t, _s, error) => void errors.push(error))
+
+    send(agent, 'go')
+    await waitForIdle(ctx, agent)
+
+    const e = [...agent.session.events]
+    // Both step/end and turn/end are present — finalization ran to completion.
+    expect(e.some(x => x.type === 'step/end')).toBe(true)
+    expect(e.some(x => x.type === 'turn/end')).toBe(true)
+    expect(e.at(-1)?.type).toBe('turn/end')
+    expect(errors.length).toBeGreaterThanOrEqual(1) // surfaced via agent/error
+
+    // loop survives.
+    send(agent, 'again')
+    await waitForIdle(ctx, agent)
+    expect(e.filter(x => x.type === 'turn/start').length).toBeGreaterThanOrEqual(1)
+  })
+
+  it('a throwing session/event listener on turn/end is contained (turn still balanced, loop survives)', async () => {
+    // closeTurn appends turn/end; Session.append pushes it BEFORE notifying
+    // session/event listeners, so a throwing listener leaves turn/end in the log
+    // (the turn is balanced) but must not escape — from the normal-path
+    // closeTurn(true) it would otherwise propagate; the append is contained so
+    // the turn/end emit + loop continue. (A throwing agent/turn-end LISTENER is
+    // a separate, already-tested path; here the session/event append notify is
+    // what throws.)
+    const adapter = new MockAdapter([textResponse('turn 1'), textResponse('turn 2')])
+    const ctx = await harness(adapter)
+    const agent = ctx.agentLoop.create('a-turnendappend', { model: 'mock' })
+
+    let threw = false
+    ctx.on('session/event', (_s, event) => {
+      if (!threw && event.type === 'turn/end') { threw = true; throw new Error('boom turn/end listener') }
+    })
+
+    send(agent, 'go')
+    await waitForIdle(ctx, agent)
+    // turn 1 is balanced despite the throwing turn/end listener.
+    const e1 = [...agent.session.events]
+    expect(e1.filter(x => x.type === 'turn/start')).toHaveLength(1)
+    expect(e1.filter(x => x.type === 'turn/end')).toHaveLength(1)
+    expect(e1.at(-1)?.type).toBe('turn/end')
+
+    // loop survives: a second turn runs to completion.
+    send(agent, 'again')
+    await waitForIdle(ctx, agent)
+    expect(adapter.requests).toHaveLength(2)
+    expect([...agent.session.events].filter(x => x.type === 'turn/end')).toHaveLength(2)
+  })
 })
 
 describe('P1-7: tool/result is logged under the originating call.id, not result.callId', () => {