|
|
@@ -0,0 +1,277 @@
|
|
|
+import { describe, expect, it, vi } from 'vitest'
|
|
|
+import { Context } from 'cordis'
|
|
|
+import type { Agent, AgentLlmTarget } from '@deepseek-ai/dsh-agent'
|
|
|
+import { Session, SessionId } from '@deepseek-ai/dsh-session'
|
|
|
+import { affectsSessionMetrics, SessionMetricsProjector } from '../src/session-metrics.ts'
|
|
|
+
|
|
|
+function assistant(
|
|
|
+ session: Session,
|
|
|
+ turn: number,
|
|
|
+ step: number,
|
|
|
+ usage: {
|
|
|
+ inputTokens: number
|
|
|
+ outputTokens: number
|
|
|
+ cacheReadTokens?: number
|
|
|
+ cacheWriteTokens?: number
|
|
|
+ },
|
|
|
+): void {
|
|
|
+ session.append('assistant/chunk', {
|
|
|
+ turn,
|
|
|
+ step,
|
|
|
+ chunk: { type: 'usage', usage },
|
|
|
+ })
|
|
|
+ session.append('assistant/message', {
|
|
|
+ turn,
|
|
|
+ step,
|
|
|
+ content: [{ type: 'text', text: `answer-${turn}-${step}` }],
|
|
|
+ provenance: { provider: 'test', model: 'alpha' },
|
|
|
+ usage,
|
|
|
+ }, { surfaceOp: 'append' })
|
|
|
+}
|
|
|
+
|
|
|
+function agent(session: Session): Agent {
|
|
|
+ return { id: session.id, session } as Agent
|
|
|
+}
|
|
|
+
|
|
|
+describe('SessionMetricsProjector', () => {
|
|
|
+ it('filters text/reasoning stream deltas while retaining usage, headers, and surface mutations', () => {
|
|
|
+ const session = new Session(SessionId('metrics-filter'))
|
|
|
+ const text = session.append('assistant/chunk', {
|
|
|
+ turn: 1,
|
|
|
+ step: 1,
|
|
|
+ chunk: { type: 'text-delta', index: 0, text: 'x' },
|
|
|
+ })
|
|
|
+ const usage = session.append('assistant/chunk', {
|
|
|
+ turn: 1,
|
|
|
+ step: 1,
|
|
|
+ chunk: { type: 'usage', usage: { inputTokens: 1, outputTokens: 1 } },
|
|
|
+ })
|
|
|
+ const header = session.append('request/header', {
|
|
|
+ header: { config: { provider: 'test', model: 'alpha' } },
|
|
|
+ reason: 'initial',
|
|
|
+ })
|
|
|
+ const surface = session.append('user/message', {
|
|
|
+ content: [{ type: 'text', text: 'question' }],
|
|
|
+ source: { kind: 'user' },
|
|
|
+ }, { surfaceOp: 'append' })
|
|
|
+ expect(affectsSessionMetrics(text)).toBe(false)
|
|
|
+ expect(affectsSessionMetrics(usage)).toBe(true)
|
|
|
+ expect(affectsSessionMetrics(header)).toBe(true)
|
|
|
+ expect(affectsSessionMetrics(surface)).toBe(true)
|
|
|
+ })
|
|
|
+
|
|
|
+ it('reconciles usage by turn:step, keeps cache writes disjoint, and survives a surface replacement', () => {
|
|
|
+ const ctx = new Context()
|
|
|
+ ctx.provide('tokenMeter', {
|
|
|
+ measure(session: Session) {
|
|
|
+ return { totalTokens: session.surface.nodes.length * 100 }
|
|
|
+ },
|
|
|
+ })
|
|
|
+ const session = new Session(SessionId('metrics-fold'))
|
|
|
+ const first = session.append('user/message', {
|
|
|
+ content: [{ type: 'text', text: 'large old surface' }],
|
|
|
+ source: { kind: 'user' },
|
|
|
+ }, { surfaceOp: 'append' })
|
|
|
+ assistant(session, 1, 1, {
|
|
|
+ inputTokens: 11,
|
|
|
+ outputTokens: 3,
|
|
|
+ cacheReadTokens: 89,
|
|
|
+ cacheWriteTokens: 8,
|
|
|
+ })
|
|
|
+
|
|
|
+ const current: AgentLlmTarget = { provider: 'test', model: 'alpha' }
|
|
|
+ const projector = new SessionMetricsProjector(ctx, () => current, () => {})
|
|
|
+ const attached = agent(session)
|
|
|
+ const before = projector.snapshot(session, attached)
|
|
|
+ expect(before).toMatchObject({
|
|
|
+ uncachedInputTokens: 11,
|
|
|
+ outputTokens: 3,
|
|
|
+ cacheReadTokens: 89,
|
|
|
+ cacheWriteTokens: 8,
|
|
|
+ contextTokens: 200,
|
|
|
+ })
|
|
|
+
|
|
|
+ const assistantSeq = session.surface.nodes.at(-1)
|
|
|
+ if (assistantSeq === undefined) throw new Error('assistant surface missing')
|
|
|
+ session.append('user/message', {
|
|
|
+ content: [{ type: 'text', text: 'compact summary' }],
|
|
|
+ source: { kind: 'plugin', plugin: 'test' },
|
|
|
+ }, {
|
|
|
+ surfaceOp: { op: 'replace', start: first.seq, end: assistantSeq },
|
|
|
+ sourceEventSeqs: [first.seq, assistantSeq],
|
|
|
+ })
|
|
|
+ // A replayed usage event for the same step replaces the settled value.
|
|
|
+ session.append('assistant/chunk', {
|
|
|
+ turn: 1,
|
|
|
+ step: 1,
|
|
|
+ chunk: {
|
|
|
+ type: 'usage',
|
|
|
+ usage: {
|
|
|
+ inputTokens: 12,
|
|
|
+ outputTokens: 4,
|
|
|
+ cacheReadTokens: 88,
|
|
|
+ cacheWriteTokens: 9,
|
|
|
+ },
|
|
|
+ },
|
|
|
+ })
|
|
|
+ const compacted = projector.snapshot(session, attached)
|
|
|
+ expect(compacted).toMatchObject({
|
|
|
+ uncachedInputTokens: 12,
|
|
|
+ outputTokens: 4,
|
|
|
+ cacheReadTokens: 88,
|
|
|
+ cacheWriteTokens: 9,
|
|
|
+ contextTokens: 100,
|
|
|
+ })
|
|
|
+ assistant(session, 1, 2, {
|
|
|
+ inputTokens: 1_000,
|
|
|
+ outputTokens: 500,
|
|
|
+ cacheReadTokens: 2_000,
|
|
|
+ cacheWriteTokens: 3_000,
|
|
|
+ })
|
|
|
+
|
|
|
+ const after = projector.snapshot(session, attached)
|
|
|
+ expect(after).toMatchObject({
|
|
|
+ logRevision: session.events.length,
|
|
|
+ projectionRevision: 2,
|
|
|
+ uncachedInputTokens: 1_012,
|
|
|
+ outputTokens: 504,
|
|
|
+ cacheReadTokens: 2_088,
|
|
|
+ cacheWriteTokens: 3_009,
|
|
|
+ contextTokens: 200,
|
|
|
+ })
|
|
|
+ expect(after.uncachedInputTokens).not.toBe(
|
|
|
+ after.uncachedInputTokens + after.cacheReadTokens + after.cacheWriteTokens,
|
|
|
+ )
|
|
|
+ })
|
|
|
+
|
|
|
+ it('publishes only the selected route capacity when asynchronous resolutions race', async () => {
|
|
|
+ const ctx = new Context()
|
|
|
+ const resolutions = new Map<string, (contextWindow: number) => void>()
|
|
|
+ ctx.provide('tokenMeter', { measure: () => ({ totalTokens: 35_000 }) })
|
|
|
+ ctx.provide('llm', {
|
|
|
+ resolveModelInfo(_provider: string, model: string) {
|
|
|
+ return new Promise<{ context: { contextWindow: number } }>((resolve) => {
|
|
|
+ resolutions.set(model, (contextWindow) => { resolve({ context: { contextWindow } }) })
|
|
|
+ })
|
|
|
+ },
|
|
|
+ })
|
|
|
+ const session = new Session(SessionId('capacity-race'))
|
|
|
+ const attached = agent(session)
|
|
|
+ let current: AgentLlmTarget = { provider: 'test', model: 'alpha' }
|
|
|
+ const resolved = vi.fn()
|
|
|
+ const projector = new SessionMetricsProjector(ctx, () => current, resolved)
|
|
|
+
|
|
|
+ expect(projector.snapshot(session, attached).contextWindow).toBeUndefined()
|
|
|
+ await vi.waitFor(() => { expect(resolutions.has('alpha')).toBe(true) })
|
|
|
+ current = { provider: 'test', model: 'beta' }
|
|
|
+ expect(projector.snapshot(session, attached).contextWindow).toBeUndefined()
|
|
|
+ await vi.waitFor(() => { expect(resolutions.has('beta')).toBe(true) })
|
|
|
+
|
|
|
+ resolutions.get('alpha')?.(64_000)
|
|
|
+ await Promise.resolve()
|
|
|
+ expect(resolved).not.toHaveBeenCalled()
|
|
|
+ resolutions.get('beta')?.(128_000)
|
|
|
+ await vi.waitFor(() => { expect(resolved).toHaveBeenCalledOnce() })
|
|
|
+ expect(projector.snapshot(session, attached)).toMatchObject({
|
|
|
+ contextTokens: 35_000,
|
|
|
+ contextWindow: 128_000,
|
|
|
+ })
|
|
|
+ })
|
|
|
+
|
|
|
+ it('omits current context fields when measurement or model metadata is unavailable', async () => {
|
|
|
+ const ctx = new Context()
|
|
|
+ ctx.provide('tokenMeter', { measure: () => { throw new Error('unmeasurable') } })
|
|
|
+ ctx.provide('llm', { resolveModelInfo: () => Promise.reject(new Error('metadata unavailable')) })
|
|
|
+ const session = new Session(SessionId('missing-metrics'))
|
|
|
+ const attached = agent(session)
|
|
|
+ const projector = new SessionMetricsProjector(
|
|
|
+ ctx,
|
|
|
+ () => ({ provider: 'test', model: 'missing' }),
|
|
|
+ () => {},
|
|
|
+ )
|
|
|
+ const metrics = projector.snapshot(session, attached)
|
|
|
+ expect(metrics.contextTokens).toBeUndefined()
|
|
|
+ expect(metrics.contextWindow).toBeUndefined()
|
|
|
+ await vi.waitFor(() => {
|
|
|
+ expect(projector.snapshot(session, attached).contextWindow).toBeUndefined()
|
|
|
+ })
|
|
|
+ })
|
|
|
+
|
|
|
+ it('keeps optional usage buckets at zero and tolerates absent host services or detached agents', async () => {
|
|
|
+ const ctx = new Context()
|
|
|
+ const session = new Session(SessionId('optional-metrics'))
|
|
|
+ assistant(session, 1, 0, { inputTokens: 7, outputTokens: 2 })
|
|
|
+ const attached = agent(session)
|
|
|
+ const projector = new SessionMetricsProjector(
|
|
|
+ ctx,
|
|
|
+ () => ({ provider: 'test', model: 'no-service' }),
|
|
|
+ () => {},
|
|
|
+ )
|
|
|
+
|
|
|
+ expect(projector.snapshot(session)).toMatchObject({
|
|
|
+ uncachedInputTokens: 7,
|
|
|
+ outputTokens: 2,
|
|
|
+ cacheReadTokens: 0,
|
|
|
+ cacheWriteTokens: 0,
|
|
|
+ })
|
|
|
+ expect(projector.snapshot(session, attached).contextWindow).toBeUndefined()
|
|
|
+ expect(projector.snapshot(session, attached).contextWindow).toBeUndefined()
|
|
|
+ await Promise.resolve()
|
|
|
+ })
|
|
|
+
|
|
|
+ it('publishes a resolved route with no advertised capacity as unknown', async () => {
|
|
|
+ const ctx = new Context()
|
|
|
+ ctx.provide('llm', { resolveModelInfo: () => Promise.resolve({}) })
|
|
|
+ const session = new Session(SessionId('no-capacity'))
|
|
|
+ const attached = agent(session)
|
|
|
+ const resolved = vi.fn()
|
|
|
+ const projector = new SessionMetricsProjector(
|
|
|
+ ctx,
|
|
|
+ () => ({ provider: 'test', model: 'metadata-without-context' }),
|
|
|
+ resolved,
|
|
|
+ )
|
|
|
+
|
|
|
+ expect(projector.snapshot(session, attached).contextWindow).toBeUndefined()
|
|
|
+ await vi.waitFor(() => { expect(resolved).toHaveBeenCalledOnce() })
|
|
|
+ expect(projector.snapshot(session, attached).contextWindow).toBeUndefined()
|
|
|
+ })
|
|
|
+
|
|
|
+ it('ignores stale resolution failures and route metadata after the target moves', async () => {
|
|
|
+ const ctx = new Context()
|
|
|
+ const resolutions = new Map<string, {
|
|
|
+ resolve(value: { context: { contextWindow: number } }): void
|
|
|
+ reject(error: Error): void
|
|
|
+ }>()
|
|
|
+ ctx.provide('llm', {
|
|
|
+ resolveModelInfo(_provider: string, model: string) {
|
|
|
+ return new Promise<{ context: { contextWindow: number } }>((resolve, reject) => {
|
|
|
+ resolutions.set(model, { resolve, reject })
|
|
|
+ })
|
|
|
+ },
|
|
|
+ })
|
|
|
+ const session = new Session(SessionId('stale-capacity'))
|
|
|
+ const attached = agent(session)
|
|
|
+ let current: AgentLlmTarget = { provider: 'test', model: 'alpha' }
|
|
|
+ const resolved = vi.fn()
|
|
|
+ const targetFor = vi.fn(() => current)
|
|
|
+ const projector = new SessionMetricsProjector(ctx, targetFor, resolved)
|
|
|
+
|
|
|
+ projector.snapshot(session, attached)
|
|
|
+ await vi.waitFor(() => { expect(resolutions.has('alpha')).toBe(true) })
|
|
|
+ current = { provider: 'test', model: 'route-moved-before-snapshot' }
|
|
|
+ resolutions.get('alpha')?.resolve({ context: { contextWindow: 64_000 } })
|
|
|
+ await vi.waitFor(() => { expect(targetFor).toHaveBeenCalledTimes(2) })
|
|
|
+ expect(resolved).not.toHaveBeenCalled()
|
|
|
+
|
|
|
+ projector.snapshot(session, attached)
|
|
|
+ await vi.waitFor(() => { expect(resolutions.has('route-moved-before-snapshot')).toBe(true) })
|
|
|
+ current = { provider: 'test', model: 'beta' }
|
|
|
+ projector.snapshot(session, attached)
|
|
|
+ await vi.waitFor(() => { expect(resolutions.has('beta')).toBe(true) })
|
|
|
+ resolutions.get('route-moved-before-snapshot')?.reject(new Error('stale failure'))
|
|
|
+ await Promise.resolve()
|
|
|
+ resolutions.get('beta')?.resolve({ context: { contextWindow: 128_000 } })
|
|
|
+ await vi.waitFor(() => { expect(resolved).toHaveBeenCalledOnce() })
|
|
|
+ expect(projector.snapshot(session, attached).contextWindow).toBe(128_000)
|
|
|
+ })
|
|
|
+})
|