ci: add manual pi-ai provider e2e

.github/workflows/pi-ai-provider-e2e.yml

@@ -0,0 +1,77 @@
+name: E2E (pi-ai OpenAI and Anthropic)
+
+# This suite spends tokens against two external providers and is intentionally
+# opt-in. It has no push, pull_request, schedule, or workflow_call trigger.
+on:
+  workflow_dispatch:
+    inputs:
+      openai_model:
+        description: OpenAI model from pi-ai's installed catalog
+        required: true
+        default: gpt-5.5
+        type: string
+      anthropic_model:
+        description: Anthropic model from pi-ai's installed catalog
+        required: true
+        default: claude-opus-4-8
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    name: OpenAI Responses + Anthropic Messages
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: Enable corepack (pnpm)
+        run: corepack enable
+
+      - name: Resolve pnpm store path
+        id: pnpm-store
+        run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/cache@v4
+        with:
+          path: ${{ steps.pnpm-store.outputs.path }}
+          key: ${{ runner.os }}-node-24-pnpm-${{ hashFiles('pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-node-24-pnpm-
+
+      - name: Install (immutable)
+        run: pnpm install --frozen-lockfile
+
+      # The tests self-skip locally when a credential is absent. A manually
+      # dispatched CI run must fail instead of reporting an all-skipped green.
+      - name: Preflight (require provider API keys)
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY_EXTERNAL }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY_EXTERNAL }}
+        run: |
+          set -euo pipefail
+          missing=0
+          for name in OPENAI_API_KEY ANTHROPIC_API_KEY; do
+            if [ -z "${!name:-}" ]; then
+              echo "::error::${name} is empty. Configure the corresponding *_EXTERNAL repository secret."
+              missing=1
+            fi
+          done
+          exit "$missing"
+
+      - name: E2E tests (real OpenAI and Anthropic APIs)
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY_EXTERNAL }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY_EXTERNAL }}
+          DSH_PI_AI_OPENAI_MODEL: ${{ inputs.openai_model }}
+          DSH_PI_AI_ANTHROPIC_MODEL: ${{ inputs.anthropic_model }}
+          DSH_E2E_MAX_WORKERS: 2
+        run: >-
+          pnpm exec vitest run --config vitest.e2e.config.ts
+          packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts

packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts

@@ -0,0 +1,146 @@
+import { afterEach, describe, expect, it } from 'vitest'
+import { Context } from 'cordis'
+import LlmService, { CallId } from '@deepseek-ai/dsh-llm'
+import type { Message, ToolSchema } from '@deepseek-ai/dsh-llm'
+import * as LlmPiAi from '@deepseek-ai/dsh-llm-pi-ai'
+import type { PiAiReplayState } from '@deepseek-ai/dsh-llm-pi-ai'
+import { assemble, type AssembledResult } from './assemble.ts'
+
+interface ProviderCase {
+  provider: 'openai' | 'anthropic'
+  api: 'openai-responses' | 'anthropic-messages'
+  model: string
+  apiKey?: string
+}
+
+const providerCases: ProviderCase[] = [
+  {
+    provider: 'openai',
+    api: 'openai-responses',
+    model: process.env.DSH_PI_AI_OPENAI_MODEL ?? 'gpt-5.5',
+    ...process.env.OPENAI_API_KEY ? { apiKey: process.env.OPENAI_API_KEY } : {},
+  },
+  {
+    provider: 'anthropic',
+    api: 'anthropic-messages',
+    model: process.env.DSH_PI_AI_ANTHROPIC_MODEL ?? 'claude-opus-4-8',
+    ...process.env.ANTHROPIC_API_KEY ? { apiKey: process.env.ANTHROPIC_API_KEY } : {},
+  },
+]
+
+const contexts: Context[] = []
+
+async function harness(): Promise<Context> {
+  const ctx = new Context()
+  contexts.push(ctx)
+  await ctx.plugin(LlmService)
+  await ctx.plugin(LlmPiAi, {
+    providers: providerCases.map(profile => ({
+      provider: profile.provider,
+      ...profile.apiKey === undefined ? {} : { apiKey: profile.apiKey },
+    })),
+  })
+  return ctx
+}
+
+afterEach(async () => {
+  await Promise.all(contexts.splice(0).map(ctx => ctx.fiber.dispose()))
+})
+
+function ask(text: string): Message[] {
+  return [{ role: 'user', content: [{ type: 'text', text }] }]
+}
+
+function textOf(result: AssembledResult): string {
+  return result.message.content
+    .filter(block => block.type === 'text')
+    .map(block => block.text)
+    .join('')
+}
+
+function expectNativeReplay(result: AssembledResult, profile: ProviderCase): PiAiReplayState {
+  const replayState = result.message.provenance?.replayState
+  expect(replayState).toMatchObject({
+    kind: 'pi-ai',
+    version: 1,
+    api: profile.api,
+    provider: profile.provider,
+    model: profile.model,
+  })
+  return replayState as PiAiReplayState
+}
+
+const lookupTool: ToolSchema = {
+  name: 'lookup_code',
+  description: 'Look up the word represented by a short code.',
+  parameters: {
+    type: 'object',
+    properties: { code: { type: 'string', description: 'The code to look up.' } },
+    required: ['code'],
+  },
+}
+
+for (const profile of providerCases) {
+  describe.skipIf(profile.apiKey === undefined)(
+    `llm-pi-ai ${profile.provider} e2e (${profile.api})`,
+    () => {
+      it('streams text with usage and native replay metadata', async () => {
+        const ctx = await harness()
+        const result = await assemble(ctx, {
+          provider: profile.provider,
+          model: profile.model,
+          messages: ask('Reply with exactly the word: pong'),
+          maxTokens: 64,
+        })
+
+        expect(result.finish.kind).toBe('stop')
+        expect(textOf(result).toLowerCase()).toContain('pong')
+        expect(result.usage?.inputTokens).toBeGreaterThan(0)
+        expect(result.usage?.outputTokens).toBeGreaterThan(0)
+        expect(expectNativeReplay(result, profile).stopReason).toBe('stop')
+      })
+
+      it('round-trips a tool call with provider-native replay metadata', async () => {
+        const ctx = await harness()
+        const prompt = ask('Use lookup_code with code "blue". Do not answer without calling the tool.')
+        const first = await assemble(ctx, {
+          provider: profile.provider,
+          model: profile.model,
+          messages: prompt,
+          tools: [lookupTool],
+          maxTokens: 256,
+        })
+
+        expect(first.finish.kind).toBe('tool-calls')
+        const call = first.message.content.find(block => block.type === 'tool-call')
+        expect(call).toBeDefined()
+        expect(call!.name).toBe('lookup_code')
+        expect(JSON.parse(call!.arguments)).toMatchObject({ code: 'blue' })
+        expect(expectNativeReplay(first, profile).stopReason).toBe('toolUse')
+
+        const second = await assemble(ctx, {
+          provider: profile.provider,
+          model: profile.model,
+          messages: [
+            ...prompt,
+            first.message,
+            {
+              role: 'user',
+              content: [{
+                type: 'tool-result',
+                toolCallId: CallId(call!.id),
+                content: [{ type: 'text', text: 'The code blue means ocean.' }],
+              }],
+            },
+          ],
+          tools: [lookupTool],
+          maxTokens: 256,
+        })
+
+        expect(second.finish.kind).toBe('stop')
+        expect(textOf(second).toLowerCase()).toContain('ocean')
+        expect(expectNativeReplay(second, profile).stopReason).toBe('stop')
+      })
+    },
+  )
+}

vitest.e2e.config.ts

@@ -2,8 +2,9 @@ import tsconfigPaths from 'vite-tsconfig-paths'
 import { defineConfig } from 'vitest/config'
 
 // Real-API suite, separate because it spends tokens. Each test self-skips without
-// DEEPSEEK_API_KEY for keyless CI; the credentialed workflow preflights the secret. Values may come
-// from the environment or gitignored root `.env`, with optional DEEPSEEK_BASE_URL.
+// its provider credential for keyless CI; credentialed workflows preflight the
+// secrets they require. Values may come from the environment or gitignored root
+// `.env`, with provider-specific endpoint overrides where supported.
 try {
   // Node >= 21.7 native; throws when the file does not exist.
   process.loadEnvFile(new URL('.env', import.meta.url).pathname)