|
|
@@ -0,0 +1,723 @@
|
|
|
+/**
|
|
|
+ * Scriptable OpenAI-compatible HTTP/SSE server for transport, protocol, and
|
|
|
+ * semantic-empty LLM recovery tests. Each accepted chat-completions request
|
|
|
+ * consumes one behavior; the server never retries or interprets harness policy.
|
|
|
+ *
|
|
|
+ * @module @deepseek-ai/dsh-llm-mock-server
|
|
|
+ */
|
|
|
+
|
|
|
+import { createServer } from 'node:http'
|
|
|
+import type { IncomingHttpHeaders, IncomingMessage, ServerResponse } from 'node:http'
|
|
|
+import { randomBytes } from 'node:crypto'
|
|
|
+import type { AddressInfo } from 'node:net'
|
|
|
+import { setTimeout as delay } from 'node:timers/promises'
|
|
|
+
|
|
|
+/** Request-scoped behaviors accepted by {@link startMockLlmServer}. */
|
|
|
+export const MOCK_LLM_BEHAVIORS = [
|
|
|
+ 'connection_reset',
|
|
|
+ 'stream_disconnect',
|
|
|
+ 'empty',
|
|
|
+ 'empty_body',
|
|
|
+ 'stream_eof',
|
|
|
+ 'partial_eof',
|
|
|
+ 'partial_disconnect',
|
|
|
+ 'stall',
|
|
|
+ 'malformed_json',
|
|
|
+ 'malformed_event',
|
|
|
+ 'wrong_content_type',
|
|
|
+ 'rate_limit',
|
|
|
+ 'server_error',
|
|
|
+ 'service_unavailable',
|
|
|
+ 'auth_error',
|
|
|
+ 'invalid_request',
|
|
|
+ 'context_overflow',
|
|
|
+ 'quota_exceeded',
|
|
|
+ 'success',
|
|
|
+ 'reasoning_success',
|
|
|
+ 'tool_call_success',
|
|
|
+ 'max_tokens',
|
|
|
+ 'slow_success',
|
|
|
+ 'random',
|
|
|
+] as const
|
|
|
+
|
|
|
+/** One scripted mock behavior name; `random` selects a concrete behavior per request. */
|
|
|
+export type MockLlmBehavior = typeof MOCK_LLM_BEHAVIORS[number]
|
|
|
+
|
|
|
+/** One concrete request behavior after resolving a `random` script entry. */
|
|
|
+export type ConcreteMockLlmBehavior = Exclude<MockLlmBehavior, 'random'>
|
|
|
+
|
|
|
+/** Relative non-negative weights for random request behavior selection. */
|
|
|
+export type MockLlmRandomWeights = Partial<Record<ConcreteMockLlmBehavior, number>>
|
|
|
+
|
|
|
+/**
|
|
|
+ * Default stress profile for `random`. Weights are configurable test pressure,
|
|
|
+ * not a claim about production incident frequency.
|
|
|
+ */
|
|
|
+export const DEFAULT_MOCK_LLM_RANDOM_WEIGHTS: Readonly<MockLlmRandomWeights> = Object.freeze({
|
|
|
+ success: 48,
|
|
|
+ slow_success: 10,
|
|
|
+ max_tokens: 2,
|
|
|
+ connection_reset: 5,
|
|
|
+ stream_disconnect: 5,
|
|
|
+ partial_disconnect: 10,
|
|
|
+ empty: 5,
|
|
|
+ stall: 2,
|
|
|
+ rate_limit: 5,
|
|
|
+ server_error: 4,
|
|
|
+ service_unavailable: 2,
|
|
|
+ partial_eof: 1,
|
|
|
+ malformed_json: 1,
|
|
|
+})
|
|
|
+
|
|
|
+/** How one accepted request ended at the mock boundary. */
|
|
|
+export type MockLlmRequestOutcome = 'completed' | 'reset' | 'stalled' | 'client_closed' | 'server_error'
|
|
|
+
|
|
|
+/** Immutable telemetry emitted when a request starts or reaches an outcome. */
|
|
|
+export type MockLlmServerEvent =
|
|
|
+ | {
|
|
|
+ readonly type: 'request'
|
|
|
+ readonly attempt: number
|
|
|
+ readonly scriptBehavior: MockLlmBehavior | 'script_exhausted'
|
|
|
+ readonly behavior: ConcreteMockLlmBehavior | 'script_exhausted'
|
|
|
+ readonly path: string
|
|
|
+ }
|
|
|
+ | {
|
|
|
+ readonly type: 'result'
|
|
|
+ readonly attempt: number
|
|
|
+ readonly scriptBehavior: MockLlmBehavior | 'script_exhausted'
|
|
|
+ readonly behavior: ConcreteMockLlmBehavior | 'script_exhausted'
|
|
|
+ readonly outcome: MockLlmRequestOutcome
|
|
|
+ readonly chunksSent: number
|
|
|
+ }
|
|
|
+
|
|
|
+/** Captured wire request and its final server-side outcome. */
|
|
|
+export interface MockLlmRequestRecord {
|
|
|
+ /** One-based accepted chat-completions request number. */
|
|
|
+ readonly attempt: number
|
|
|
+ /** Script entry consumed for this request before random resolution. */
|
|
|
+ readonly scriptBehavior: MockLlmBehavior | 'script_exhausted'
|
|
|
+ /** Concrete behavior selected for this request, or exhaustion after the configured script. */
|
|
|
+ readonly behavior: ConcreteMockLlmBehavior | 'script_exhausted'
|
|
|
+ /** Original request path, including a `/v1` prefix when the client supplied one. */
|
|
|
+ readonly path: string
|
|
|
+ /** Detached request headers. */
|
|
|
+ readonly headers: Readonly<IncomingHttpHeaders>
|
|
|
+ /** Parsed JSON request body. */
|
|
|
+ readonly body: unknown
|
|
|
+ /** Number of SSE `data:` events handed to Node before the outcome. */
|
|
|
+ chunksSent: number
|
|
|
+ /** Final server-side outcome; absent while a stalled request remains open. */
|
|
|
+ outcome?: MockLlmRequestOutcome
|
|
|
+}
|
|
|
+
|
|
|
+/** Configuration for one mock server instance. */
|
|
|
+export interface MockLlmServerOptions {
|
|
|
+ /** Loopback host by default. */
|
|
|
+ readonly host?: string
|
|
|
+ /** TCP port; zero requests an OS-assigned port. */
|
|
|
+ readonly port?: number
|
|
|
+ /** Optional exact bearer token; omission accepts any authorization header. */
|
|
|
+ readonly apiKey?: string
|
|
|
+ /** Ordered request behaviors; exhaustion fails loud unless `repeatLast` is true. */
|
|
|
+ readonly sequence: readonly MockLlmBehavior[]
|
|
|
+ /** Reuse the final behavior after the sequence is consumed. */
|
|
|
+ readonly repeatLast?: boolean
|
|
|
+ /** Optional deterministic unsigned 32-bit seed; omission generates and exposes one. */
|
|
|
+ readonly randomSeed?: number
|
|
|
+ /** Relative weights used whenever a script entry is `random`. */
|
|
|
+ readonly randomWeights?: Readonly<MockLlmRandomWeights>
|
|
|
+ /** Complete text returned by success-shaped behaviors. */
|
|
|
+ readonly successText?: string
|
|
|
+ /** Text emitted before partial EOF/reset behaviors terminate. */
|
|
|
+ readonly partialText?: string
|
|
|
+ /** Reasoning text emitted by `reasoning_success`. */
|
|
|
+ readonly reasoningText?: string
|
|
|
+ /** Unicode code-point count per text or reasoning SSE delta. */
|
|
|
+ readonly chunkSize?: number
|
|
|
+ /** Inter-chunk delay for `slow_success`, in milliseconds. */
|
|
|
+ readonly chunkDelayMs?: number
|
|
|
+ /** Delay after headers/deltas before a forced disconnect, in milliseconds. */
|
|
|
+ readonly disconnectDelayMs?: number
|
|
|
+ /** Provider retry delay; the wire `Retry-After` value rounds up to whole seconds. */
|
|
|
+ readonly retryAfterMs?: number
|
|
|
+ /** Optional provider request id returned on HTTP failures. */
|
|
|
+ readonly requestId?: string
|
|
|
+ /** Tool name emitted by `tool_call_success`. */
|
|
|
+ readonly toolName?: string
|
|
|
+ /** Raw JSON arguments emitted by `tool_call_success`. */
|
|
|
+ readonly toolArguments?: string
|
|
|
+ /** Optional observer for JSONL CLI telemetry; observer failures never affect wire behavior. */
|
|
|
+ readonly onEvent?: (event: MockLlmServerEvent) => void
|
|
|
+}
|
|
|
+
|
|
|
+/** Running mock server and captured request state. */
|
|
|
+export interface MockLlmServer {
|
|
|
+ /** Base URL without `/v1`; both root and `/v1` chat-completions paths are accepted. */
|
|
|
+ readonly baseURL: string
|
|
|
+ /** Actual bound port, including an OS-assigned value. */
|
|
|
+ readonly port: number
|
|
|
+ /** Seed used for random behavior selection, including the generated default. */
|
|
|
+ readonly randomSeed: number
|
|
|
+ /** Live request records in arrival order. */
|
|
|
+ readonly requests: readonly MockLlmRequestRecord[]
|
|
|
+ /** Stop accepting requests and force-close stalled/streaming connections; idempotent. */
|
|
|
+ close(): Promise<void>
|
|
|
+}
|
|
|
+
|
|
|
+interface ResolvedOptions {
|
|
|
+ readonly host: string
|
|
|
+ readonly port: number
|
|
|
+ readonly apiKey?: string
|
|
|
+ readonly sequence: readonly MockLlmBehavior[]
|
|
|
+ readonly lastBehavior: MockLlmBehavior
|
|
|
+ readonly repeatLast: boolean
|
|
|
+ readonly randomSeed: number
|
|
|
+ readonly randomWeights: readonly (readonly [ConcreteMockLlmBehavior, number])[]
|
|
|
+ readonly successText: string
|
|
|
+ readonly partialText: string
|
|
|
+ readonly reasoningText: string
|
|
|
+ readonly chunkSize: number
|
|
|
+ readonly chunkDelayMs: number
|
|
|
+ readonly disconnectDelayMs: number
|
|
|
+ readonly retryAfterMs: number
|
|
|
+ readonly requestId?: string
|
|
|
+ readonly toolName: string
|
|
|
+ readonly toolArguments: string
|
|
|
+ readonly onEvent?: (event: MockLlmServerEvent) => void
|
|
|
+}
|
|
|
+
|
|
|
+const MAX_TIMER_DELAY_MS = 2_147_483_647
|
|
|
+const DEFAULT_SUCCESS_TEXT = 'mock response recovered'
|
|
|
+const DEFAULT_PARTIAL_TEXT = 'discarded partial response'
|
|
|
+const DEFAULT_REASONING_TEXT = 'mock reasoning'
|
|
|
+const CONCRETE_BEHAVIORS = new Set<string>(MOCK_LLM_BEHAVIORS.filter(behavior => behavior !== 'random'))
|
|
|
+
|
|
|
+function boundedInteger(name: string, value: number, min: number, max: number): number {
|
|
|
+ if (!Number.isInteger(value) || value < min || value > max) {
|
|
|
+ throw new Error(`llm-mock-server: ${name} must be an integer between ${min} and ${max}`)
|
|
|
+ }
|
|
|
+ return value
|
|
|
+}
|
|
|
+
|
|
|
+function resolveOptions(options: MockLlmServerOptions): ResolvedOptions {
|
|
|
+ const host = options.host ?? '127.0.0.1'
|
|
|
+ const port = boundedInteger('port', options.port ?? 0, 0, 65_535)
|
|
|
+ const chunkSize = boundedInteger('chunkSize', options.chunkSize ?? 8, 1, Number.MAX_SAFE_INTEGER)
|
|
|
+ const chunkDelayMs = boundedInteger('chunkDelayMs', options.chunkDelayMs ?? 25, 0, MAX_TIMER_DELAY_MS)
|
|
|
+ const disconnectDelayMs = boundedInteger(
|
|
|
+ 'disconnectDelayMs',
|
|
|
+ options.disconnectDelayMs ?? 10,
|
|
|
+ 0,
|
|
|
+ MAX_TIMER_DELAY_MS,
|
|
|
+ )
|
|
|
+ const retryAfterMs = boundedInteger('retryAfterMs', options.retryAfterMs ?? 1_000, 1, MAX_TIMER_DELAY_MS)
|
|
|
+ const randomSeed = boundedInteger(
|
|
|
+ 'randomSeed',
|
|
|
+ options.randomSeed ?? randomBytes(4).readUInt32LE(0),
|
|
|
+ 0,
|
|
|
+ 0xffff_ffff,
|
|
|
+ )
|
|
|
+ const successText = options.successText ?? DEFAULT_SUCCESS_TEXT
|
|
|
+ const partialText = options.partialText ?? DEFAULT_PARTIAL_TEXT
|
|
|
+ const reasoningText = options.reasoningText ?? DEFAULT_REASONING_TEXT
|
|
|
+ const toolName = options.toolName ?? 'mock_tool'
|
|
|
+ const toolArguments = options.toolArguments ?? '{"value":"mock"}'
|
|
|
+
|
|
|
+ if (host.length === 0) throw new Error('llm-mock-server: host must not be empty')
|
|
|
+ if (options.sequence.length === 0) throw new Error('llm-mock-server: sequence must not be empty')
|
|
|
+ const lastBehavior = options.sequence.reduce((_previous, behavior) => behavior)
|
|
|
+ if (options.apiKey === '') throw new Error('llm-mock-server: apiKey must not be empty')
|
|
|
+ if (successText.length === 0) throw new Error('llm-mock-server: successText must not be empty')
|
|
|
+ if (partialText.length === 0) throw new Error('llm-mock-server: partialText must not be empty')
|
|
|
+ if (reasoningText.length === 0) throw new Error('llm-mock-server: reasoningText must not be empty')
|
|
|
+ if (toolName.length === 0) throw new Error('llm-mock-server: toolName must not be empty')
|
|
|
+ if (options.requestId === '') throw new Error('llm-mock-server: requestId must not be empty')
|
|
|
+ try {
|
|
|
+ JSON.parse(toolArguments)
|
|
|
+ } catch {
|
|
|
+ throw new Error('llm-mock-server: toolArguments must be valid JSON')
|
|
|
+ }
|
|
|
+
|
|
|
+ const configuredWeights = options.randomWeights ?? DEFAULT_MOCK_LLM_RANDOM_WEIGHTS
|
|
|
+ const randomWeights: Array<readonly [ConcreteMockLlmBehavior, number]> = []
|
|
|
+ for (const [behavior, weight] of Object.entries(configuredWeights)) {
|
|
|
+ if (!CONCRETE_BEHAVIORS.has(behavior)) {
|
|
|
+ throw new Error(`llm-mock-server: randomWeights contains unknown concrete behavior ${JSON.stringify(behavior)}`)
|
|
|
+ }
|
|
|
+ if (!Number.isFinite(weight) || weight < 0) {
|
|
|
+ throw new Error(`llm-mock-server: random weight for ${behavior} must be a non-negative finite number`)
|
|
|
+ }
|
|
|
+ if (weight > 0) randomWeights.push([behavior as ConcreteMockLlmBehavior, weight])
|
|
|
+ }
|
|
|
+ if (randomWeights.length === 0) {
|
|
|
+ throw new Error('llm-mock-server: randomWeights must contain at least one positive weight')
|
|
|
+ }
|
|
|
+
|
|
|
+ return {
|
|
|
+ host,
|
|
|
+ port,
|
|
|
+ ...options.apiKey === undefined ? {} : { apiKey: options.apiKey },
|
|
|
+ sequence: [...options.sequence],
|
|
|
+ lastBehavior,
|
|
|
+ repeatLast: options.repeatLast ?? false,
|
|
|
+ randomSeed,
|
|
|
+ randomWeights,
|
|
|
+ successText,
|
|
|
+ partialText,
|
|
|
+ reasoningText,
|
|
|
+ chunkSize,
|
|
|
+ chunkDelayMs,
|
|
|
+ disconnectDelayMs,
|
|
|
+ retryAfterMs,
|
|
|
+ ...options.requestId === undefined ? {} : { requestId: options.requestId },
|
|
|
+ toolName,
|
|
|
+ toolArguments,
|
|
|
+ ...options.onEvent === undefined ? {} : { onEvent: options.onEvent },
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+function emit(options: ResolvedOptions, event: MockLlmServerEvent): void {
|
|
|
+ try {
|
|
|
+ options.onEvent?.(Object.freeze(event))
|
|
|
+ } catch (_telemetryObserverFailure) {
|
|
|
+ // Test telemetry is observational; a broken observer cannot change provider wire behavior.
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+async function readJsonBody(request: IncomingMessage): Promise<unknown> {
|
|
|
+ let body = ''
|
|
|
+ for await (const chunk of request) body += Buffer.from(chunk).toString('utf8')
|
|
|
+ return body.length === 0 ? undefined : JSON.parse(body)
|
|
|
+}
|
|
|
+
|
|
|
+function splitText(text: string, size: number): string[] {
|
|
|
+ const points = Array.from(text)
|
|
|
+ const chunks: string[] = []
|
|
|
+ for (let index = 0; index < points.length; index += size) chunks.push(points.slice(index, index + size).join(''))
|
|
|
+ return chunks
|
|
|
+}
|
|
|
+
|
|
|
+function openSse(response: ServerResponse, contentType = 'text/event-stream; charset=utf-8'): void {
|
|
|
+ response.writeHead(200, {
|
|
|
+ 'content-type': contentType,
|
|
|
+ 'cache-control': 'no-cache',
|
|
|
+ 'connection': 'keep-alive',
|
|
|
+ })
|
|
|
+ response.flushHeaders()
|
|
|
+}
|
|
|
+
|
|
|
+function writeSse(record: MockLlmRequestRecord, response: ServerResponse, payload: unknown): void {
|
|
|
+ response.write(`data: ${typeof payload === 'string' ? payload : JSON.stringify(payload)}\n\n`)
|
|
|
+ record.chunksSent += 1
|
|
|
+}
|
|
|
+
|
|
|
+function writeDone(record: MockLlmRequestRecord, response: ServerResponse): void {
|
|
|
+ writeSse(record, response, '[DONE]')
|
|
|
+}
|
|
|
+
|
|
|
+function finishRecord(
|
|
|
+ options: ResolvedOptions,
|
|
|
+ record: MockLlmRequestRecord,
|
|
|
+ outcome: MockLlmRequestOutcome,
|
|
|
+): void {
|
|
|
+ record.outcome = outcome
|
|
|
+ emit(options, {
|
|
|
+ type: 'result',
|
|
|
+ attempt: record.attempt,
|
|
|
+ scriptBehavior: record.scriptBehavior,
|
|
|
+ behavior: record.behavior,
|
|
|
+ outcome,
|
|
|
+ chunksSent: record.chunksSent,
|
|
|
+ })
|
|
|
+}
|
|
|
+
|
|
|
+function httpError(
|
|
|
+ options: ResolvedOptions,
|
|
|
+ record: MockLlmRequestRecord,
|
|
|
+ response: ServerResponse,
|
|
|
+ status: number,
|
|
|
+ message: string,
|
|
|
+ code: string,
|
|
|
+ type = 'mock_error',
|
|
|
+): void {
|
|
|
+ const headers: Record<string, string> = { 'content-type': 'application/json' }
|
|
|
+ if (record.behavior === 'rate_limit') {
|
|
|
+ headers['retry-after'] = String(Math.ceil(options.retryAfterMs / 1_000))
|
|
|
+ }
|
|
|
+ if (options.requestId !== undefined) headers['x-request-id'] = options.requestId
|
|
|
+ response.writeHead(status, headers)
|
|
|
+ response.end(JSON.stringify({ error: { message, type, code } }))
|
|
|
+ finishRecord(options, record, 'completed')
|
|
|
+}
|
|
|
+
|
|
|
+function terminalChunk(reason: string, outputTokens: number): unknown {
|
|
|
+ return {
|
|
|
+ choices: [{ index: 0, delta: { content: '' }, finish_reason: reason }],
|
|
|
+ usage: { prompt_tokens: 3, completion_tokens: outputTokens },
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+async function pause(milliseconds: number, response: ServerResponse): Promise<boolean> {
|
|
|
+ if (milliseconds === 0) return !response.destroyed
|
|
|
+ const controller = new AbortController()
|
|
|
+ const stop = (): void => { controller.abort() }
|
|
|
+ response.once('close', stop)
|
|
|
+ try {
|
|
|
+ await delay(milliseconds, undefined, { signal: controller.signal })
|
|
|
+ return true
|
|
|
+ } catch (_responseClosed) {
|
|
|
+ // The timer only receives this response-owned abort signal; closing the response cancels its wait.
|
|
|
+ return false
|
|
|
+ } finally {
|
|
|
+ response.off('close', stop)
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+async function streamText(
|
|
|
+ options: ResolvedOptions,
|
|
|
+ record: MockLlmRequestRecord,
|
|
|
+ response: ServerResponse,
|
|
|
+ text: string,
|
|
|
+ delayMs: number,
|
|
|
+): Promise<boolean> {
|
|
|
+ for (const chunk of splitText(text, options.chunkSize)) {
|
|
|
+ writeSse(record, response, { choices: [{ index: 0, delta: { content: chunk }, finish_reason: null }] })
|
|
|
+ if (!await pause(delayMs, response)) return false
|
|
|
+ }
|
|
|
+ return true
|
|
|
+}
|
|
|
+
|
|
|
+async function completeText(
|
|
|
+ options: ResolvedOptions,
|
|
|
+ record: MockLlmRequestRecord,
|
|
|
+ response: ServerResponse,
|
|
|
+ reason: 'stop' | 'length',
|
|
|
+ delayMs: number,
|
|
|
+): Promise<void> {
|
|
|
+ if (!await streamText(options, record, response, options.successText, delayMs)) {
|
|
|
+ finishRecord(options, record, 'client_closed')
|
|
|
+ return
|
|
|
+ }
|
|
|
+ writeSse(record, response, terminalChunk(reason, Array.from(options.successText).length))
|
|
|
+ writeDone(record, response)
|
|
|
+ response.end()
|
|
|
+ finishRecord(options, record, 'completed')
|
|
|
+}
|
|
|
+
|
|
|
+async function disconnect(
|
|
|
+ options: ResolvedOptions,
|
|
|
+ record: MockLlmRequestRecord,
|
|
|
+ response: ServerResponse,
|
|
|
+): Promise<void> {
|
|
|
+ if (!await pause(options.disconnectDelayMs, response)) {
|
|
|
+ finishRecord(options, record, 'client_closed')
|
|
|
+ return
|
|
|
+ }
|
|
|
+ finishRecord(options, record, 'reset')
|
|
|
+ response.destroy()
|
|
|
+}
|
|
|
+
|
|
|
+function toolCallChunks(options: ResolvedOptions): readonly unknown[] {
|
|
|
+ const midpoint = Math.max(1, Math.floor(options.toolArguments.length / 2))
|
|
|
+ return [
|
|
|
+ {
|
|
|
+ choices: [{
|
|
|
+ index: 0,
|
|
|
+ delta: {
|
|
|
+ tool_calls: [{
|
|
|
+ index: 0,
|
|
|
+ id: 'mock-call-1',
|
|
|
+ type: 'function',
|
|
|
+ function: { name: options.toolName, arguments: options.toolArguments.slice(0, midpoint) },
|
|
|
+ }],
|
|
|
+ },
|
|
|
+ finish_reason: null,
|
|
|
+ }],
|
|
|
+ },
|
|
|
+ {
|
|
|
+ choices: [{
|
|
|
+ index: 0,
|
|
|
+ delta: { tool_calls: [{ index: 0, function: { arguments: options.toolArguments.slice(midpoint) } }] },
|
|
|
+ finish_reason: null,
|
|
|
+ }],
|
|
|
+ },
|
|
|
+ ]
|
|
|
+}
|
|
|
+
|
|
|
+async function runBehavior(
|
|
|
+ options: ResolvedOptions,
|
|
|
+ record: MockLlmRequestRecord,
|
|
|
+ request: IncomingMessage,
|
|
|
+ response: ServerResponse,
|
|
|
+): Promise<void> {
|
|
|
+ switch (record.behavior) {
|
|
|
+ case 'script_exhausted':
|
|
|
+ httpError(options, record, response, 500, 'mock script exhausted', 'MOCK_SCRIPT_EXHAUSTED')
|
|
|
+ return
|
|
|
+ case 'connection_reset':
|
|
|
+ finishRecord(options, record, 'reset')
|
|
|
+ request.socket.destroy()
|
|
|
+ return
|
|
|
+ case 'stream_disconnect':
|
|
|
+ openSse(response)
|
|
|
+ await disconnect(options, record, response)
|
|
|
+ return
|
|
|
+ case 'empty':
|
|
|
+ openSse(response)
|
|
|
+ writeSse(record, response, terminalChunk('stop', 0))
|
|
|
+ writeDone(record, response)
|
|
|
+ response.end()
|
|
|
+ finishRecord(options, record, 'completed')
|
|
|
+ return
|
|
|
+ case 'empty_body':
|
|
|
+ openSse(response)
|
|
|
+ response.end()
|
|
|
+ finishRecord(options, record, 'completed')
|
|
|
+ return
|
|
|
+ case 'stream_eof':
|
|
|
+ openSse(response)
|
|
|
+ writeSse(record, response, { choices: [{ index: 0, delta: { role: 'assistant' }, finish_reason: null }] })
|
|
|
+ response.end()
|
|
|
+ finishRecord(options, record, 'completed')
|
|
|
+ return
|
|
|
+ case 'partial_eof':
|
|
|
+ openSse(response)
|
|
|
+ await streamText(options, record, response, options.partialText, 0)
|
|
|
+ response.end()
|
|
|
+ finishRecord(options, record, 'completed')
|
|
|
+ return
|
|
|
+ case 'partial_disconnect':
|
|
|
+ openSse(response)
|
|
|
+ if (!await streamText(options, record, response, options.partialText, options.chunkDelayMs)) return
|
|
|
+ await disconnect(options, record, response)
|
|
|
+ return
|
|
|
+ case 'stall':
|
|
|
+ openSse(response)
|
|
|
+ finishRecord(options, record, 'stalled')
|
|
|
+ return
|
|
|
+ case 'malformed_json':
|
|
|
+ openSse(response)
|
|
|
+ writeSse(record, response, '{not-json')
|
|
|
+ writeDone(record, response)
|
|
|
+ response.end()
|
|
|
+ finishRecord(options, record, 'completed')
|
|
|
+ return
|
|
|
+ case 'malformed_event':
|
|
|
+ openSse(response)
|
|
|
+ writeSse(record, response, { choices: [null] })
|
|
|
+ writeDone(record, response)
|
|
|
+ response.end()
|
|
|
+ finishRecord(options, record, 'completed')
|
|
|
+ return
|
|
|
+ case 'wrong_content_type':
|
|
|
+ openSse(response, 'application/json')
|
|
|
+ await completeText(options, record, response, 'stop', 0)
|
|
|
+ return
|
|
|
+ case 'rate_limit':
|
|
|
+ httpError(options, record, response, 429, 'mock rate limit', 'rate_limit')
|
|
|
+ return
|
|
|
+ case 'server_error':
|
|
|
+ httpError(options, record, response, 500, 'mock server error', 'server_error')
|
|
|
+ return
|
|
|
+ case 'service_unavailable':
|
|
|
+ httpError(options, record, response, 503, 'mock service unavailable', 'service_unavailable')
|
|
|
+ return
|
|
|
+ case 'auth_error':
|
|
|
+ httpError(options, record, response, 401, 'mock authentication failed', 'invalid_api_key')
|
|
|
+ return
|
|
|
+ case 'invalid_request':
|
|
|
+ httpError(options, record, response, 400, 'mock invalid request', 'invalid_request')
|
|
|
+ return
|
|
|
+ case 'context_overflow':
|
|
|
+ httpError(
|
|
|
+ options,
|
|
|
+ record,
|
|
|
+ response,
|
|
|
+ 400,
|
|
|
+ 'mock input exceeds the model context window',
|
|
|
+ 'context_length_exceeded',
|
|
|
+ 'invalid_request_error',
|
|
|
+ )
|
|
|
+ return
|
|
|
+ case 'quota_exceeded':
|
|
|
+ httpError(options, record, response, 429, 'mock insufficient quota', 'insufficient_quota')
|
|
|
+ return
|
|
|
+ case 'success':
|
|
|
+ openSse(response)
|
|
|
+ await completeText(options, record, response, 'stop', 0)
|
|
|
+ return
|
|
|
+ case 'reasoning_success':
|
|
|
+ openSse(response)
|
|
|
+ for (const chunk of splitText(options.reasoningText, options.chunkSize)) {
|
|
|
+ writeSse(record, response, {
|
|
|
+ choices: [{ index: 0, delta: { reasoning_content: chunk }, finish_reason: null }],
|
|
|
+ })
|
|
|
+ }
|
|
|
+ await completeText(options, record, response, 'stop', 0)
|
|
|
+ return
|
|
|
+ case 'tool_call_success':
|
|
|
+ openSse(response)
|
|
|
+ for (const chunk of toolCallChunks(options)) writeSse(record, response, chunk)
|
|
|
+ writeSse(record, response, terminalChunk('tool_calls', 2))
|
|
|
+ writeDone(record, response)
|
|
|
+ response.end()
|
|
|
+ finishRecord(options, record, 'completed')
|
|
|
+ return
|
|
|
+ case 'max_tokens':
|
|
|
+ openSse(response)
|
|
|
+ await completeText(options, record, response, 'length', 0)
|
|
|
+ return
|
|
|
+ case 'slow_success':
|
|
|
+ openSse(response)
|
|
|
+ await completeText(options, record, response, 'stop', options.chunkDelayMs)
|
|
|
+ return
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+function seededRandom(seed: number): () => number {
|
|
|
+ let state = seed
|
|
|
+ return () => {
|
|
|
+ state = (state + 0x6d2b_79f5) >>> 0
|
|
|
+ let mixed = state
|
|
|
+ mixed = Math.imul(mixed ^ mixed >>> 15, mixed | 1)
|
|
|
+ mixed ^= mixed + Math.imul(mixed ^ mixed >>> 7, mixed | 61)
|
|
|
+ return ((mixed ^ mixed >>> 14) >>> 0) / 0x1_0000_0000
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+function chooseRandomBehavior(
|
|
|
+ weights: readonly (readonly [ConcreteMockLlmBehavior, number])[],
|
|
|
+ random: () => number,
|
|
|
+): ConcreteMockLlmBehavior {
|
|
|
+ const total = weights.reduce((sum, entry) => sum + entry[1], 0)
|
|
|
+ let draw = random() * total
|
|
|
+ for (const [behavior, weight] of weights) {
|
|
|
+ if (draw < weight) return behavior
|
|
|
+ draw -= weight
|
|
|
+ }
|
|
|
+ // Floating-point subtraction can only leave a rounding residue at the upper boundary.
|
|
|
+ /* v8 ignore next -- seededRandom is strictly less than one; this guards floating-point residue only */
|
|
|
+ return (weights.at(-1) as readonly [ConcreteMockLlmBehavior, number])[0]
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * Start a local chat-completions server that consumes one configured behavior
|
|
|
+ * per accepted request. Only a `POST` path ending in `/chat/completions` consumes the script;
|
|
|
+ * invalid routes, methods, authorization, and JSON receive ordinary 4xx
|
|
|
+ * responses. Closing the handle terminates stalled connections.
|
|
|
+ *
|
|
|
+ * @param options - listener, script, response content, timing, and telemetry options.
|
|
|
+ * @returns the listening handle after the port is bound.
|
|
|
+ */
|
|
|
+export async function startMockLlmServer(options: MockLlmServerOptions): Promise<MockLlmServer> {
|
|
|
+ const resolved = resolveOptions(options)
|
|
|
+ const requests: MockLlmRequestRecord[] = []
|
|
|
+ const random = seededRandom(resolved.randomSeed)
|
|
|
+ let cursor = 0
|
|
|
+
|
|
|
+ const selectBehavior = (): {
|
|
|
+ scriptBehavior: MockLlmBehavior | 'script_exhausted'
|
|
|
+ behavior: ConcreteMockLlmBehavior | 'script_exhausted'
|
|
|
+ } => {
|
|
|
+ const selected = resolved.sequence[cursor]
|
|
|
+ cursor += 1
|
|
|
+ const scriptBehavior = selected
|
|
|
+ ?? (resolved.repeatLast ? resolved.lastBehavior : 'script_exhausted')
|
|
|
+ return {
|
|
|
+ scriptBehavior,
|
|
|
+ behavior: scriptBehavior === 'random'
|
|
|
+ ? chooseRandomBehavior(resolved.randomWeights, random)
|
|
|
+ : scriptBehavior,
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ const handle = async (request: IncomingMessage, response: ServerResponse): Promise<void> => {
|
|
|
+ /* v8 ignore next -- node:http server requests always carry a URL despite the shared optional type */
|
|
|
+ const path = new URL(request.url ?? '/', 'http://mock.invalid').pathname
|
|
|
+ if (request.method !== 'POST') {
|
|
|
+ response.writeHead(405, { allow: 'POST' }).end()
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if (!path.endsWith('/chat/completions')) {
|
|
|
+ response.writeHead(404).end()
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if (resolved.apiKey !== undefined && request.headers.authorization !== `Bearer ${resolved.apiKey}`) {
|
|
|
+ response.writeHead(401, { 'content-type': 'application/json' })
|
|
|
+ response.end(JSON.stringify({ error: { message: 'invalid mock bearer token', code: 'invalid_api_key' } }))
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ let body: unknown
|
|
|
+ try {
|
|
|
+ body = await readJsonBody(request)
|
|
|
+ } catch {
|
|
|
+ response.writeHead(400, { 'content-type': 'application/json' })
|
|
|
+ response.end(JSON.stringify({ error: { message: 'request body must be valid JSON', code: 'invalid_json' } }))
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ const selected = selectBehavior()
|
|
|
+ const record: MockLlmRequestRecord = {
|
|
|
+ attempt: requests.length + 1,
|
|
|
+ scriptBehavior: selected.scriptBehavior,
|
|
|
+ behavior: selected.behavior,
|
|
|
+ path,
|
|
|
+ headers: { ...request.headers },
|
|
|
+ body,
|
|
|
+ chunksSent: 0,
|
|
|
+ }
|
|
|
+ requests.push(record)
|
|
|
+ response.once('close', () => {
|
|
|
+ if (!response.writableFinished && record.outcome === undefined) {
|
|
|
+ finishRecord(resolved, record, 'client_closed')
|
|
|
+ }
|
|
|
+ })
|
|
|
+ emit(resolved, {
|
|
|
+ type: 'request',
|
|
|
+ attempt: record.attempt,
|
|
|
+ scriptBehavior: record.scriptBehavior,
|
|
|
+ behavior: record.behavior,
|
|
|
+ path,
|
|
|
+ })
|
|
|
+ await runBehavior(resolved, record, request, response)
|
|
|
+ }
|
|
|
+
|
|
|
+ const server = createServer((request, response) => {
|
|
|
+ /* v8 ignore start -- last-resort containment for Node response failures after validated test inputs */
|
|
|
+ handle(request, response).catch((error: unknown) => {
|
|
|
+ const record = requests.at(-1)
|
|
|
+ if (record !== undefined) finishRecord(resolved, record, 'server_error')
|
|
|
+ if (response.headersSent) {
|
|
|
+ response.destroy(error instanceof Error ? error : new Error(String(error)))
|
|
|
+ return
|
|
|
+ }
|
|
|
+ response.writeHead(500, { 'content-type': 'application/json' })
|
|
|
+ response.end(JSON.stringify({ error: { message: 'mock server handler failed', code: 'MOCK_HANDLER_FAILED' } }))
|
|
|
+ })
|
|
|
+ /* v8 ignore stop */
|
|
|
+ })
|
|
|
+
|
|
|
+ let closing: Promise<void> | undefined
|
|
|
+ const close = (): Promise<void> => (closing ??= new Promise((resolveClose) => {
|
|
|
+ server.close(() => { resolveClose() })
|
|
|
+ server.closeAllConnections()
|
|
|
+ }))
|
|
|
+
|
|
|
+ await new Promise<void>((resolveListen, rejectListen) => {
|
|
|
+ server.once('error', rejectListen)
|
|
|
+ server.listen(resolved.port, resolved.host, () => {
|
|
|
+ server.off('error', rejectListen)
|
|
|
+ resolveListen()
|
|
|
+ })
|
|
|
+ })
|
|
|
+
|
|
|
+ const address = server.address() as AddressInfo
|
|
|
+ return {
|
|
|
+ baseURL: `http://${resolved.host}:${address.port}`,
|
|
|
+ port: address.port,
|
|
|
+ randomSeed: resolved.randomSeed,
|
|
|
+ requests,
|
|
|
+ close,
|
|
|
+ }
|
|
|
+}
|
Yichen Jiang