|
@@ -1,12 +1,12 @@
|
|
|
/**
|
|
/**
|
|
|
* Host-side session-log download: streams one ZIP archive whose files are the
|
|
* Host-side session-log download: streams one ZIP archive whose files are the
|
|
|
- * sessions' stored artifact text verbatim plus every referenced media object.
|
|
|
|
|
|
|
+ * sessions' stored artifact text verbatim plus every referenced attachment.
|
|
|
* The root artifact sits under its original base name (`session.jsonl`); each
|
|
* The root artifact sits under its original base name (`session.jsonl`); each
|
|
|
* subagent descendant under `subagents/<id>/<filename>`; each image referenced
|
|
* subagent descendant under `subagents/<id>/<filename>`; each image referenced
|
|
|
* by any included log under `media/<attachmentId>.<ext>` (content-addressed,
|
|
* by any included log under `media/<attachmentId>.<ext>` (content-addressed,
|
|
|
- * so one archive never duplicates a shared image). No manifest is written —
|
|
|
|
|
- * every file is byte-identical to the backend's durable artifact or attachment
|
|
|
|
|
- * store and self-describing through its own header line or media type. Before
|
|
|
|
|
|
|
+ * so one archive never duplicates a shared image); each file under
|
|
|
|
|
+ * `files/<prefix>/<digest>/<name>`. No manifest is written. Every entry is
|
|
|
|
|
+ * byte-identical to the backend's durable artifact or attachment store. Before
|
|
|
* each live session's artifact read, the SessionStore flush barrier makes the
|
|
* each live session's artifact read, the SessionStore flush barrier makes the
|
|
|
* current in-memory log durable; cold sessions need no barrier. Request abort
|
|
* current in-memory log durable; cold sessions need no barrier. Request abort
|
|
|
* and response-consumer cancellation share one producer signal and terminate
|
|
* and response-consumer cancellation share one producer signal and terminate
|
|
@@ -21,7 +21,9 @@
|
|
|
|
|
|
|
|
import { Zip, ZipDeflate } from 'fflate'
|
|
import { Zip, ZipDeflate } from 'fflate'
|
|
|
import type { Context } from '@deepseek-ai/cordis'
|
|
import type { Context } from '@deepseek-ai/cordis'
|
|
|
-import type { AttachmentStore, ImageAttachmentRef } from '@deepseek-ai/dsh-attachment'
|
|
|
|
|
|
|
+import type {
|
|
|
|
|
+ AttachmentStore, FileAttachmentRef, ImageAttachmentRef,
|
|
|
|
|
+} from '@deepseek-ai/dsh-attachment'
|
|
|
import type { SessionLineageNode, SessionQueryEngine } from '@deepseek-ai/dsh-session-query'
|
|
import type { SessionLineageNode, SessionQueryEngine } from '@deepseek-ai/dsh-session-query'
|
|
|
import type { SessionId, SessionStore } from '@deepseek-ai/dsh-session'
|
|
import type { SessionId, SessionStore } from '@deepseek-ai/dsh-session'
|
|
|
import type { SessionPersistence, SessionRawArtifact } from '@deepseek-ai/dsh-session-persistence'
|
|
import type { SessionPersistence, SessionRawArtifact } from '@deepseek-ai/dsh-session-persistence'
|
|
@@ -84,10 +86,11 @@ export async function flushLiveSessionLog(
|
|
|
signal?.throwIfAborted()
|
|
signal?.throwIfAborted()
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-/** One exported file: a stored artifact text or one referenced media object. */
|
|
|
|
|
|
|
+/** One exported artifact or referenced attachment object. */
|
|
|
export type SessionLogZipEntry =
|
|
export type SessionLogZipEntry =
|
|
|
| { readonly path: string; readonly content: string }
|
|
| { readonly path: string; readonly content: string }
|
|
|
| { readonly path: string; readonly data: Uint8Array }
|
|
| { readonly path: string; readonly data: Uint8Array }
|
|
|
|
|
+ | { readonly path: string; readonly chunks: AsyncIterable<Uint8Array> }
|
|
|
|
|
|
|
|
/** Zip extension for each accepted raster media type. */
|
|
/** Zip extension for each accepted raster media type. */
|
|
|
const MEDIA_TYPE_EXTENSIONS: Record<ImageAttachmentRef['mediaType'], string> = {
|
|
const MEDIA_TYPE_EXTENSIONS: Record<ImageAttachmentRef['mediaType'], string> = {
|
|
@@ -108,13 +111,26 @@ function mediaEntryPath(ref: ImageAttachmentRef): string {
|
|
|
return `media/${String(ref.attachmentId)}.${MEDIA_TYPE_EXTENSIONS[ref.mediaType]}`
|
|
return `media/${String(ref.attachmentId)}.${MEDIA_TYPE_EXTENSIONS[ref.mediaType]}`
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+/** Archive path that preserves one stored file reference's digest and name. */
|
|
|
|
|
+function fileEntryPath(ref: FileAttachmentRef): string {
|
|
|
|
|
+ const digest = String(ref.attachmentId).replace(/^sha256:/u, '')
|
|
|
|
|
+ const name = ref.name.replace(/[\\/\u0000-\u001f\u007f]/gu, '_')
|
|
|
|
|
+ const safeName = name === '.' || name === '..' || name === '' ? 'file' : name
|
|
|
|
|
+ return `files/${digest.slice(0, 2)}/${digest}/${safeName}`
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
/**
|
|
/**
|
|
|
- * Collect every image reference inside one content array, descending into
|
|
|
|
|
|
|
+ * Collect every attachment reference inside one content array, descending into
|
|
|
* nested tool results the way the live attachment route does.
|
|
* nested tool results the way the live attachment route does.
|
|
|
* @param content - an event content array (or nested tool-result content).
|
|
* @param content - an event content array (or nested tool-result content).
|
|
|
- * @param refs - the dedupe map being filled (keyed by attachment id).
|
|
|
|
|
|
|
+ * @param images - image dedupe map keyed by attachment id.
|
|
|
|
|
+ * @param files - file dedupe map keyed by attachment id and stored name.
|
|
|
*/
|
|
*/
|
|
|
-function collectImageRefs(content: unknown, refs: Map<string, ImageAttachmentRef>): void {
|
|
|
|
|
|
|
+function collectAttachmentRefs(
|
|
|
|
|
+ content: unknown,
|
|
|
|
|
+ images: Map<string, ImageAttachmentRef>,
|
|
|
|
|
+ files: Map<string, FileAttachmentRef>,
|
|
|
|
|
+): void {
|
|
|
if (!Array.isArray(content)) return
|
|
if (!Array.isArray(content)) return
|
|
|
const pending: unknown[] = []
|
|
const pending: unknown[] = []
|
|
|
for (const item of content) pending.push(item)
|
|
for (const item of content) pending.push(item)
|
|
@@ -124,7 +140,11 @@ function collectImageRefs(content: unknown, refs: Map<string, ImageAttachmentRef
|
|
|
const block = value as { type?: unknown; attachment?: unknown; content?: unknown }
|
|
const block = value as { type?: unknown; attachment?: unknown; content?: unknown }
|
|
|
if (block.type === 'image' && typeof block.attachment === 'object' && block.attachment !== null) {
|
|
if (block.type === 'image' && typeof block.attachment === 'object' && block.attachment !== null) {
|
|
|
const ref = block.attachment as ImageAttachmentRef
|
|
const ref = block.attachment as ImageAttachmentRef
|
|
|
- refs.set(String(ref.attachmentId), ref)
|
|
|
|
|
|
|
+ images.set(String(ref.attachmentId), ref)
|
|
|
|
|
+ }
|
|
|
|
|
+ if (block.type === 'file' && typeof block.attachment === 'object' && block.attachment !== null) {
|
|
|
|
|
+ const ref = block.attachment as FileAttachmentRef
|
|
|
|
|
+ files.set(`${String(ref.attachmentId)}\u0000${ref.name}`, ref)
|
|
|
}
|
|
}
|
|
|
if (Array.isArray(block.content)) {
|
|
if (Array.isArray(block.content)) {
|
|
|
for (const item of block.content) pending.push(item)
|
|
for (const item of block.content) pending.push(item)
|
|
@@ -133,13 +153,18 @@ function collectImageRefs(content: unknown, refs: Map<string, ImageAttachmentRef
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
|
- * Collect every image reference one session event carries, across the same
|
|
|
|
|
|
|
+ * Collect every attachment reference one session event carries, across the same
|
|
|
* carriers the live attachment route scans (direct content, message content,
|
|
* carriers the live attachment route scans (direct content, message content,
|
|
|
* inserted messages, and completed assistant chunk blocks).
|
|
* inserted messages, and completed assistant chunk blocks).
|
|
|
* @param event - one parsed JSONL event object.
|
|
* @param event - one parsed JSONL event object.
|
|
|
- * @param refs - the dedupe map being filled (keyed by attachment id).
|
|
|
|
|
|
|
+ * @param images - image dedupe map keyed by attachment id.
|
|
|
|
|
+ * @param files - file dedupe map keyed by attachment id and stored name.
|
|
|
*/
|
|
*/
|
|
|
-function collectEventImageRefs(event: unknown, refs: Map<string, ImageAttachmentRef>): void {
|
|
|
|
|
|
|
+function collectEventAttachmentRefs(
|
|
|
|
|
+ event: unknown,
|
|
|
|
|
+ images: Map<string, ImageAttachmentRef>,
|
|
|
|
|
+ files: Map<string, FileAttachmentRef>,
|
|
|
|
|
+): void {
|
|
|
const data = (event as { data?: unknown }).data
|
|
const data = (event as { data?: unknown }).data
|
|
|
if (typeof data !== 'object' || data === null) return
|
|
if (typeof data !== 'object' || data === null) return
|
|
|
const carrier = data as {
|
|
const carrier = data as {
|
|
@@ -148,23 +173,27 @@ function collectEventImageRefs(event: unknown, refs: Map<string, ImageAttachment
|
|
|
inserted?: Array<{ content?: unknown }>
|
|
inserted?: Array<{ content?: unknown }>
|
|
|
chunk?: { type?: unknown; block?: unknown }
|
|
chunk?: { type?: unknown; block?: unknown }
|
|
|
}
|
|
}
|
|
|
- collectImageRefs(carrier.content, refs)
|
|
|
|
|
- if (carrier.message !== undefined) collectImageRefs(carrier.message.content, refs)
|
|
|
|
|
|
|
+ collectAttachmentRefs(carrier.content, images, files)
|
|
|
|
|
+ if (carrier.message !== undefined) collectAttachmentRefs(carrier.message.content, images, files)
|
|
|
if (carrier.inserted !== undefined) {
|
|
if (carrier.inserted !== undefined) {
|
|
|
- for (const message of carrier.inserted) collectImageRefs(message.content, refs)
|
|
|
|
|
|
|
+ for (const message of carrier.inserted) collectAttachmentRefs(message.content, images, files)
|
|
|
}
|
|
}
|
|
|
- if (carrier.chunk?.type === 'block-end') collectImageRefs([carrier.chunk.block], refs)
|
|
|
|
|
|
|
+ if (carrier.chunk?.type === 'block-end') collectAttachmentRefs([carrier.chunk.block], images, files)
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
|
- * Collect the distinct media references one stored artifact text names.
|
|
|
|
|
- * Lines that fail to parse cannot reference media and are skipped (the
|
|
|
|
|
|
|
+ * Collect the distinct attachment references one stored artifact text names.
|
|
|
|
|
+ * Lines that fail to parse cannot reference attachments and are skipped (the
|
|
|
* artifact text itself is exported verbatim regardless).
|
|
* artifact text itself is exported verbatim regardless).
|
|
|
* @param content - the stored artifact text.
|
|
* @param content - the stored artifact text.
|
|
|
- * @returns the dedupe map keyed by attachment id.
|
|
|
|
|
|
|
+ * @returns image and file dedupe maps.
|
|
|
*/
|
|
*/
|
|
|
-function imageRefsInArtifact(content: string): Map<string, ImageAttachmentRef> {
|
|
|
|
|
- const refs = new Map<string, ImageAttachmentRef>()
|
|
|
|
|
|
|
+function attachmentRefsInArtifact(content: string): {
|
|
|
|
|
+ readonly images: Map<string, ImageAttachmentRef>
|
|
|
|
|
+ readonly files: Map<string, FileAttachmentRef>
|
|
|
|
|
+} {
|
|
|
|
|
+ const images = new Map<string, ImageAttachmentRef>()
|
|
|
|
|
+ const files = new Map<string, FileAttachmentRef>()
|
|
|
for (const line of content.split('\n')) {
|
|
for (const line of content.split('\n')) {
|
|
|
if (line === '') continue
|
|
if (line === '') continue
|
|
|
let event: unknown
|
|
let event: unknown
|
|
@@ -173,9 +202,9 @@ function imageRefsInArtifact(content: string): Map<string, ImageAttachmentRef> {
|
|
|
} catch {
|
|
} catch {
|
|
|
continue
|
|
continue
|
|
|
}
|
|
}
|
|
|
- collectEventImageRefs(event, refs)
|
|
|
|
|
|
|
+ collectEventAttachmentRefs(event, images, files)
|
|
|
}
|
|
}
|
|
|
- return refs
|
|
|
|
|
|
|
+ return { images, files }
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
@@ -204,10 +233,10 @@ export function sessionLogZipFilename(sessionId: string): string {
|
|
|
* Yield the export entries in zip order: the preloaded root artifact first,
|
|
* Yield the export entries in zip order: the preloaded root artifact first,
|
|
|
* then every subagent descendant in lineage order (each flushed when live,
|
|
* then every subagent descendant in lineage order (each flushed when live,
|
|
|
* read from the persistence backend right before it is yielded, and dropped
|
|
* read from the persistence backend right before it is yielded, and dropped
|
|
|
- * after the consumer moves on), then every distinct media object referenced by any of
|
|
|
|
|
- * the included logs (read and verified from the attachment store, one archive
|
|
|
|
|
- * entry per attachment id). The host holds at most one descendant's artifact
|
|
|
|
|
- * text and one media object at a time beyond the root.
|
|
|
|
|
|
|
+ * after the consumer moves on), then every distinct attachment referenced by
|
|
|
|
|
+ * the included logs. Images are read and verified as bounded stored objects;
|
|
|
|
|
+ * generic files remain streamed through the ZIP writer. The host holds at most
|
|
|
|
|
+ * one descendant artifact, one image, and one file chunk beyond the root.
|
|
|
* @param deps - the mounted export services (the caller answered 500 before this runs).
|
|
* @param deps - the mounted export services (the caller answered 500 before this runs).
|
|
|
* @param root - the already-read root artifact (read by the caller so the
|
|
* @param root - the already-read root artifact (read by the caller so the
|
|
|
* missing-session path can answer cleanly before streaming starts).
|
|
* missing-session path can answer cleanly before streaming starts).
|
|
@@ -224,10 +253,13 @@ export async function* sessionLogZipEntries(
|
|
|
signal?: AbortSignal,
|
|
signal?: AbortSignal,
|
|
|
): AsyncGenerator<SessionLogZipEntry> {
|
|
): AsyncGenerator<SessionLogZipEntry> {
|
|
|
const media = new Map<string, ImageAttachmentRef>()
|
|
const media = new Map<string, ImageAttachmentRef>()
|
|
|
- const rememberMedia = (content: string): void => {
|
|
|
|
|
- for (const [id, ref] of imageRefsInArtifact(content)) media.set(id, ref)
|
|
|
|
|
|
|
+ const files = new Map<string, FileAttachmentRef>()
|
|
|
|
|
+ const rememberAttachments = (content: string): void => {
|
|
|
|
|
+ const refs = attachmentRefsInArtifact(content)
|
|
|
|
|
+ for (const [id, ref] of refs.images) media.set(id, ref)
|
|
|
|
|
+ for (const [id, ref] of refs.files) files.set(id, ref)
|
|
|
}
|
|
}
|
|
|
- rememberMedia(root.content)
|
|
|
|
|
|
|
+ rememberAttachments(root.content)
|
|
|
yield { path: root.filename, content: root.content }
|
|
yield { path: root.filename, content: root.content }
|
|
|
if (includeDescendants) {
|
|
if (includeDescendants) {
|
|
|
const seen = new Set<SessionId>([sessionId])
|
|
const seen = new Set<SessionId>([sessionId])
|
|
@@ -245,7 +277,7 @@ export async function* sessionLogZipEntries(
|
|
|
if (raw === undefined) {
|
|
if (raw === undefined) {
|
|
|
throw new Error(`subagent "${id}" has no stored log artifact`)
|
|
throw new Error(`subagent "${id}" has no stored log artifact`)
|
|
|
}
|
|
}
|
|
|
- rememberMedia(raw.content)
|
|
|
|
|
|
|
+ rememberAttachments(raw.content)
|
|
|
yield {
|
|
yield {
|
|
|
path: `subagents/${safeSessionIdSegment(id)}/${raw.filename}`,
|
|
path: `subagents/${safeSessionIdSegment(id)}/${raw.filename}`,
|
|
|
content: raw.content,
|
|
content: raw.content,
|
|
@@ -263,6 +295,13 @@ export async function* sessionLogZipEntries(
|
|
|
signal?.throwIfAborted()
|
|
signal?.throwIfAborted()
|
|
|
yield { path: mediaEntryPath(ref), data: stored.data }
|
|
yield { path: mediaEntryPath(ref), data: stored.data }
|
|
|
}
|
|
}
|
|
|
|
|
+ for (const ref of files.values()) {
|
|
|
|
|
+ signal?.throwIfAborted()
|
|
|
|
|
+ yield {
|
|
|
|
|
+ path: fileEntryPath(ref),
|
|
|
|
|
+ chunks: deps.attachments.readFileStream(ref, signal),
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/** How many code units of artifact text one zip push carries (bounded encode memory). */
|
|
/** How many code units of artifact text one zip push carries (bounded encode memory). */
|
|
@@ -334,6 +373,25 @@ async function pushBinaryChunks(
|
|
|
} while (offset < data.byteLength)
|
|
} while (offset < data.byteLength)
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+/** Push one streamed file entry without retaining its complete byte sequence. */
|
|
|
|
|
+async function pushStreamChunks(
|
|
|
|
|
+ deflate: ZipDeflate,
|
|
|
|
|
+ chunks: AsyncIterable<Uint8Array>,
|
|
|
|
|
+ controller: ReadableStreamDefaultController<Uint8Array>,
|
|
|
|
|
+ capacity: ResponseCapacityGate,
|
|
|
|
|
+ signal: AbortSignal,
|
|
|
|
|
+): Promise<void> {
|
|
|
|
|
+ for await (const chunk of chunks) {
|
|
|
|
|
+ signal.throwIfAborted()
|
|
|
|
|
+ if (chunk.byteLength === 0) continue
|
|
|
|
|
+ deflate.push(chunk, false)
|
|
|
|
|
+ await capacity.wait(controller, signal)
|
|
|
|
|
+ }
|
|
|
|
|
+ signal.throwIfAborted()
|
|
|
|
|
+ deflate.push(new Uint8Array(), true)
|
|
|
|
|
+ await capacity.wait(controller, signal)
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
/**
|
|
/**
|
|
|
* Push one artifact's text into a deflate stream in bounded chunks, never
|
|
* Push one artifact's text into a deflate stream in bounded chunks, never
|
|
|
* splitting a surrogate pair across a chunk boundary (a lone high surrogate
|
|
* splitting a surrogate pair across a chunk boundary (a lone high surrogate
|
|
@@ -427,8 +485,10 @@ export function streamSessionLogZip(
|
|
|
archive.add(deflate)
|
|
archive.add(deflate)
|
|
|
if ('content' in entry) {
|
|
if ('content' in entry) {
|
|
|
await pushArtifactChunks(deflate, entry.content, controller, capacity, producerSignal)
|
|
await pushArtifactChunks(deflate, entry.content, controller, capacity, producerSignal)
|
|
|
- } else {
|
|
|
|
|
|
|
+ } else if ('data' in entry) {
|
|
|
await pushBinaryChunks(deflate, entry.data, controller, capacity, producerSignal)
|
|
await pushBinaryChunks(deflate, entry.data, controller, capacity, producerSignal)
|
|
|
|
|
+ } else {
|
|
|
|
|
+ await pushStreamChunks(deflate, entry.chunks, controller, capacity, producerSignal)
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
archive.end()
|
|
archive.end()
|