stream.ts 2.6 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586
  1. import { openBucket } from '../lib/bucket.js';
  2. import type { StorageFailure, UploadTelemetry } from './types.js';
  3. export interface StoredObject {
  4. key: string;
  5. bytes: number;
  6. contentType: string;
  7. }
  8. /**
  9. * Stream a request body into object storage without buffering it in memory.
  10. * The body is piped through a counting transform so the byte total is known
  11. * by the time the put resolves.
  12. */
  13. export async function streamBodyToStorage(
  14. body: ReadableStream<Uint8Array>,
  15. key: string,
  16. contentType: string,
  17. ): Promise<StoredObject> {
  18. const bucket = openBucket();
  19. const counter = createByteCounter();
  20. const piped = body.pipeThrough(counter.transform, { preventClose: false });
  21. await bucket.put(key, piped, { httpMetadata: { contentType } });
  22. return { key, bytes: counter.total(), contentType };
  23. }
  24. /**
  25. * A transform stream that counts the bytes flowing through it. Separated from
  26. * the pipe above so the byte total can be read after the stream settles.
  27. */
  28. export function createByteCounter() {
  29. let total = 0;
  30. const transform = new TransformStream<Uint8Array, Uint8Array>({
  31. transform(chunk, controller) {
  32. total += chunk.byteLength;
  33. controller.enqueue(chunk);
  34. },
  35. });
  36. return { transform, total: () => total };
  37. }
  38. /**
  39. * Read a stored object back out of the bucket as a stream, for the download
  40. * path. Mirrors the upload side so both directions live in one module.
  41. */
  42. export async function readObjectStream(key: string): Promise<ReadableStream<Uint8Array> | null> {
  43. const bucket = openBucket();
  44. const object = await bucket.get(key);
  45. if (!object) return null;
  46. return object.body;
  47. }
  48. /** Timing/retry record for one stored object, handed to the metrics sink. */
  49. export function telemetryFor(stored: StoredObject, durationMs: number): UploadTelemetry {
  50. return { key: stored.key, bytes: stored.bytes, durationMs, retries: 0 };
  51. }
  52. /** Describe a failed stage so the caller can report it without re-deriving it. */
  53. export function storageFailure(
  54. key: string,
  55. stage: StorageFailure['stage'],
  56. message: string,
  57. ): StorageFailure {
  58. return { key, stage, message };
  59. }
  60. /** Cap a stream at `limit` bytes, erroring out rather than storing an overrun. */
  61. export function limitStream(
  62. source: ReadableStream<Uint8Array>,
  63. limit: number,
  64. ): ReadableStream<Uint8Array> {
  65. let seen = 0;
  66. const guard = new TransformStream<Uint8Array, Uint8Array>({
  67. transform(chunk, controller) {
  68. seen += chunk.byteLength;
  69. if (seen > limit) {
  70. controller.error(new Error(`upload exceeded ${limit} bytes`));
  71. return;
  72. }
  73. controller.enqueue(chunk);
  74. },
  75. });
  76. return source.pipeThrough(guard);
  77. }