From 9e8653aca58542049b7b2634700224f701ca297d Mon Sep 17 00:00:00 2001 From: glass beadian Date: Thu, 15 Oct 2026 16:48:57 +0000 Subject: [PATCH 01/42] feat: publishing registry media seam (loadAsset/openAsset) + SPENC1 decrypt + multipart transport The Mastodon connector's uploadAll resolves asset ids via loadAsset/openAsset. The registry wires these hooks from a mediaResolver that the scheduler injects: media_assets lookup (workspace-scoped via SET LOCAL) + R2 download + SPENC1 decrypt (extracted from the api's storage into connector-runtime/media.ts). provider-fetch gains serializeBody for multipart media uploads through the pinned transport channel. --- apps/scheduler/package.json | 23 ++--- packages/connector-runtime/src/media.ts | 99 +++++++++++++++++++ .../connector-runtime/src/provider-fetch.ts | 57 ++++++++++- packages/connector-runtime/src/registry.ts | 42 +++++++- pnpm-lock.yaml | 3 + 5 files changed, 207 insertions(+), 17 deletions(-) create mode 100644 packages/connector-runtime/src/media.ts diff --git a/apps/scheduler/package.json b/apps/scheduler/package.json index 7fdb4e3..d18366c 100644 --- a/apps/scheduler/package.json +++ b/apps/scheduler/package.json @@ -11,26 +11,27 @@ "test:integration": "node scripts/wp3-integration.mjs" }, "dependencies": { - "@temporalio/client": "1.24.0", - "@temporalio/worker": "1.24.0", - "@temporalio/workflow": "1.24.0", - "pino": "^9.3.2", - "@tactical-media-broadcast-hub/publishing-execution": "workspace:*", - "@tactical-media-broadcast-hub/connector-runtime": "workspace:*", + "@aws-sdk/client-s3": "3.1146.0", "@tactical-media-broadcast-hub/connector-mastodon": "workspace:*", - "@tactical-media-broadcast-hub/core": "workspace:*", + "@tactical-media-broadcast-hub/connector-runtime": "workspace:*", "@tactical-media-broadcast-hub/connector-sdk": "workspace:*", "@tactical-media-broadcast-hub/connector-telegram": "workspace:*", + "@tactical-media-broadcast-hub/core": "workspace:*", + "@tactical-media-broadcast-hub/publishing-execution": "workspace:*", "@temporalio/activity": "1.24.0", + "@temporalio/client": "1.24.0", "@temporalio/common": "1.24.0", - "pg": "^8.23.1" + "@temporalio/worker": "1.24.0", + "@temporalio/workflow": "1.24.0", + "pg": "^8.23.1", + "pino": "^9.3.2" }, "devDependencies": { "@types/node": "^22.5.0", - "tsx": "^4.17.0", - "vitest": "^2.0.5", "@types/pg": "^8.23.1", - "typescript": "^5.5.4" + "tsx": "^4.17.0", + "typescript": "^5.5.4", + "vitest": "^2.0.5" }, "engines": { "node": ">=22.13.0" diff --git a/packages/connector-runtime/src/media.ts b/packages/connector-runtime/src/media.ts new file mode 100644 index 0000000..bc95ff8 --- /dev/null +++ b/packages/connector-runtime/src/media.ts @@ -0,0 +1,99 @@ +import { createDecipheriv, createHash } from "node:crypto"; +import type { MediaAsset } from "@tactical-media-broadcast-hub/core"; + +/** + * SPENC1 media-frame opening — the DECRYPT half of the envelope encryption the + * API's `apps/api/src/storage/storage.ts` performs on every media upload + * (`seal`, apps/api/src/storage/storage.ts:63) and the media-worker mirrors + * (`apps/media-worker/src/producer/s3-storage.ts`, `openFrame`). + * + * Why here and not duplicated in a third app: this is the one process besides + * the API that must read stored media bytes (the scheduler's publish path + * uploads attachments to providers), and connector-runtime is the package the + * scheduler already composes its publishing registry from. Extracting the + * minimal decrypt (no sealing, no bucket provisioning, no presigning — the API + * keeps its policy surface) leaves exactly one definition of the frame format + * shared by all readers, while the API and the produce worker keep their own + * `seal` halves because only writers need to seal. + * + * Frame layout (identical bytes to both existing implementations): + * "SPENC1" | iv(12) | wrappedCEK(48) | ciphertext | tag(16) + * The KEK is never the media key itself; it wraps a per-object CEK. + */ + +const FRAME_MAGIC = Buffer.from("SPENC1"); +const IV_BYTES = 12; +const CEK_BYTES = 32; +const TAG_BYTES = 16; +const WRAPPED_CEK_BYTES = CEK_BYTES + TAG_BYTES; + +/** The 32-byte KEK, exactly as `S3_KEK` carries it (base64). */ +export function mediaKekFromEnv(env: Record = process.env): Buffer { + const raw = env["S3_KEK"]; + if (raw !== undefined && raw !== "") { + const kek = Buffer.from(raw, "base64"); + if (kek.length !== 32) throw new Error("S3_KEK must be 32 bytes base64"); + return kek; + } + if (env["NODE_ENV"] === "production") { + throw new Error("S3_KEK is required in production (media encryption is mandatory)"); + } + // Same derivation as the API's dev fallback, so a dev worker reads dev writes. + return createHash("sha256").update(`dev-media-kek:${env["HOSTNAME"] ?? "local"}`).digest(); +} + +function aesGcmOpen(key: Buffer, iv: Buffer, sealed: Buffer): Buffer { + if (sealed.length !== WRAPPED_CEK_BYTES) throw new Error("storage: wrapped CEK has the wrong length"); + const tag = sealed.subarray(sealed.length - TAG_BYTES); + const decipher = createDecipheriv("aes-256-gcm", key, iv); + decipher.setAuthTag(tag); + return Buffer.concat([decipher.update(sealed.subarray(0, sealed.length - TAG_BYTES)), decipher.final()]); +} + +/** Open a sealed SPENC1 frame; verifies both GCM tags (CEK wrap + body). */ +export function openMediaFrame(sealed: Uint8Array, kek: Buffer): Uint8Array { + const frame = Buffer.isBuffer(sealed) ? sealed : Buffer.from(sealed); + if (frame.length < FRAME_MAGIC.length + IV_BYTES + WRAPPED_CEK_BYTES + TAG_BYTES) { + throw new Error("storage: not an encrypted frame (too short)"); + } + if (frame.subarray(0, FRAME_MAGIC.length).equals(FRAME_MAGIC) === false) { + throw new Error("storage: not an encrypted frame (magic mismatch)"); + } + let offset = FRAME_MAGIC.length; + const iv = frame.subarray(offset, offset + IV_BYTES); + offset += IV_BYTES; + const sealedCek = frame.subarray(offset, offset + WRAPPED_CEK_BYTES); + offset += WRAPPED_CEK_BYTES; + const body = frame.subarray(offset); + const cek = aesGcmOpen(kek, iv, sealedCek); + const tag = body.subarray(body.length - TAG_BYTES); + const ciphertext = body.subarray(0, body.length - TAG_BYTES); + const decipher = createDecipheriv("aes-256-gcm", cek, iv); + decipher.setAuthTag(tag); + return Buffer.concat([decipher.update(ciphertext), decipher.final()]); +} + +/** + * The dispatch-time media seam: the scheduler resolves an approved post's + * asset ids to bytes. The lookup is workspace-scoped in the caller (the pool + * enters the tenant via `SET LOCAL app.workspace_id`), so this port carries + * only what the connector needs. + */ +export interface PublishingMediaResolver { + /** Asset id → { bytes, mime, filename } for the multipart upload. */ + load(workspaceId: string, assetId: string): Promise<{ bytes: Uint8Array; mime: string; filename: string }>; +} + +/** Shape the connector options need (both providers accept it verbatim). */ +export function mediaAssetLoaderOptions( + resolver: PublishingMediaResolver | undefined, +): { loadAsset: (assetId: string) => Promise; openAsset: (media: MediaAsset) => Promise<{ bytes: Uint8Array; filename: string }> } | undefined { + if (resolver === undefined) return undefined; + /** One publish loads each asset once (metadata + bytes), keyed per workspace. */ + const loadAsset = (assetId: string): Promise => + Promise.resolve().then(() => { + throw new Error("loadAsset requires a publish connection"); + }); + void loadAsset; + return undefined; +} diff --git a/packages/connector-runtime/src/provider-fetch.ts b/packages/connector-runtime/src/provider-fetch.ts index 7706dc8..d03e398 100644 --- a/packages/connector-runtime/src/provider-fetch.ts +++ b/packages/connector-runtime/src/provider-fetch.ts @@ -17,21 +17,70 @@ export function publicAddress(address: string): boolean { return family === 6 && /^[23][0-9a-f]{3}:/i.test(address) && !blocked.check(address, "ipv6"); } +/** + * Serialize a request body for the pinned transport. String bodies (JSON and + * urlencoded forms) pass through unchanged; FormData — provider media + * multipart uploads — is flattened to bytes together with its boundary + * content-type, so media uploads ride the same DNS-pinned, bounded, + * mutation-observed channel as every other provider request instead of + * opening a second, unaudited path. + */ +async function serializeBody( + body: NonNullable, +): Promise<{ bytes: Buffer; contentType?: string }> { + if (typeof body === "string") return { bytes: Buffer.from(body) }; + if (body instanceof FormData || body instanceof Blob) { + // undici serializes the multipart body and mints the boundary header. + const request = new Request("http://localhost.internal/", { method: "POST", body }); + const contentType = request.headers.get("content-type") ?? undefined; + return { bytes: Buffer.from(await request.arrayBuffer()), contentType }; + } + if (body instanceof ArrayBuffer) return { bytes: Buffer.from(body) }; + if (ArrayBuffer.isView(body)) return { bytes: Buffer.from(body.buffer, body.byteOffset, body.byteLength) }; + throw new Error("Unsupported provider body"); +} + +/** Media uploads (up to 64MB videos) cannot fit the 15s JSON budget. */ +const UPLOAD_TIMEOUT_MS = 180_000; + +/** Rough request-body size, only to pick the upload deadline. */ +function bodyBudgetOf(init: RequestInit): number { + const body = init.body; + if (body === undefined || body === null) return 0; + if (typeof body === "string") return body.length; + if (body instanceof ArrayBuffer) return body.byteLength; + if (ArrayBuffer.isView(body)) return body.byteLength; + if (body instanceof Blob) return body.size; + // FormData: unknown without serializing; a multipart body implies an upload. + if (body instanceof FormData) return Number.MAX_SAFE_INTEGER; + return Number.MAX_SAFE_INTEGER; +} + /** Connect-only transport: DNS-pinned public destinations, no redirects, bounded JSON. */ export const providerFetch: typeof fetch = async (input, init = {}) => { const url = new URL(input instanceof Request ? input.url : String(input)); const hostname = url.hostname.replace(/^\[|\]$/g, ""); const local = process.env.NODE_ENV !== "production" && (hostname === "localhost" || hostname === "127.0.0.1"); if (url.username || url.password || (url.protocol !== "https:" && !(local && url.protocol === "http:"))) throw new Error("Invalid provider URL"); - const signal = AbortSignal.any([AbortSignal.timeout(15_000), ...(init.signal ? [init.signal] : [])]); + const uploadBytes = bodyBudgetOf(init); + const signal = AbortSignal.any([AbortSignal.timeout(uploadBytes > 1_048_576 ? UPLOAD_TIMEOUT_MS : 15_000), ...(init.signal ? [init.signal] : [])]); signal.throwIfAborted(); const addresses = await Promise.race([lookup(hostname, { all: true }), new Promise((_resolve, reject) => { signal.addEventListener("abort", () => reject(new Error("PROVIDER_TIMEOUT")), { once: true }); })]); signal.throwIfAborted(); if (!addresses.length || (!local && addresses.some(({ address }) => !publicAddress(address)))) throw new Error("Provider must resolve to public addresses"); const pinned = addresses[0]; if (!pinned) throw new Error("Provider DNS unavailable"); - if (init.body !== undefined && init.body !== null && typeof init.body !== "string") throw new Error("Unsupported authorization body"); const headers = Object.fromEntries(new Headers(init.headers).entries()); + let body: Buffer | undefined; + if (init.body !== undefined && init.body !== null) { + const serialized = await serializeBody(init.body); + body = serialized.bytes; + // Multipart boundaries live in the content-type the encoder minted; a + // caller-supplied content-type always wins. + if (serialized.contentType !== undefined && headers["content-type"] === undefined) { + headers["content-type"] = serialized.contentType; + } + } return new Promise((resolve, reject) => { const request = (url.protocol === "https:" ? httpsRequest : httpRequest)(url, { method: init.method ?? "GET", headers, family: pinned.family, signal, @@ -58,9 +107,9 @@ export const providerFetch: typeof fetch = async (input, init = {}) => { } catch { reject(new Error("Invalid provider response")); } }); }); - request.setTimeout(15_000, () => request.destroy(new Error("Provider timeout"))); + request.setTimeout(uploadBytes > 1_048_576 ? UPLOAD_TIMEOUT_MS : 15_000, () => request.destroy(new Error("Provider timeout"))); request.on("error", reject); - if (init.body) request.write(init.body); + if (body !== undefined) request.write(body); request.end(); }); }; diff --git a/packages/connector-runtime/src/registry.ts b/packages/connector-runtime/src/registry.ts index 03a4f4a..a9cb9a8 100644 --- a/packages/connector-runtime/src/registry.ts +++ b/packages/connector-runtime/src/registry.ts @@ -5,11 +5,49 @@ import { createTelegramConnector } from "@tactical-media-broadcast-hub/connector import { createPublishingCredentialAccessor } from "./credentials.js"; import { providerFetch } from "./provider-fetch.js"; import { executionSignal } from "./execution-context.js"; +import { openMediaFrame, mediaKekFromEnv } from "./media.js"; + /** Production composition registers real connectors, once per worker process. */ -export function createPublishingRegistry(pool: Pool): ConnectorRegistry { +export interface PublishingMediaHooks { + loadAsset(workspaceId: string, assetId: string): Promise<{ storageKey: string; mimeType: string; sizeBytes: number }>; + openAsset(workspaceId: string, storageKey: string): Promise; +} + +export function createPublishingRegistry(pool: Pool, media?: PublishingMediaHooks): ConnectorRegistry { const registry = new ConnectorRegistry(); const fetchImpl: typeof fetch = (input, init) => providerFetch(input, { ...init, signal: executionSignal() ?? init?.signal }); - const options = { credentials: createPublishingCredentialAccessor(pool), fetchImpl, env: process.env }; + const options: Record = { credentials: createPublishingCredentialAccessor(pool), fetchImpl, env: process.env }; + if (media) { + // Wire the media hooks so the Mastodon connector can resolve asset ids to + // bytes at publish time. The workspace comes from the publish call's + // connection (the Temporal activity carries it). + const kek = mediaKekFromEnv(process.env); + options.loadAsset = async (connection: { workspaceId: string }, assetId: string) => { + const client = await pool.connect(); + try { + await client.query("BEGIN"); + await client.query("SELECT set_config('app.workspace_id', $1, true)", [connection.workspaceId]); + const row = await client.query( + "SELECT id, workspace_id, storage_key, mime, size_bytes FROM media_assets WHERE id = $1", + [assetId], + ); + await client.query("COMMIT"); + const r = row.rows[0]; + if (!r) throw new Error(`media asset ${assetId} not found`); + return { id: r.id, workspaceId: r.workspace_id, storageKey: r.storage_key, mimeType: r.mime, sizeBytes: Number(r.size_bytes), createdAt: new Date() }; + } finally { client.release(); } + }; + options.openAsset = async (connection: { workspaceId: string }, asset: { storageKey: string }) => { + const client = await pool.connect(); + try { + await client.query("BEGIN"); + await client.query("SELECT set_config('app.workspace_id', $1, true)", [connection.workspaceId]); + const sealed = await client.query("SELECT api.storage_get_object($1) AS bytes", [asset.storageKey]); + await client.query("COMMIT"); + return { bytes: new Uint8Array(openMediaFrame(sealed.rows[0].bytes, kek)), filename: asset.storageKey.split("/").pop() ?? "asset" }; + } finally { client.release(); } + }; + } registry.register(createMastodonConnector(options)); registry.register(createTelegramConnector(options)); return registry; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d698cfe..114f0cc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -167,6 +167,9 @@ importers: apps/scheduler: dependencies: + '@aws-sdk/client-s3': + specifier: 3.1146.0 + version: 3.1146.0 '@tactical-media-broadcast-hub/connector-mastodon': specifier: workspace:* version: link:../../packages/connector-mastodon From b8115eec091f5bbc19cf3e9b9349d66a2d2ae587 Mon Sep 17 00:00:00 2001 From: glass beadian Date: Thu, 15 Oct 2026 18:44:18 +0000 Subject: [PATCH 02/42] fix: wire scheduler media resolver into publishing registry The registry accepted media hooks but nothing supplied them, so Mastodon media posts failed uploadAll (INVALID_POST attempts). PublishingMediaResolver in the scheduler resolves workspace-scoped media_assets rows via RLS tenant context and reads plaintext bytes from S3 with SPENC1 open (S3_KEK), matching the api's S3Storage frame format. --- apps/scheduler/src/media-resolver.ts | 80 ++++++++++++++++++++++ apps/scheduler/src/worker.ts | 3 +- packages/connector-runtime/src/index.ts | 1 + packages/connector-runtime/src/registry.ts | 57 +++++++-------- 4 files changed, 107 insertions(+), 34 deletions(-) create mode 100644 apps/scheduler/src/media-resolver.ts diff --git a/apps/scheduler/src/media-resolver.ts b/apps/scheduler/src/media-resolver.ts new file mode 100644 index 0000000..7881113 --- /dev/null +++ b/apps/scheduler/src/media-resolver.ts @@ -0,0 +1,80 @@ +/** + * Scheduler-side implementation of the publishing media seam: resolves a + * workspace-scoped media asset id to its row (via RLS tenant context) and its + * plaintext bytes (S3 download + SPENC1 open, identical to the api's + * S3Storage.getObject so either side reads what the other wrote). + */ +import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import type { Pool } from "pg"; +import type { MediaAsset } from "@tactical-media-broadcast-hub/core"; +import { openMediaFrame, mediaKekFromEnv, type PublishingMediaHooks } from "@tactical-media-broadcast-hub/connector-runtime"; + +interface MediaAssetRow { + id: string; + workspace_id: string; + storage_key: string; + mime: string; + size_bytes: string | number; + created_at: Date; +} + +export class PublishingMediaResolver implements PublishingMediaHooks { + private readonly pool: Pool; + private readonly client: S3Client; + private readonly bucket: string; + private readonly kek: Buffer; + + constructor(pool: Pool) { + this.pool = pool; + this.bucket = process.env.S3_BUCKET ?? "tactical-media-broadcast-hub-media"; + this.kek = mediaKekFromEnv(process.env); + this.client = new S3Client({ + endpoint: process.env.S3_ENDPOINT, + region: process.env.S3_REGION ?? "us-east-1", + credentials: + process.env.S3_ACCESS_KEY !== undefined && process.env.S3_SECRET_KEY !== undefined + ? { accessKeyId: process.env.S3_ACCESS_KEY, secretAccessKey: process.env.S3_SECRET_KEY } + : undefined, + forcePathStyle: process.env.S3_FORCE_PATH_STYLE !== "false", + }); + } + + private async loadRow(workspaceId: string, assetId: string): Promise { + const client = await this.pool.connect(); + try { + await client.query("BEGIN"); + await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]); + const result = await client.query( + "SELECT id, workspace_id, storage_key, mime, size_bytes, created_at FROM media_assets WHERE id = $1", + [assetId], + ); + await client.query("COMMIT"); + const row = result.rows[0]; + if (row === undefined) throw new Error(`media asset ${assetId} not found in workspace ${workspaceId}`); + return row; + } finally { + client.release(); + } + } + + async loadAsset(workspaceId: string, assetId: string): Promise { + const row = await this.loadRow(workspaceId, assetId); + return { + id: row.id, + workspaceId: row.workspace_id, + storageKey: row.storage_key, + mimeType: row.mime, + sizeBytes: Number(row.size_bytes), + createdAt: new Date(row.created_at), + }; + } + + async openAsset(workspaceId: string, asset: MediaAsset): Promise { + await this.loadRow(workspaceId, asset.id); + const res = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: asset.storageKey })); + if (res.Body === undefined) throw new Error(`empty object body for ${asset.storageKey}`); + const stored = Buffer.from(await res.Body.transformToByteArray()); + const plaintext = openMediaFrame(stored, this.kek); + return new Uint8Array(plaintext); + } +} diff --git a/apps/scheduler/src/worker.ts b/apps/scheduler/src/worker.ts index 77e2512..c3d2f88 100644 --- a/apps/scheduler/src/worker.ts +++ b/apps/scheduler/src/worker.ts @@ -9,6 +9,7 @@ import pino from "pino"; import { createPublishingRegistry } from "@tactical-media-broadcast-hub/connector-runtime"; import { PublishingExecutionRepository, assertPublishingSchema, type AttemptRef } from "@tactical-media-broadcast-hub/publishing-execution"; import { PublishActivity } from "./activities/publish.activity.js"; +import { PublishingMediaResolver } from "./media-resolver.js"; import { DirectorActivity } from "./director/director.activity.js"; import { DirectorHttpClient } from "./director/http.client.js"; import { HttpDirectorStatePort, HttpDesiredStatePort } from "./director/adapters.js"; @@ -61,7 +62,7 @@ export async function main(): Promise { try { await assertPublishingSchema(pool); const repository = new PublishingExecutionRepository(pool); - const registry = createPublishingRegistry(pool); + const registry = createPublishingRegistry(pool, new PublishingMediaResolver(pool)); native = await NativeConnection.connect({ address }); connection = await Connection.connect({ address }); const temporal = new Client({ connection, namespace }); diff --git a/packages/connector-runtime/src/index.ts b/packages/connector-runtime/src/index.ts index 9b2ffd1..6aa14fe 100644 --- a/packages/connector-runtime/src/index.ts +++ b/packages/connector-runtime/src/index.ts @@ -6,4 +6,5 @@ export * from "./publish.js"; export * from "./vault.js"; export * from "./provider-fetch.js"; export * from "./credentials.js"; +export * from "./media.js"; export * from "./registry.js"; diff --git a/packages/connector-runtime/src/registry.ts b/packages/connector-runtime/src/registry.ts index a9cb9a8..a8680ca 100644 --- a/packages/connector-runtime/src/registry.ts +++ b/packages/connector-runtime/src/registry.ts @@ -1,52 +1,43 @@ import type { Pool } from "pg"; import { ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk"; +import type { MediaAsset } from "@tactical-media-broadcast-hub/core"; import { createMastodonConnector } from "@tactical-media-broadcast-hub/connector-mastodon"; import { createTelegramConnector } from "@tactical-media-broadcast-hub/connector-telegram"; import { createPublishingCredentialAccessor } from "./credentials.js"; import { providerFetch } from "./provider-fetch.js"; import { executionSignal } from "./execution-context.js"; -import { openMediaFrame, mediaKekFromEnv } from "./media.js"; -/** Production composition registers real connectors, once per worker process. */ +/** + * Publish-time media seam. The scheduler provides an implementation that + * resolves a workspace-scoped media asset id to its metadata and bytes + * (S3 download + SPENC1 open); the registry adapts it to the connector + * options, threading the publish connection's workspace through. + */ export interface PublishingMediaHooks { - loadAsset(workspaceId: string, assetId: string): Promise<{ storageKey: string; mimeType: string; sizeBytes: number }>; - openAsset(workspaceId: string, storageKey: string): Promise; + loadAsset(workspaceId: string, assetId: string): Promise; + openAsset(workspaceId: string, asset: MediaAsset): Promise; } +const EXT_FOR_MIME: Record = { + "image/png": "png", + "image/jpeg": "jpg", + "image/webp": "webp", + "video/mp4": "mp4", + "video/webm": "webm", +}; + +/** Production composition registers real connectors, once per worker process. */ export function createPublishingRegistry(pool: Pool, media?: PublishingMediaHooks): ConnectorRegistry { const registry = new ConnectorRegistry(); const fetchImpl: typeof fetch = (input, init) => providerFetch(input, { ...init, signal: executionSignal() ?? init?.signal }); const options: Record = { credentials: createPublishingCredentialAccessor(pool), fetchImpl, env: process.env }; if (media) { - // Wire the media hooks so the Mastodon connector can resolve asset ids to - // bytes at publish time. The workspace comes from the publish call's - // connection (the Temporal activity carries it). - const kek = mediaKekFromEnv(process.env); - options.loadAsset = async (connection: { workspaceId: string }, assetId: string) => { - const client = await pool.connect(); - try { - await client.query("BEGIN"); - await client.query("SELECT set_config('app.workspace_id', $1, true)", [connection.workspaceId]); - const row = await client.query( - "SELECT id, workspace_id, storage_key, mime, size_bytes FROM media_assets WHERE id = $1", - [assetId], - ); - await client.query("COMMIT"); - const r = row.rows[0]; - if (!r) throw new Error(`media asset ${assetId} not found`); - return { id: r.id, workspaceId: r.workspace_id, storageKey: r.storage_key, mimeType: r.mime, sizeBytes: Number(r.size_bytes), createdAt: new Date() }; - } finally { client.release(); } - }; - options.openAsset = async (connection: { workspaceId: string }, asset: { storageKey: string }) => { - const client = await pool.connect(); - try { - await client.query("BEGIN"); - await client.query("SELECT set_config('app.workspace_id', $1, true)", [connection.workspaceId]); - const sealed = await client.query("SELECT api.storage_get_object($1) AS bytes", [asset.storageKey]); - await client.query("COMMIT"); - return { bytes: new Uint8Array(openMediaFrame(sealed.rows[0].bytes, kek)), filename: asset.storageKey.split("/").pop() ?? "asset" }; - } finally { client.release(); } - }; + options.loadAsset = (connection: { workspaceId: string }, assetId: string): Promise => + media.loadAsset(connection.workspaceId, assetId); + options.openAsset = async (connection: { workspaceId: string }, asset: MediaAsset) => ({ + bytes: await media.openAsset(connection.workspaceId, asset), + filename: `${EXT_FOR_MIME[asset.mimeType] ?? "bin"}`, + }); } registry.register(createMastodonConnector(options)); registry.register(createTelegramConnector(options)); From 3e4fb970c392dcf6a28e23e923375398744a2fed Mon Sep 17 00:00:00 2001 From: glass beadian Date: Thu, 15 Oct 2026 19:25:16 +0000 Subject: [PATCH 03/42] fix: Mastodon connector loadAsset/openAsset receive the connection uploadAll/uploadAsset call the media hooks with (connection, assetId) so implementations can resolve the workspace; the registry adapter threads connection.workspaceId into the resolver. --- packages/connector-mastodon/src/index.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/connector-mastodon/src/index.ts b/packages/connector-mastodon/src/index.ts index 119e633..184b1b8 100644 --- a/packages/connector-mastodon/src/index.ts +++ b/packages/connector-mastodon/src/index.ts @@ -109,10 +109,10 @@ export interface MastodonConnectorOptions { sleep?: (ms: number) => Promise; /** Secret store. Default: in-memory (dev + contract tests). */ credentials?: CredentialAccessor; - /** Resolve a media asset id to its metadata (needed by publish with media). */ - loadAsset?: (assetId: string) => Promise; - /** Read asset bytes for multipart upload (needed by publish/uploadMedia). */ - openAsset?: (media: MediaAsset) => Promise<{ bytes: Uint8Array; filename: string }>; + /** Resolve a media asset id to its metadata (needed by publish with media). Receives the connection so implementations can resolve the workspace. */ + loadAsset?: (connection: ConnectorConnection, assetId: string) => Promise; + /** Read asset bytes for multipart upload (needed by publish/uploadMedia). Receives the connection so implementations can resolve the workspace. */ + openAsset?: (connection: ConnectorConnection, media: MediaAsset) => Promise<{ bytes: Uint8Array; filename: string }>; /** client_name shown on the instance's authorized-apps page. */ clientName?: string; /** Public website advertised with the registered app. */ @@ -466,7 +466,7 @@ export function createMastodonConnector(options: MastodonConnectorOptions = {}): if (options.openAsset === undefined) { throw new ConnectorError("media upload needs an openAsset option", PROVIDER); } - const { bytes, filename } = await options.openAsset(media); + const { bytes, filename } = await options.openAsset(connection, media); const body = new FormData(); body.append("file", new Blob([new Uint8Array(bytes)], { type: media.mimeType }), filename); if (description !== undefined) body.append("description", description); @@ -496,7 +496,7 @@ export function createMastodonConnector(options: MastodonConnectorOptions = {}): } const ids: string[] = []; for (const assetId of assetIds) { - const asset = await options.loadAsset(assetId); + const asset = await options.loadAsset(connection, assetId); ids.push((await uploadAsset(connection, asset, str(altText[assetId]))).remoteId); } return ids; From ef8d28bc2624e2125c7df3a4fd94341c39ca6232 Mon Sep 17 00:00:00 2001 From: glass beadian Date: Thu, 15 Oct 2026 19:37:31 +0000 Subject: [PATCH 04/42] fix: declare social.occupymedia.cc as verified Mastodon dedupe origin Media posts are two mutations (media upload + status create). The dispatch gate fences the second mutation unless the origin is deployment-verified via MASTODON_PUBLISH_DEDUPE_ORIGINS, so without it every media post died as PUBLISH_OUTCOME_UNKNOWN after a successful upload. --- infra/occupymedia/docker-compose.app.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infra/occupymedia/docker-compose.app.yml b/infra/occupymedia/docker-compose.app.yml index 9113fc4..34a5117 100644 --- a/infra/occupymedia/docker-compose.app.yml +++ b/infra/occupymedia/docker-compose.app.yml @@ -98,6 +98,10 @@ services: # Publishing workflows and the scheduler worker MUST share one queue, # else approved publishes sit unconsumed. TEMPORAL_TASK_QUEUE: ${TEMPORAL_TASK_QUEUE:-tmbh-publishing} + # Deployment-verified Mastodon Idempotency-Key origins (policy.ts). Media + # posts are two mutations (media upload + status create); the dispatch + # gate allows the second only for origins listed here. + MASTODON_PUBLISH_DEDUPE_ORIGINS: ${MASTODON_PUBLISH_DEDUPE_ORIGINS:-https://social.occupymedia.cc} # Hub-as-OIDC-issuer signing keys (see docs/design/2026-10-13-files-oidc-sso.md). OIDC_KEY_DIR: /var/lib/tmbh/oidc volumes: From 613aba236bc9d27e32a844c9af93640cb38bdffc Mon Sep 17 00:00:00 2001 From: glass beadian Date: Thu, 15 Oct 2026 20:49:46 +0000 Subject: [PATCH 05/42] fix: raise Files group upload cap 20 MiB -> 2 GiB (api + web) The Files tab dropzone still enforced the old 20 MiB cap client- and server-side while every other path had moved to 2 GiB. Also corrects the stale '100 MiB' message on the video upload rejection. --- apps/api/src/group-files/group-files.controller.ts | 4 ++-- apps/api/src/group-files/ocis-webdav.ts | 2 +- apps/api/src/video/video.service.ts | 2 +- apps/web/src/components/files/upload-dropzone.tsx | 13 +++++++------ 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/apps/api/src/group-files/group-files.controller.ts b/apps/api/src/group-files/group-files.controller.ts index 84afe64..8c2233e 100644 --- a/apps/api/src/group-files/group-files.controller.ts +++ b/apps/api/src/group-files/group-files.controller.ts @@ -7,7 +7,7 @@ import { GroupFilesService, type FilePage } from "./group-files.service.js"; import type { GroupFile } from "@tactical-media-broadcast-hub/core"; import { verifyWhileStreaming } from "./ocis-webdav.js"; -const MAX_FILE_UPLOAD_BYTES = 20 * 1024 * 1024; +const MAX_FILE_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024; interface FileUpload { buffer: Buffer; @@ -20,7 +20,7 @@ interface FileUpload { * * GET /files list/search, keyset (groupId, senderMxid, * mimeClass, origin, name, from, to, cursor, limit) - * POST /files/groups/:groupId/upload direct web upload (≤20 MiB, multipart) + * POST /files/groups/:groupId/upload direct web upload (≤2 GiB, multipart) * GET /files/:id one GroupFile * GET /files/:id/download stream payload (sha256 re-verified) * POST /files/:id/retention set retention (keep/until/legalHold) diff --git a/apps/api/src/group-files/ocis-webdav.ts b/apps/api/src/group-files/ocis-webdav.ts index 8db8678..af576c4 100644 --- a/apps/api/src/group-files/ocis-webdav.ts +++ b/apps/api/src/group-files/ocis-webdav.ts @@ -184,7 +184,7 @@ export class OcisWebdavClient { async upload(groupId: string, safeName: string, body: Readable, contentType: string): Promise { await this.ensureGroupFolder(groupId); // reva rejects chunked PUTs, so the source drains into a bounded buffer - // (the upload cap is 20 MiB) and goes out with an explicit content-length. + // (the upload cap is 2 GiB) and goes out with an explicit content-length. const hash = createHash("sha256"); const chunks: Buffer[] = []; let sizeBytes = 0; diff --git a/apps/api/src/video/video.service.ts b/apps/api/src/video/video.service.ts index 72cfd68..5e56a97 100644 --- a/apps/api/src/video/video.service.ts +++ b/apps/api/src/video/video.service.ts @@ -453,7 +453,7 @@ function validateUpload(storageKey: string, mimeType: string, sizeBytes: number) if (!Number.isSafeInteger(sizeBytes) || sizeBytes < 1) { throw new BadRequestException("sizeBytes must be a positive safe integer"); } - if (sizeBytes > MAX_VIDEO_UPLOAD_BYTES) throw new BadRequestException("video exceeds the 100 MiB upload limit"); + if (sizeBytes > MAX_VIDEO_UPLOAD_BYTES) throw new BadRequestException("video exceeds the 2 GiB upload limit"); } function requiredRow(row: unknown, description: string): Record { diff --git a/apps/web/src/components/files/upload-dropzone.tsx b/apps/web/src/components/files/upload-dropzone.tsx index 1150d1b..a5c564c 100644 --- a/apps/web/src/components/files/upload-dropzone.tsx +++ b/apps/web/src/components/files/upload-dropzone.tsx @@ -10,11 +10,12 @@ import type { GroupFile } from "./types"; /** * Drag-drop + picker for POST /files/groups/:groupId/upload. The API accepts a - * single `file` field capped at 20 MiB; the cap is enforced here too so an + * single `file` field capped at 2 GiB; the cap is enforced here too so an * oversized pick fails fast with a friendly message instead of a 413. */ -const MAX_UPLOAD_BYTES = 20 * 1024 * 1024; +const MAX_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024; +const MAX_LABEL = "2 GiB"; export function UploadDropzone({ groupId, live, onUploaded }: { groupId: string; @@ -41,8 +42,8 @@ export function UploadDropzone({ groupId, live, onUploaded }: { if (file.size > MAX_UPLOAD_BYTES) { setError( t({ - en: `${file.name} is ${formatBytes(file.size)} — the limit is 20 MiB.`, - es: `${file.name} pesa ${formatBytes(file.size)} — el límite es 20 MiB.`, + en: `${file.name} is ${formatBytes(file.size)} — the limit is ${MAX_LABEL}.`, + es: `${file.name} pesa ${formatBytes(file.size)} — el límite es ${MAX_LABEL}.`, }), ); return; @@ -73,8 +74,8 @@ export function UploadDropzone({ groupId, live, onUploaded }: { : groupId === "" ? t({ en: "Choose a group first.", es: "Elige un grupo primero." }) : t({ - en: "Drop a file, paste one, or browse. Up to 20 MiB; stored with re-verified bytes.", - es: "Arrastra un archivo, pega uno o búscalo. Hasta 20 MiB; guardado con bytes reverificados.", + en: `Drop a file, paste one, or browse. Up to ${MAX_LABEL}; stored with re-verified bytes.`, + es: `Arrastra un archivo, pega uno o búscalo. Hasta ${MAX_LABEL}; guardado con bytes reverificados.`, }); return ( From 5808e89dded43f9640ac73b8d597195b65307e5e Mon Sep 17 00:00:00 2001 From: glass beadian Date: Thu, 15 Oct 2026 21:27:09 +0000 Subject: [PATCH 06/42] fix: synth silence bed for audio-less produce sources Audio-less inputs no longer emit [0:a] maps that ffmpeg rejects; the worker branches to an anullsrc bed configured at the output preset's rate/layout. Producer tests 42/42. --- .../src/producer/command-builder.test.ts | 25 +++++++++++++ .../src/producer/command-builder.ts | 29 +++++++++++++-- .../src/producer/pg-store.test.ts | 27 ++++++++++++++ apps/media-worker/src/producer/pg-store.ts | 13 +++++-- apps/media-worker/src/producer/runner.test.ts | 37 +++++++++++++++++++ apps/media-worker/src/producer/runner.ts | 7 +++- 6 files changed, 130 insertions(+), 8 deletions(-) diff --git a/apps/media-worker/src/producer/command-builder.test.ts b/apps/media-worker/src/producer/command-builder.test.ts index a091018..c4b25bb 100644 --- a/apps/media-worker/src/producer/command-builder.test.ts +++ b/apps/media-worker/src/producer/command-builder.test.ts @@ -169,6 +169,31 @@ test("a source without audio mixes the voiceover over a synthetic silence bed", assert.ok(g.includes("duration=longest")); }); +test("an audio-less source never references [0:a]: the bed is a valid source-filter chain", () => { + // No voiceover: the synthetic bed alone is the output's audio. + const bare = graph({ input: { path: `${SCRATCH}/${HASH}.mp4`, durationSec: 2, audioBed: "none" } }); + const bareChains = bare.split(";"); + // anullsrc is a source filter, so aformat follows a COMMA — the missing comma + // produced `anullsrc=...aformat=...`, which ffmpeg rejects with "Option + // 'sample_rates' not found". + assert.equal(bareChains.at(-1), `anullsrc=r=${48000}:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[aout]`); + assert.ok(!bare.includes("[0:a]"), "an audio-less source must never be referenced as [0:a]"); + + // Ducking over silence: bedRaw gets the same comma-separated form. + const duck = graph({ input: { path: `${SCRATCH}/${HASH}.mp4`, durationSec: 2, audioBed: "none" }, + voiceover: { path: `${SCRATCH}/${VO_HASH}.m4a`, gainDb: -6, duck: true } }); + const duckChains = duck.split(";"); + const bedRaw = duckChains.find((c) => c.endsWith("[bedRaw]")); + assert.ok(bedRaw !== undefined, "ducking path must still produce [bedRaw]"); + assert.ok(bedRaw.startsWith(`anullsrc=r=${48000}:cl=stereo,aformat=`), "synthetic bed + aformat must be comma-separated"); + assert.ok(duck.includes("duration=longest"), "a synthetic bed mixes to longest, bounded by -t"); + assert.ok(!duck.includes("[0:a]")); + + // An audio-bearing source keeps the input-pad form (no comma after [0:a]). + const withAudio = graph({ input: { path: `${SCRATCH}/${HASH}.mp4`, durationSec: 2, audioBed: "source" } }); + assert.ok(withAudio.includes("[0:a]aformat=sample_fmts=fltp"), "the source bed stays an input pad chain"); +}); + test("a synthetic bed must be bounded by the source duration", () => { assert.throws(() => graph({ input: { path: `${SCRATCH}/${HASH}.mp4`, durationSec: null, audioBed: "none" }, diff --git a/apps/media-worker/src/producer/command-builder.ts b/apps/media-worker/src/producer/command-builder.ts index 81f3a64..60d8682 100644 --- a/apps/media-worker/src/producer/command-builder.ts +++ b/apps/media-worker/src/producer/command-builder.ts @@ -265,12 +265,33 @@ export interface ProducePlan { /** ffmpeg input indexes: source 0, then each logo, then the voiceover — argv order. */ export interface ProduceInputIndex { logos: number[]; voiceover: number | null } +/** `aformat` as a filter step (a real input's chain: comma-separated, after a comma). */ function audioFormat(): string { return `aformat=sample_fmts=fltp:sample_rates=${PRODUCE_PRESET.sampleRate}:channel_layouts=stereo`; } +/** + * The synthetic silence bed for an audio-less source. anullsrc is a source + * filter, so it cannot be followed by `aformat` — it is configured directly with + * the output preset's rate and stereo layout, and the encoder's `-ar`/channel + * handling downstream already guarantees the format. + */ +function silenceBed(): string { + return `anullsrc=r=${PRODUCE_PRESET.sampleRate}:cl=stereo`; +} + function gain(gainDb: number): string { return `volume=${gainDb}dB`; } +/** + * One bed chain: `[0:a]` is an input pad, so `aformat` follows it directly, but + * the synthetic bed is a source filter, so it needs the `,` separator before + * `aformat` (the missing comma is what made the audio-less path emit + * `anullsrc=...aformat=...`, which ffmpeg rejects outright). + */ +function bedChain(bed: string, label: string): string { + return `${bed}${bed.startsWith("[") ? "" : ","}${audioFormat()}[${label}]`; +} + /** * The `-filter_complex` string: video chain (scale → overlays in request order) and * the audio mix. `min(px,W-tw)`-style expressions are single-quoted because `min` @@ -312,10 +333,10 @@ export function buildProduceFilter(plan: ProducePlan, index: ProduceInputIndex): // on the bed. `duration=first` ends with the source bed; a synthetic bed is // unbounded, so it mixes to `longest` and only the source's `-t` can end the encode. const vo = plan.voiceover; - const bed = plan.input.audioBed === "source" ? "[0:a]" : `anullsrc=r=${PRODUCE_PRESET.sampleRate}:cl=stereo`; + const bed = plan.input.audioBed === "source" ? "[0:a]" : silenceBed(); if (vo === null) { - chains.push(`${bed}${audioFormat()}[aout]`); + chains.push(bedChain(bed, "aout")); return chains.join(";"); } const voIndex = index.voiceover ?? fail("voiceover input index required"); @@ -324,13 +345,13 @@ export function buildProduceFilter(plan: ProducePlan, index: ProduceInputIndex): if (vo.duck) { // asplit feeds the same voiceover to the compressor key and to the mix. chains.push(`${voIn}${audioFormat()},asplit=2[voMix][voKey]`); - chains.push(`${bed}${audioFormat()}[bedRaw]`); + chains.push(bedChain(bed, "bedRaw")); chains.push(`[bedRaw][voKey]sidechaincompress=threshold=0.03:ratio=8:attack=20:release=300:makeup=1:link=average[duck]`); if (vo.gainDb !== null && vo.gainDb !== 0) chains.push(`[voMix]${gain(vo.gainDb)}[voGain]`); const mixIn = vo.gainDb !== null && vo.gainDb !== 0 ? "voGain" : "voMix"; chains.push(`[duck][${mixIn}]amix=inputs=2:duration=${amixDuration}:dropout_transition=2:normalize=0[aout]`); } else { - chains.push(`${bed}${audioFormat()}[bedFmt]`); + chains.push(bedChain(bed, "bedFmt")); chains.push(`${voIn}${audioFormat()}${vo.gainDb === null || vo.gainDb === 0 ? "" : `,${gain(vo.gainDb)}`}[voGain]`); chains.push(`[bedFmt][voGain]amix=inputs=2:duration=${amixDuration}:dropout_transition=2:normalize=0[aout]`); } diff --git a/apps/media-worker/src/producer/pg-store.test.ts b/apps/media-worker/src/producer/pg-store.test.ts index c598c21..65ad864 100644 --- a/apps/media-worker/src/producer/pg-store.test.ts +++ b/apps/media-worker/src/producer/pg-store.test.ts @@ -140,6 +140,33 @@ test("claim goes through the definer function with typed lane + lease params", a assert.equal(job!.request.overlays[0]!.kind, "lower_third"); }); +test("a probed audio-less asset reports hasAudio:false so the runner uses a silent bed", async () => { + // The probe writes duration_ms and audio_codec together: audio_codec NULL with + // a duration means probed-and-silent (muted screen recording, GIF conversion). + // Before the fix this mapped to hasAudio:null → audioBed "source" → ffmpeg + // "Stream specifier ':a' matches no streams". + const pool = new FakePool(); + pool.expectClient({ rows: [{ ...assetRow(), audio_codec: null }] }); + pool.reply([claimRow()]); + + const job = await storeOf(pool).claim("local"); + + assert.ok(job !== null); + assert.equal(job.source.hasAudio, false, "probed + audio_codec NULL must be hasAudio:false"); + // The runner turns hasAudio===false into audioBed "none" (synthetic silence). +}); + +test("an unprobed asset (NULL duration AND NULL codec) stays hasAudio:null", async () => { + const pool = new FakePool(); + pool.expectClient({ rows: [{ ...assetRow(), audio_codec: null, duration_ms: null }] }); + pool.reply([claimRow()]); + + const job = await storeOf(pool).claim("local"); + + assert.ok(job !== null); + assert.equal(job.source.hasAudio, null, "unprobed is unknown, not audio-less — the source bed stays"); +}); + test("claim passes the lane through: a remote worker binds lane='remote'", async () => { const { pool } = healthyClaim(); await storeOf(pool).claim("remote"); diff --git a/apps/media-worker/src/producer/pg-store.ts b/apps/media-worker/src/producer/pg-store.ts index 7358ab3..ab414b2 100644 --- a/apps/media-worker/src/producer/pg-store.ts +++ b/apps/media-worker/src/producer/pg-store.ts @@ -47,6 +47,11 @@ function uuid(value: unknown, field: string): string { return text; } +/** True once 009's probe job has run: the probe writes duration + codecs together, so duration≠NULL means probed. */ +function probed(row: Record): boolean { + return row.duration_ms !== null && row.duration_ms !== undefined; +} + function nullableNumber(value: unknown): number | null { if (value === null || value === undefined) return null; const n = typeof value === "number" ? value : Number(value); @@ -63,9 +68,11 @@ function sourceFromRow(row: Record): ProduceSourceAsset { height: nullableNumber(row.height), durationMs: row.duration_ms === null || row.duration_ms === undefined ? null : Math.round(Number(row.duration_ms)), frameRate: nullableNumber(row.frame_rate), - // Probed "no audio track" (009 stores an empty codec) → a silent bed, so the - // output always has an audio stream. - hasAudio: row.audio_codec === null || row.audio_codec === undefined ? null : String(row.audio_codec) !== "", + // An unprobed asset has NULL codec AND NULL duration (009's probe fills all + // of them at once); a probed one always has a duration. So the probed + // audio-less case is duration≠NULL + audio_codec NULL/'' — which must become + // a silent bed, or the graph references [0:a] on a file with no audio track. + hasAudio: probed(row) ? String(row.audio_codec ?? "") !== "" : null, }; } diff --git a/apps/media-worker/src/producer/runner.test.ts b/apps/media-worker/src/producer/runner.test.ts index 112f585..ffbda83 100644 --- a/apps/media-worker/src/producer/runner.test.ts +++ b/apps/media-worker/src/producer/runner.test.ts @@ -43,6 +43,17 @@ function makeClip(path: string): void { function key(bytes: Buffer): string { return `sha256/${createHash("sha256").update(bytes).digest("hex")}.mp4`; } +/** + * The audio-less class from the bug report: video only, no lavfi sine, no audio + * track at all (muted screen recording / GIF conversion / synthetic test clip). + */ +function makeSilentClip(path: string): void { + const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", + "-f", "lavfi", "-i", "testsrc2=s=320x180:r=30:d=2", + "-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", "-movflags", "+faststart", path]); + if (r.status !== 0) throw new Error(`silent fixture encode failed: ${r.stderr?.toString() ?? ""}`); +} + function doubles() { const events: Array<{ type: string; jobId?: string; detail?: string }> = []; const jobs: Array<{ jobId: string; state: string; reason?: string; output?: { storageKey: string; producedAssetId: string } }> = []; @@ -132,6 +143,32 @@ test("compose job: encode, publish, derived asset, complete", { skip: !hasFfmpeg assert.ok(body !== undefined && body.subarray(4, 8).toString("latin1") === "ftyp"); }); +test("an audio-less source produces output WITH audio (synthetic bed), not a [0:a] crash", { skip: !hasFfmpeg() || !hasFont() ? "needs ffmpeg + a ttf font" : false }, async () => { + const d = doubles(); + const clip = join(SCRATCH, "source-silent.mp4"); + makeSilentClip(clip); + const bytes = await import("node:fs/promises").then((fs) => fs.readFile(clip)); + const inputKey = key(bytes); + d.objects.set(inputKey, bytes); + // hasAudio:false is what the pg store reports for a probed audio-less asset. + d.setQueue([job(inputKey, "compose", + { overlays: [{ kind: "lower_third", text: "Ada Lovelace", position: { x: 8, y: 85 } }], lane: "local" }, + { hasAudio: false, durationMs: 2000 })]); + + await runner(d).pollOnce(); + + assert.equal(d.jobs[0]?.state, "complete", d.jobs[0]?.reason); + assert.ok(d.jobs[0]?.output?.storageKey.startsWith("sha256/")); + // The produced mp4 must actually carry an audio stream — the whole point of the bed. + const body = d.objects.get(d.published()!.key); + assert.ok(body !== undefined && body.subarray(4, 8).toString("latin1") === "ftyp"); + const probe = spawnSync("ffprobe", ["-v", "error", "-print_format", "json", "-show_streams", "-i", `pipe:0`], + { input: body, encoding: "buffer" }); + const streams = JSON.parse(probe.stdout.toString("utf8")).streams as Array<{ codec_type: string }>; + assert.ok(streams.some((s) => s.codec_type === "audio"), "output must have an audio stream"); + assert.ok(streams.some((s) => s.codec_type === "video"), "output must keep its video stream"); +}); + test("a mix job with a voiceover runs the ducking path end to end", { skip: !hasFfmpeg() || !hasFont() ? "needs ffmpeg + a ttf font" : false }, async () => { const d = doubles(); const clip = join(SCRATCH, "source-mix.mp4"); diff --git a/apps/media-worker/src/producer/runner.ts b/apps/media-worker/src/producer/runner.ts index 4302f9e..2f0c4ba 100644 --- a/apps/media-worker/src/producer/runner.ts +++ b/apps/media-worker/src/producer/runner.ts @@ -171,7 +171,12 @@ export class ProduceRunner { }; } const durationSec = job.source.durationMs === null ? null : job.source.durationMs / 1000; - // A probed asset without an audio track gets a silent bed, so the output always has audio. + // An audio-less source (muted screen recording, GIF conversion, synthetic + // test clip) must not feed [0:a] into the graph — ffmpeg dies with "Stream + // specifier ':a' matches no streams". hasAudio===false means probed and + // audio-less; null means the probe never ran, where the source bed stays + // (the [0:a] reference then fails loudly rather than silently dropping audio + // from a file that may well have a track). const audioBed = job.source.hasAudio === false ? "none" : "source"; return { input: { path: source, durationSec, audioBed }, From 18afff5e494b83e0e16fe03766aa1f3d461731b3 Mon Sep 17 00:00:00 2001 From: glass beadian Date: Thu, 15 Oct 2026 22:11:07 +0000 Subject: [PATCH 07/42] feat: Files payloads move from oCIS WebDAV to R2 object storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Files tab was the only payload path not on object storage, violating the 2026-10-06 directive (object storage is the ONLY media I/O path) — and oCIS's WebDAV surface is currently 500ing server-side, which made every upload fail. Uploads now seal + dedupe into R2 via StorageService (SPENC1, content- addressed); downloads read R2 when storage_key is set, falling back to oCIS for legacy rows. Share-into-library copies oCIS Drive sources into R2 and records the real sha256 instead of a placeholder. Migration 041 adds chat_media_archive.storage_key. --- .../migrations/041_archive_storage_key.sql | 4 + .../group-files/collective-share.service.ts | 112 +++++++++--------- .../src/group-files/group-files.service.ts | 68 ++++++++--- packages/core/src/shared-files.ts | 2 +- 4 files changed, 114 insertions(+), 72 deletions(-) create mode 100644 apps/api/migrations/041_archive_storage_key.sql diff --git a/apps/api/migrations/041_archive_storage_key.sql b/apps/api/migrations/041_archive_storage_key.sql new file mode 100644 index 0000000..6a3c25d --- /dev/null +++ b/apps/api/migrations/041_archive_storage_key.sql @@ -0,0 +1,4 @@ +-- Files uploads move to R2 object storage (SPENC1-sealed, content-addressed), +-- matching the user directive that object storage is the only media path. +-- Legacy rows keep their oCIS resource ids; storage_key is read first when set. +ALTER TABLE chat_media_archive ADD COLUMN storage_key text; diff --git a/apps/api/src/group-files/collective-share.service.ts b/apps/api/src/group-files/collective-share.service.ts index fc581a4..24a9618 100644 --- a/apps/api/src/group-files/collective-share.service.ts +++ b/apps/api/src/group-files/collective-share.service.ts @@ -31,10 +31,8 @@ import { mimeClassOf, noLibraryTargetMessage, openRoomNoticeContent, - suffixedName, wantsLibrary, wantsOpenRoom, - MAX_COLLISION_ATTEMPTS, type CollectiveRoomPoster, type CollectiveShareInput, type CollectiveShareItem, @@ -42,7 +40,8 @@ import { type CollectiveShareResult, type ShareErrorCode, } from "./collective-share.js"; -import { OcisNotFoundError, OcisPreconditionFailedError, OcisWebdavClient, ocisConfigFromEnv, type FetchLike } from "./ocis-webdav.js"; +import { OcisWebdavClient, ocisConfigFromEnv, type FetchLike } from "./ocis-webdav.js"; +import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js"; interface CollectiveRow { id: string; @@ -72,9 +71,24 @@ export class CollectiveShareService { constructor( @Optional() @Inject(COLLECTIVE_ROOM_POSTER) private readonly poster?: CollectiveRoomPoster, - @Optional() @Inject(COLLECTIVE_SHARE_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike }, + @Optional() @Inject(COLLECTIVE_SHARE_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike; storage?: StorageService }, ) {} + private store: StorageService | null = null; + + /** R2 object storage — the library payload store (041). */ + private storage(): StorageService { + if (this.store === null) { + this.store = this.deps?.storage ?? new S3Storage(s3ConfigFromEnv()); + } + return this.store; + } + + private libraryUrlFor(resourceId: string): string { + const base = (process.env.FILES_PUBLIC_BASE ?? "").replace(/\/+$/, ""); + return base === "" ? resourceId : `${base}/f/${encodeURIComponent(resourceId)}`; + } + /** Lazily bound: the oCIS env may be absent in dev/test contexts. */ private webdav(): OcisWebdavClient { if (this.client === null) { @@ -265,63 +279,50 @@ export class CollectiveShareService { }; } - const baseName = safeFileName(segments[segments.length - 1] ?? "attachment"); - let copied: Awaited> | null = null; - let finalName = baseName; - for (let attempt = 0; attempt <= MAX_COLLISION_ATTEMPTS; attempt += 1) { - const candidate = suffixedName(baseName, attempt); - try { - copied = await this.webdav().copyIntoGroup(collective.id, segments, folder, candidate); - finalName = candidate; - break; - } catch (err) { - if (err instanceof OcisPreconditionFailedError) continue; // try the next suffix - if (err instanceof OcisNotFoundError) { - return { - title: titleFromPath(item.path), - error: { - code: "source-not-found", - message: collective.language === "es" - ? `No se encontró el archivo de origen (${item.path}).` - : `The source file was not found (${item.path}).`, - }, - }; - } - return { - title: titleFromPath(item.path), - error: { - code: "copy-failed", - message: collective.language === "es" - ? "La copia en la biblioteca del colectivo falló." - : "Copying into the collective's library failed.", - }, - }; - } - } - if (copied === null) { + const name = safeFileName(segments[segments.length - 1] ?? "attachment"); + // oCIS stays the Drive source; the library destination is R2 (041). + const source = await this.webdav().download(segments.join("/")).catch(() => null); + if (source === null || source.stream === null) { return { title: titleFromPath(item.path), + error: { + code: "source-not-found", + message: collective.language === "es" + ? `No se encontró el archivo de origen (${item.path}).` + : `The source file was not found (${item.path}).`, + }, + }; + } + const chunks: Buffer[] = []; + for await (const chunk of source.stream as unknown as AsyncIterable) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array)); + } + const bytes = Buffer.concat(chunks); + const stored = await this.storage().putContent(bytes, source.contentType ?? "application/octet-stream").catch(() => null); + if (stored === null) { + return { + title: name, error: { code: "copy-failed", message: collective.language === "es" - ? `Ya existe un archivo llamado ${baseName} (y hasta 100 variantes); renómbralo primero.` - : `A file named ${baseName} already exists (plus 100 variants); rename it first.`, + ? "La copia en la biblioteca del colectivo falló." + : "Copying into the collective's library failed.", }, }; } const indexed = await this.indexCopy(workspaceId, collective, actor, { folder, - name: finalName, - resourceId: copied.resourceId, - version: copied.version, - sizeBytes: copied.sizeBytes, - contentType: copied.contentType, + name, + storageKey: stored.key, + sha256: stored.hash, + sizeBytes: stored.sizeBytes, + contentType: source.contentType ?? "application/octet-stream", sourcePath: item.path, senderMxid, }); - if ("error" in indexed) return { title: finalName, error: indexed.error }; - return { title: finalName, result: indexed.result }; + if ("error" in indexed) return { title: name, error: indexed.error }; + return { title: name, result: indexed.result }; } /** @@ -336,8 +337,8 @@ export class CollectiveShareService { copy: { folder: readonly string[]; name: string; - resourceId: string; - version: string | null; + storageKey: string; + sha256: string; sizeBytes: number; contentType: string; sourcePath: string; @@ -345,7 +346,7 @@ export class CollectiveShareService { }, ): Promise<{ result: { itemId: string; name: string; url: string } } | { error: { code: ShareErrorCode; message: string } }> { const mime = copy.contentType === "" ? "application/octet-stream" : copy.contentType; - const libraryUrl = this.webdav().libraryUrl(copy.resourceId); + const libraryUrl = this.libraryUrlFor(copy.storageKey); const folderPath = copy.folder.length === 0 ? "" : `${copy.folder.join("/")}/`; const origin = { kind: "upload", @@ -357,8 +358,8 @@ export class CollectiveShareService { `INSERT INTO chat_media_archive (workspace_id, group_id, room_id, event_id, origin, sender_mxid, matrix_event_json, original_name, safe_name, mime, mime_class, byte_size, sha256, ocis_resource_id, - ocis_resource_version, created_at, indexed_at) - VALUES ($1,$2,NULL,NULL,'upload',$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,now(),now()) + ocis_resource_version, created_at, indexed_at, storage_key) + VALUES ($1,$2,NULL,NULL,'upload',$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,now(),now(),$13) RETURNING id`, [ workspaceId, @@ -370,9 +371,10 @@ export class CollectiveShareService { mime, mimeClassOf(mime), copy.sizeBytes, - digestPlaceholder(copy.resourceId), - copy.resourceId, - copy.version, + copy.sha256, + copy.storageKey, + null, + copy.storageKey, ], ); return row.rows[0]?.id; diff --git a/apps/api/src/group-files/group-files.service.ts b/apps/api/src/group-files/group-files.service.ts index 186c617..02f38e1 100644 --- a/apps/api/src/group-files/group-files.service.ts +++ b/apps/api/src/group-files/group-files.service.ts @@ -29,6 +29,7 @@ import { withSystem, withTenant } from "../db/client.js"; import { isUuid, parseLimit } from "../http/validation.js"; import { decodeFileListCursor, encodeFileListCursor } from "./file-cursor.js"; import { ocisConfigFromEnv, OcisWebdavClient, type FetchLike, type OcisDownload } from "./ocis-webdav.js"; +import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js"; import { hashShareToken, mintShareToken } from "./share-token.js"; import { MIME_CLASSES, ORIGIN_KINDS, parseRetentionInput, type MimeClass, type OriginKind } from "./group-files.validation.js"; @@ -87,6 +88,7 @@ interface ArchiveRow { sha256: string; ocis_resource_id: string; ocis_resource_version: string | null; + storage_key: string | null; created_at: Date; indexed_at: Date; available: boolean; @@ -169,11 +171,12 @@ function rowToGroupFile(row: ArchiveRow, libraryUrl: (resourceId: string) => str indexedAt: iso(row.indexed_at), file: { spaceId: row.group_id, - resourceId: row.ocis_resource_id, + resourceId: row.storage_key ?? row.ocis_resource_id, version: row.ocis_resource_version ?? row.ocis_resource_id, + ...(row.storage_key === null ? {} : { storageKey: row.storage_key }), }, shareLink: { - url: libraryUrl(row.ocis_resource_id), + url: libraryUrl(row.storage_key ?? row.ocis_resource_id), access: row.share_access === "public" ? "public" : "members", expiresAt: row.share_expires_at === null ? null : iso(row.share_expires_at), }, @@ -185,10 +188,11 @@ function rowToGroupFile(row: ArchiveRow, libraryUrl: (resourceId: string) => str @Injectable() export class GroupFilesService { private client: OcisWebdavClient | null = null; + private store: StorageService | null = null; - constructor(@Optional() @Inject(GROUP_FILES_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike }) {} + constructor(@Optional() @Inject(GROUP_FILES_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike; storage?: StorageService }) {} - /** Lazily bound: the oCIS env may be absent in dev/test contexts. */ + /** Lazily bound: the oCIS env may be absent in dev/test contexts. Legacy rows only. */ private webdav(): OcisWebdavClient { if (this.client === null) { const fetcher: FetchLike = this.deps?.fetcher ?? ((url, init) => fetch(url, init)); @@ -197,6 +201,20 @@ export class GroupFilesService { return this.client; } + /** R2 object storage — the primary payload store (041). */ + private storage(): StorageService { + if (this.store === null) { + this.store = this.deps?.storage ?? new S3Storage(s3ConfigFromEnv()); + } + return this.store; + } + + /** Authenticated library link; never a persisted presigned R2 URL. */ + private libraryUrlFor(resourceId: string): string { + const base = (process.env.FILES_PUBLIC_BASE ?? "").replace(/\/+$/, ""); + return base === "" ? resourceId : `${base}/f/${encodeURIComponent(resourceId)}`; + } + // ------------------------------------------------------------------------- // Read paths // ------------------------------------------------------------------------- @@ -271,7 +289,7 @@ export class GroupFilesService { const result = await client.query(sql, values); const page = result.rows.slice(0, limit); const last = page.at(-1); - const lib = (resourceId: string): string => this.webdav().libraryUrl(resourceId); + const lib = (resourceId: string): string => this.libraryUrlFor(resourceId); return { items: page.map((row) => rowToGroupFile(row, lib)), nextCursor: result.rows.length > limit && last !== undefined @@ -292,7 +310,7 @@ export class GroupFilesService { const row = await this.requireRow(client, itemId); await this.requireAccess(client, userId, role, row.group_id); if (!row.available) throw new GoneException("file is revoked"); - return rowToGroupFile(row, (resourceId) => this.webdav().libraryUrl(resourceId)); + return rowToGroupFile(row, (resourceId) => this.libraryUrlFor(resourceId)); }); } @@ -300,12 +318,22 @@ export class GroupFilesService { async openDownload(workspaceId: string, userId: string, role: string, itemId: string): Promise { const item = await this.getFile(workspaceId, userId, role, itemId); if (item.status !== "available") throw new GoneException(`file is ${item.status}`); - const download = await this.webdav().download(item.file.resourceId).catch(() => { + const download = await this.openPayload(item.file.storageKey ?? null, item.file.resourceId, item.mimeType).catch(() => { throw new ServiceUnavailableException("file storage is unavailable"); }); return { item, download }; } + /** R2 for rows with a storage key (041+); oCIS WebDAV for legacy rows. */ + private async openPayload(storageKey: string | null | undefined, resourceId: string, mime: string): Promise { + const key = storageKey ?? null; + if (key !== null) { + const bytes = await this.storage().getObject(key); + return { stream: Readable.from(bytes), sizeBytes: bytes.byteLength, contentType: mime }; + } + return this.webdav().download(resourceId); + } + // ------------------------------------------------------------------------- // Retention / availability // ------------------------------------------------------------------------- @@ -441,8 +469,8 @@ export class GroupFilesService { if (row.retention_mode === "until" && row.retention_until !== null && row.retention_until.getTime() <= Date.now()) { throw new GoneException("file retention has expired"); } - const item = rowToGroupFile(row, (resourceId) => this.webdav().libraryUrl(resourceId)); - const download = await this.webdav().download(item.file.resourceId).catch(() => { + const item = rowToGroupFile(row, (resourceId) => this.libraryUrlFor(resourceId)); + const download = await this.openPayload(row.storage_key, item.file.resourceId, item.mimeType).catch(() => { throw new ServiceUnavailableException("file storage is unavailable"); }); return { item, download }; @@ -473,13 +501,14 @@ export class GroupFilesService { `INSERT INTO chat_media_archive (workspace_id, group_id, room_id, event_id, origin, sender_mxid, matrix_event_json, original_name, safe_name, mime, mime_class, byte_size, sha256, - ocis_resource_id, ocis_resource_version, created_at) - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) + ocis_resource_id, ocis_resource_version, created_at, storage_key) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17) RETURNING id`, [ workspaceId, entry.groupId, entry.roomId, entry.eventId, entry.originKind, entry.senderMxid, JSON.stringify(entry.origin), entry.originalName, entry.safeName, entry.mime, entry.mimeClass, entry.byteSize, entry.sha256, entry.ocisResourceId, entry.ocisResourceVersion, entry.createdAt, + entry.storageKey ?? null, ], ); } catch (err) { @@ -559,7 +588,12 @@ export class GroupFilesService { } const sniff = sniffMime(file.buffer.subarray(0, SNIFF_BYTES)); const safe = safeFileName(file.originalName, sniff.extension); - const uploaded = await this.webdav().upload(groupId, safe, Readable.from(file.buffer), sniff.mime); + // Object storage is the only payload path (user directive 2026-10-06): + // SPENC1-sealed, content-addressed, deduplicated. ocis_resource_id keeps + // carrying the resource handle (now the R2 key) for legacy readers. + const stored = await this.storage().putContent(file.buffer, sniff.mime).catch(() => { + throw new ServiceUnavailableException("file storage is unavailable"); + }); const username = await this.usernameFor(workspaceId, actor.userId); const { id } = await this.registerArchiveEntry(workspaceId, { groupId, @@ -572,10 +606,11 @@ export class GroupFilesService { safeName: safe, mime: sniff.mime, mimeClass: sniff.mimeClass, - byteSize: uploaded.sizeBytes, - sha256: uploaded.sha256, - ocisResourceId: uploaded.resourceId, - ocisResourceVersion: uploaded.version, + byteSize: stored.sizeBytes, + sha256: stored.hash, + ocisResourceId: stored.key, + ocisResourceVersion: null, + storageKey: stored.key, createdAt: new Date(), }); return this.getFile(workspaceId, actor.userId, actor.role, id); @@ -610,6 +645,7 @@ export interface ArchiveEntryInput { sha256: string; ocisResourceId: string; ocisResourceVersion: string | null; + storageKey?: string | null; createdAt: Date; } diff --git a/packages/core/src/shared-files.ts b/packages/core/src/shared-files.ts index a55f968..7c40c7a 100644 --- a/packages/core/src/shared-files.ts +++ b/packages/core/src/shared-files.ts @@ -43,7 +43,7 @@ export interface GroupFile { checksum: { algorithm: "sha256"; value: string }; createdAt: string; indexedAt: string; - file: { spaceId: string; resourceId: string; version: string }; + file: { spaceId: string; resourceId: string; version: string; storageKey?: string }; /** Authenticated library link by default; never a persisted presigned R2 URL. */ shareLink: { url: string; access: "members" | "public"; expiresAt: string | null }; retention: FileRetention; From 93edeab5ca9d0cc2a13110f488280b277ec93d40 Mon Sep 17 00:00:00 2001 From: glass beadian Date: Thu, 15 Oct 2026 22:28:45 +0000 Subject: [PATCH 08/42] feat: chunked video ingest API (init / chunk / complete) Cloudflare's proxy plan rejects bodies above ~100 MB, so whole-body video uploads cannot exceed it no matter the origin limits. Clients now slice into 10 MB chunks: POST /video/upload/init, PUT /video/upload/:id/chunk/:index, POST /video/upload/:id/complete reassembles to disk, seals, stores to R2 and runs the normal probe + job pipeline. Max chunked upload: 100 GiB. Tests 17/17. --- apps/api/src/app.module.ts | 3 +- apps/api/src/video/chunked-upload.test.ts | 327 ++++++++++++++++++++++ apps/api/src/video/chunked-upload.ts | 299 ++++++++++++++++++++ apps/api/src/video/upload-limits.ts | 39 +++ apps/api/src/video/video.controller.ts | 138 ++++++++- apps/api/src/video/video.service.ts | 42 ++- 6 files changed, 843 insertions(+), 5 deletions(-) create mode 100644 apps/api/src/video/chunked-upload.test.ts create mode 100644 apps/api/src/video/chunked-upload.ts diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts index ee1dd29..985d1ff 100644 --- a/apps/api/src/app.module.ts +++ b/apps/api/src/app.module.ts @@ -20,6 +20,7 @@ import { GroupsController } from "./groups/groups.controller"; import { GroupsService } from "./groups/groups.service"; import { VideoController } from "./video/video.controller"; import { VideoService } from "./video/video.service"; +import { ChunkedUploadService } from "./video/chunked-upload.js"; import { LiveMonitorModule } from "./live-monitor/live-monitor.module"; import { MediaControlModule } from "./media-control/media-control.module"; import { ProductionModule } from "./production/production.module"; @@ -66,6 +67,6 @@ import { MediaModule } from "./media/media.module.js"; { provide: PublishingExecutionRepository, useFactory: () => new PublishingExecutionRepository(pool) }, TemporalWorkflowStarter, { provide: PUBLISHING_DEPENDENCIES, inject: [PublishingExecutionRepository, TemporalWorkflowStarter, ApprovalGateService], useFactory: (repository: PublishingExecutionRepository, workflowStarter: TemporalWorkflowStarter, approvalGate: ApprovalGateService) => ({ repository, workflowStarter, approvalGate, tenantTransaction: withTenant, assertReady: () => assertPublishingSchema(pool) }) }, - PublishingService, ApprovalGateService, AuthService, ConnectionsService, TeamService, InviteLinkService, AuditQueryService, VideoService, GroupsService, SnapshotService, SettingsService, BrandingService], + PublishingService, ApprovalGateService, AuthService, ConnectionsService, TeamService, InviteLinkService, AuditQueryService, VideoService, ChunkedUploadService, GroupsService, SnapshotService, SettingsService, BrandingService], }) export class AppModule {} diff --git a/apps/api/src/video/chunked-upload.test.ts b/apps/api/src/video/chunked-upload.test.ts new file mode 100644 index 0000000..f91a560 --- /dev/null +++ b/apps/api/src/video/chunked-upload.test.ts @@ -0,0 +1,327 @@ +import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + ChunkedUploadService, + chunkOffset, + type ChunkedUploadSession, +} from "./chunked-upload.js"; +import { + CHUNK_SIZE_BYTES, + MAX_CHUNKED_UPLOAD_BYTES, + chunkBytesFor, + totalChunksFor, +} from "./upload-limits.js"; +import { VideoService } from "./video.service.js"; +import { assetId, serviceWithDatabase, workspaceId, type FakeVideoDatabase } from "./test-fixtures.js"; + +const otherWorkspace = "90000000-0000-4000-8000-00000000000f"; +let directory: string; +let service: ChunkedUploadService; +let counter = 0; + +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), "chunked-test-")); + counter += 1; + service = new ChunkedUploadService( + directory, + Number.MAX_SAFE_INTEGER, + () => `00000000-0000-4000-8000-${String(counter).padStart(12, "0")}`, + ); +}); + +afterEach(async () => { + await service.onModuleDestroy(); + await rm(directory, { recursive: true, force: true }); +}); + +function init(totalBytes: number, overrides: Partial<{ filename: string; mimeType: string }> = {}): Promise { + return service.init(workspaceId, { + filename: overrides.filename ?? "raw-capture.mp4", + mimeType: overrides.mimeType ?? "video/mp4", + totalBytes, + }); +} + +/** Deterministic pseudo-random chunk bytes; offset-tagged so mis-writes are visible. */ +function chunkBytes(totalBytes: number, index: number): Buffer { + return Buffer.alloc(chunkBytesFor(totalBytes, index), (index % 251) + 1); +} + +describe("chunk plan math (pure)", () => { + it("splits files into 10 MiB chunks with a short final chunk", () => { + expect(CHUNK_SIZE_BYTES).toBe(10 * 1024 * 1024); + expect(totalChunksFor(CHUNK_SIZE_BYTES)).toBe(1); + expect(totalChunksFor(CHUNK_SIZE_BYTES + 1)).toBe(2); + expect(totalChunksFor(3 * CHUNK_SIZE_BYTES)).toBe(3); + expect(totalChunksFor(3 * CHUNK_SIZE_BYTES + 123)).toBe(4); + // 20 GB target size stays within the 100 GiB ceiling. + expect(totalChunksFor(20 * 1024 ** 3)).toBe(2048); + expect(MAX_CHUNKED_UPLOAD_BYTES).toBe(100 * 1024 ** 3); + }); + + it("computes chunk sizes and byte offsets", () => { + const total = 2 * CHUNK_SIZE_BYTES + 5; + expect(chunkBytesFor(total, 0)).toBe(CHUNK_SIZE_BYTES); + expect(chunkBytesFor(total, 1)).toBe(CHUNK_SIZE_BYTES); + expect(chunkBytesFor(total, 2)).toBe(5); + expect(chunkOffset(0)).toBe(0); + expect(chunkOffset(1)).toBe(CHUNK_SIZE_BYTES); + expect(chunkOffset(2047)).toBe(2047 * CHUNK_SIZE_BYTES); + expect(() => chunkBytesFor(total, 3)).toThrow("outside the 3-chunk upload"); + expect(() => chunkOffset(-1)).toThrow("non-negative"); + }); +}); + +describe("init", () => { + it("creates a 0600 temp file under /tmp/uploads/.tmp and tracks the session", async () => { + const session = await init(2 * CHUNK_SIZE_BYTES); + expect(session.uploadId).toMatch(/^[0-9a-f-]{36}$/); + expect(session.totalChunks).toBe(2); + expect(session.tempPath).toBe(join(directory, `${session.uploadId}.tmp`)); + expect((await stat(session.tempPath)).size).toBe(0); + expect((await stat(session.tempPath)).mode & 0o777).toBe(0o600); + expect(service.activeSessions.get(session.uploadId)).toBe(session); + expect(session.workspaceId).toBe(workspaceId); + }); + + it("rejects non-video MIME types, bad filenames, and bad byte counts", async () => { + await expect(init(1, { mimeType: "image/png" })).rejects.toMatchObject({ status: 400 }); + await expect(init(1, { filename: " " })).rejects.toMatchObject({ status: 400 }); + await expect(init(0)).rejects.toMatchObject({ status: 400, message: expect.stringContaining("positive safe integer") }); + await expect(init(1.5)).rejects.toMatchObject({ status: 400 }); + await expect(init(Number.MAX_SAFE_INTEGER + 1)).rejects.toMatchObject({ status: 400 }); + // Spec ceiling: totalBytes ≤ 100 GiB. + await expect(init(MAX_CHUNKED_UPLOAD_BYTES + 1)).rejects.toMatchObject({ + status: 400, + message: expect.stringContaining("100 GiB"), + }); + }); +}); + +describe("appendChunk — correct offset writes", () => { + it("writes chunks at index × 10 MiB regardless of arrival order (sparse middle-out)", async () => { + const total = 3 * CHUNK_SIZE_BYTES; + await init(total); + const uploadId = firstUploadId(); + // Deliberately out of order: last, first, middle. + const third = chunkBytes(total, 2); + await service.appendChunk(uploadId, workspaceId, 2, third); + await service.appendChunk(uploadId, workspaceId, 0, chunkBytes(total, 0)); + const receipt = await service.appendChunk(uploadId, workspaceId, 1, chunkBytes(total, 1)); + expect(receipt).toEqual({ received: true, bytesSoFar: total }); + const assembled = await readFile(join(directory, `${uploadId}.tmp`)); + expect(assembled.length).toBe(total); + expect(assembled.subarray(0, CHUNK_SIZE_BYTES)).toEqual(chunkBytes(total, 0)); + expect(assembled.subarray(CHUNK_SIZE_BYTES, 2 * CHUNK_SIZE_BYTES)).toEqual(chunkBytes(total, 1)); + expect(assembled.subarray(2 * CHUNK_SIZE_BYTES)).toEqual(third); + }); + + it("fills holes with zero bytes before their chunk arrives, and reports progressive bytesSoFar", async () => { + const total = 3 * CHUNK_SIZE_BYTES; + await init(total); + const uploadId = firstUploadId(); + const afterFirst = await service.appendChunk(uploadId, workspaceId, 0, chunkBytes(total, 0)); + expect(afterFirst.bytesSoFar).toBe(CHUNK_SIZE_BYTES); + const afterLast = await service.appendChunk(uploadId, workspaceId, 2, chunkBytes(total, 2)); + expect(afterLast.bytesSoFar).toBe(2 * CHUNK_SIZE_BYTES); + // Chunk 1 has not arrived: its range must read as zeros, not garbage. + const partial = await readFile(join(directory, `${uploadId}.tmp`)); + expect(partial.subarray(CHUNK_SIZE_BYTES, 2 * CHUNK_SIZE_BYTES)).toEqual(Buffer.alloc(CHUNK_SIZE_BYTES, 0)); + // The file is already full length (sparse). + expect(partial.length).toBe(total); + }); + + it("is idempotent for a retransmitted chunk and rejects wrong-sized bodies", async () => { + const total = 2 * CHUNK_SIZE_BYTES; + await init(total); + const uploadId = firstUploadId(); + await service.appendChunk(uploadId, workspaceId, 0, chunkBytes(total, 0)); + await service.appendChunk(uploadId, workspaceId, 0, chunkBytes(total, 0)); + expect((await service.verifyComplete(uploadId, workspaceId)).bytesSoFar).toBe(CHUNK_SIZE_BYTES); + // Too short. + await expect(service.appendChunk(uploadId, workspaceId, 1, Buffer.alloc(CHUNK_SIZE_BYTES - 1, 7))) + .rejects.toMatchObject({ status: 400, message: expect.stringContaining("exactly") }); + // Too long. + await expect(service.appendChunk(uploadId, workspaceId, 1, Buffer.alloc(CHUNK_SIZE_BYTES + 1, 7))) + .rejects.toMatchObject({ status: 400 }); + // Out of range index. + await expect(service.appendChunk(uploadId, workspaceId, 2, Buffer.alloc(1))) + .rejects.toMatchObject({ status: 400, message: expect.stringContaining("between 0 and 1") }); + await expect(service.appendChunk(uploadId, workspaceId, -1, Buffer.alloc(1))).rejects.toMatchObject({ status: 400 }); + }); + + it("enforces tenant isolation and rejects unknown/finished uploads", async () => { + const session = await init(CHUNK_SIZE_BYTES); + await expect(service.appendChunk(session.uploadId, otherWorkspace, 0, Buffer.alloc(CHUNK_SIZE_BYTES))) + .rejects.toMatchObject({ status: 404 }); + await expect(service.appendChunk("not-a-uuid", workspaceId, 0, Buffer.alloc(1))).rejects.toMatchObject({ status: 400 }); + await expect(service.appendChunk("11111111-1111-4111-8111-111111111111", workspaceId, 0, Buffer.alloc(1))) + .rejects.toMatchObject({ status: 404 }); + await service.appendChunk(session.uploadId, workspaceId, 0, Buffer.alloc(CHUNK_SIZE_BYTES)); + service.complete(session.uploadId); + await expect(service.appendChunk(session.uploadId, workspaceId, 0, Buffer.alloc(CHUNK_SIZE_BYTES))) + .rejects.toMatchObject({ status: 409 }); + expect(service.activeSessions.has(session.uploadId)).toBe(true); + }); +}); + +describe("verifyComplete / assertAssembled — missing chunk + size verification", () => { + it("reports missing chunks and under-length files until every chunk lands", async () => { + const total = 3 * CHUNK_SIZE_BYTES; + await init(total); + const uploadId = firstUploadId(); + const first = await service.verifyComplete(uploadId, workspaceId); + expect(first.missing).toEqual([0, 1, 2]); + expect(first.bytesSoFar).toBe(0); + // Sparse file is already totalBytes long on disk, so size alone would lie — + // the chunk-count check is what catches the gap. + expect(first.fileBytes).toBe(0); // nothing written yet + expect(() => service.assertAssembled(first)).toThrow("missing 3 chunk(s)"); + + await service.appendChunk(uploadId, workspaceId, 1, chunkBytes(total, 1)); + const middle = await service.verifyComplete(uploadId, workspaceId); + expect(middle.missing).toEqual([0, 2]); + expect(() => service.assertAssembled(middle)).toThrow("missing 2 chunk(s): 0, 2"); + + await service.appendChunk(uploadId, workspaceId, 0, chunkBytes(total, 0)); + await service.appendChunk(uploadId, workspaceId, 2, chunkBytes(total, 2)); + const done = await service.verifyComplete(uploadId, workspaceId); + expect(done.missing).toEqual([]); + expect(done.bytesSoFar).toBe(total); + expect(done.fileBytes).toBe(total); + expect(() => service.assertAssembled(done)).not.toThrow(); + }); + + it("detects a declared size that disagrees with the assembled file", async () => { + const total = 2 * CHUNK_SIZE_BYTES; + const session = await init(total); + await service.appendChunk(session.uploadId, workspaceId, 0, chunkBytes(total, 0)); + await service.appendChunk(session.uploadId, workspaceId, 1, chunkBytes(total, 1)); + // Simulate a truncated/oversized file behind the assembler's back. + await writeFile(session.tempPath, Buffer.alloc(total - 1, 9)); + const check = await service.verifyComplete(session.uploadId, workspaceId); + expect(check.missing).toEqual([]); + expect(check.fileBytes).toBe(total - 1); + expect(() => service.assertAssembled(check)).toThrow("bytes on disk"); + expect(() => service.assertAssembled(check)).toThrow(`expected ${total}`); + }); + + it("detects byte-accounting drift when a received set over/under-counts totalBytes", async () => { + const total = 2 * CHUNK_SIZE_BYTES; + const session = await init(total); + session.received.add(0); + session.received.add(1); + session.received.add(1); // Set: no duplicate effect, but simulate accounting drift below. + const check = await service.verifyComplete(session.uploadId, workspaceId); + expect(check.bytesSoFar).toBe(2 * CHUNK_SIZE_BYTES); + // Forcing a mismatched declaration is caught by the service-level check. + const drifted: typeof check = { ...check, session: { ...session, totalBytes: total + 5 } }; + expect(() => service.assertAssembled(drifted)).toThrow(/received \d+ bytes but declared/); + }); +}); + +describe("abort, cleanup, and expiry sweeper", () => { + it("abort removes the temp file and session, and is idempotent per-uploadId", async () => { + const session = await init(CHUNK_SIZE_BYTES); + await service.appendChunk(session.uploadId, workspaceId, 0, Buffer.alloc(CHUNK_SIZE_BYTES)); + await service.abort(session.uploadId, workspaceId); + await expect(stat(session.tempPath)).rejects.toMatchObject({ code: "ENOENT" }); + expect(service.activeSessions.has(session.uploadId)).toBe(false); + await expect(service.abort(session.uploadId, workspaceId)).rejects.toMatchObject({ status: 404 }); + await expect(service.verifyComplete(session.uploadId, workspaceId)).rejects.toMatchObject({ status: 404 }); + }); + + it("cleanup drops the entry after complete; failure paths can still see the file", async () => { + const session = await init(CHUNK_SIZE_BYTES); + await service.appendChunk(session.uploadId, workspaceId, 0, Buffer.alloc(CHUNK_SIZE_BYTES)); + service.complete(session.uploadId); + await service.cleanup(session.uploadId); + await expect(stat(session.tempPath)).rejects.toMatchObject({ code: "ENOENT" }); + expect(service.activeSessions.has(session.uploadId)).toBe(false); + await expect(service.cleanup(session.uploadId)).resolves.toBeUndefined(); + }); + + it("sweeps sessions idle past the 24h TTL along with their temp files", async () => { + const session = await init(CHUNK_SIZE_BYTES); + await service.appendChunk(session.uploadId, workspaceId, 0, Buffer.alloc(CHUNK_SIZE_BYTES)); + const now = Date.now(); + // Fresh session is retained. + expect(await service.sweepExpired(now)).toEqual([]); + // 24h minus a second: still retained. + expect(await service.sweepExpired(now + 24 * 60 * 60 * 1000 - 1000)).toEqual([]); + expect(service.activeSessions.has(session.uploadId)).toBe(true); + // TTL reached: swept. + const swept = await service.sweepExpired(now + 24 * 60 * 60 * 1000 + 1); + expect(swept).toEqual([session.uploadId]); + await expect(stat(session.tempPath)).rejects.toMatchObject({ code: "ENOENT" }); + expect(service.activeSessions.has(session.uploadId)).toBe(false); + }); +}); + +describe("end-to-end assemble → seal → register → probe (VideoService tail end)", () => { + it("uploadVideoFromPath stores, probes, and returns the same VideoUploadResult shape as POST /video/upload", async () => { + const database: FakeVideoDatabase = {}; + const bytes = Buffer.concat([Buffer.from("MP4DATA-"), Buffer.alloc(512, 3)]); + const probe = vi.fn(async (path: string) => { + expect(await readFile(path)).toEqual(bytes); + expect(database.job?.state).toBe("probing"); + return { width: 1920, height: 1080, durationMs: 90_000, frameRate: 24, codec: "hevc" }; + }); + const { service: video, storage, appendAudit } = serviceWithDatabase(database, probe); + + // Assemble through the real chunk pipeline (small single-chunk file). + const session = await service.init(workspaceId, { + filename: "capture.mp4", + mimeType: "video/mp4", + totalBytes: bytes.length, + }); + await service.appendChunk(session.uploadId, workspaceId, 0, bytes); + const check = await service.verifyComplete(session.uploadId, workspaceId); + expect(() => service.assertAssembled(check)).not.toThrow(); + + const uploaded = await video.uploadVideoFromPath(workspaceId, session.tempPath, "video/mp4", bytes.length); + expect(storage.putContent).toHaveBeenCalledWith(bytes, "video/mp4"); + expect(uploaded.videoAsset).toMatchObject({ + storageKey: expect.stringMatching(/^sha256\/[a-f0-9]{64}\.mp4$/), + sizeBytes: bytes.length, + width: 1920, + height: 1080, + durationMs: 90_000, + codec: "hevc", + }); + // Identical shape/outcome to the single-shot path: a completed probe job. + expect(uploaded.processingJob).toMatchObject({ state: "complete", progressPercent: 100, outputKey: uploaded.videoAsset.storageKey }); + expect(database.transitions).toEqual(["queued", "probing", "transcoding", "complete"]); + expect(appendAudit).toHaveBeenCalledWith(workspaceId, expect.objectContaining({ action: "video.uploaded", targetId: assetId })); + // Cleanup is the caller's (controller's) job, verified separately. + expect(probe).toHaveBeenCalledTimes(1); + }); + + it("rejects a declared size that disagrees with the file on disk", async () => { + const { service: video } = serviceWithDatabase({}, async () => ({})); + const path = join(directory, "mismatch.tmp"); + await writeFile(path, Buffer.alloc(16, 1)); + await expect(video.uploadVideoFromPath(workspaceId, path, "video/mp4", 15)) + .rejects.toMatchObject({ status: 400, message: expect.stringContaining("16 bytes on disk but declared 15") }); + await expect(video.uploadVideoFromPath(workspaceId, join(directory, "missing.tmp"), "video/mp4", 16)) + .rejects.toMatchObject({ status: 400, message: expect.stringContaining("missing on disk") }); + }); + + it("refuses non-video MIME types and empty files", async () => { + const { service: video } = serviceWithDatabase({}, async () => ({})); + const path = join(directory, "notvideo.tmp"); + await writeFile(path, Buffer.alloc(4, 1)); + await expect(video.uploadVideoFromPath(workspaceId, path, "image/png", 4)) + .rejects.toMatchObject({ status: 400, message: expect.stringContaining("video media type") }); + await expect(video.uploadVideoFromPath(workspaceId, path, "video/mp4", 0)) + .rejects.toMatchObject({ status: 400, message: expect.stringContaining("positive safe integer") }); + }); +}); + +function firstUploadId(): string { + const ids = [...service.activeSessions.keys()]; + if (ids.length !== 1) throw new Error(`expected exactly one session, found ${ids.length}`); + return ids[0] as string; +} diff --git a/apps/api/src/video/chunked-upload.ts b/apps/api/src/video/chunked-upload.ts new file mode 100644 index 0000000..00bbadc --- /dev/null +++ b/apps/api/src/video/chunked-upload.ts @@ -0,0 +1,299 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, open, rm, stat } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { BadRequestException, ConflictException, Injectable, NotFoundException, OnModuleDestroy } from "@nestjs/common"; +import { + CHUNK_SIZE_BYTES, + CHUNKED_UPLOAD_TTL_MS, + MAX_CHUNKED_UPLOAD_BYTES, + chunkBytesFor, + totalChunksFor, +} from "./upload-limits.js"; + +/** + * Chunked (resumable-style) large-file ingest. + * + * Cloudflare's free plan rejects proxied request bodies above 100 MB, so a 20 GB + * camera original can never arrive in one POST /video/upload. The client instead + * slices the file into CHUNK_SIZE_BYTES (10 MiB) chunks — each far under the + * limit — and PUTs them one at a time. Each chunk is written at its fixed offset + * (`index × 10 MiB`) into a sparse temp file, so chunks may arrive in ANY order + * and may be retransmitted after a network failure without corrupting the file. + * + * On complete, the assembled file goes through the SAME SPENC1 seal → S3 + * putContent path as every other upload. + * + * ── MEMORY REQUIREMENT (read this) ──────────────────────────────────────────── + * The seal function (apps/api/src/storage/storage.ts) takes a whole Buffer, and + * S3 PutObject is likewise handed one body. Assembling therefore peaks at + * roughly 2× file size of RAM: one Buffer of plaintext for the seal plus the + * sealed copy the S3 client transmits (plus ~1× transiently for + * Buffer.concat in seal). There is no way to bound that without streaming + * encryption + multipart S3, which would change the storage contract for every + * other caller — out of scope here. + * + * Practical sizing: a 20 GB upload needs a host with ≥ 48 GB RAM + * (MAX_OLD_SPACE_SIZE raised, or --max_old-space-size=32768+, plus headroom for + * the sealed copy). Up to ~2 GiB the ordinary multipart path is fine and cheaper. + * Disk also matters: /tmp must hold the assembled file (see SP_CHUNK_UPLOAD_DIR, + * default /tmp/uploads) on top of whatever S3 buffers. + * ───────────────────────────────────────────────────────────────────────────── + */ + +export interface ChunkedUploadSession { + uploadId: string; + /** Owning workspace; chunks and complete are rejected for other tenants. */ + workspaceId: string; + filename: string; + mimeType: string; + totalBytes: number; + totalChunks: number; + /** Indices of chunks whose bytes are durably on disk. */ + received: Set; + createdAt: number; + updatedAt: number; + /** Set once /complete has been accepted; the entry is dropped afterwards. */ + completed?: boolean; + tempPath: string; +} + +export interface InitChunkedUploadRequest { + filename: string; + mimeType: string; + totalBytes: number; +} + +export interface ChunkReceipt { + received: true; + bytesSoFar: number; +} + +export interface CompletionCheck { + session: ChunkedUploadSession; + /** Missing chunk indices, ascending — empty when the upload is complete. */ + missing: number[]; + /** Bytes durably written so far (sum of received chunk sizes). */ + bytesSoFar: number; + /** Size of the assembled file on disk (sparse holes read as zero bytes). */ + fileBytes: number; +} + +/** A pure function that makes the on-disk layout testable without touching a disk. */ +export function chunkOffset(index: number, chunkSize: number = CHUNK_SIZE_BYTES): number { + if (!Number.isSafeInteger(index) || index < 0) throw new Error("index must be a non-negative integer"); + return index * chunkSize; +} + +/** Max chunk byte length a client may PUT for the given session. */ +export function maxChunkBytesFor(totalBytes: number, index: number, chunkSize: number = CHUNK_SIZE_BYTES): number { + return chunkBytesFor(totalBytes, index, chunkSize); +} + +const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +@Injectable() +export class ChunkedUploadService implements OnModuleDestroy { + /** uploadId → session. Module-level by design: one assembler per API process. */ + private readonly sessions = new Map(); + private sweeper: NodeJS.Timeout | undefined; + + constructor( + private readonly directory: string = process.env.SP_CHUNK_UPLOAD_DIR ?? join(tmpdir(), "uploads"), + private readonly sweeperIntervalMs: number = 30 * 60 * 1000, + private readonly generateId: () => string = randomUUID, + ) { + // Only arm the sweeper when running under a live event loop (not unit tests + // that construct the service directly and would hold the process open). + if (process.env.VITEST === undefined) { + this.sweeper = setInterval(() => void this.sweepExpired(), this.sweeperIntervalMs); + this.sweeper.unref?.(); + } + } + + /** Test seam: the live sessions map. */ + get activeSessions(): Map { + return this.sessions; + } + + async init(workspaceId: string, request: InitChunkedUploadRequest): Promise { + const filename = request.filename.trim(); + if (filename === "" || filename.length > 512) { + throw new BadRequestException("filename must be a non-empty string of at most 512 characters"); + } + const mimeType = request.mimeType.trim(); + if (!mimeType.toLowerCase().startsWith("video/")) { + throw new BadRequestException("mimeType must be a video media type"); + } + if (!Number.isSafeInteger(request.totalBytes) || request.totalBytes < 1) { + throw new BadRequestException("totalBytes must be a positive safe integer"); + } + if (request.totalBytes > MAX_CHUNKED_UPLOAD_BYTES) { + throw new BadRequestException("totalBytes exceeds the 100 GiB chunked upload limit"); + } + const totalChunks = totalChunksFor(request.totalBytes); + const uploadId = this.generateId(); + await mkdir(this.directory, { recursive: true }); + const tempPath = join(this.directory, `${uploadId}.tmp`); + // Create the file eagerly (and exclusively per uploadId) so a later chunk + // never races a first-time create, and so the path is known to be writable. + const handle = await open(tempPath, "wx", 0o600).catch((error: NodeJS.ErrnoException) => { + if (error.code === "EEXIST") throw new ConflictException("temp upload path already exists; retry init"); + throw error; + }); + await handle.close(); + const now = Date.now(); + const session: ChunkedUploadSession = { + uploadId, + workspaceId, + filename, + mimeType, + totalBytes: request.totalBytes, + totalChunks, + received: new Set(), + createdAt: now, + updatedAt: now, + tempPath, + }; + this.sessions.set(uploadId, session); + return session; + } + + async appendChunk(uploadId: string, workspaceId: string, index: number, bytes: Buffer): Promise { + const session = this.requireSession(uploadId, workspaceId); + if (!Number.isSafeInteger(index) || index < 0 || index >= session.totalChunks) { + throw new BadRequestException(`index must be an integer between 0 and ${session.totalChunks - 1}`); + } + const expected = chunkBytesFor(session.totalBytes, index); + if (bytes.length !== expected) { + throw new BadRequestException( + `chunk ${index} must be exactly ${expected} bytes (got ${bytes.length}); the final chunk carries the remainder`, + ); + } + // Positional write at index × 10 MiB: arrival order does not matter and a + // retransmitted chunk is idempotent (same bytes, same offset). + const handle = await open(session.tempPath, "r+").catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") { + throw new NotFoundException(`upload ${uploadId} temp file is gone; start a new upload`); + } + throw error; + }); + try { + const { bytesWritten } = await handle.write(bytes, 0, bytes.length, chunkOffset(index)); + if (bytesWritten !== expected) { + throw new Error(`short write for chunk ${index}: ${bytesWritten} of ${expected} bytes`); + } + // Durability before acknowledgement: a chunk reported received must + // actually be on disk if the API restarts mid-upload. + await handle.datasync(); + } finally { + await handle.close(); + } + session.received.add(index); + session.updatedAt = Date.now(); + return { received: true, bytesSoFar: this.bytesSoFar(session) }; + } + + /** + * Verifies every chunk landed before ingest: exact chunk-count check against + * the session, byte accounting against the assembled file on disk, and — the + * authoritative check — that the file size equals the declared totalBytes. + */ + async verifyComplete(uploadId: string, workspaceId: string): Promise { + const session = this.requireSession(uploadId, workspaceId); + const missing: number[] = []; + for (let index = 0; index < session.totalChunks; index += 1) { + if (!session.received.has(index)) missing.push(index); + } + const fileBytes = await stat(session.tempPath).then((info) => info.size).catch(() => -1); + return { session, missing, bytesSoFar: this.bytesSoFar(session), fileBytes }; + } + + /** Size verification for tests / callers: the file must be exactly totalBytes. */ + assertAssembled(check: CompletionCheck): void { + if (check.missing.length > 0) { + throw new BadRequestException( + `upload ${check.session.uploadId} is missing ${check.missing.length} chunk(s): ${describeIndices(check.missing)}`, + ); + } + if (check.bytesSoFar !== check.session.totalBytes) { + throw new BadRequestException( + `upload ${check.session.uploadId} received ${check.bytesSoFar} bytes but declared ${check.session.totalBytes}`, + ); + } + if (check.fileBytes !== check.session.totalBytes) { + throw new BadRequestException( + `assembled file for ${check.session.uploadId} is ${check.fileBytes} bytes on disk; expected ${check.session.totalBytes}`, + ); + } + } + + /** Marks the session consumed so no further chunk can be appended. */ + complete(uploadId: string): void { + const session = this.sessions.get(uploadId); + if (session === undefined) throw new NotFoundException(`upload ${uploadId} not found`); + session.completed = true; + } + + /** Removes the temp file and the session entry. Idempotent (abort + sweep). */ + async abort(uploadId: string, workspaceId: string): Promise { + const session = this.requireSession(uploadId, workspaceId); + this.sessions.delete(uploadId); + await rm(session.tempPath, { force: true }); + } + + /** Drops the temp file and entry after complete; keeps failures visible for retry. */ + async cleanup(uploadId: string): Promise { + const session = this.sessions.get(uploadId); + if (session === undefined) return; + this.sessions.delete(uploadId); + await rm(session.tempPath, { force: true }); + } + + /** Removes sessions idle past the TTL along with their temp files. */ + async sweepExpired(now: number = Date.now()): Promise { + const expired: string[] = []; + for (const [uploadId, session] of this.sessions) { + if (now - session.updatedAt <= CHUNKED_UPLOAD_TTL_MS) continue; + expired.push(uploadId); + this.sessions.delete(uploadId); + await rm(session.tempPath, { force: true }); + } + return expired; + } + + async onModuleDestroy(): Promise { + if (this.sweeper !== undefined) clearInterval(this.sweeper); + // In-flight sessions are resumable in principle, but the process is going + // away: drop the entries so nothing can half-append later. + for (const [uploadId, session] of this.sessions) { + this.sessions.delete(uploadId); + await rm(session.tempPath, { force: true }); + } + } + + private requireSession(uploadId: string, workspaceId: string): ChunkedUploadSession { + if (!UUID_PATTERN.test(uploadId)) throw new BadRequestException("uploadId must be a UUID"); + const session = this.sessions.get(uploadId); + if (session === undefined) throw new NotFoundException(`upload ${uploadId} not found or expired`); + if (session.workspaceId !== workspaceId) { + // Do not leak existence across tenants. + throw new NotFoundException(`upload ${uploadId} not found or expired`); + } + if (session.completed === true) { + throw new ConflictException(`upload ${uploadId} has already been completed`); + } + return session; + } + + private bytesSoFar(session: ChunkedUploadSession): number { + let total = 0; + for (const index of session.received) total += chunkBytesFor(session.totalBytes, index); + return total; + } +} + +function describeIndices(indices: number[]): string { + const shown = indices.slice(0, 8).map(String).join(", "); + return indices.length > 8 ? `${shown}, … (+${indices.length - 8} more)` : shown; +} diff --git a/apps/api/src/video/upload-limits.ts b/apps/api/src/video/upload-limits.ts index 0cfa028..c0207a6 100644 --- a/apps/api/src/video/upload-limits.ts +++ b/apps/api/src/video/upload-limits.ts @@ -1,2 +1,41 @@ // StorageService seals whole Buffers; bound multipart memory use per request. export const MAX_VIDEO_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024; + +/** + * Chunked ingest: POST /video/upload/init → PUT /video/upload/:uploadId/chunk/:index + * → POST /video/upload/:uploadId/complete (see chunked-upload.ts). + * + * Cloudflare's free plan rejects proxied request bodies above 100 MB, so the + * client slices any file into CHUNK_SIZE_BYTES chunks — each comfortably under + * the limit — and the server reassembles them on disk before the normal + * SPENC1 seal → S3 putContent path. This removes the practical size limit: + * only MAX_CHUNKED_UPLOAD_BYTES (and server RAM, documented in chunked-upload.ts) + * bounds an upload. + */ +export const CHUNK_SIZE_BYTES = 10 * 1024 * 1024; + +/** Hard ceiling for a single chunked upload (spec: validate totalBytes ≤ 100 GiB). */ +export const MAX_CHUNKED_UPLOAD_BYTES = 100 * 1024 * 1024 * 1024; + +/** An upload session that receives nothing for this long is swept (file + entry). */ +export const CHUNKED_UPLOAD_TTL_MS = 24 * 60 * 60 * 1000; + +/** Number of 10 MiB chunks a client must send for a file of totalBytes. */ +export function totalChunksFor(totalBytes: number, chunkSize: number = CHUNK_SIZE_BYTES): number { + if (!Number.isSafeInteger(totalBytes) || totalBytes < 1) { + throw new Error("totalBytes must be a positive safe integer"); + } + return Math.ceil(totalBytes / chunkSize); +} + +/** + * Exact byte length of chunk `index`: every chunk is a full `chunkSize` except + * the last, which carries the remainder. Enforced on the server so the + * assembled file can never be short or over-long. + */ +export function chunkBytesFor(totalBytes: number, index: number, chunkSize: number = CHUNK_SIZE_BYTES): number { + if (!Number.isSafeInteger(index) || index < 0) throw new Error("index must be a non-negative integer"); + const totalChunks = totalChunksFor(totalBytes, chunkSize); + if (index >= totalChunks) throw new Error(`index ${index} is outside the ${totalChunks}-chunk upload`); + return Math.min(chunkSize, totalBytes - index * chunkSize); +} diff --git a/apps/api/src/video/video.controller.ts b/apps/api/src/video/video.controller.ts index 5fd8233..01468d2 100644 --- a/apps/api/src/video/video.controller.ts +++ b/apps/api/src/video/video.controller.ts @@ -6,7 +6,9 @@ import { Get, Inject, Param, + ParseIntPipe, Post, + Put, Query, Req, UseGuards, @@ -25,7 +27,17 @@ import { type VideoUploadResult, } from "./video.service.js"; import { parseProduceRequest } from "./produce-request.js"; -import { MAX_VIDEO_UPLOAD_BYTES } from "./upload-limits.js"; +import { + CHUNK_SIZE_BYTES, + MAX_CHUNKED_UPLOAD_BYTES, + MAX_VIDEO_UPLOAD_BYTES, + totalChunksFor, +} from "./upload-limits.js"; +import { + ChunkedUploadService, + type ChunkReceipt, + type InitChunkedUploadRequest, +} from "./chunked-upload.js"; import { loadDriveBytes, loadMatrixBytes, parseMxcUri } from "./import-sources.js"; interface VideoFile { @@ -42,7 +54,80 @@ interface UploadVideoDto { @Controller("video") @UseGuards(RbacGuard) export class VideoController { - constructor(@Inject(VideoService) private readonly video: VideoService) {} + constructor( + @Inject(VideoService) private readonly video: VideoService, + private readonly chunked: ChunkedUploadService, + ) {} + + /** + * Chunked ingest for files that cannot clear Cloudflare's 100 MB proxied-body + * cap (or the 2 GiB whole-body limit). The client slices the file into 10 MiB + * chunks and PUTs them sequentially; the server reassembles on disk, then runs + * the SAME SPENC1 seal → S3 putContent path as POST /video/upload. + * + * MEMORY: reading the assembled file for the seal peaks around 2× file size of + * RAM (see chunked-upload.ts) — a 20 GB upload needs a ≥ 48 GB host. + */ + @Post("upload/init") + @RequirePermission("media.manage") + async initChunkedUpload( + @Req() request: Request & AuthenticatedRequest, + @Body() body: unknown, + ): Promise<{ uploadId: string; chunkSizeBytes: number; totalChunks: number }> { + const session = requireSession(request); + const dto = parseInitUpload(body); + const created = await this.chunked.init(session.workspaceId, dto); + return { + uploadId: created.uploadId, + chunkSizeBytes: CHUNK_SIZE_BYTES, + totalChunks: created.totalChunks, + }; + } + + @Put("upload/:uploadId/chunk/:index") + @RequirePermission("media.manage") + async uploadChunk( + @Req() request: Request & AuthenticatedRequest, + @Param("uploadId") uploadId: string, + @Param("index", ParseIntPipe) index: number, + ): Promise { + const session = requireSession(request); + const bytes = await readRawBody(request); + return this.chunked.appendChunk(uploadId, session.workspaceId, index, bytes); + } + + @Post("upload/:uploadId/complete") + @RequirePermission("media.manage") + async completeChunkedUpload( + @Req() request: Request & AuthenticatedRequest, + @Param("uploadId") uploadId: string, + ): Promise { + const session = requireSession(request); + const check = await this.chunked.verifyComplete(uploadId, session.workspaceId); + this.chunked.assertAssembled(check); + this.chunked.complete(uploadId); + try { + return await this.video.uploadVideoFromPath( + session.workspaceId, + check.session.tempPath, + check.session.mimeType, + check.session.totalBytes, + ); + } finally { + // Never leave an assembled multi-GB temp file behind — success or failure. + await this.chunked.cleanup(uploadId); + } + } + + @Delete("upload/:uploadId") + @RequirePermission("media.manage") + async abortChunkedUpload( + @Req() request: Request & AuthenticatedRequest, + @Param("uploadId") uploadId: string, + ): Promise { + const session = requireSession(request); + await this.chunked.abort(uploadId, session.workspaceId); + } @Post("upload") @RequirePermission("media.manage") @@ -138,3 +223,52 @@ function parseUploadVideo(value: unknown): UploadVideoDto { sizeBytes, }; } + +/** + * Raw octet-stream body reader for chunk PUTs. Express leaves a non-multipart + * body unread until something consumes it; read exactly Content-Length bytes so + * a truncated client upload is rejected instead of silently padding the file. + */ +async function readRawBody(request: Request): Promise { + const declared = Number(request.headers["content-length"] ?? NaN); + const declaredBytes = Number.isSafeInteger(declared) ? declared : undefined; + const chunks: Buffer[] = []; + let received = 0; + for await (const part of request) { + const chunk = Buffer.isBuffer(part) ? part : Buffer.from(String(part)); + received += chunk.length; + if (declaredBytes !== undefined && received > declaredBytes) { + throw new BadRequestException("request body exceeded its Content-Length"); + } + if (received > CHUNK_SIZE_BYTES) { + throw new BadRequestException(`chunk exceeds the ${CHUNK_SIZE_BYTES} byte chunk size`); + } + chunks.push(chunk); + } + const body = Buffer.concat(chunks); + if (declaredBytes !== undefined && body.length !== declaredBytes) { + throw new BadRequestException("request body was shorter than its Content-Length"); + } + return body; +} + +function parseInitUpload(value: unknown): InitChunkedUploadRequest { + const body = requireObject(value, "body"); + const totalBytes = body.totalBytes; + if (typeof totalBytes !== "number" || !Number.isSafeInteger(totalBytes) || totalBytes < 1) { + throw new BadRequestException("totalBytes must be a positive safe integer"); + } + if (totalBytes > MAX_CHUNKED_UPLOAD_BYTES) { + throw new BadRequestException("totalBytes exceeds the 100 GiB chunked upload limit"); + } + return { + filename: requireString(body.filename, "filename"), + mimeType: requireString(body.mimeType, "mimeType"), + totalBytes, + }; +} + +/** Client-facing chunk math (kept server-side so a stale client cannot lie). */ +export function chunkPlan(totalBytes: number): { chunkSizeBytes: number; totalChunks: number } { + return { chunkSizeBytes: CHUNK_SIZE_BYTES, totalChunks: totalChunksFor(totalBytes) }; +} diff --git a/apps/api/src/video/video.service.ts b/apps/api/src/video/video.service.ts index 5e56a97..251c09b 100644 --- a/apps/api/src/video/video.service.ts +++ b/apps/api/src/video/video.service.ts @@ -1,4 +1,4 @@ -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { BadRequestException, Inject, Injectable, NotFoundException, Optional, ServiceUnavailableException } from "@nestjs/common"; @@ -8,7 +8,7 @@ import { withTenant } from "../db/client.js"; import { probeVideoFile, type VideoProbeResult } from "./ffprobe.js"; import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js"; import { type ProduceRequest } from "./produce-request.js"; -import { MAX_VIDEO_UPLOAD_BYTES } from "./upload-limits.js"; +import { MAX_CHUNKED_UPLOAD_BYTES, MAX_VIDEO_UPLOAD_BYTES } from "./upload-limits.js"; const NO_OP_TRANSCODE_NOTE = "Transcode skipped: original stored object retained; rendering belongs to the media-worker compositor."; @@ -139,6 +139,44 @@ export class VideoService { return result; } + /** + * Chunked-ingest tail end: seal + store + probe an already-assembled file on + * local disk (see chunked-upload.ts). The bytes never sat in a request body, + * so the 2 GiB whole-body cap does not apply — MAX_CHUNKED_UPLOAD_BYTES does. + * + * MEMORY: reading the assembled file into one Buffer peaks around 2× file + * size of RAM (seal takes a Buffer and S3 takes the sealed copy). A 20 GB + * upload therefore needs a host with ≥ 48 GB RAM; /tmp must also hold the + * file itself. + */ + async uploadVideoFromPath( + workspaceId: string, + inputPath: string, + mimeType: string, + sizeBytes: number, + ): Promise { + validateUpload("pending", mimeType, sizeBytes); + if (sizeBytes > MAX_CHUNKED_UPLOAD_BYTES) { + throw new BadRequestException("video exceeds the 100 GiB chunked upload limit"); + } + const onDisk = await stat(inputPath).catch(() => { + throw new BadRequestException("assembled upload file is missing on disk"); + }); + if (onDisk.size !== sizeBytes) { + throw new BadRequestException( + `assembled upload is ${onDisk.size} bytes on disk but declared ${sizeBytes}`, + ); + } + const storage = this.requireStorage(); + const bytes = await readFile(inputPath).catch(() => { + throw new ServiceUnavailableException("could not read the assembled upload from disk"); + }); + const stored = await storage.putContent(bytes, mimeType).catch(() => { + throw new ServiceUnavailableException("Video storage unavailable: could not store encrypted upload in S3/MinIO."); + }); + return this.registerAndProbe(workspaceId, stored.key, stored.contentType, stored.sizeBytes); + } + async uploadVideo( workspaceId: string, storageKey: string, From 4cbb63920ac9cc052ebddd49ce7b19b11091d35d Mon Sep 17 00:00:00 2001 From: glass beadian Date: Thu, 15 Oct 2026 22:48:40 +0000 Subject: [PATCH 09/42] fix: mark ChunkedUploadService constructor params @Optional Nest tried to resolve String/Number/Function as DI tokens and the api crashed at bootstrap. --- apps/api/src/video/chunked-upload.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/api/src/video/chunked-upload.ts b/apps/api/src/video/chunked-upload.ts index 00bbadc..69fdcfe 100644 --- a/apps/api/src/video/chunked-upload.ts +++ b/apps/api/src/video/chunked-upload.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { mkdir, open, rm, stat } from "node:fs/promises"; import { join } from "node:path"; import { tmpdir } from "node:os"; -import { BadRequestException, ConflictException, Injectable, NotFoundException, OnModuleDestroy } from "@nestjs/common"; +import { BadRequestException, ConflictException, Injectable, NotFoundException, OnModuleDestroy, Optional } from "@nestjs/common"; import { CHUNK_SIZE_BYTES, CHUNKED_UPLOAD_TTL_MS, @@ -99,9 +99,9 @@ export class ChunkedUploadService implements OnModuleDestroy { private sweeper: NodeJS.Timeout | undefined; constructor( - private readonly directory: string = process.env.SP_CHUNK_UPLOAD_DIR ?? join(tmpdir(), "uploads"), - private readonly sweeperIntervalMs: number = 30 * 60 * 1000, - private readonly generateId: () => string = randomUUID, + @Optional() private readonly directory: string = process.env.SP_CHUNK_UPLOAD_DIR ?? join(tmpdir(), "uploads"), + @Optional() private readonly sweeperIntervalMs: number = 30 * 60 * 1000, + @Optional() private readonly generateId: () => string = randomUUID, ) { // Only arm the sweeper when running under a live event loop (not unit tests // that construct the service directly and would hold the process open). From e7bf870de954d2a579a8a68defcd2fa65e97fd9b Mon Sep 17 00:00:00 2001 From: glass beadian Date: Fri, 16 Oct 2026 01:32:56 +0000 Subject: [PATCH 10/42] =?UTF-8?q?feat:=20retire=20oCIS=20=E2=80=94=20the?= =?UTF-8?q?=20hub=20runs=20entirely=20on=20R2=20object=20storage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Files payloads moved to R2 in 041; this removes the remaining oCIS coupling: - delete the oCIS WebDAV client; verifyWhileStreaming moves to its own storage-agnostic module (verify-stream.ts) - group-files: storage_key is the only payload source; legacy rows answer 'retired storage' instead of touching oCIS - collective share: the ocis-resource item kind is gone (peertube links stay); share-into-library was R2-backed since the previous commit - import-drive endpoint removed (no web caller); import-chat (homeserver) stays - web: Drive tab + embedded oCIS page removed; Library is the Files view Legacy e2e rows (2) deleted; every remaining archive row is R2-backed. --- .../group-files/collective-share.service.ts | 213 +----------- apps/api/src/group-files/collective-share.ts | 25 +- .../src/group-files/group-files.controller.ts | 2 +- .../group-files/group-files.service.test.ts | 20 +- .../src/group-files/group-files.service.ts | 40 +-- apps/api/src/group-files/group-files.test.ts | 56 +-- apps/api/src/group-files/ocis-webdav.ts | 323 ------------------ apps/api/src/group-files/verify-stream.ts | 44 +++ apps/api/src/video/import-sources.ts | 14 - apps/api/src/video/video.controller.ts | 12 +- apps/web/src/app/(main)/files/drive/page.tsx | 79 ----- apps/web/src/app/(main)/files/layout.tsx | 2 +- apps/web/src/components/files/files-tabs.tsx | 8 +- 13 files changed, 88 insertions(+), 750 deletions(-) delete mode 100644 apps/api/src/group-files/ocis-webdav.ts create mode 100644 apps/api/src/group-files/verify-stream.ts delete mode 100644 apps/web/src/app/(main)/files/drive/page.tsx diff --git a/apps/api/src/group-files/collective-share.service.ts b/apps/api/src/group-files/collective-share.service.ts index 24a9618..657af3c 100644 --- a/apps/api/src/group-files/collective-share.service.ts +++ b/apps/api/src/group-files/collective-share.service.ts @@ -40,8 +40,6 @@ import { type CollectiveShareResult, type ShareErrorCode, } from "./collective-share.js"; -import { OcisWebdavClient, ocisConfigFromEnv, type FetchLike } from "./ocis-webdav.js"; -import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js"; interface CollectiveRow { id: string; @@ -67,36 +65,10 @@ export const COLLECTIVE_SHARE_DEPENDENCIES: unique symbol = Symbol("COLLECTIVE_S @Injectable() export class CollectiveShareService { - private client: OcisWebdavClient | null = null; - constructor( @Optional() @Inject(COLLECTIVE_ROOM_POSTER) private readonly poster?: CollectiveRoomPoster, - @Optional() @Inject(COLLECTIVE_SHARE_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike; storage?: StorageService }, ) {} - private store: StorageService | null = null; - - /** R2 object storage — the library payload store (041). */ - private storage(): StorageService { - if (this.store === null) { - this.store = this.deps?.storage ?? new S3Storage(s3ConfigFromEnv()); - } - return this.store; - } - - private libraryUrlFor(resourceId: string): string { - const base = (process.env.FILES_PUBLIC_BASE ?? "").replace(/\/+$/, ""); - return base === "" ? resourceId : `${base}/f/${encodeURIComponent(resourceId)}`; - } - - /** Lazily bound: the oCIS env may be absent in dev/test contexts. */ - private webdav(): OcisWebdavClient { - if (this.client === null) { - const fetcher: FetchLike = this.deps?.fetcher ?? ((url, init) => fetch(url, init)); - this.client = new OcisWebdavClient(ocisConfigFromEnv(), fetcher); - } - return this.client; - } private roomPoster(): CollectiveRoomPoster { if (this.poster === undefined) { @@ -167,10 +139,9 @@ export class CollectiveShareService { ); } - const senderMxid = `@${actor.username}:tmbh.local`; const results: CollectiveShareItemResult[] = []; for (const [index, item] of input.items.entries()) { - results.push(await this.shareOne(workspaceId, collective, actor, input, item, index, senderMxid, doLibrary, doRoom)); + results.push(await this.shareOne(workspaceId, collective, actor, input, item, index, doLibrary, doRoom)); } await appendTenantAudit(workspaceId, { @@ -199,12 +170,11 @@ export class CollectiveShareService { input: CollectiveShareInput, item: CollectiveShareItem, index: number, - senderMxid: string, doLibrary: boolean, doRoom: boolean, ): Promise { - if (item.kind === "peertube-video" && doLibrary && !doRoom) { - // A video has no library bytes to copy; the target combination is invalid for it. + if (doLibrary && !doRoom) { + // A video link has no library bytes to copy; the target combination is invalid for it. return { index, kind: item.kind, @@ -215,182 +185,21 @@ export class CollectiveShareService { }; } - let library: CollectiveShareItemResult["library"] = null; - if (doLibrary && item.kind === "ocis-resource") { - const copied = await this.copyIntoLibrary(workspaceId, collective, actor, item, input.folder, senderMxid); - if ("error" in copied) { - return { index, kind: item.kind, title: copied.title, library: null, openRoom: null, error: copied.error }; - } - library = copied.result; - } - let openRoom: CollectiveShareItemResult["openRoom"] = null; let failure: { code: ShareErrorCode; message: string } | null = null; if (doRoom) { - const title = item.kind === "peertube-video" ? item.title : library?.name ?? titleFromPath(item.path); - const url = item.kind === "peertube-video" - ? item.url - : library?.url ?? libraryUrlFor(webdavSafePath(input.folder, item.path)); - const posted = await this.postToOpenRoom(collective, { title, url, note: input.note, linkKind: item.kind === "peertube-video" ? "video" : "library" }); + const posted = await this.postToOpenRoom(collective, { title: item.title, url: item.url, note: input.note, linkKind: "video" }); if ("error" in posted) failure = posted.error; else openRoom = posted.result; } - if (failure !== null && library === null) { - return { index, kind: item.kind, title: item.kind === "peertube-video" ? item.title : titleFromPath(item.path), library: null, openRoom: null, error: failure }; + if (failure !== null) { + return { index, kind: item.kind, title: item.title, library: null, openRoom: null, error: failure }; } - return { - index, - kind: item.kind, - title: item.kind === "peertube-video" ? item.title : library?.name ?? titleFromPath(item.path), - library, - openRoom, - error: failure, - }; + return { index, kind: item.kind, title: item.title, library: null, openRoom, error: failure }; } - /** - * COPY + index for one oCIS resource. Collision naming (stem-1, stem-2 …) - * is applied here; the caller's chosen destination folder is honored - * relative to the collective's group root. - */ - private async copyIntoLibrary( - workspaceId: string, - collective: ResolvedCollective, - actor: { userId: string; role: string }, - item: { kind: "ocis-resource"; path: string }, - folder: readonly string[], - senderMxid: string, - ): Promise< - | { title: string; result: { itemId: string; name: string; url: string } } - | { title: string; error: { code: ShareErrorCode; message: string } } - > { - const segments = davSegments(item.path, "path"); - const stat = await this.webdav().statPath(segments).catch(() => null); - if (stat === null || !stat.exists) { - return { - title: titleFromPath(item.path), - error: { - code: "source-not-found", - message: collective.language === "es" - ? `No se encontró el archivo de origen (${item.path}).` - : `The source file was not found (${item.path}).`, - }, - }; - } - const name = safeFileName(segments[segments.length - 1] ?? "attachment"); - // oCIS stays the Drive source; the library destination is R2 (041). - const source = await this.webdav().download(segments.join("/")).catch(() => null); - if (source === null || source.stream === null) { - return { - title: titleFromPath(item.path), - error: { - code: "source-not-found", - message: collective.language === "es" - ? `No se encontró el archivo de origen (${item.path}).` - : `The source file was not found (${item.path}).`, - }, - }; - } - const chunks: Buffer[] = []; - for await (const chunk of source.stream as unknown as AsyncIterable) { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array)); - } - const bytes = Buffer.concat(chunks); - const stored = await this.storage().putContent(bytes, source.contentType ?? "application/octet-stream").catch(() => null); - if (stored === null) { - return { - title: name, - error: { - code: "copy-failed", - message: collective.language === "es" - ? "La copia en la biblioteca del colectivo falló." - : "Copying into the collective's library failed.", - }, - }; - } - - const indexed = await this.indexCopy(workspaceId, collective, actor, { - folder, - name, - storageKey: stored.key, - sha256: stored.hash, - sizeBytes: stored.sizeBytes, - contentType: source.contentType ?? "application/octet-stream", - sourcePath: item.path, - senderMxid, - }); - if ("error" in indexed) return { title: name, error: indexed.error }; - return { title: name, result: indexed.result }; - } - - /** - * Index the copy in chat_media_archive (origin 'upload' — the caller put it - * there themselves) and mint the members library link row in - * group_file_shares so the collective's files index lists it. - */ - private async indexCopy( - workspaceId: string, - collective: ResolvedCollective, - actor: { userId: string; role: string }, - copy: { - folder: readonly string[]; - name: string; - storageKey: string; - sha256: string; - sizeBytes: number; - contentType: string; - sourcePath: string; - senderMxid: string; - }, - ): Promise<{ result: { itemId: string; name: string; url: string } } | { error: { code: ShareErrorCode; message: string } }> { - const mime = copy.contentType === "" ? "application/octet-stream" : copy.contentType; - const libraryUrl = this.libraryUrlFor(copy.storageKey); - const folderPath = copy.folder.length === 0 ? "" : `${copy.folder.join("/")}/`; - const origin = { - kind: "upload", - sharedFrom: copy.sourcePath, - sharedInto: `hub-groups/${collective.id}/${folderPath}${copy.name}`, - }; - const inserted = await withTenant(workspaceId, async (client: PoolClient) => { - const row = await client.query<{ id: string }>( - `INSERT INTO chat_media_archive - (workspace_id, group_id, room_id, event_id, origin, sender_mxid, matrix_event_json, - original_name, safe_name, mime, mime_class, byte_size, sha256, ocis_resource_id, - ocis_resource_version, created_at, indexed_at, storage_key) - VALUES ($1,$2,NULL,NULL,'upload',$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,now(),now(),$13) - RETURNING id`, - [ - workspaceId, - collective.id, - copy.senderMxid, - JSON.stringify(origin), - copy.name, - copy.name, - mime, - mimeClassOf(mime), - copy.sizeBytes, - copy.sha256, - copy.storageKey, - null, - copy.storageKey, - ], - ); - return row.rows[0]?.id; - }).catch(() => undefined); - if (inserted === undefined) { - return { - error: { - code: "index-failed", - message: collective.language === "es" - ? "La copia se realizó pero no se pudo registrar en el índice." - : "The copy succeeded but indexing it failed.", - }, - }; - } - return { result: { itemId: inserted, name: copy.name, url: libraryUrl } }; - } /** * JOIN-if-not-in-room, then one m.text notice. A closed room is refused @@ -470,12 +279,4 @@ function digestPlaceholder(resourceId: string): string { return `copied:${resourceId}`; } -function webdavSafePath(folder: readonly string[], path: string): string { - const segments = davSegments(path, "path"); - return folder.length === 0 ? segments.join("/") : `${folder.join("/")}/${segments.join("/")}`; -} -function libraryUrlFor(path: string): string { - const base = process.env.FILES_PUBLIC_BASE ?? ""; - return base === "" ? path : `${base.replace(/\/+$/, "")}/f/${encodeURIComponent(path)}`; -} diff --git a/apps/api/src/group-files/collective-share.ts b/apps/api/src/group-files/collective-share.ts index ec70dbd..13cf246 100644 --- a/apps/api/src/group-files/collective-share.ts +++ b/apps/api/src/group-files/collective-share.ts @@ -31,26 +31,13 @@ import type { MimeClass as ValidationMimeClass } from "./group-files.validation. export const SHARE_TARGETS = ["library", "open-room", "both"] as const; export type ShareTarget = (typeof SHARE_TARGETS)[number]; -export const SHARE_ITEM_KINDS = ["ocis-resource", "peertube-video"] as const; +export const SHARE_ITEM_KINDS = ["peertube-video"] as const; export type ShareItemKind = (typeof SHARE_ITEM_KINDS)[number]; /** Per-request caps; a share is a person acting on a handful of items, not a bulk ingest. */ export const MAX_SHARE_ITEMS = 20; export const MAX_NOTE_LENGTH = 500; export const MAX_TITLE_LENGTH = 200; -/** Collision suffix ceiling: stem-1 … stem-100, then a per-item error. */ -export const MAX_COLLISION_ATTEMPTS = 100; - -export interface OcisResourceItem { - readonly kind: "ocis-resource"; - /** - * dav path RELATIVE to /remote.php/dav/files// — the hub's - * own conventions: `hub-resources/` (a row of the group files - * index) or a group folder path `hub-groups//`. - */ - readonly path: string; -} - export interface PeertubeVideoItem { readonly kind: "peertube-video"; /** Canonical video URL (…/w/:id, /videos/watch/:id or /videos/embed/:id). */ @@ -58,7 +45,7 @@ export interface PeertubeVideoItem { readonly title: string; } -export type CollectiveShareItem = OcisResourceItem | PeertubeVideoItem; +export type CollectiveShareItem = PeertubeVideoItem; export interface CollectiveShareInput { readonly target: ShareTarget; @@ -148,14 +135,6 @@ function parseItem(value: unknown, index: number): CollectiveShareItem { } const record = value as Record; const kind = record.kind; - if (kind === "ocis-resource") { - if (typeof record.path !== "string" || record.path.trim() === "") { - throw new BadRequestException(`items[${index}].path must be a non-empty string`); - } - // Segment shape is validated here so a traversal attempt never reaches the service. - davSegments(record.path, `items[${index}].path`); - return { kind, path: record.path.trim() }; - } if (kind === "peertube-video") { const title = boundedText(record.title, `items[${index}].title`, MAX_TITLE_LENGTH); if (title === null) throw new BadRequestException(`items[${index}].title must be a non-empty string`); diff --git a/apps/api/src/group-files/group-files.controller.ts b/apps/api/src/group-files/group-files.controller.ts index 8c2233e..49fe0d7 100644 --- a/apps/api/src/group-files/group-files.controller.ts +++ b/apps/api/src/group-files/group-files.controller.ts @@ -5,7 +5,7 @@ import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac import { requireObject, requirePositiveInt, requireSession, requireString } from "../http/validation.js"; import { GroupFilesService, type FilePage } from "./group-files.service.js"; import type { GroupFile } from "@tactical-media-broadcast-hub/core"; -import { verifyWhileStreaming } from "./ocis-webdav.js"; +import { verifyWhileStreaming } from "./verify-stream.js"; const MAX_FILE_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024; diff --git a/apps/api/src/group-files/group-files.service.test.ts b/apps/api/src/group-files/group-files.service.test.ts index e30eae5..86bb36d 100644 --- a/apps/api/src/group-files/group-files.service.test.ts +++ b/apps/api/src/group-files/group-files.service.test.ts @@ -322,12 +322,20 @@ describe("GroupFilesService invariants", () => { const { createHash } = await import("node:crypto"); const digest = createHash("sha256").update(payload).digest("hex"); const { Readable } = await import("node:stream"); - const fetcher = (async () => - new Response(Readable.toWeb(Readable.from([payload])) as unknown as ReadableStream, { - status: 200, - headers: { "content-type": "application/octet-stream", "content-length": String(payload.byteLength) }, - })) as unknown as typeof fetch; - const service = new GroupFilesService({ fetcher }); + // The R2-backed payload path: the storage stub serves the exact bytes. + const storage = { + putContent: async () => { throw new Error("not used in this test"); }, + getObject: async () => payload, + hasContent: async () => true, + presign: async () => "", + }; + behavior = (sql) => { + if (sql.includes("share_for_public_claim")) { + return { rows: [{ id: "s1", workspace_id: row.workspace_id, item_id: row.id, access: "public", expires_at: new Date(Date.now() + 3600_000), revoked_at: null }], rowCount: 1 }; + } + return { rows: [{ ...row, sha256: digestOf("payload-bytes"), storage_key: "sha256/testing" }], rowCount: 1 }; + }; + const service = new GroupFilesService({ storage: storage as never }); const handle = await service.claimPublicDownload("some-token"); expect(handle.item.id).toBe(row.id); expect(handle.item.checksum.value).toBe(digest); diff --git a/apps/api/src/group-files/group-files.service.ts b/apps/api/src/group-files/group-files.service.ts index 02f38e1..123fdf5 100644 --- a/apps/api/src/group-files/group-files.service.ts +++ b/apps/api/src/group-files/group-files.service.ts @@ -28,8 +28,8 @@ import { appendTenantAudit, type AuditEvent } from "../audit/audit.service.js"; import { withSystem, withTenant } from "../db/client.js"; import { isUuid, parseLimit } from "../http/validation.js"; import { decodeFileListCursor, encodeFileListCursor } from "./file-cursor.js"; -import { ocisConfigFromEnv, OcisWebdavClient, type FetchLike, type OcisDownload } from "./ocis-webdav.js"; import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js"; +import type { PayloadDownload } from "./verify-stream.js"; import { hashShareToken, mintShareToken } from "./share-token.js"; import { MIME_CLASSES, ORIGIN_KINDS, parseRetentionInput, type MimeClass, type OriginKind } from "./group-files.validation.js"; @@ -38,7 +38,7 @@ export const GROUP_FILES_DEPENDENCIES: unique symbol = Symbol("GROUP_FILES_DEPEN /** Wraps the raw payload handle the controller streams out. */ export interface FileDownloadHandle { item: GroupFile; - download: OcisDownload; + download: PayloadDownload; } export interface ShareCreated { @@ -187,21 +187,11 @@ function rowToGroupFile(row: ArchiveRow, libraryUrl: (resourceId: string) => str @Injectable() export class GroupFilesService { - private client: OcisWebdavClient | null = null; private store: StorageService | null = null; - constructor(@Optional() @Inject(GROUP_FILES_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike; storage?: StorageService }) {} + constructor(@Optional() @Inject(GROUP_FILES_DEPENDENCIES) private readonly deps?: { storage?: StorageService }) {} - /** Lazily bound: the oCIS env may be absent in dev/test contexts. Legacy rows only. */ - private webdav(): OcisWebdavClient { - if (this.client === null) { - const fetcher: FetchLike = this.deps?.fetcher ?? ((url, init) => fetch(url, init)); - this.client = new OcisWebdavClient(ocisConfigFromEnv(), fetcher); - } - return this.client; - } - - /** R2 object storage — the primary payload store (041). */ + /** R2 object storage — the only payload store (041; oCIS retired). */ private storage(): StorageService { if (this.store === null) { this.store = this.deps?.storage ?? new S3Storage(s3ConfigFromEnv()); @@ -318,20 +308,18 @@ export class GroupFilesService { async openDownload(workspaceId: string, userId: string, role: string, itemId: string): Promise { const item = await this.getFile(workspaceId, userId, role, itemId); if (item.status !== "available") throw new GoneException(`file is ${item.status}`); - const download = await this.openPayload(item.file.storageKey ?? null, item.file.resourceId, item.mimeType).catch(() => { + const download = await this.openPayload(item.file.storageKey, item.mimeType).catch(() => { throw new ServiceUnavailableException("file storage is unavailable"); }); return { item, download }; } - /** R2 for rows with a storage key (041+); oCIS WebDAV for legacy rows. */ - private async openPayload(storageKey: string | null | undefined, resourceId: string, mime: string): Promise { - const key = storageKey ?? null; - if (key !== null) { - const bytes = await this.storage().getObject(key); - return { stream: Readable.from(bytes), sizeBytes: bytes.byteLength, contentType: mime }; + private async openPayload(storageKey: string | null | undefined, mime: string): Promise { + if (storageKey === null || storageKey === undefined) { + throw new ServiceUnavailableException("file was stored on retired storage and is no longer served"); } - return this.webdav().download(resourceId); + const bytes = await this.storage().getObject(storageKey); + return { stream: Readable.from(bytes), sizeBytes: bytes.byteLength, contentType: mime }; } // ------------------------------------------------------------------------- @@ -358,7 +346,7 @@ export class GroupFilesService { }, }); const after = await this.requireRow(client, itemId); - return rowToGroupFile(after, (resourceId) => this.webdav().libraryUrl(resourceId)); + return rowToGroupFile(after, (resourceId) => this.libraryUrlFor(resourceId)); }); } @@ -375,7 +363,7 @@ export class GroupFilesService { diff: { name: before.safe_name, roomId: before.room_id, eventId: before.event_id }, }); const after = await this.requireRow(client, itemId); - return rowToGroupFile(after, (resourceId) => this.webdav().libraryUrl(resourceId)); + return rowToGroupFile(after, (resourceId) => this.libraryUrlFor(resourceId)); }); } @@ -470,7 +458,7 @@ export class GroupFilesService { throw new GoneException("file retention has expired"); } const item = rowToGroupFile(row, (resourceId) => this.libraryUrlFor(resourceId)); - const download = await this.openPayload(row.storage_key, item.file.resourceId, item.mimeType).catch(() => { + const download = await this.openPayload(row.storage_key, item.mimeType).catch(() => { throw new ServiceUnavailableException("file storage is unavailable"); }); return { item, download }; @@ -590,7 +578,7 @@ export class GroupFilesService { const safe = safeFileName(file.originalName, sniff.extension); // Object storage is the only payload path (user directive 2026-10-06): // SPENC1-sealed, content-addressed, deduplicated. ocis_resource_id keeps - // carrying the resource handle (now the R2 key) for legacy readers. + // carrying the resource handle (the R2 key) for legacy readers. const stored = await this.storage().putContent(file.buffer, sniff.mime).catch(() => { throw new ServiceUnavailableException("file storage is unavailable"); }); diff --git a/apps/api/src/group-files/group-files.test.ts b/apps/api/src/group-files/group-files.test.ts index 8fe713b..8d05a78 100644 --- a/apps/api/src/group-files/group-files.test.ts +++ b/apps/api/src/group-files/group-files.test.ts @@ -16,7 +16,7 @@ import { requireRoomId, requireSha256, } from "./group-files.validation.js"; -import { OcisVerificationError, ocisConfigFromEnv, OcisWebdavClient, verifyWhileStreaming } from "./ocis-webdav.js"; +import { OcisVerificationError, verifyWhileStreaming } from "./verify-stream.js"; import { hashShareToken, isPlausibleShareToken, mintShareToken, sameToken } from "./share-token.js"; /** Deterministic 64-hex helper: repeat a pattern to exactly 64 chars. */ @@ -185,59 +185,7 @@ describe("share tokens", () => { }); }); -describe("ocis-webdav config + streaming verification", () => { - it("reads env lazily and throws 503 when unconfigured", () => { - expect(() => ocisConfigFromEnv({})).toThrow(ServiceUnavailableException); - expect(() => ocisConfigFromEnv({ FILES_OCID_WEBDAV_URL: "https://files.example.org" })).toThrow(ServiceUnavailableException); - const config = ocisConfigFromEnv({ - FILES_OCID_WEBDAV_URL: "https://files.example.org/", - FILES_OCID_USER: "svc", - FILES_OCID_PASSWORD: "pw", - FILES_PUBLIC_BASE: "https://files.example.org/", - }); - expect(config.webdavUrl).toBe("https://files.example.org"); - expect(config.publicBase).toBe("https://files.example.org"); - }); - - it("uploads hash the actual streamed bytes and surface oc-fileid/oc-etag", async () => { - const bytes = Buffer.from("group file payload"); - const calls: string[] = []; - const fetcher = (async (url: string, init: RequestInit) => { - calls.push(`${init.method} ${url}`); - return new Response(null, { - status: 201, - headers: init.method === "MKCOL" ? new Headers() : { "oc-fileid": "res-1", "oc-etag": "v-9" }, - }); - }) as unknown as typeof fetch; - const client = new OcisWebdavClient( - { webdavUrl: "https://files.example.org", user: "svc", password: "pw", publicBase: "https://files.example.org" }, - fetcher, - ); - const result = await client.upload("group-1", "clip.mp4", Readable.from([bytes]), "video/mp4"); - const { createHash } = await import("node:crypto"); - expect(result.sha256).toBe(createHash("sha256").update(bytes).digest("hex")); - expect(result.sizeBytes).toBe(bytes.byteLength); - expect(result.resourceId).toBe("res-1"); - expect(result.version).toBe("v-9"); - expect(calls[0]).toBe("MKCOL https://files.example.org/remote.php/dav/files/svc/hub-groups"); - expect(calls[1]).toBe("MKCOL https://files.example.org/remote.php/dav/files/svc/hub-groups/group-1"); - expect(calls[2]).toBe("PUT https://files.example.org/remote.php/dav/files/svc/hub-groups/group-1/clip.mp4"); - expect(client.uploadUrl("g", "n.mp4")).toContain("/remote.php/dav/files/svc/hub-groups/g/n.mp4"); - expect(client.libraryUrl("res-1")).toBe("https://files.example.org/f/res-1"); - }); - - it("fails the upload when the group folder cannot be created", async () => { - const fetcher = (async (url: string, init: RequestInit) => - new Response(null, { status: init.method === "MKCOL" ? 500 : 201 })) as unknown as typeof fetch; - const client = new OcisWebdavClient( - { webdavUrl: "https://files.example.org", user: "svc", password: "pw", publicBase: "https://files.example.org" }, - fetcher, - ); - await expect( - client.upload("group-1", "clip.mp4", Readable.from(["x"]), "text/plain"), - ).rejects.toThrow("oCIS folder creation failed with 500"); - }); - +describe("verify-stream (storage-agnostic streaming verification)", () => { it("verifyWhileStreaming passes on match and aborts on tamper", async () => { const { createHash } = await import("node:crypto"); const payload = Buffer.from("0123456789"); diff --git a/apps/api/src/group-files/ocis-webdav.ts b/apps/api/src/group-files/ocis-webdav.ts deleted file mode 100644 index af576c4..0000000 --- a/apps/api/src/group-files/ocis-webdav.ts +++ /dev/null @@ -1,323 +0,0 @@ -/** - * oCIS WebDAV port — the ONLY payload I/O path for group files. - * - * The API streams bytes through itself on both upload and download: it NEVER - * mints or persists a presigned object URL (docs/occupymedia-storage.md - * "Persist library links/share IDs and expiry, not presigned object URLs"). - * Downloads re-verify the SHA-256 while streaming so storage-level tampering - * is detected instead of silently relayed. - * - * Server-side COPY ("share into a collective's library") moves no bytes - * through the API at all: oCIS performs the copy in the storage backend and - * the API only learns the resulting file id / etag / length, which is exactly - * the metadata the group files index stores. The copy is verified by oCIS - * itself; the index records what the server committed, never a claimed length. - * - * ENV CONTRACT (lazy getters — read per call so tests/parallel deployments - * can reconfigure without a rebuild; nothing is read at import time): - * - * FILES_OCID_WEBDAV_URL oCIS WebDAV root, e.g. https://files.example.org - * FILES_OCID_USER service account username (the scoped oCIS identity) - * FILES_OCID_PASSWORD service account password - * FILES_PUBLIC_BASE public base of the file library, used to build the - * persistent (authenticated) library link returned in - * GroupFile.shareLink.url — a stable resource URL, - * NEVER a presigned object URL. - * - * The oCIS deployment lands in parallel; when the env is unset the port - * reports unconfigured and routes fail 503 rather than silently no-op. - * - * WebDAV shape: PUT returns the stable resource id (oc-fileid) and ETag - * (oc-etag / etag) this index needs; plain PUT/GET carry the bytes. The - * transport is injectable (OCIS_FETCH token) so tests stub it without HTTP. - */ -import { ServiceUnavailableException } from "@nestjs/common"; -import { createHash } from "node:crypto"; -import { Readable } from "node:stream"; - -export interface OcisEnvConfig { - webdavUrl: string; - user: string; - password: string; - publicBase: string; -} - -export function ocisConfigFromEnv(env: Readonly> = process.env): OcisEnvConfig { - const webdavUrl = env.FILES_OCID_WEBDAV_URL; - const user = env.FILES_OCID_USER; - const password = env.FILES_OCID_PASSWORD; - if (webdavUrl === undefined || webdavUrl === "") throw new ServiceUnavailableException("FILES_OCID_WEBDAV_URL is not configured"); - if (user === undefined || user === "") throw new ServiceUnavailableException("FILES_OCID_USER is not configured"); - if (password === undefined || password === "") throw new ServiceUnavailableException("FILES_OCID_PASSWORD is not configured"); - return { - webdavUrl: webdavUrl.replace(/\/+$/, ""), - user, - password, - publicBase: (env.FILES_PUBLIC_BASE ?? "").replace(/\/+$/, ""), - }; -} - -/** Result of a committed upload: stable identity + digest over bytes sent. */ -export interface OcisUploadResult { - resourceId: string; - version: string | null; - sha256: string; - sizeBytes: number; -} - -/** An opened download: verifyWhileStreaming checks the digest before relay. */ -export interface OcisDownload { - stream: Readable; - sizeBytes: number | null; - contentType: string | null; -} - -/** A server-side COPY result: identity + size, with the digest oCIS reports. */ -export interface OcisCopyResult { - resourceId: string; - version: string | null; - sizeBytes: number; - sha256: string | null; - contentType: string; -} - -export class OcisVerificationError extends Error { - constructor(expected: string, actual: string) { - super(`oCIS payload verification failed: expected sha256 ${expected}, got ${actual}`); - this.name = "OcisVerificationError"; - } -} - -/** Minimal fetch surface so tests can stub transport without HTTP. */ -export type FetchLike = (url: string, init: RequestInit) => Promise; - -export const OCIS_FETCH: unique symbol = Symbol("OCIS_FETCH"); - -/** Bytes allowed in flight to the transport before the pump pauses. */ -export const UPLOAD_BUFFER_CAP = 8 * 1024 * 1024; - -/** Node's undici fetch needs this hint when a stream body is supplied. */ -type FetchInitWithDuplex = RequestInit & { duplex?: "half" }; - -/** Distinguishing marker for "the source path does not exist (404)". */ -export class OcisNotFoundError extends Error { - constructor(message: string) { - super(message); - this.name = "OcisNotFoundError"; - } -} - -/** Distinguishing marker for "destination exists and Overwrite:F was sent (412)". */ -export class OcisPreconditionFailedError extends Error { - constructor(message: string) { - super(message); - this.name = "OcisPreconditionFailedError"; - } -} - -function encodeSegments(segments: readonly string[]): string { - return segments.map((segment) => encodeURIComponent(segment)).join("/"); -} - -/** - * WebDAV client. `fetcher` defaults to the global fetch. - */ -export class OcisWebdavClient { - constructor( - private readonly config: OcisEnvConfig, - private readonly fetcher: FetchLike = (url, init) => fetch(url, init), - ) {} - - /** Group folder path under the service identity's WebDAV root. */ - uploadUrl(groupId: string, safeName: string): string { - return `${this.config.webdavUrl}/remote.php/dav/files/${encodeURIComponent(this.config.user)}/hub-groups/${encodeURIComponent(groupId)}/${encodeURIComponent(safeName)}`; - } - - downloadUrl(resourceId: string): string { - return `${this.config.webdavUrl}/remote.php/dav/files/${encodeURIComponent(this.config.user)}/hub-resources/${encodeURIComponent(resourceId)}`; - } - - /** Any path, as sanitized segments, under the service identity's WebDAV root. */ - private rootUrl(segments: readonly string[]): string { - return `${this.config.webdavUrl}/remote.php/dav/files/${encodeURIComponent(this.config.user)}/${encodeSegments(segments)}`; - } - - /** MKCOL the group folder; 405 (already exists) is fine. */ - private async ensureGroupFolder(groupId: string): Promise { - const base = `${this.config.webdavUrl}/remote.php/dav/files/${encodeURIComponent(this.config.user)}`; - // MKCOL returns 409 when the parent collection is missing — create the - // hub-groups root (405 = already exists), then the per-group folder. - const root = await this.fetcher(`${base}/hub-groups`, { method: "MKCOL", headers: this.authHeaders() }); - if (!root.ok && root.status !== 405) { - throw new Error(`oCIS folder creation failed with ${root.status}`); - } - const url = `${base}/hub-groups/${encodeURIComponent(groupId)}`; - const response = await this.fetcher(url, { method: "MKCOL", headers: this.authHeaders() }); - if (response.ok || response.status === 405) return; - throw new Error(`oCIS folder creation failed with ${response.status}`); - } - - /** MKCOL a folder path (segments already sanitized); 405 = already there. */ - async ensureFolderPath(segments: readonly string[]): Promise { - if (segments.length === 0) return; - const response = await this.fetcher(this.rootUrl(segments), { method: "MKCOL", headers: this.authHeaders() }); - if (response.ok || response.status === 405) return; - throw new Error(`oCIS folder creation failed with ${response.status}`); - } - - private authHeaders(): Record { - const basic = Buffer.from(`${this.config.user}:${this.config.password}`, "utf8").toString("base64"); - return { authorization: `Basic ${basic}` }; - } - - /** - * Stream-to-storage PUT, hashing while streaming (contract: "Hash actual - * bytes while streaming"). The digest covers the exact bytes handed to the - * store — never a claimed Content-Length. - * - * Backpressure: the pump pauses when more than UPLOAD_BUFFER_CAP bytes are - * queued for the transport, so RAM stays bounded for large payloads. The - * pump is driven by our own loop, so the digest completes even if the - * transport returns without draining a small body (a 2xx store that never - * reads its request body is broken and will surface as an integrity error). - */ - async upload(groupId: string, safeName: string, body: Readable, contentType: string): Promise { - await this.ensureGroupFolder(groupId); - // reva rejects chunked PUTs, so the source drains into a bounded buffer - // (the upload cap is 2 GiB) and goes out with an explicit content-length. - const hash = createHash("sha256"); - const chunks: Buffer[] = []; - let sizeBytes = 0; - for await (const chunk of body) { - const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array | string); - hash.update(buffer); - sizeBytes += buffer.byteLength; - chunks.push(buffer); - } - const payload = Buffer.concat(chunks); - const response = await this.fetcher(this.uploadUrl(groupId, safeName), { - method: "PUT", - headers: { ...this.authHeaders(), "content-type": contentType, "content-length": String(payload.byteLength) }, - body: payload, - }); - if (!response.ok) throw new Error(`oCIS upload failed with ${response.status}`); - const etag = response.headers.get("oc-etag") ?? response.headers.get("etag"); - const resourceId = response.headers.get("oc-fileid"); - return { - resourceId: resourceId ?? safeName, - version: etag, - sha256: hash.digest("hex"), - sizeBytes, - }; - } - - /** Open a GET. The digest is re-checked by verifyWhileStreaming downstream. */ - async download(resourceId: string): Promise { - const response = await this.fetcher(this.downloadUrl(resourceId), { method: "GET", headers: this.authHeaders() }); - if (!response.ok || response.body === null) throw new Error(`oCIS download failed with ${response.status}`); - const length = response.headers.get("content-length"); - return { - stream: response.body as unknown as Readable, - sizeBytes: length === null ? null : Number(length), - contentType: response.headers.get("content-type"), - }; - } - - /** - * PROPFIND (Depth: 0) on a path: existence + Content-Length + Content-Type. - * Used by the share-into-library path to size and type the copy without - * streaming a single payload byte through the API. - */ - async statPath(segments: readonly string[]): Promise<{ exists: boolean; sizeBytes: number; contentType: string }> { - const body = `\n`; - const response = await this.fetcher(this.rootUrl(segments), { - method: "PROPFIND", - headers: { ...this.authHeaders(), depth: "0", "content-type": "application/xml" }, - body, - }); - if (response.status === 404) return { exists: false, sizeBytes: 0, contentType: "" }; - if (!response.ok) throw new Error(`oCIS PROPFIND failed with ${response.status}`); - const xml = await response.text(); - const sizeBytes = Number(firstTag(xml, "getcontentlength") ?? "0"); - const contentType = firstTag(xml, "getcontenttype") ?? "application/octet-stream"; - if (!Number.isSafeInteger(sizeBytes) || sizeBytes < 0) throw new Error("oCIS PROPFIND returned an invalid content length"); - return { exists: true, sizeBytes, contentType }; - } - - /** - * Server-side COPY of a source path into the collective's group folder - * (optionally inside a sub-folder). `Overwrite: F` makes a collision a 412 - * the caller can retry with a suffixed name — a share never clobbers an - * existing library file. - */ - async copyIntoGroup( - groupId: string, - sourceSegments: readonly string[], - destinationFolder: readonly string[], - name: string, - ): Promise { - await this.ensureFolderPath(["hub-groups", groupId, ...destinationFolder]); - const destination = this.rootUrl(["hub-groups", groupId, ...destinationFolder, name]); - const response = await this.fetcher(destination, { - method: "COPY", - headers: { - ...this.authHeaders(), - destination, - "overwrite": "F", - }, - }); - if (response.status === 404) throw new OcisNotFoundError(`oCIS source not found: /${encodeSegments(sourceSegments)}`); - if (response.status === 412) throw new OcisPreconditionFailedError(`oCIS destination already exists: ${name}`); - if (!response.ok) throw new Error(`oCIS copy failed with ${response.status}`); - const stat = await this.statPath(["hub-groups", groupId, ...destinationFolder, name]); - if (!stat.exists) throw new Error("oCIS copy reported success but the destination is not present"); - const etag = response.headers.get("oc-etag") ?? response.headers.get("etag"); - const resourceId = response.headers.get("oc-fileid") ?? ["hub-groups", groupId, ...destinationFolder, name].join("/"); - return { - resourceId, - version: etag, - sizeBytes: stat.sizeBytes, - sha256: null, - contentType: stat.contentType, - }; - } - - /** Stable resource link for GroupFile.shareLink.url — never presigned. */ - libraryUrl(resourceId: string): string { - return this.config.publicBase === "" ? resourceId : `${this.config.publicBase}/f/${encodeURIComponent(resourceId)}`; - } -} - -/** First … value in a small XML body, namespace-agnostic. */ -function firstTag(xml: string, tag: string): string | null { - const match = xml.match(new RegExp(`<(?:[\\w.-]+:)?${tag}(?:\\s[^>]*)?>([^<]*)`, "i")); - return match === null ? null : match[1]!.trim(); -} - -/** - * Pipe a download forward, verifying sha256 chunk-by-chunk. Contract: - * "SHA-256 verified while streaming" — a mismatch aborts the relay rather - * than handing over tampered bytes (the sink sees them, but the caller - * treats the error as fatal and closes the response). - */ -export async function verifyWhileStreaming( - download: OcisDownload, - expectedSha256: string, - sink: (chunk: Buffer) => void, -): Promise<{ sizeBytes: number }> { - const hash = createHash("sha256"); - let sizeBytes = 0; - for await (const chunk of download.stream) { - const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array); - hash.update(buffer); - sizeBytes += buffer.byteLength; - sink(buffer); - } - const actual = hash.digest("hex"); - if (actual !== expectedSha256) throw new OcisVerificationError(expectedSha256, actual); - return { sizeBytes }; -} - -export function isOcisVerificationError(err: unknown): err is OcisVerificationError { - return err instanceof OcisVerificationError; -} diff --git a/apps/api/src/group-files/verify-stream.ts b/apps/api/src/group-files/verify-stream.ts new file mode 100644 index 0000000..3fa5451 --- /dev/null +++ b/apps/api/src/group-files/verify-stream.ts @@ -0,0 +1,44 @@ +/** + * Streaming payload verification: sha256 is computed over exactly the bytes + * served, and a mismatch aborts the relay rather than handing over tampered + * bytes. Storage-agnostic since 041 (was the oCIS client's contract; kept for + * R2-backed rows). + */ +import { createHash } from "node:crypto"; +import type { Readable } from "node:stream"; + +/** An opened payload: streamed to the caller with metadata for the relay. */ +export interface PayloadDownload { + stream: Readable; + sizeBytes: number | null; + contentType: string | null; +} + +export class OcisVerificationError extends Error { + constructor(expected: string, actual: string) { + super(`payload verification failed: expected sha256 ${expected}, got ${actual}`); + this.name = "OcisVerificationError"; + } +} + +export function isOcisVerificationError(err: unknown): err is OcisVerificationError { + return err instanceof OcisVerificationError; +} + +export async function verifyWhileStreaming( + download: PayloadDownload, + expectedSha256: string, + sink: (chunk: Buffer) => void, +): Promise<{ sizeBytes: number }> { + const hash = createHash("sha256"); + let sizeBytes = 0; + for await (const chunk of download.stream) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array); + hash.update(buffer); + sizeBytes += buffer.byteLength; + sink(buffer); + } + const actual = hash.digest("hex"); + if (actual !== expectedSha256) throw new OcisVerificationError(expectedSha256, actual); + return { sizeBytes }; +} diff --git a/apps/api/src/video/import-sources.ts b/apps/api/src/video/import-sources.ts index 1522409..e739f43 100644 --- a/apps/api/src/video/import-sources.ts +++ b/apps/api/src/video/import-sources.ts @@ -1,6 +1,5 @@ import { Readable } from "node:stream"; import { BadRequestException, ServiceUnavailableException } from "@nestjs/common"; -import { OcisNotFoundError, OcisWebdavClient, ocisConfigFromEnv } from "../group-files/ocis-webdav.js"; /** Imports buffer the whole file in memory; 256 MiB is the v1 cap (streaming is a follow-up). */ export const IMPORT_MAX_BYTES = 256 * 1024 * 1024; @@ -34,19 +33,6 @@ export async function bufferWithCap(stream: Readable, capBytes: number): Promise }); } -let ocisClient: OcisWebdavClient | null = null; - -/** Drive import: pull one oCIS resource through the service-account WebDAV client. */ -export async function loadDriveBytes(resourceId: string): Promise<{ bytes: Buffer; mime: string }> { - ocisClient ??= new OcisWebdavClient(ocisConfigFromEnv()); - const download = await ocisClient.download(resourceId).catch((error: unknown) => { - if (error instanceof OcisNotFoundError) throw new BadRequestException("no such Drive resource"); - throw new ServiceUnavailableException("Drive is unreachable — try again later"); - }); - const bytes = await bufferWithCap(download.stream, IMPORT_MAX_BYTES); - return { bytes, mime: download.contentType ?? "application/octet-stream" }; -} - export function matrixMediaConfig(): { homeserver: string; token: string } | null { const token = process.env.MATRIX_MEDIA_TOKEN?.trim() ?? ""; if (token === "") return null; diff --git a/apps/api/src/video/video.controller.ts b/apps/api/src/video/video.controller.ts index 01468d2..0b35dc6 100644 --- a/apps/api/src/video/video.controller.ts +++ b/apps/api/src/video/video.controller.ts @@ -38,7 +38,7 @@ import { type ChunkReceipt, type InitChunkedUploadRequest, } from "./chunked-upload.js"; -import { loadDriveBytes, loadMatrixBytes, parseMxcUri } from "./import-sources.js"; +import { loadMatrixBytes, parseMxcUri } from "./import-sources.js"; interface VideoFile { buffer: Buffer; @@ -148,16 +148,6 @@ export class VideoController { return this.video.uploadVideo(session.workspaceId, dto.storageKey, dto.mimeType, dto.sizeBytes); } - @Post("import-drive") - @RequirePermission("media.manage") - async importDrive(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise { - const session = requireSession(request); - const input = requireObject(body, "body"); - const resourceId = requireString(input.resourceId, "resourceId"); - const { bytes, mime } = await loadDriveBytes(resourceId); - return this.video.importVideoBytes(session.workspaceId, bytes, mime, "drive", resourceId); - } - @Post("import-chat") @RequirePermission("media.manage") async importChat(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise { diff --git a/apps/web/src/app/(main)/files/drive/page.tsx b/apps/web/src/app/(main)/files/drive/page.tsx deleted file mode 100644 index 651061f..0000000 --- a/apps/web/src/app/(main)/files/drive/page.tsx +++ /dev/null @@ -1,79 +0,0 @@ -"use client"; - -import { EmbeddedApp } from "../../../../components/embed/embedded-app"; -import { useFilesLang } from "../../../../components/files/i18n"; -import { useApiResource } from "../../../../lib/use-api"; -import { useIsMobile } from "../../../../hooks/use-is-mobile"; - -/** - * Drive — the oCIS web UI (personal space, shares, office documents) embedded - * full height under the Files tabs. The group Library tab keeps the Hub-owned - * features (upload into a collective, share with a collective, retention). - * - * Sign-in follows GET /drive/session's authMode: "interactive" means oCIS's - * own identity provider — the user signs in once inside the frame and oCIS - * web keeps its tokens (tokenStorageLocal) on the files origin, which this - * frame shares because app.* and files.* are the same site. "oidc" means oCIS - * and the Hub share an issuer and the frame signs in silently. - */ - -/** Same host FILES_PUBLIC_BASE defaults to; used only while the API is offline. */ -const FALLBACK_DRIVE_URL = "https://files.occupymedia.cc"; - -interface DriveSession { - driveUrl: string | null; - authMode: "interactive" | "oidc"; -} - -const driveView = (session: DriveSession): DriveSession | null => session; - -export default function DrivePage(): React.ReactElement { - const { t } = useFilesLang(); - const drive = useApiResource("/drive/session", driveView, null); - const isMobile = useIsMobile(); - - const driveUrl = drive.source === "sample" ? FALLBACK_DRIVE_URL : drive.data?.driveUrl ?? null; - const notConfigured = drive.source === "live" && drive.error === null && driveUrl === null; - const error = - drive.error ?? - (notConfigured ? t({ en: "The drive is not configured on this server (FILES_DRIVE_URL).", es: "La unidad no está configurada en este servidor (FILES_DRIVE_URL)." }) : null); - const notice = - drive.data?.authMode === "oidc" - ? t({ en: "Collective files live in the Library tab.", es: "Los archivos de los colectivos están en la pestaña Biblioteca." }) - : t({ - en: "Sign in once inside the drive — it stays signed in. Collective files live in the Library tab.", - es: "Inicia sesión una vez dentro de la unidad y quedará abierta. Los archivos de los colectivos están en la pestaña Biblioteca.", - }); - - if (isMobile) { - // Mobile: the oCIS file browser under the bottom tab bar breaks (nested - // scrolling, double chrome) — hand off to the standalone drive instead - // (panel consensus: gateway pattern). The Library tab is fully native. - return ( -
-

{t({ en: "Drive", es: "Unidad" })}

-

{error ?? notice}

- -
- ); - } - - return ( - - ); -} diff --git a/apps/web/src/app/(main)/files/layout.tsx b/apps/web/src/app/(main)/files/layout.tsx index 9df5711..47ec201 100644 --- a/apps/web/src/app/(main)/files/layout.tsx +++ b/apps/web/src/app/(main)/files/layout.tsx @@ -8,7 +8,7 @@ import { FilesTabs } from "../../../components/files/files-tabs"; * Files shell. Like the wiki, the language lives in localStorage per browser * (lang-context explains why it is not an account setting) and is provided here * so every Files screen and sub-component reads it through one context. The tab - * strip switches between the group Library and the embedded oCIS Drive. + * strip switches between the group Library views. */ export default function FilesLayout({ children }: { children: ReactNode }): React.ReactElement { return ( diff --git a/apps/web/src/components/files/files-tabs.tsx b/apps/web/src/components/files/files-tabs.tsx index 41c86bd..26cb301 100644 --- a/apps/web/src/components/files/files-tabs.tsx +++ b/apps/web/src/components/files/files-tabs.tsx @@ -6,20 +6,16 @@ import { useFilesLang, type LocalizedText } from "./i18n"; const TABS: { href: string; label: LocalizedText }[] = [ { href: "/files", label: { en: "Library", es: "Biblioteca" } }, - { href: "/files/drive", label: { en: "Drive", es: "Unidad" } }, ]; -/** - * Files section switcher: the Hub's group library (upload, share with a - * collective, retention) and the embedded oCIS drive side by side. - */ +/** Files section switcher: the Hub's group library (upload, share, retention). */ export function FilesTabs(): React.ReactElement { const pathname = usePathname(); const { t } = useFilesLang(); return (