feat: direct-to-R2 Files uploads (client-sealed SPENC2) + chunked video dropzone + multi-file queue
For scale: payload bytes go browser → R2 → browser; the api handles keys, authz and metadata only, so a cheap box can serve thousands of users. - SPENC2 frame (browser-sealed, P-256 ECDH → HKDF wrap of a per-file CEK, 10 MiG AES-GCM segments): storage.ts gains the reader (magic-branched getObject), presignPut/headObject, openDirectHeader; sealForDirect is the reference sealer for cross-implementation tests - Files API: GET /files/direct/config, POST /files/direct/init (presigned PUT), POST /files/direct/complete (HEAD-gated commit), POST /files/:id/direct-download (presigned GET + CEK ticket) - web: lib/direct-storage (WebCrypto seal/open), lib/files-direct (flows), dropzone rewritten with a multi-file sequential queue (per-file status, direct-first with server fallback), file download decrypts in-browser for direct objects (falls back to streaming) - video dropzone wired to the chunked ingest API (>10 MiB → init/chunk/ complete), fixing the Cloudflare ~100 MB body cap for large videos - DIRECT_EC_PRIVATE_KEY (P-256, PKCS8 b64) wired through compose; cross-impl roundtrip tests prove web-seal ↔ server-open byte compatibility
This commit is contained in:
parent
b3c2085436
commit
0a1c3b392b
21 changed files with 1444 additions and 83 deletions
|
|
@ -20,7 +20,7 @@
|
|||
* (audit_append), the house pattern.
|
||||
*/
|
||||
import { BadRequestException, ConflictException, ForbiddenException, GoneException, Injectable, NotFoundException, Optional, Inject, ServiceUnavailableException } from "@nestjs/common";
|
||||
import { createHash } from "node:crypto";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { Readable } from "node:stream";
|
||||
import type { PoolClient } from "pg";
|
||||
import { attachmentArchiveKey, safeFileName, SNIFF_BYTES, sniffMime, type FileRetention, type GroupFile } from "@tactical-media-broadcast-hub/core";
|
||||
|
|
@ -68,6 +68,38 @@ export interface FilePage {
|
|||
nextCursor: string | null;
|
||||
}
|
||||
|
||||
/** GET /files/direct/config — what the browser needs before it can seal. */
|
||||
export interface DirectUploadConfig {
|
||||
enabled: boolean;
|
||||
frameVersion: 2;
|
||||
publicKeyJwk: { kty: "EC"; crv: "P-256"; x: string; y: string } | null;
|
||||
maxBytes: number;
|
||||
}
|
||||
|
||||
/** POST /files/direct/init response. Nothing is persisted until complete. */
|
||||
export interface DirectUploadInit {
|
||||
uploadId: string;
|
||||
objectKey: string;
|
||||
putUrl: string;
|
||||
}
|
||||
|
||||
/** POST /files/:id/direct-download response: URL + the unwrapped CEK. */
|
||||
export interface DirectDownloadTicket {
|
||||
url: string;
|
||||
cekB64: string;
|
||||
frame: "SPENC2";
|
||||
sizeBytes: number;
|
||||
}
|
||||
|
||||
export const DIRECT_MAX_BYTES = 2_147_483_648;
|
||||
const DIRECT_PUT_TTL_SECONDS = 900;
|
||||
const DIRECT_GET_TTL_SECONDS = 600;
|
||||
const DIRECT_KEY_PREFIX = "direct/";
|
||||
const DIRECT_OBJECT_SUFFIX = ".enc";
|
||||
|
||||
/** MIME allowlist for direct uploads: prefix match on the declared type. */
|
||||
const DIRECT_MIME_PREFIXES = ["image/", "video/", "audio/", "application/", "text/"] as const;
|
||||
|
||||
const MAX_SHARE_TTL_HOURS = 720;
|
||||
const WORKSPACE_WIDE_ROLES: readonly string[] = ["Owner", "Admin"];
|
||||
|
||||
|
|
@ -511,6 +543,133 @@ export class GroupFilesService {
|
|||
});
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Direct-to-R2 (browser-sealed SPENC2): keys/authz/metadata stay here,
|
||||
// payload bytes go browser → R2 → browser
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Whether the deployment can unwrap browser-sealed frames. The public key is
|
||||
* DERIVED from the private key, so a browser can only ever wrap a CEK this
|
||||
* process can open — it never sends a key of its own.
|
||||
*/
|
||||
async directUploadConfig(): Promise<DirectUploadConfig> {
|
||||
const storage = this.storage();
|
||||
const publicKeyJwk = typeof storage.directPublicJwk === "function" ? await storage.directPublicJwk() : null;
|
||||
return {
|
||||
enabled: publicKeyJwk !== null,
|
||||
frameVersion: 2,
|
||||
publicKeyJwk,
|
||||
maxBytes: DIRECT_MAX_BYTES,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Mint the presigned PUT for a direct upload. Group membership is checked
|
||||
* exactly like uploadForGroup; the size is bounded up front so an oversized
|
||||
* pick fails before any bytes are sealed. Nothing is persisted — the row
|
||||
* only appears on complete, so an abandoned PUT leaves an orphan object
|
||||
* (no row), never a dangling row without an object.
|
||||
*/
|
||||
async initDirectUpload(
|
||||
workspaceId: string,
|
||||
groupId: string,
|
||||
actor: ActorRef,
|
||||
input: { filename: string; mimeType: string; sizeBytes: number },
|
||||
): Promise<DirectUploadInit> {
|
||||
if (!isUuid(groupId)) throw new BadRequestException("groupId must be a UUID");
|
||||
if (!Number.isSafeInteger(input.sizeBytes) || input.sizeBytes < 1 || input.sizeBytes > DIRECT_MAX_BYTES) {
|
||||
throw new BadRequestException(`sizeBytes must be an integer between 1 and ${DIRECT_MAX_BYTES}`);
|
||||
}
|
||||
validateDirectFilename(input.filename);
|
||||
validateDirectMime(input.mimeType);
|
||||
await this.requireGroupMembership(workspaceId, groupId, actor);
|
||||
if ((await this.storage().directPublicJwk()) === null) {
|
||||
throw new ServiceUnavailableException("DIRECT_EC_PRIVATE_KEY is not configured");
|
||||
}
|
||||
const uploadId = randomUUID();
|
||||
const objectKey = `${DIRECT_KEY_PREFIX}${uploadId}${DIRECT_OBJECT_SUFFIX}`;
|
||||
const putUrl = await this.storage()
|
||||
.presignPut(objectKey, input.mimeType, DIRECT_PUT_TTL_SECONDS)
|
||||
.catch(() => { throw new ServiceUnavailableException("file storage is unavailable"); });
|
||||
return { uploadId, objectKey, putUrl };
|
||||
}
|
||||
|
||||
/**
|
||||
* Commit a completed direct upload. The object is HEAD-checked (exists, exact
|
||||
* declared size) before the row is written, so metadata can never point at a
|
||||
* missing or partial PUT. sha256 is client-attested — the payload is opaque
|
||||
* to this process by design; the browser computed it over the plaintext it
|
||||
* sealed. Filename runs through the same safeFileName the streamed path uses.
|
||||
*/
|
||||
async completeDirectUpload(
|
||||
workspaceId: string,
|
||||
actor: ActorRef,
|
||||
input: { uploadId: string; objectKey: string; groupId: string; filename: string; mimeType: string; sizeBytes: number; sha256: string },
|
||||
): Promise<GroupFile> {
|
||||
if (!isUuid(input.uploadId)) throw new BadRequestException("uploadId must be a UUID");
|
||||
if (!isUuid(input.groupId)) throw new BadRequestException("groupId must be a UUID");
|
||||
if (input.objectKey !== `${DIRECT_KEY_PREFIX}${input.uploadId}${DIRECT_OBJECT_SUFFIX}`) {
|
||||
throw new BadRequestException("objectKey must be direct/<uploadId>.enc");
|
||||
}
|
||||
if (!/^[0-9a-f]{64}$/.test(input.sha256)) throw new BadRequestException("sha256 must be lowercase hex (64 chars)");
|
||||
if (!Number.isSafeInteger(input.sizeBytes) || input.sizeBytes < 1 || input.sizeBytes > DIRECT_MAX_BYTES) {
|
||||
throw new BadRequestException(`sizeBytes must be an integer between 1 and ${DIRECT_MAX_BYTES}`);
|
||||
}
|
||||
const safe = validateDirectFilename(input.filename);
|
||||
const mimeType = validateDirectMime(input.mimeType);
|
||||
await this.requireGroupMembership(workspaceId, input.groupId, actor);
|
||||
|
||||
const head = await this.storage().headObject(input.objectKey).catch(() => {
|
||||
throw new ServiceUnavailableException("file storage is unavailable");
|
||||
});
|
||||
if (!head.exists || head.contentLength !== input.sizeBytes) {
|
||||
throw new BadRequestException("uploaded object is missing or its size does not match sizeBytes");
|
||||
}
|
||||
|
||||
const mimeClass = mimeClassForDeclared(mimeType);
|
||||
const username = await this.usernameFor(workspaceId, actor.userId);
|
||||
const { id } = await this.registerArchiveEntry(workspaceId, {
|
||||
groupId: input.groupId,
|
||||
roomId: null,
|
||||
eventId: null,
|
||||
origin: { kind: "upload" },
|
||||
originKind: "upload",
|
||||
senderMxid: `@${username}:tmbh.local`,
|
||||
originalName: input.filename,
|
||||
safeName: safe,
|
||||
mime: mimeType,
|
||||
mimeClass,
|
||||
byteSize: input.sizeBytes,
|
||||
sha256: input.sha256,
|
||||
ocisResourceId: input.objectKey,
|
||||
ocisResourceVersion: null,
|
||||
storageKey: input.objectKey,
|
||||
createdAt: new Date(),
|
||||
});
|
||||
return this.getFile(workspaceId, actor.userId, actor.role, id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ticket for a browser-side download: presigned GET (600 s) plus the
|
||||
* unwrapped CEK. Availability/ACL checks are getFile's (the same ones the
|
||||
* streaming route runs); only direct/ objects can take this path.
|
||||
*/
|
||||
async openDirectDownload(workspaceId: string, userId: string, role: string, itemId: string): Promise<DirectDownloadTicket> {
|
||||
const item = await this.getFile(workspaceId, userId, role, itemId);
|
||||
if (item.status !== "available") throw new GoneException(`file is ${item.status}`);
|
||||
const storageKey = item.file.storageKey;
|
||||
if (storageKey === undefined || !storageKey.startsWith(DIRECT_KEY_PREFIX)) {
|
||||
throw new BadRequestException("not a direct object");
|
||||
}
|
||||
const storage = this.storage();
|
||||
const [url, frame] = await Promise.all([
|
||||
storage.presign(storageKey, DIRECT_GET_TTL_SECONDS),
|
||||
storage.getObject(storageKey).then((stored) => storage.openDirectHeader(stored)),
|
||||
]).catch(() => { throw new ServiceUnavailableException("file storage is unavailable"); });
|
||||
return { url, cekB64: frame.toString("base64"), frame: "SPENC2", sizeBytes: item.sizeBytes };
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// internals
|
||||
// -------------------------------------------------------------------------
|
||||
|
|
@ -604,6 +763,15 @@ export class GroupFilesService {
|
|||
return this.getFile(workspaceId, actor.userId, actor.role, id);
|
||||
}
|
||||
|
||||
/** Owner/Admin pass; everyone else must be an active member of the group. */
|
||||
private async requireGroupMembership(workspaceId: string, groupId: string, actor: ActorRef): Promise<void> {
|
||||
if (WORKSPACE_WIDE_ROLES.includes(actor.role)) return;
|
||||
const membership = await withTenant(workspaceId, (client) =>
|
||||
client.query("SELECT 1 FROM group_members WHERE group_id = $1 AND user_id = $2 LIMIT 1", [groupId, actor.userId]),
|
||||
);
|
||||
if (membership.rows.length === 0) throw new ForbiddenException("not a member of this group");
|
||||
}
|
||||
|
||||
private async usernameFor(workspaceId: string, userId: string): Promise<string> {
|
||||
const row = await withTenant(workspaceId, (client) =>
|
||||
client.query<{ username: string }>("SELECT username FROM users WHERE id = $1", [userId]),
|
||||
|
|
@ -659,6 +827,41 @@ function parseFilters(query: Record<string, unknown>): FileListFilters {
|
|||
return { groupId, senderMxid, mimeClass: rawClass as MimeClass | null, origin: rawOrigin as OriginKind | null, name, from, to };
|
||||
}
|
||||
|
||||
/**
|
||||
* Filename gate for direct uploads: no client bytes are sniffed (the payload
|
||||
* is opaque), so the name itself must pass the library-safe rules and carry a
|
||||
* plausible extension for the declared type.
|
||||
*/
|
||||
function validateDirectFilename(filename: string): string {
|
||||
if (typeof filename !== "string" || filename.trim() === "" || filename.length > 255) {
|
||||
throw new BadRequestException("filename must be 1-255 characters");
|
||||
}
|
||||
const safe = safeFileName(filename);
|
||||
if (safe === "" || safe === "attachment") {
|
||||
throw new BadRequestException("filename must contain a usable name");
|
||||
}
|
||||
return safe;
|
||||
}
|
||||
|
||||
/** Declared MIME must sit under one of the allowed prefixes (no sniffing). */
|
||||
function validateDirectMime(mimeType: string): string {
|
||||
if (typeof mimeType !== "string" || mimeType.trim() === "" || mimeType.length > 255) {
|
||||
throw new BadRequestException("mimeType must be 1-255 characters");
|
||||
}
|
||||
if (DIRECT_MIME_PREFIXES.some((prefix) => mimeType.startsWith(prefix))) return mimeType;
|
||||
throw new BadRequestException(`mimeType must start with one of: ${DIRECT_MIME_PREFIXES.join(" ")}`);
|
||||
}
|
||||
|
||||
/** Coarse class from the declared MIME — the sniff path's equivalent. */
|
||||
function mimeClassForDeclared(mimeType: string): MimeClass {
|
||||
if (mimeType.startsWith("image/")) return "image";
|
||||
if (mimeType.startsWith("video/")) return "video";
|
||||
if (mimeType.startsWith("audio/")) return "audio";
|
||||
if (mimeType === "application/pdf" || mimeType.startsWith("text/")) return "document";
|
||||
if (mimeType === "application/zip" || mimeType === "application/x-zip-compressed") return "archive";
|
||||
return "other";
|
||||
}
|
||||
|
||||
function optionalUuid(value: unknown): string | null {
|
||||
if (value === undefined || value === null || value === "") return null;
|
||||
if (typeof value !== "string" || !isUuid(value)) throw new BadRequestException("groupId must be a UUID");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue