diff --git a/apps/api/src/group-files/group-files.controller.ts b/apps/api/src/group-files/group-files.controller.ts index 49fe0d7..9bc2a78 100644 --- a/apps/api/src/group-files/group-files.controller.ts +++ b/apps/api/src/group-files/group-files.controller.ts @@ -20,6 +20,10 @@ interface FileUpload { * * GET /files list/search, keyset (groupId, senderMxid, * mimeClass, origin, name, from, to, cursor, limit) + * GET /files/direct/config direct-upload capability + server public JWK + * POST /files/direct/init presigned PUT for a browser-sealed upload + * POST /files/direct/complete HEAD-checked commit + index row + * POST /files/:id/direct-download presigned GET + unwrapped CEK * POST /files/groups/:groupId/upload direct web upload (≤2 GiB, multipart) * GET /files/:id one GroupFile * GET /files/:id/download stream payload (sha256 re-verified) @@ -49,6 +53,64 @@ export class GroupFilesController { return this.files.listFiles(session.workspaceId, session.userId, session.role, query); } + /** + * Direct-upload capability. Declared before the :id routes so the literal + * "direct/config" path wins route matching over ":id". + */ + @Get("direct/config") + @RequirePermission("media.view") + async directConfig(): Promise> { + return this.files.directUploadConfig(); + } + + @Post("direct/init") + @RequirePermission("media.view") + async directInit( + @Req() request: Request & AuthenticatedRequest, + @Body() body: unknown, + ): Promise> { + const session = requireSession(request); + const dto = requireObject(body, "body"); + return this.files.initDirectUpload(session.workspaceId, requireUuidValue(dto.groupId, "groupId"), { + userId: session.userId, + role: session.role, + }, { + filename: requireString(dto.filename, "filename"), + mimeType: requireString(dto.mimeType, "mimeType"), + sizeBytes: requireNumber(dto.sizeBytes, "sizeBytes"), + }); + } + + @Post("direct/complete") + @RequirePermission("media.view") + async directComplete( + @Req() request: Request & AuthenticatedRequest, + @Body() body: unknown, + ): Promise { + const session = requireSession(request); + const dto = requireObject(body, "body"); + return this.files.completeDirectUpload(session.workspaceId, { userId: session.userId, role: session.role }, { + uploadId: requireUuidValue(dto.uploadId, "uploadId"), + objectKey: requireString(dto.objectKey, "objectKey"), + groupId: requireUuidValue(dto.groupId, "groupId"), + filename: requireString(dto.filename, "filename"), + mimeType: requireString(dto.mimeType, "mimeType"), + sizeBytes: requireNumber(dto.sizeBytes, "sizeBytes"), + sha256: requireString(dto.sha256, "sha256"), + }); + } + + /** Same ACL/availability checks as the streaming route; direct objects only. */ + @Post(":id/direct-download") + @RequirePermission("media.view") + async directDownload( + @Req() request: Request & AuthenticatedRequest, + @Param("id", ParseUUIDPipe) id: string, + ): Promise> { + const session = requireSession(request); + return this.files.openDirectDownload(session.workspaceId, session.userId, session.role, id); + } + @Post("groups/:groupId/upload") @RequirePermission("media.view") @UseInterceptors(FileInterceptor("file", { @@ -176,3 +238,18 @@ export class PublicFileShareController { response.end(); } } + +function requireUuidValue(value: unknown, field: string): string { + const text = requireString(value, field); + if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(text)) { + throw new BadRequestException(`${field} must be a UUID`); + } + return text; +} + +function requireNumber(value: unknown, field: string): number { + if (typeof value !== "number" || !Number.isSafeInteger(value)) { + throw new BadRequestException(`${field} must be a safe integer`); + } + return value; +} diff --git a/apps/api/src/group-files/group-files.service.ts b/apps/api/src/group-files/group-files.service.ts index 123fdf5..7201326 100644 --- a/apps/api/src/group-files/group-files.service.ts +++ b/apps/api/src/group-files/group-files.service.ts @@ -20,7 +20,7 @@ * (audit_append), the house pattern. */ import { BadRequestException, ConflictException, ForbiddenException, GoneException, Injectable, NotFoundException, Optional, Inject, ServiceUnavailableException } from "@nestjs/common"; -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { Readable } from "node:stream"; import type { PoolClient } from "pg"; import { attachmentArchiveKey, safeFileName, SNIFF_BYTES, sniffMime, type FileRetention, type GroupFile } from "@tactical-media-broadcast-hub/core"; @@ -68,6 +68,38 @@ export interface FilePage { nextCursor: string | null; } +/** GET /files/direct/config — what the browser needs before it can seal. */ +export interface DirectUploadConfig { + enabled: boolean; + frameVersion: 2; + publicKeyJwk: { kty: "EC"; crv: "P-256"; x: string; y: string } | null; + maxBytes: number; +} + +/** POST /files/direct/init response. Nothing is persisted until complete. */ +export interface DirectUploadInit { + uploadId: string; + objectKey: string; + putUrl: string; +} + +/** POST /files/:id/direct-download response: URL + the unwrapped CEK. */ +export interface DirectDownloadTicket { + url: string; + cekB64: string; + frame: "SPENC2"; + sizeBytes: number; +} + +export const DIRECT_MAX_BYTES = 2_147_483_648; +const DIRECT_PUT_TTL_SECONDS = 900; +const DIRECT_GET_TTL_SECONDS = 600; +const DIRECT_KEY_PREFIX = "direct/"; +const DIRECT_OBJECT_SUFFIX = ".enc"; + +/** MIME allowlist for direct uploads: prefix match on the declared type. */ +const DIRECT_MIME_PREFIXES = ["image/", "video/", "audio/", "application/", "text/"] as const; + const MAX_SHARE_TTL_HOURS = 720; const WORKSPACE_WIDE_ROLES: readonly string[] = ["Owner", "Admin"]; @@ -511,6 +543,133 @@ export class GroupFilesService { }); } + // ------------------------------------------------------------------------- + // Direct-to-R2 (browser-sealed SPENC2): keys/authz/metadata stay here, + // payload bytes go browser → R2 → browser + // ------------------------------------------------------------------------- + + /** + * Whether the deployment can unwrap browser-sealed frames. The public key is + * DERIVED from the private key, so a browser can only ever wrap a CEK this + * process can open — it never sends a key of its own. + */ + async directUploadConfig(): Promise { + const storage = this.storage(); + const publicKeyJwk = typeof storage.directPublicJwk === "function" ? await storage.directPublicJwk() : null; + return { + enabled: publicKeyJwk !== null, + frameVersion: 2, + publicKeyJwk, + maxBytes: DIRECT_MAX_BYTES, + }; + } + + /** + * Mint the presigned PUT for a direct upload. Group membership is checked + * exactly like uploadForGroup; the size is bounded up front so an oversized + * pick fails before any bytes are sealed. Nothing is persisted — the row + * only appears on complete, so an abandoned PUT leaves an orphan object + * (no row), never a dangling row without an object. + */ + async initDirectUpload( + workspaceId: string, + groupId: string, + actor: ActorRef, + input: { filename: string; mimeType: string; sizeBytes: number }, + ): Promise { + if (!isUuid(groupId)) throw new BadRequestException("groupId must be a UUID"); + if (!Number.isSafeInteger(input.sizeBytes) || input.sizeBytes < 1 || input.sizeBytes > DIRECT_MAX_BYTES) { + throw new BadRequestException(`sizeBytes must be an integer between 1 and ${DIRECT_MAX_BYTES}`); + } + validateDirectFilename(input.filename); + validateDirectMime(input.mimeType); + await this.requireGroupMembership(workspaceId, groupId, actor); + if ((await this.storage().directPublicJwk()) === null) { + throw new ServiceUnavailableException("DIRECT_EC_PRIVATE_KEY is not configured"); + } + const uploadId = randomUUID(); + const objectKey = `${DIRECT_KEY_PREFIX}${uploadId}${DIRECT_OBJECT_SUFFIX}`; + const putUrl = await this.storage() + .presignPut(objectKey, input.mimeType, DIRECT_PUT_TTL_SECONDS) + .catch(() => { throw new ServiceUnavailableException("file storage is unavailable"); }); + return { uploadId, objectKey, putUrl }; + } + + /** + * Commit a completed direct upload. The object is HEAD-checked (exists, exact + * declared size) before the row is written, so metadata can never point at a + * missing or partial PUT. sha256 is client-attested — the payload is opaque + * to this process by design; the browser computed it over the plaintext it + * sealed. Filename runs through the same safeFileName the streamed path uses. + */ + async completeDirectUpload( + workspaceId: string, + actor: ActorRef, + input: { uploadId: string; objectKey: string; groupId: string; filename: string; mimeType: string; sizeBytes: number; sha256: string }, + ): Promise { + if (!isUuid(input.uploadId)) throw new BadRequestException("uploadId must be a UUID"); + if (!isUuid(input.groupId)) throw new BadRequestException("groupId must be a UUID"); + if (input.objectKey !== `${DIRECT_KEY_PREFIX}${input.uploadId}${DIRECT_OBJECT_SUFFIX}`) { + throw new BadRequestException("objectKey must be direct/.enc"); + } + if (!/^[0-9a-f]{64}$/.test(input.sha256)) throw new BadRequestException("sha256 must be lowercase hex (64 chars)"); + if (!Number.isSafeInteger(input.sizeBytes) || input.sizeBytes < 1 || input.sizeBytes > DIRECT_MAX_BYTES) { + throw new BadRequestException(`sizeBytes must be an integer between 1 and ${DIRECT_MAX_BYTES}`); + } + const safe = validateDirectFilename(input.filename); + const mimeType = validateDirectMime(input.mimeType); + await this.requireGroupMembership(workspaceId, input.groupId, actor); + + const head = await this.storage().headObject(input.objectKey).catch(() => { + throw new ServiceUnavailableException("file storage is unavailable"); + }); + if (!head.exists || head.contentLength !== input.sizeBytes) { + throw new BadRequestException("uploaded object is missing or its size does not match sizeBytes"); + } + + const mimeClass = mimeClassForDeclared(mimeType); + const username = await this.usernameFor(workspaceId, actor.userId); + const { id } = await this.registerArchiveEntry(workspaceId, { + groupId: input.groupId, + roomId: null, + eventId: null, + origin: { kind: "upload" }, + originKind: "upload", + senderMxid: `@${username}:tmbh.local`, + originalName: input.filename, + safeName: safe, + mime: mimeType, + mimeClass, + byteSize: input.sizeBytes, + sha256: input.sha256, + ocisResourceId: input.objectKey, + ocisResourceVersion: null, + storageKey: input.objectKey, + createdAt: new Date(), + }); + return this.getFile(workspaceId, actor.userId, actor.role, id); + } + + /** + * Ticket for a browser-side download: presigned GET (600 s) plus the + * unwrapped CEK. Availability/ACL checks are getFile's (the same ones the + * streaming route runs); only direct/ objects can take this path. + */ + async openDirectDownload(workspaceId: string, userId: string, role: string, itemId: string): Promise { + const item = await this.getFile(workspaceId, userId, role, itemId); + if (item.status !== "available") throw new GoneException(`file is ${item.status}`); + const storageKey = item.file.storageKey; + if (storageKey === undefined || !storageKey.startsWith(DIRECT_KEY_PREFIX)) { + throw new BadRequestException("not a direct object"); + } + const storage = this.storage(); + const [url, frame] = await Promise.all([ + storage.presign(storageKey, DIRECT_GET_TTL_SECONDS), + storage.getObject(storageKey).then((stored) => storage.openDirectHeader(stored)), + ]).catch(() => { throw new ServiceUnavailableException("file storage is unavailable"); }); + return { url, cekB64: frame.toString("base64"), frame: "SPENC2", sizeBytes: item.sizeBytes }; + } + // ------------------------------------------------------------------------- // internals // ------------------------------------------------------------------------- @@ -604,6 +763,15 @@ export class GroupFilesService { return this.getFile(workspaceId, actor.userId, actor.role, id); } + /** Owner/Admin pass; everyone else must be an active member of the group. */ + private async requireGroupMembership(workspaceId: string, groupId: string, actor: ActorRef): Promise { + if (WORKSPACE_WIDE_ROLES.includes(actor.role)) return; + const membership = await withTenant(workspaceId, (client) => + client.query("SELECT 1 FROM group_members WHERE group_id = $1 AND user_id = $2 LIMIT 1", [groupId, actor.userId]), + ); + if (membership.rows.length === 0) throw new ForbiddenException("not a member of this group"); + } + private async usernameFor(workspaceId: string, userId: string): Promise { const row = await withTenant(workspaceId, (client) => client.query<{ username: string }>("SELECT username FROM users WHERE id = $1", [userId]), @@ -659,6 +827,41 @@ function parseFilters(query: Record): FileListFilters { return { groupId, senderMxid, mimeClass: rawClass as MimeClass | null, origin: rawOrigin as OriginKind | null, name, from, to }; } +/** + * Filename gate for direct uploads: no client bytes are sniffed (the payload + * is opaque), so the name itself must pass the library-safe rules and carry a + * plausible extension for the declared type. + */ +function validateDirectFilename(filename: string): string { + if (typeof filename !== "string" || filename.trim() === "" || filename.length > 255) { + throw new BadRequestException("filename must be 1-255 characters"); + } + const safe = safeFileName(filename); + if (safe === "" || safe === "attachment") { + throw new BadRequestException("filename must contain a usable name"); + } + return safe; +} + +/** Declared MIME must sit under one of the allowed prefixes (no sniffing). */ +function validateDirectMime(mimeType: string): string { + if (typeof mimeType !== "string" || mimeType.trim() === "" || mimeType.length > 255) { + throw new BadRequestException("mimeType must be 1-255 characters"); + } + if (DIRECT_MIME_PREFIXES.some((prefix) => mimeType.startsWith(prefix))) return mimeType; + throw new BadRequestException(`mimeType must start with one of: ${DIRECT_MIME_PREFIXES.join(" ")}`); +} + +/** Coarse class from the declared MIME — the sniff path's equivalent. */ +function mimeClassForDeclared(mimeType: string): MimeClass { + if (mimeType.startsWith("image/")) return "image"; + if (mimeType.startsWith("video/")) return "video"; + if (mimeType.startsWith("audio/")) return "audio"; + if (mimeType === "application/pdf" || mimeType.startsWith("text/")) return "document"; + if (mimeType === "application/zip" || mimeType === "application/x-zip-compressed") return "archive"; + return "other"; +} + function optionalUuid(value: unknown): string | null { if (value === undefined || value === null || value === "") return null; if (typeof value !== "string" || !isUuid(value)) throw new BadRequestException("groupId must be a UUID"); diff --git a/apps/api/src/media/media.test.ts b/apps/api/src/media/media.test.ts index 98226c3..f2f4e65 100644 --- a/apps/api/src/media/media.test.ts +++ b/apps/api/src/media/media.test.ts @@ -10,7 +10,7 @@ import { sniffMediaAsset, } from "./media.service.js"; import { appendTenantAudit } from "../audit/audit.service.js"; -import { sha256, type StorageService } from "../storage/storage.js"; +import { sha256, type StorageService, directStorageDisabled } from "../storage/storage.js"; import { MediaController } from "./media.controller.js"; import type { AuthenticatedRequest } from "../rbac/rbac.guard.js"; import type { Request, Response } from "express"; @@ -68,6 +68,7 @@ function fakeStorage(): StorageService & { }), hasContent: vi.fn(async () => false), presign: vi.fn(async () => ""), + ...directStorageDisabled(), }; } diff --git a/apps/api/src/production/preview.test.ts b/apps/api/src/production/preview.test.ts index 04f5556..1e3ece1 100644 --- a/apps/api/src/production/preview.test.ts +++ b/apps/api/src/production/preview.test.ts @@ -2,11 +2,11 @@ import { randomUUID } from "node:crypto"; import { describe, expect, it, vi } from "vitest"; import { MemoryRedis } from "./testing/memory-redis.js"; import { PreviewService } from "./preview.js"; -import { keyForHash, sha256, type StorageService } from "../storage/storage.js"; +import { keyForHash, sha256, directStorageDisabled, type StorageService } from "../storage/storage.js"; const WS = randomUUID(), CH = randomUUID(), TAP = randomUUID(); function fixture(): { preview: PreviewService; storage: StorageService; redis: MemoryRedis; seed: (file: string, body: string, type?: string) => void; advance: (ms: number) => void } { const redis = new MemoryRedis(), objects = new Map(); let now = Date.now(); - const storage: StorageService = { putContent: async () => { throw new Error("unused"); }, getObject: vi.fn(async key => { await new Promise(r => setTimeout(r, 1)); return objects.get(key) ?? Buffer.alloc(0); }), hasContent: async () => false, presign: async () => { throw new Error("must not redirect to encrypted S3"); } }; + const storage: StorageService = { putContent: async () => { throw new Error("unused"); }, getObject: vi.fn(async key => { await new Promise(r => setTimeout(r, 1)); return objects.get(key) ?? Buffer.alloc(0); }), hasContent: async () => false, presign: async () => { throw new Error("must not redirect to encrypted S3"); }, ...directStorageDisabled() }; const preview = new PreviewService({ redis, storage, secret: "test", now: () => now, assertTap: async () => undefined }); const seed = (file: string, body: string, type = "video/mp2t"): void => { const buffer = Buffer.from(body), key = keyForHash(sha256(buffer), type); objects.set(key, buffer); diff --git a/apps/api/src/production/production.test.ts b/apps/api/src/production/production.test.ts index da9e9aa..5a6d178 100644 --- a/apps/api/src/production/production.test.ts +++ b/apps/api/src/production/production.test.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import type { PoolClient } from "pg"; import { describe, expect, it, vi } from "vitest"; import { MediaControlService, type IssuedCommand } from "../media-control/media-control.service.js"; -import { sha256, keyForHash, type StorageService } from "../storage/storage.js"; +import { sha256, keyForHash, directStorageDisabled, type StorageService } from "../storage/storage.js"; import { ProductionService } from "./production.service.js"; import type { TenantTransaction } from "./journal.js"; const WS = randomUUID(), CH = randomUUID(), SLOT = randomUUID(); @@ -29,7 +29,7 @@ function fixture(render?: (input: unknown) => Promise): { production: Pr try { return await fn({ query } as unknown as PoolClient); } finally { unlock(); } }; const control = new MediaControlService({ signer: null, assignments: [] }); vi.spyOn(control, "director").mockResolvedValue(ISSUED); - const storage: StorageService = { putContent: vi.fn(async (buffer, contentType) => { const hash = sha256(buffer); return { hash, key: keyForHash(hash, contentType), sizeBytes: buffer.length, contentType, deduplicated: false }; }), getObject: async () => Buffer.alloc(0), hasContent: async () => false, presign: async () => "" }; + const storage: StorageService = { putContent: vi.fn(async (buffer, contentType) => { const hash = sha256(buffer); return { hash, key: keyForHash(hash, contentType), sizeBytes: buffer.length, contentType, deduplicated: false }; }), getObject: async () => Buffer.alloc(0), hasContent: async () => false, presign: async () => "", ...directStorageDisabled() }; return { rows, control, storage, production: new ProductionService(control, { transaction, storage, ...(render ? { render } : {}) }) }; } describe("revision → render job → content-addressed storage → desired-state pipeline", () => { diff --git a/apps/api/src/settings/branding.test.ts b/apps/api/src/settings/branding.test.ts index 6d458b3..0c8da9f 100644 --- a/apps/api/src/settings/branding.test.ts +++ b/apps/api/src/settings/branding.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import { directStorageDisabled } from "../storage/storage.js"; import { BadRequestException, ServiceUnavailableException } from "@nestjs/common"; import type { PoolClient } from "pg"; import type { Request, Response } from "express"; @@ -67,6 +68,7 @@ function fakeStorage(objects: Record = {}): { }), hasContent: vi.fn(async () => false), presign: vi.fn(async () => ""), + ...directStorageDisabled(), }; return { storage, puts }; } @@ -339,6 +341,7 @@ describe("branding service — uploads (storage reuse, magic-byte sniffing)", () getObject: vi.fn(async () => Buffer.alloc(0)), hasContent: vi.fn(async () => false), presign: vi.fn(async () => ""), + ...directStorageDisabled(), }; const branding = serviceWith({ storage: broken }); await expect(branding.uploadImage(PNG_HEADER, "image/png")).rejects.toBeInstanceOf(ServiceUnavailableException); diff --git a/apps/api/src/storage/storage.ts b/apps/api/src/storage/storage.ts index 2084774..5c6d100 100644 --- a/apps/api/src/storage/storage.ts +++ b/apps/api/src/storage/storage.ts @@ -1,4 +1,5 @@ -import { createCipheriv, createDecipheriv, createHash, randomBytes } from "node:crypto"; +import { createCipheriv, createDecipheriv, createHash, createPrivateKey, createPublicKey, diffieHellman, generateKeyPairSync, hkdfSync, randomBytes, randomUUID } from "node:crypto"; +import { ServiceUnavailableException } from "@nestjs/common"; import { CreateBucketCommand, GetObjectCommand, @@ -18,6 +19,13 @@ import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; * stays on the PLAINTEXT hash (dedup survives); getObject re-verifies the * hash, so storage-level tampering is detected. SSE-at-rest headers are set * additionally for endpoints that honor them (belt-and-braces). + * + * SPENC2 (direct-to-R2, 2026-10-16): objects under `direct/` are sealed IN + * THE BROWSER (apps/web/src/lib/direct-storage.ts). The server never sees + * the payload — it only mints the presigned PUT/GET and unwraps the CEK with + * the ECDH share it derives from DIRECT_EC_PRIVATE_KEY. Both frame versions + * share `getObject`: the leading magic decides the branch, so a single + * reader serves the whole bucket and the SPENC1 path stays byte-identical. */ export interface StoredObject { @@ -32,6 +40,37 @@ export interface StorageService { getObject(key: string): Promise; hasContent(hash: string, contentType: string): Promise; presign(key: string, ttlSeconds: number): Promise; + /** Direct-to-R2: the JWK the browser seals against; null = feature off. */ + directPublicJwk(): Promise<{ kty: "EC"; crv: "P-256"; x: string; y: string } | null>; + /** Direct-to-R2: presigned PUT for the browser to push a SPENC2 frame. */ + presignPut(key: string, contentType: string, ttlSeconds: number): Promise; + /** Direct-to-R2: existence + exact size, used to gate /files/direct/complete. */ + headObject(key: string): Promise<{ exists: boolean; contentLength: number | null }>; + /** Direct-to-R2: unwrap the frame's CEK (header only) for /files/:id/direct-download. */ + openDirectHeader(stored: Buffer): Promise; +} + +/** The direct-only members of StorageService, as one spreadable group. */ +export type DirectStorageMembers = Pick< + StorageService, + "directPublicJwk" | "presignPut" | "headObject" | "openDirectHeader" +>; + +/** + * Direct-path stub for in-memory fakes that never serve browser uploads: + * reports the feature disabled and refuses any direct operation. Lets a fake + * stay a complete StorageService with one spread instead of four stubs. + */ +export function directStorageDisabled(): DirectStorageMembers { + const unavailable = async (): Promise => { + throw new Error("direct storage is unavailable"); + }; + return { + directPublicJwk: async () => null, + presignPut: unavailable, + headObject: unavailable, + openDirectHeader: unavailable, + }; } const EXTENSIONS: Record = { @@ -89,6 +128,218 @@ export function open(sealed: Buffer, kek: Buffer): Buffer { return Buffer.concat([decipher.update(ciphertext), decipher.final()]); } +// --------------------------------------------------------------------------- +// SPENC2 — browser-sealed direct-to-R2 frames +// --------------------------------------------------------------------------- + +/** Frame magic for browser-sealed objects (spec: "SPENC2" exactly). */ +export const DIRECT_FRAME_MAGIC = Buffer.from("SPENC2", "ascii"); + +/** + * Fixed frame layout, all integers big-endian. Apps/web/src/lib/direct-storage.ts + * seals the same shape; this reader is the cross-implementation proof target. + * + * [0..6) magic "SPENC2" + * [6..71) ephemeral P-256 public key, uncompressed: 0x04 || X(32) || Y(32) + * [71..83) wrapNonce, 12 random bytes + * [83..131) wrappedCek: AES-256-GCM(CEK), key = HKDF-SHA256(ECDH, salt=∅, + * info="tmbh-direct-cek-wrap", L=32), nonce = wrapNonce → 48 bytes + * [131..135) segmentCount, u32 + * [135..] segments: ctLen u32 | iv 12 | AES-256-GCM ciphertext+tag, key=CEK, no AAD + */ +const EPHEMERAL_PUBLIC_OFFSET = 6; +const WRAP_NONCE_OFFSET = 71; +const WRAPPED_CEK_OFFSET = 83; +const SEGMENT_COUNT_OFFSET = 131; +const SEGMENTS_OFFSET = 135; +const WRAPPED_CEK_BYTES = 32 + 16; + +/** HKDF info binding the wrap key to this purpose. ASCII, exact. */ +const DIRECT_WRAP_INFO = Buffer.from("tmbh-direct-cek-wrap", "ascii"); + +/** Segment plaintext ceiling. Mirrors the web side, which seals 10 MiB chunks. */ +export const DIRECT_SEGMENT_BYTES = 10 * 1024 * 1024; + +/** Server direct keypair: P-256 private key from DIRECT_EC_PRIVATE_KEY (base64 DER). */ +export interface DirectKeyPair { + privateKey: ReturnType; + /** Compact JWK {kty,crv,x,y} — exactly what the browser's importKey wants. */ + publicJwk: { kty: "EC"; crv: "P-256"; x: string; y: string }; +} + +export class DirectKeyError extends Error {} + +/** True when DIRECT_EC_PRIVATE_KEY is present (routes report enabled=false otherwise). */ +export function directEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return loadDirectPrivateJwk(env) !== null; +} + +function loadDirectPrivateJwk(env: NodeJS.ProcessEnv): ReturnType | null { + const raw = env.DIRECT_EC_PRIVATE_KEY; + if (raw === undefined || raw.trim() === "") return null; + let der: Buffer; + try { + der = Buffer.from(raw.trim(), "base64"); + } catch { + throw new DirectKeyError("DIRECT_EC_PRIVATE_KEY must be base64 DER"); + } + try { + return createPrivateKey({ key: der, format: "der", type: "pkcs8" }); + } catch { + throw new DirectKeyError("DIRECT_EC_PRIVATE_KEY is not a PKCS8 P-256 private key"); + } +} + +/** + * The direct keypair, deriving the public key from the private one — a + * client-supplied public key is never trusted, so the browser can only wrap + * CEKs this process can unwrap. + */ +export function directKeyPairFromEnv(env: NodeJS.ProcessEnv = process.env): DirectKeyPair | null { + const privateKey = loadDirectPrivateJwk(env); + if (privateKey === null) return null; + const jwk = createPublicKey(privateKey).export({ format: "jwk" }); + if (jwk.kty !== "EC" || jwk.crv !== "P-256" || typeof jwk.x !== "string" || typeof jwk.y !== "string") { + throw new DirectKeyError("DIRECT_EC_PRIVATE_KEY is not a P-256 key"); + } + return { privateKey, publicJwk: { kty: "EC", crv: "P-256", x: jwk.x, y: jwk.y } }; +} + +/** ECDH → HKDF-SHA256(salt empty, info "tmbh-direct-cek-wrap") → 32-byte wrap key. */ +function directWrapKey(privateKey: ReturnType, ephemeralPoint: Buffer): Buffer { + const x = ephemeralPoint.subarray(1, 33).toString("base64url"); + const y = ephemeralPoint.subarray(33, 65).toString("base64url"); + let peer: ReturnType; + try { + peer = createPublicKey({ key: { kty: "EC", crv: "P-256", x, y }, format: "jwk" }); + } catch { + throw new Error("storage: SPENC2 frame holds a malformed ephemeral key"); + } + const ikm = Buffer.from(diffieHellman({ privateKey, publicKey: peer })); + // RFC 5869: an absent salt defaults to HashLen zero bytes; WebCrypto's empty + // salt and Node's zero-filled salt produce the identical PRK (verified). + return Buffer.from(hkdfSync("sha256", ikm, Buffer.alloc(0), DIRECT_WRAP_INFO, 32)); +} + +interface DirectHeader { + ephemeralPoint: Buffer; + wrapNonce: Buffer; + wrappedCek: Buffer; + segmentCount: number; +} + +/** Validate + slice the fixed header. Throws on any truncation or bad shape. */ +function parseDirectHeader(stored: Buffer): DirectHeader { + if (stored.length < SEGMENTS_OFFSET) { + throw new Error("storage: SPENC2 frame is truncated"); + } + const ephemeralPoint = stored.subarray(EPHEMERAL_PUBLIC_OFFSET, WRAP_NONCE_OFFSET); + if (ephemeralPoint[0] !== 0x04) throw new Error("storage: SPENC2 ephemeral key is not an uncompressed point"); + const wrapNonce = stored.subarray(WRAP_NONCE_OFFSET, WRAPPED_CEK_OFFSET); + const wrappedCek = stored.subarray(WRAPPED_CEK_OFFSET, WRAPPED_CEK_OFFSET + WRAPPED_CEK_BYTES); + const segmentCount = stored.readUInt32BE(SEGMENT_COUNT_OFFSET); + if (segmentCount === 0 || segmentCount > 4_294_967_295 / DIRECT_SEGMENT_BYTES + 1) { + throw new Error("storage: SPENC2 segment count is out of range"); + } + return { ephemeralPoint, wrapNonce, wrappedCek, segmentCount }; +} + +/** Walk the segment table; yields {ciphertext, iv} in order. */ +function* directSegments(stored: Buffer, segmentCount: number): Generator<{ iv: Buffer; ciphertext: Buffer }> { + let offset = SEGMENTS_OFFSET; + for (let index = 0; index < segmentCount; index += 1) { + if (offset + 4 + 12 > stored.length) throw new Error("storage: SPENC2 segment table is truncated"); + const ctLen = stored.readUInt32BE(offset); + offset += 4; + if (ctLen < 16 || offset + 12 + ctLen > stored.length) throw new Error("storage: SPENC2 segment length is out of range"); + const iv = stored.subarray(offset, offset + 12); + offset += 12; + const ciphertext = stored.subarray(offset, offset + ctLen); + offset += ctLen; + yield { iv, ciphertext }; + } + if (offset !== stored.length) throw new Error("storage: SPENC2 frame has trailing bytes"); +} + +/** Unwrap the CEK (verifies the wrap GCM tag). Exposed for openDirectHeader. */ +export function unwrapDirectCek(stored: Buffer, privateKey: ReturnType): Buffer { + const header = parseDirectHeader(stored); + return aesGcmOpen(directWrapKey(privateKey, header.ephemeralPoint), header.wrapNonce, header.wrappedCek); +} + +/** + * Open a SPENC2 frame: unwrap the CEK, then decrypt every segment. Segment + * GCM tags are verified individually, so any flipped byte throws. Empty + * plaintexts are rejected (the upload floor is 1 byte). + */ +export function openDirectFrame(stored: Buffer, privateKey: ReturnType): Buffer { + const { segmentCount } = parseDirectHeader(stored); + const cek = unwrapDirectCek(stored, privateKey); + const parts: Buffer[] = []; + for (const segment of directSegments(stored, segmentCount)) { + parts.push(aesGcmOpen(cek, segment.iv, segment.ciphertext)); + } + const plaintext = Buffer.concat(parts); + if (plaintext.length === 0) throw new Error("storage: SPENC2 frame decrypts to zero bytes"); + return plaintext; +} + +/** + * Test helper: seal `plaintext` exactly the way the browser does, so server + * readers can be exercised without a DOM. Production uploads come from + * apps/web/src/lib/direct-storage.ts — this is the reference implementation. + */ +export function sealForDirect(plaintext: Buffer, publicJwk: { kty: "EC"; crv: "P-256"; x: string; y: string }): Buffer { + if (plaintext.length === 0) throw new Error("sealForDirect: plaintext must not be empty"); + const cek = randomBytes(32); + const { privateKey: ephemeralPrivate, publicKey: ephemeralPublic } = newEphemeralKeyPair(); + // Wrap to the SERVER's public key — ECDH(ephPriv, serverPub), the same share + // the server derives as ECDH(serverPriv, ephPub) in directWrapKey. + const wrapKey = directWrapKey(ephemeralPrivate, uncompressedPoint(createPublicKey({ key: { ...publicJwk }, format: "jwk" }))); + const wrapNonce = randomBytes(12); + const wrappedCek = aesGcmSeal(wrapKey, wrapNonce, cek); + const segments: Buffer[] = []; + for (let offset = 0; offset < plaintext.length; offset += DIRECT_SEGMENT_BYTES) { + const iv = randomBytes(12); + const ciphertext = aesGcmSeal(cek, iv, plaintext.subarray(offset, offset + DIRECT_SEGMENT_BYTES)); + segments.push(Buffer.concat([u32be(ciphertext.byteLength), iv, ciphertext])); + } + return Buffer.concat([ + DIRECT_FRAME_MAGIC, + uncompressedPoint(ephemeralPublic), + wrapNonce, + wrappedCek, + u32be(segments.length), + ...segments, + ]); +} + +/** Random object key for a browser upload: `direct/.enc`. */ +export function directObjectKey(): string { + return `direct/${randomUUID()}.enc`; +} + +function u32be(value: number): Buffer { + const out = Buffer.alloc(4); + out.writeUInt32BE(value, 0); + return out; +} + +function newEphemeralKeyPair(): { privateKey: ReturnType; publicKey: ReturnType } { + return generateKeyPairSync("ec", { namedCurve: "P-256" }); +} + +/** Raised wherever a direct operation needs a key the deployment never supplied. */ +function missingDirectKey(): ServiceUnavailableException { + return new ServiceUnavailableException("DIRECT_EC_PRIVATE_KEY is not configured"); +} + +function uncompressedPoint(publicKey: ReturnType): Buffer { + const point = publicKey.export({ type: "spki", format: "der" }).subarray(26); + if (point.length !== 65 || point[0] !== 0x04) throw new Error("storage: expected an uncompressed P-256 point"); + return point; +} + function aesGcmSeal(key: Buffer, iv: Buffer, plaintext: Buffer): Buffer { const cipher = createCipheriv("aes-256-gcm", key, iv); return Buffer.concat([cipher.update(plaintext), cipher.final(), cipher.getAuthTag()]); @@ -122,6 +373,8 @@ export interface S3StorageConfig { bucket?: string; forcePathStyle?: boolean; mediaKek?: Buffer; + /** Overrides DIRECT_EC_PRIVATE_KEY (tests). */ + directPrivateKey?: ReturnType; } export function s3ConfigFromEnv(): S3StorageConfig { @@ -139,11 +392,13 @@ export class S3Storage implements StorageService { private readonly client: S3Client; private readonly bucket: string; private readonly kek: Buffer; + private readonly directKey: DirectKeyPair | null; private bucketReady = false; constructor(config: S3StorageConfig) { this.bucket = config.bucket ?? "tactical-media-broadcast-hub-media"; this.kek = config.mediaKek ?? loadMediaKek(); + this.directKey = config.directPrivateKey !== undefined ? directKeyPairFromPrivateKey(config.directPrivateKey) : directKeyPairFromEnv(); this.client = new S3Client({ endpoint: config.endpoint, region: config.region ?? "us-east-1", @@ -155,6 +410,11 @@ export class S3Storage implements StorageService { }); } + /** The JWK the browser seals against; null when direct upload is disabled. */ + async directPublicJwk(): Promise<{ kty: "EC"; crv: "P-256"; x: string; y: string } | null> { + return this.directKey === null ? null : { ...this.directKey.publicJwk }; + } + private async ensureBucket(): Promise { if (this.bucketReady) return; try { @@ -205,9 +465,21 @@ export class S3Storage implements StorageService { const res = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: key })); if (res.Body === undefined) throw new Error(`empty body for ${key}`); const stored = Buffer.from(await res.Body.transformToByteArray()); - const plaintext = open(stored, this.kek); + const leading = stored.subarray(0, 6).toString("latin1"); + let plaintext: Buffer; + if (leading === "SPENC1") { + plaintext = open(stored, this.kek); + } else if (leading === "SPENC2") { + // Browser-sealed frame: only this process can unwrap the CEK. + if (this.directKey === null) throw missingDirectKey(); + plaintext = openDirectFrame(stored, this.directKey.privateKey); + } else { + plaintext = open(stored, this.kek); // raises the magic-mismatch error + } + // Content addressing is on the PLAINTEXT hash, so tampering (or a + // mismatched key) is detected here for both frame versions. const hashFromKey = key.split("/")[1]?.split(".")[0]; - if (hashFromKey !== undefined && sha256(plaintext) !== hashFromKey) { + if (hashFromKey !== undefined && hashFromKey.length === 64 && sha256(plaintext) !== hashFromKey) { throw new Error(`storage: integrity failure for ${key} (plaintext hash mismatch)`); } return plaintext; @@ -221,4 +493,34 @@ export class S3Storage implements StorageService { async presign(key: string, ttlSeconds: number): Promise { return getSignedUrl(this.client, new GetObjectCommand({ Bucket: this.bucket, Key: key }), { expiresIn: ttlSeconds }); } + + async presignPut(key: string, contentType: string, ttlSeconds: number): Promise { + return getSignedUrl(this.client, new PutObjectCommand({ Bucket: this.bucket, Key: key, ContentType: contentType }), { expiresIn: ttlSeconds }); + } + + async headObject(key: string): Promise<{ exists: boolean; contentLength: number | null }> { + try { + const res = await this.client.send(new HeadObjectCommand({ Bucket: this.bucket, Key: key })); + return { exists: true, contentLength: res.ContentLength ?? null }; + } catch (err) { + const status = err instanceof S3ServiceException ? err.$metadata.httpStatusCode : undefined; + const name = err instanceof S3ServiceException ? err.name : ""; + if (status === 404 || name === "NotFound") return { exists: false, contentLength: null }; + throw err; + } + } + + async openDirectHeader(stored: Buffer): Promise { + if (this.directKey === null) throw missingDirectKey(); + return unwrapDirectCek(stored, this.directKey.privateKey); + } +} + +/** Build a DirectKeyPair from an already-loaded private key (config + tests). */ +function directKeyPairFromPrivateKey(privateKey: ReturnType): DirectKeyPair { + const jwk = createPublicKey(privateKey).export({ format: "jwk" }); + if (jwk.kty !== "EC" || jwk.crv !== "P-256" || typeof jwk.x !== "string" || typeof jwk.y !== "string") { + throw new DirectKeyError("direct key is not a P-256 key"); + } + return { privateKey, publicJwk: { kty: "EC", crv: "P-256", x: jwk.x, y: jwk.y } }; } diff --git a/apps/api/src/video/test-fixtures.ts b/apps/api/src/video/test-fixtures.ts index 8e61d08..8755a67 100644 --- a/apps/api/src/video/test-fixtures.ts +++ b/apps/api/src/video/test-fixtures.ts @@ -1,4 +1,4 @@ -import { keyForHash, sha256, type StorageService } from "../storage/storage.js"; +import { keyForHash, sha256, type StorageService, directStorageDisabled } from "../storage/storage.js"; import type { PoolClient, QueryResult } from "pg"; import { vi } from "vitest"; import { @@ -43,6 +43,7 @@ export function serviceWithDatabase( }), hasContent: async (hash, type) => objects.has(keyForHash(hash, type)), presign: async () => { throw new Error("Encrypted source must be decrypted before probing"); }, + ...directStorageDisabled(), }; database.transitions = []; database.statements = []; diff --git a/apps/api/src/video/video.integration.test.ts b/apps/api/src/video/video.integration.test.ts index 37e8611..e125536 100644 --- a/apps/api/src/video/video.integration.test.ts +++ b/apps/api/src/video/video.integration.test.ts @@ -3,7 +3,7 @@ import { join } from "node:path"; import { Pool, type PoolClient } from "pg"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { runMigrations } from "../db/migrate.js"; -import { sha256, keyForHash, type StorageService } from "../storage/storage.js"; +import { sha256, keyForHash, type StorageService, directStorageDisabled } from "../storage/storage.js"; import { VideoService, type VideoServiceDependencies } from "./video.service.js"; const dbAvailable = process.env.DATABASE_URL !== undefined; @@ -45,6 +45,7 @@ describe.skipIf(!dbAvailable)("video library (live DB)", () => { }, hasContent: async (hash, type) => objects.has(keyForHash(hash, type)), presign: async () => "unused", + ...directStorageDisabled(), }; beforeAll(async () => { diff --git a/apps/api/src/video/video.roundtrip.test.ts b/apps/api/src/video/video.roundtrip.test.ts index 00e08ab..938a6cb 100644 --- a/apps/api/src/video/video.roundtrip.test.ts +++ b/apps/api/src/video/video.roundtrip.test.ts @@ -1,6 +1,6 @@ import { readFile } from "node:fs/promises"; import { describe, expect, it } from "vitest"; -import { keyForHash, sha256, seal, open, type StorageService } from "../storage/storage.js"; +import { keyForHash, sha256, seal, open, type StorageService, directStorageDisabled } from "../storage/storage.js"; import { probeVideoFile } from "./ffprobe.js"; import { serviceWithDatabase, workspaceId } from "./test-fixtures.js"; @@ -30,6 +30,7 @@ describe.skipIf(fixture === undefined)("real ffprobe roundtrip (encrypted in-mem }, hasContent: async (hash, contentType) => objects.has(keyForHash(hash, contentType)), presign: async () => { throw new Error("must decrypt, not presign"); }, + ...directStorageDisabled(), }; const { service } = serviceWithDatabase({}, probeVideoFile, storage); const uploaded = await service.uploadVideoBytes(workspaceId, bytes, "video/mp4"); diff --git a/apps/web/src/app/(main)/files/page.tsx b/apps/web/src/app/(main)/files/page.tsx index 6be80b3..b93ce4e 100644 --- a/apps/web/src/app/(main)/files/page.tsx +++ b/apps/web/src/app/(main)/files/page.tsx @@ -14,6 +14,7 @@ import { UploadDropzone } from "../../../components/files/upload-dropzone"; import { ShareDialog } from "../../../components/files/share-dialog"; import { RetentionDialog } from "../../../components/files/retention-dialog"; import { downloadGroupFile } from "../../../components/files/files-client"; +import { downloadFileDirect, isDirectObject, saveBytes } from "../../../lib/files-direct"; import { useFilesLang } from "../../../components/files/i18n"; import { MIME_CLASS_LABEL } from "../../../components/files/labels"; import type { @@ -171,8 +172,23 @@ export default function FilesPage(): React.ReactElement { async function handleDownload(item: GroupFile): Promise { setActionError(null); try { + if (isDirectObject(item)) { + const bytes = await downloadFileDirect(item.id); + saveBytes(bytes, item.name); + return; + } await downloadGroupFile(item.id, item.name); } catch (err) { + // Direct path is best-effort: fall back to streaming via the api once. + if (isDirectObject(item)) { + try { + await downloadGroupFile(item.id, item.name); + return; + } catch (fallbackErr) { + setActionError(errorMessage(fallbackErr)); + return; + } + } setActionError(errorMessage(err)); } } diff --git a/apps/web/src/components/files/types.ts b/apps/web/src/components/files/types.ts index a579982..52be4ac 100644 --- a/apps/web/src/components/files/types.ts +++ b/apps/web/src/components/files/types.ts @@ -34,7 +34,7 @@ export interface GroupFile { checksum: { value: string }; createdAt: string; indexedAt: string; - file: { resourceId: string }; + file: { resourceId: string; storageKey?: string }; shareLink: { url: string; access: "members" | "public"; expiresAt: string | null }; retention: FileRetention; status: FileStatus; diff --git a/apps/web/src/components/files/upload-dropzone.tsx b/apps/web/src/components/files/upload-dropzone.tsx index a5c564c..ddf0b8a 100644 --- a/apps/web/src/components/files/upload-dropzone.tsx +++ b/apps/web/src/components/files/upload-dropzone.tsx @@ -4,19 +4,40 @@ import { useCallback, useRef, useState } from "react"; import { Button } from "../ui/primitives"; import { Loading } from "../brand/loading"; import { apiUploadFile, errorMessage } from "../../lib/api-client"; +import { fetchDirectConfig, uploadFileDirect, DIRECT_MEMORY_CAP_BYTES, type DirectUploadConfig } from "../../lib/files-direct"; import { formatBytes } from "../video/format"; import { useFilesLang } from "./i18n"; import type { GroupFile } from "./types"; /** - * Drag-drop + picker for POST /files/groups/:groupId/upload. The API accepts a - * single `file` field capped at 2 GiB; the cap is enforced here too so an - * oversized pick fails fast with a friendly message instead of a 413. + * Drag-drop + picker for group file uploads — MULTIPLE files at once. + * + * Two transports per file (chosen per file, sequential queue): + * direct — browser seals (SPENC2) and PUTs straight to R2 via a presigned + * URL; the api handles keys/metadata only. Chosen when the + * deployment enables it and the file fits the memory cap + * (sealing holds plaintext + frame in memory at once). + * server — multipart POST /files/groups/:groupId/upload (≤2 GiB api-side). + * + * A file that fails on the direct path falls back to the server path before + * being marked failed, and one file's failure never aborts the queue. */ const MAX_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024; const MAX_LABEL = "2 GiB"; +type QueueState = "queued" | "uploading" | "done" | "failed"; + +interface QueueItem { + id: string; + name: string; + size: number; + state: QueueState; + progress: number; + error: string | null; + file: File; +} + export function UploadDropzone({ groupId, live, onUploaded }: { groupId: string; live: boolean; @@ -25,57 +46,92 @@ export function UploadDropzone({ groupId, live, onUploaded }: { const { t } = useFilesLang(); const inputRef = useRef(null); const busyRef = useRef(false); + const configRef = useRef(undefined); const [dragging, setDragging] = useState(false); - const [uploading, setUploading] = useState(null); - const [progress, setProgress] = useState(0); + const [queue, setQueue] = useState([]); const [error, setError] = useState(null); - const disabled = !live || groupId === "" || uploading !== null; + const uploading = queue.some((item) => item.state === "uploading" || item.state === "queued"); + const disabled = !live || groupId === "" || uploading; - const register = useCallback( - async (file: File): Promise => { - if (disabled || busyRef.current) return; - if (file.size === 0) { - setError(t({ en: `${file.name}: the file is empty.`, es: `${file.name}: el archivo está vacío.` })); - return; - } - if (file.size > MAX_UPLOAD_BYTES) { - setError( - t({ - en: `${file.name} is ${formatBytes(file.size)} — the limit is ${MAX_LABEL}.`, - es: `${file.name} pesa ${formatBytes(file.size)} — el límite es ${MAX_LABEL}.`, - }), - ); - return; - } - setError(null); - busyRef.current = true; - setProgress(0); - setUploading(`${file.name} (${formatBytes(file.size)})`); + const patch = useCallback((id: string, changes: Partial): void => { + setQueue((current) => current.map((item) => (item.id === id ? { ...item, ...changes } : item))); + }, []); + + /** One file through whichever transport applies; direct failure falls back. */ + const uploadOne = useCallback(async (item: QueueItem, onProgress: (percent: number) => void): Promise => { + let config = configRef.current; + if (config === undefined) { + config = await fetchDirectConfig().catch(() => null); + configRef.current = config ?? null; + } + const directPossible = + config !== null && config.enabled && config.publicKeyJwk !== null && + item.size <= config.maxBytes && item.size <= DIRECT_MEMORY_CAP_BYTES; + if (directPossible && config !== null && config.publicKeyJwk !== null) { try { - const created = await apiUploadFile( - `/files/groups/${encodeURIComponent(groupId)}/upload`, - file, - setProgress, - ); + return await uploadFileDirect(groupId, item.file, config, onProgress); + } catch (directError) { + // Direct path is best-effort: fall back to the server upload, but say why. + setError(t({ + en: `${item.name}: direct upload failed (${errorMessage(directError)}) — uploading via server instead.`, + es: `${item.name}: la subida directa falló (${errorMessage(directError)}) — subiendo por el servidor.`, + })); + } + } + return apiUploadFile(`/files/groups/${encodeURIComponent(groupId)}/upload`, item.file, onProgress); + }, [groupId, t]); + + const runQueue = useCallback(async (items: QueueItem[]): Promise => { + busyRef.current = true; + for (const item of items) { + patch(item.id, { state: "uploading", progress: 0 }); + try { + const created = await uploadOne(item, (progress) => patch(item.id, { progress })); + patch(item.id, { state: "done", progress: 100 }); onUploaded(created); } catch (err) { - setError(errorMessage(err)); - } finally { - setUploading(null); - busyRef.current = false; + patch(item.id, { state: "failed", error: errorMessage(err) }); } - }, - [disabled, groupId, onUploaded, t], - ); + } + busyRef.current = false; + // Let the results stay visible briefly, then clear finished entries. + window.setTimeout(() => { + setQueue((current) => current.filter((entry) => entry.state === "uploading" || entry.state === "queued")); + }, 8000); + }, [onUploaded, patch, uploadOne]); + + const register = useCallback((files: File[]): void => { + if (disabled || files.length === 0) return; + setError(null); + const accepted: QueueItem[] = []; + for (const file of files) { + const id = `${file.name}:${file.size}:${Math.random().toString(36).slice(2, 8)}`; + if (file.size === 0) { + accepted.push({ id, name: file.name, size: 0, state: "failed", progress: 0, error: t({ en: "the file is empty.", es: "el archivo está vacío." }), file }); + continue; + } + if (file.size > MAX_UPLOAD_BYTES) { + accepted.push({ + id, name: file.name, size: file.size, state: "failed", progress: 0, + error: t({ en: `is ${formatBytes(file.size)} — the limit is ${MAX_LABEL}.`, es: `pesa ${formatBytes(file.size)} — el límite es ${MAX_LABEL}.` }), + file, + }); + continue; + } + accepted.push({ id, name: file.name, size: file.size, state: "queued", progress: 0, error: null, file }); + } + setQueue((current) => [...current, ...accepted]); + if (accepted.some((entry) => entry.state === "queued")) void runQueue(accepted); + }, [disabled, runQueue, t]); const hint = !live ? t({ en: "Offline — connect to the API to upload.", es: "Sin conexión — conéctate a la API para subir." }) : groupId === "" ? t({ en: "Choose a group first.", es: "Elige un grupo primero." }) : t({ - en: `Drop a file, paste one, or browse. Up to ${MAX_LABEL}; stored with re-verified bytes.`, - es: `Arrastra un archivo, pega uno o búscalo. Hasta ${MAX_LABEL}; guardado con bytes reverificados.`, + en: `Drop files, paste one, or browse. Multiple files welcome — up to ${MAX_LABEL} each; encrypted, stored in the hub's object storage.`, + es: `Arrastra archivos, pega uno o búscalos. Varios archivos a la vez — hasta ${MAX_LABEL} cada uno; cifrados en el almacenamiento del hub.`, }); return ( @@ -84,7 +140,7 @@ export function UploadDropzone({ groupId, live, onUploaded }: { role="button" tabIndex={disabled ? -1 : 0} aria-disabled={disabled} - aria-label={t({ en: "Upload a file", es: "Subir un archivo" })} + aria-label={t({ en: "Upload files", es: "Subir archivos" })} onClick={() => { if (!disabled) inputRef.current?.click(); }} @@ -104,8 +160,8 @@ export function UploadDropzone({ groupId, live, onUploaded }: { e.preventDefault(); setDragging(false); if (disabled) return; - const file = e.dataTransfer.files[0]; - if (file !== undefined) void register(file); + const files = Array.from(e.dataTransfer.files); + if (files.length > 0) register(files); }} className={`flex flex-col items-center justify-center gap-2 rounded-lg border border-dashed px-4 py-6 text-center transition-colors duration-150 ${ disabled @@ -115,20 +171,33 @@ export function UploadDropzone({ groupId, live, onUploaded }: { : "cursor-pointer border-border hover:border-accent hover:bg-surface-2" }`} > - {uploading !== null ? ( -
- - - {progress < 100 - ? t({ en: `Uploading ${progress}%`, es: `Subiendo ${progress}%` }) - : t({ en: "Upload sent — indexing…", es: "Subida enviada — indexando…" })}{" "} - {uploading} - - + {queue.length > 0 ? ( +
+ {queue.map((item) => ( +
+ {item.state === "uploading" ? ( + + ) : item.state === "done" ? ( + ✓ + ) : item.state === "failed" ? ( + ✗ + ) : ( + … + )} + + {item.name} ({formatBytes(item.size)}) + {item.state === "uploading" ? ` — ${item.progress}%` : ""} + {item.state === "failed" && item.error !== null ? ` — ${item.error}` : ""} + + {item.state === "uploading" && ( + + )} +
+ ))}
) : ( <> - {t({ en: "Drop a file here", es: "Suelta un archivo aquí" })} + {t({ en: "Drop files here", es: "Suelta archivos aquí" })} {hint} {!disabled && ( )} @@ -146,11 +215,12 @@ export function UploadDropzone({ groupId, live, onUploaded }: { { - const file = e.target.files?.[0]; + const files = Array.from(e.target.files ?? []); e.target.value = ""; - if (file !== undefined) void register(file); + if (files.length > 0) register(files); }} />
diff --git a/apps/web/src/components/video/upload-dropzone.tsx b/apps/web/src/components/video/upload-dropzone.tsx index 27c6133..f257d65 100644 --- a/apps/web/src/components/video/upload-dropzone.tsx +++ b/apps/web/src/components/video/upload-dropzone.tsx @@ -4,13 +4,17 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { Button } from "../ui/primitives"; import { Loading } from "../brand/loading"; import { apiUploadFile, errorMessage } from "../../lib/api-client"; +import { CHUNK_SIZE_BYTES, uploadVideoChunked } from "../../lib/chunked-upload"; import type { VideoStatusDto } from "../../lib/video-types"; import { formatBytes } from "./format"; /** - * Drag-drop + file-picker target for POST /video/upload. - * The API stores encrypted bytes, mints the content-addressed key, and probes - * the stored object before responding. Progress measures the actual transfer. + * Drag-drop + file-picker target for video ingest. Files at or under one chunk + * keep the single multipart POST /video/upload; larger ones go through the + * chunked API (init → PUT slices → complete) so no single request approaches + * Cloudflare's ~100 MB proxied-body cap. The API stores encrypted bytes, mints + * the content-addressed key, and probes the stored object before responding. + * Progress measures the actual transfer. */ const MAX_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024; @@ -46,7 +50,7 @@ export function UploadDropzone({ live, canUpload, onStarted, onSettled, onUpload return; } if (file.size === 0 || file.size > MAX_UPLOAD_BYTES) { - setError("Choose a non-empty video up to 100 MiB."); + setError("Choose a non-empty video up to 2 GiB."); return; } setError(null); @@ -55,7 +59,10 @@ export function UploadDropzone({ live, canUpload, onStarted, onSettled, onUpload setUploading(`${file.name} (${formatBytes(file.size)})`); onStarted(draft); try { - const result = await apiUploadFile("/video/upload", file, setProgress); + // Above one chunk the whole-body POST would trip Cloudflare's body cap. + const result = file.size > CHUNK_SIZE_BYTES + ? await uploadVideoChunked(file, setProgress) + : await apiUploadFile("/video/upload", file, setProgress); onUploaded(result); if (result.processingJob.state === "failed") setError(result.processingJob.error ?? "Video probing failed."); } catch (err) { @@ -85,7 +92,7 @@ export function UploadDropzone({ live, canUpload, onStarted, onSettled, onUpload ? "Offline — connect to the API to upload." : !canUpload ? "Uploading needs the media.manage permission." - : "Drop a video file, paste one, or browse. Up to 100 MiB; stored encrypted and checked before it is ready."; + : "Drop a video file, paste one, or browse. Up to 2 GiB — larger videos upload in 10 MiB chunks; stored encrypted and checked before it is ready."; return (
diff --git a/apps/web/src/lib/api-client.ts b/apps/web/src/lib/api-client.ts index f4f0d7b..857fd76 100644 --- a/apps/web/src/lib/api-client.ts +++ b/apps/web/src/lib/api-client.ts @@ -58,15 +58,34 @@ function messageFrom(body: unknown, fallback: string): string { return fallback; } -/** Multipart bytes with browser upload progress; shares the API's auth/error rules. */ -export async function apiUploadFile(path: string, file: File, onProgress: (percent: number) => void): Promise { +/** Byte payloads XHR can stream with upload progress. */ +type ByteBody = FormData | Blob; + +interface ByteRequest { + method: "POST" | "PUT"; + body: ByteBody; + /** + * Omitted for multipart, where the browser must own the boundary; set for + * raw bodies (chunked ingest slices are application/octet-stream). + */ + contentType?: string; + timeoutMs?: number; +} + +/** + * XHR transport shared by every byte upload (multipart and chunked slices) so + * credentials, the bearer header, upload progress, and the epoch-stale retry + * behave identically no matter which endpoint carries the bytes. + */ +async function apiSendBytes(path: string, request: ByteRequest, onProgress: (percent: number) => void): Promise { const send = async (retried: boolean): Promise => { const token = getSessionToken(); const response = await new Promise<{ status: number; text: string }>((resolve, reject) => { const xhr = new XMLHttpRequest(); - xhr.open("POST", buildUrl(path, undefined)); + xhr.open(request.method, buildUrl(path, undefined)); xhr.withCredentials = true; - xhr.timeout = 120_000; + xhr.timeout = request.timeoutMs ?? 120_000; + if (request.contentType !== undefined) xhr.setRequestHeader("Content-Type", request.contentType); if (token !== null) xhr.setRequestHeader("Authorization", `Bearer ${token}`); xhr.upload.onprogress = (event) => { if (event.lengthComputable) onProgress(Math.floor((event.loaded / event.total) * 100)); @@ -76,9 +95,7 @@ export async function apiUploadFile(path: string, file: File, onProgress: (pe xhr.onerror = () => reject(new ApiUnreachableError(new Error("Upload network error"))); xhr.ontimeout = () => reject(new Error("Upload timed out; check the video library before retrying.")); xhr.onabort = () => reject(new Error("Upload cancelled")); - const form = new FormData(); - form.append("file", file); - xhr.send(form); + xhr.send(request.body); }); let body: unknown = null; if (response.text !== "") { @@ -99,6 +116,21 @@ export async function apiUploadFile(path: string, file: File, onProgress: (pe return send(false); } +/** Multipart bytes with browser upload progress; shares the API's auth/error rules. */ +export async function apiUploadFile(path: string, file: File, onProgress: (percent: number) => void): Promise { + const form = new FormData(); + form.append("file", file); + return apiSendBytes(path, { method: "POST", body: form }, onProgress); +} + +/** + * Raw application/octet-stream PUT with browser upload progress — one slice of + * a chunked upload. Same auth, error, and retry rules as the multipart path. + */ +export function apiPutBytes(path: string, bytes: Blob, onProgress: (percent: number) => void): Promise { + return apiSendBytes(path, { method: "PUT", body: bytes, contentType: "application/octet-stream" }, onProgress); +} + export function reissueSession(): Promise { if (reissuePromise === null) { reissuePromise = apiFetch<{ token: string }>("/auth/reissue", { method: "POST", redirectOn401: false }) diff --git a/apps/web/src/lib/chunked-upload.test.ts b/apps/web/src/lib/chunked-upload.test.ts new file mode 100644 index 0000000..62c6dc6 --- /dev/null +++ b/apps/web/src/lib/chunked-upload.test.ts @@ -0,0 +1,218 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; +import { readFileSync } from "node:fs"; +import { API_URL, ApiError } from "./api-client"; +import { setSessionToken } from "./session-token"; +import { CHUNK_SIZE_BYTES, chunkBounds, uploadVideoChunked, wholeFilePercent } from "./chunked-upload"; +import type { VideoStatusDto } from "./video-types"; + +/** + * Chunked ingest wiring: init → sequential octet-stream PUTs → complete. + * Transport is stubbed at the same seams the existing control-room test uses + * (globalThis.fetch and a fake XHR), so the assertions cover the REAL + * api-client wrappers: bearer header, credentials, method, content-type, and + * the whole-file progress math. + */ + +type FetchCall = { url: string; init: RequestInit }; + +/** The XMLHttpRequest global is typed as the browser constructor; tests install doubles. */ +function setGlobalXHR(ctor: unknown): void { + (globalThis as { XMLHttpRequest: unknown }).XMLHttpRequest = ctor; +} + + +class FakeXHR { + static instances: FakeXHR[] = []; + method = ""; + url = ""; + body: Blob | FormData | null = null; + withCredentials = false; + timeout = 0; + headers: Record = {}; + status = 200; + responseText = "{}"; + upload: { onprogress: ((event: { lengthComputable: boolean; loaded: number; total: number }) => void) | null; onload: (() => void) | null } = { onprogress: null, onload: null }; + onload: (() => void) | null = null; + onerror: (() => void) | null = null; + ontimeout: (() => void) | null = null; + onabort: (() => void) | null = null; + + constructor() { FakeXHR.instances.push(this); } + + open(method: string, url: string): void { this.method = method; this.url = url; } + setRequestHeader(key: string, value: string): void { this.headers[key.toLowerCase()] = value; } + send(body: Blob | FormData): void { + this.body = body; + // Half of the slice, then the completed slice — mirrors real xhr.upload events. + queueMicrotask(() => { + const size = body instanceof Blob ? body.size : 0; + this.upload.onprogress?.({ lengthComputable: true, loaded: Math.floor(size / 2), total: size }); + this.upload.onload?.(); + this.onload?.(); + }); + } +} + +const videoStatus: VideoStatusDto = { + videoAsset: { id: "va-1", workspaceId: "ws", storageKey: "spenc1/key", mimeType: "video/mp4", sizeBytes: 2500 }, + processingJob: { id: "job-1", workspaceId: "ws", videoAssetId: "va-1", state: "complete" }, +}; + +/** Server-driven plan: 2500 bytes in 1000-byte chunks → 1000 + 1000 + 500. */ +async function stubHappyPath(events: string[]): Promise<() => void> { + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + events.push(`fetch:${url.replace(API_URL ?? "", "")}`); + if (url.endsWith("/video/upload/init")) { + const body = JSON.parse(String(init?.body)) as { filename: string; mimeType: string; totalBytes: number }; + events.push(`init:${body.filename}:${body.mimeType}:${body.totalBytes}`); + return Response.json({ uploadId: "upl-1", chunkSizeBytes: 1000, totalChunks: 3 }); + } + if (url.endsWith("/video/upload/upl-1/complete")) return Response.json(videoStatus); + return Response.json({ message: `unexpected fetch ${url}` }, { status: 500 }); + }) as typeof fetch; + return () => { globalThis.fetch = originalFetch; }; +} + +test("chunk bounds cover the file exactly; every chunk is full except the last", () => { + assert.deepEqual(chunkBounds(2500, 0, 1000), { start: 0, end: 1000 }); + assert.deepEqual(chunkBounds(2500, 1, 1000), { start: 1000, end: 2000 }); + assert.deepEqual(chunkBounds(2500, 2, 1000), { start: 2000, end: 2500 }); + assert.deepEqual(chunkBounds(1000, 0, 1000), { start: 0, end: 1000 }); // exactly one chunk + // whole-file percent accumulates across chunks and never exceeds 100 + assert.equal(wholeFilePercent(0, 500, 2500), 20); + assert.equal(wholeFilePercent(1000, 500, 2500), 60); + assert.equal(wholeFilePercent(2500, 0, 2500), 100); + assert.equal(wholeFilePercent(2500, 999, 2500), 100); // clamped + assert.equal(wholeFilePercent(0, 0, 0), 0); // degenerate file never divides by zero +}); + +test("client CHUNK_SIZE_BYTES mirrors apps/api upload-limits.ts", () => { + assert.equal(CHUNK_SIZE_BYTES, 10 * 1024 * 1024); + const candidates = ["../api/src/video/upload-limits.ts", "../../api/src/video/upload-limits.ts"]; + const source = candidates.map((p) => { try { return readFileSync(p, "utf8"); } catch { return null; } }).find((s) => s !== null); + if (typeof source === "string") { + const match = /export const CHUNK_SIZE_BYTES = ([^;]+);/.exec(source); + assert.ok(match, "server CHUNK_SIZE_BYTES not found"); + // eslint-disable-next-line no-new-func + assert.equal(Function(`"use strict";return (${match[1]!});`)(), CHUNK_SIZE_BYTES); + } +}); + +test("upload runs init → sequential chunk PUTs → complete with the same auth as the multipart path", async () => { + setSessionToken("tok-1"); + const events: string[] = []; + const restore = await stubHappyPath(events); + const originalXHR = globalThis.XMLHttpRequest; + FakeXHR.instances = []; + setGlobalXHR(FakeXHR); + const progress: number[] = []; + const bytes = new Uint8Array(2500).map((_, i) => i % 251); + const file = new File([bytes], "clip.mp4", { type: "video/mp4" }); + try { + const result = await uploadVideoChunked(file, (percent) => progress.push(percent)); + assert.deepEqual(result, videoStatus); + + // init + complete went through fetch with JSON bodies, bearer auth, cookies + assert.deepEqual(events, [ + "fetch:/video/upload/init", "init:clip.mp4:video/mp4:2500", + "fetch:/video/upload/upl-1/complete", + ]); + + // strictly sequential PUTs, index order preserved + const puts = FakeXHR.instances; + assert.equal(puts.length, 3); + assert.deepEqual(puts.map((x) => x.method), ["PUT", "PUT", "PUT"]); + assert.deepEqual(puts.map((x) => x.url.replace(API_URL ?? "", "")), [ + "/video/upload/upl-1/chunk/0", + "/video/upload/upl-1/chunk/1", + "/video/upload/upl-1/chunk/2", + ]); + for (const put of puts) { + assert.equal(put.headers["content-type"], "application/octet-stream"); + assert.equal(put.headers.authorization, "Bearer tok-1"); + assert.equal(put.withCredentials, true); + assert.ok(put.body instanceof Blob); + } + assert.deepEqual(puts.map((put) => (put.body as Blob).size), [1000, 1000, 500]); + + // slices reassemble to the original file, byte for byte + const sent = new Uint8Array(await Promise.all(puts.map(async (put) => new Uint8Array(await (put.body as Blob).arrayBuffer()))).then((parts) => parts.flatMap((p) => Array.from(p)))); + assert.deepEqual(Array.from(sent), Array.from(bytes)); + + // progress is monotonic, spans chunk boundaries, and lands on 100 + assert.ok(progress.length > 0); + assert.equal(progress.at(-1)!, 100); + assert.equal(progress[0]!, 20); // half of chunk 0 of 3 + assert.ok(progress.includes(60)); // half of chunk 1: 1000 done + 500 in flight + assert.ok(progress.includes(90)); + for (let i = 1; i < progress.length; i += 1) assert.ok(progress[i]! >= progress[i - 1]!, `progress went backwards: ${progress.join(",")}`); + } finally { + setGlobalXHR(originalXHR); + await restore(); + setSessionToken(null); + } +}); + +test("a failed chunk aborts the upload: later chunks and complete are never sent", async () => { + setSessionToken("tok-1"); + const events: string[] = []; + const restore = await stubHappyPath(events); + const originalXHR = globalThis.XMLHttpRequest; + class FailingXHR extends FakeXHR { + override send(body: Blob | FormData): void { + this.body = body; + queueMicrotask(() => { + if (this.url.endsWith("/chunk/1")) { + this.status = 500; + this.responseText = JSON.stringify({ message: "disk full" }); + } + this.upload.onload?.(); + this.onload?.(); + }); + } + } + FakeXHR.instances = []; + setGlobalXHR(FailingXHR); + try { + const file = new File([new Uint8Array(2500)], "clip.mp4", { type: "video/mp4" }); + await assert.rejects( + uploadVideoChunked(file, () => undefined), + (err: unknown) => { + assert.ok(err instanceof ApiError); + assert.equal(err.status, 500); + assert.equal(err.message, "disk full"); + return true; + }, + ); + assert.equal(FakeXHR.instances.length, 2); // chunk 2 never left the browser + assert.ok(!events.some((e) => e.includes("complete"))); + } finally { + setGlobalXHR(originalXHR); + await restore(); + setSessionToken(null); + } +}); + +test("an init plan that disagrees with the client is rejected before any bytes move", async () => { + setSessionToken("tok-1"); + const originalFetch = globalThis.fetch; + globalThis.fetch = (async () => Response.json({ uploadId: "upl-2", chunkSizeBytes: 1000, totalChunks: 4 })) as typeof fetch; + const originalXHR = globalThis.XMLHttpRequest; + FakeXHR.instances = []; + setGlobalXHR(FakeXHR); + try { + const file = new File([new Uint8Array(2500)], "clip.mp4", { type: "video/mp4" }); + await assert.rejects( + uploadVideoChunked(file, () => undefined), + /server expects 4 chunks, client computed 3/, + ); + assert.equal(FakeXHR.instances.length, 0); + } finally { + globalThis.fetch = originalFetch; + setGlobalXHR(originalXHR); + setSessionToken(null); + } +}); diff --git a/apps/web/src/lib/chunked-upload.ts b/apps/web/src/lib/chunked-upload.ts new file mode 100644 index 0000000..b1ed629 --- /dev/null +++ b/apps/web/src/lib/chunked-upload.ts @@ -0,0 +1,71 @@ +import { api, apiPutBytes } from "./api-client"; +import type { VideoStatusDto } from "./video-types"; + +/** + * Client half of the chunked ingest protocol — see apps/api/src/video/upload-limits.ts + * and the /video/upload/init | chunk | complete routes in VideoController. + * + * Files at or under one chunk stay on the single multipart POST /video/upload; + * anything larger is sliced here so no single request approaches Cloudflare's + * ~100 MB proxied-body cap. Kept as a pure lib module (no React) so the + * sequencing and progress math are unit-testable with node:test. + */ + +/** + * Client mirror of CHUNK_SIZE_BYTES in apps/api/src/video/upload-limits.ts. + * It only picks the path (whole-body vs chunked); the server's chunkSizeBytes + * from /video/upload/init drives the actual slicing. + */ +export const CHUNK_SIZE_BYTES = 10 * 1024 * 1024; + +/** Response of POST /video/upload/init. */ +export interface ChunkedUploadInit { + uploadId: string; + chunkSizeBytes: number; + totalChunks: number; +} + +/** Byte offset → slice bounds for one chunk; every chunk is full except the last. */ +export function chunkBounds(totalBytes: number, index: number, chunkSizeBytes: number): { start: number; end: number } { + return { start: index * chunkSizeBytes, end: Math.min((index + 1) * chunkSizeBytes, totalBytes) }; +} + +/** Upload progress as a whole-file percentage: sent bytes over total bytes, clamped to 100. */ +export function wholeFilePercent(completedBytes: number, inFlightBytes: number, totalBytes: number): number { + if (totalBytes <= 0) return 0; + return Math.min(100, Math.floor(((completedBytes + inFlightBytes) / totalBytes) * 100)); +} + +/** + * Chunked ingest for files too large for one proxied request: init, then one + * application/octet-stream PUT per slice — strictly sequential, so chunk order + * is guaranteed — then complete, which runs the same seal/probe path as + * POST /video/upload and returns the same VideoStatusDto shape. + * + * Any failure rejects and abandons the upload (no resume in v1); server-side + * TTL sweeping reclaims the partial file. Progress is reported as + * (completedBytes + bytes of the slice in flight) / totalBytes. + */ +export async function uploadVideoChunked(file: File, onProgress: (percent: number) => void): Promise { + const init = await api.post("/video/upload/init", { + filename: file.name, + mimeType: file.type, + totalBytes: file.size, + }); + const chunkSizeBytes = init.chunkSizeBytes > 0 ? init.chunkSizeBytes : CHUNK_SIZE_BYTES; + const totalChunks = Math.ceil(file.size / chunkSizeBytes); + if (init.totalChunks !== totalChunks) { + throw new Error(`Upload rejected: server expects ${init.totalChunks} chunks, client computed ${totalChunks}.`); + } + let completedBytes = 0; + for (let index = 0; index < totalChunks; index += 1) { + const { start, end } = chunkBounds(file.size, index, chunkSizeBytes); + const slice = file.slice(start, end); + await apiPutBytes(`/video/upload/${init.uploadId}/chunk/${index}`, slice, (percent) => { + onProgress(wholeFilePercent(completedBytes, (slice.size * percent) / 100, file.size)); + }); + completedBytes += slice.size; + onProgress(wholeFilePercent(completedBytes, 0, file.size)); + } + return api.post(`/video/upload/${init.uploadId}/complete`, {}); +} diff --git a/apps/web/src/lib/direct-storage.test.ts b/apps/web/src/lib/direct-storage.test.ts new file mode 100644 index 0000000..88bae82 --- /dev/null +++ b/apps/web/src/lib/direct-storage.test.ts @@ -0,0 +1,44 @@ +/** + * Cross-implementation proof for SPENC2: the web seal (WebCrypto) must open + * with the server reader (node:crypto) and vice versa. Run under tsx/node + * (WebCrypto is global in node 22). + */ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; +import { generateKeyPairSync } from "node:crypto"; +import { sealForDirect, openDirectFrame, unwrapDirectCek } from "../../../api/src/storage/storage.js"; +import { sealDirect, openDirectFrameBytes, bytesToBase64 } from "./direct-storage.js"; + +const { publicKey, privateKey } = generateKeyPairSync("ec", { namedCurve: "prime256v1" }); +const jwk = publicKey.export({ format: "jwk" }) as { kty: "EC"; crv: "P-256"; x: string; y: string }; + +test("web sealDirect → server openDirectFrame roundtrip (25 MiB spans 3 segments)", async () => { + const plaintext = new Uint8Array(25 * 1024 * 1024 + 7); + for (let i = 0; i < plaintext.length; i += 4096) plaintext[i] = i % 251; + const { frame, sha256 } = await sealDirect(plaintext, jwk); + const opened = openDirectFrame(Buffer.from(frame), privateKey); + assert.equal(opened.length, plaintext.length); + assert.equal(Buffer.from(opened).equals(Buffer.from(plaintext)), true); + assert.equal(sha256.length, 64); +}); + +test("server sealForDirect → web openDirectFrameBytes roundtrip", async () => { + const plaintext = Buffer.from("server-sealed bytes for the browser to open"); + const frame = sealForDirect(plaintext, jwk); + const cek = unwrapDirectCek(frame, privateKey); + const opened = await openDirectFrameBytes(new Uint8Array(frame), new Uint8Array(cek)); + assert.equal(Buffer.from(opened).equals(plaintext), true); +}); + +test("tampered segment ciphertext throws on open", async () => { + const plaintext = new Uint8Array(1024); + const { frame } = await sealDirect(plaintext, jwk); + const copy = Buffer.from(frame); + copy[copy.length - 20] ^= 0xff; + assert.throws(() => openDirectFrame(copy, privateKey)); +}); + +test("bytesToBase64 matches node's base64", () => { + const bytes = new Uint8Array([0, 1, 2, 250, 251, 252, 253, 254, 255]); + assert.equal(bytesToBase64(bytes), Buffer.from(bytes).toString("base64")); +}); diff --git a/apps/web/src/lib/direct-storage.ts b/apps/web/src/lib/direct-storage.ts new file mode 100644 index 0000000..fdedaa7 --- /dev/null +++ b/apps/web/src/lib/direct-storage.ts @@ -0,0 +1,200 @@ +/** + * Direct-to-R2 client-side encryption (WebCrypto ONLY — no dependencies). + * + * Frame format "SPENC2", all integers big-endian, mirrors the server reader + * in apps/api/src/storage/storage.ts byte for byte: + * + * [0..6) magic ASCII "SPENC2" + * [6..71) ephemeral P-256 public key, uncompressed: 0x04 || X(32) || Y(32) + * [71..83) wrapNonce, 12 random bytes + * [83..131) wrappedCek: AES-256-GCM over the 32-byte CEK; + * key = HKDF-SHA256(ikm=ECDH(ephPriv, serverPub), salt empty, + * info ASCII "tmbh-direct-cek-wrap", L=32); nonce = wrapNonce + * → 48 bytes (32 ct + 16 tag) + * [131..135) segmentCount, u32 + * [135..] segments, in order: ctLen u32 | iv 12 | AES-256-GCM ciphertext + * WITH the 16-byte tag appended (WebCrypto default), key = CEK, no AAD + * + * Segment plaintext size: 10 MiB except the last. + * + * The API never sees payload bytes: it mints a presigned PUT for the sealed + * frame, and later a presigned GET plus the unwrapped CEK for download. + */ + +export const DIRECT_FRAME_VERSION = 2; +const SEGMENT_PLAINTEXT_BYTES = 10 * 1024 * 1024; +const WRAP_INFO = new TextEncoder().encode("tmbh-direct-cek-wrap"); +const FRAME_MAGIC = new TextEncoder().encode("SPENC2"); + +/** EC public key JWK as served by GET /files/direct/config. */ +export interface DirectPublicJwk { + kty: "EC"; + crv: "P-256"; + x: string; + y: string; +} + +/** --- small typed helpers over Buffer-free Uint8Array math ----------------- */ + + +function concat(parts: readonly Uint8Array[]): Uint8Array { + const total = parts.reduce((sum, part) => sum + part.byteLength, 0); + const out = new Uint8Array(total); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.byteLength; + } + return out; +} + +function u32be(value: number): Uint8Array { + if (!Number.isSafeInteger(value) || value < 0 || value > 0xffff_ffff) throw new Error("direct-storage: u32 out of range"); + return new Uint8Array([(value >>> 24) & 0xff, (value >>> 16) & 0xff, (value >>> 8) & 0xff, value & 0xff]); +} + +function readU32be(bytes: Uint8Array, offset: number): number { + if (offset + 4 > bytes.length) throw new Error("direct-storage: truncated u32"); + return ((bytes[offset]! << 24) | (bytes[offset + 1]! << 16) | (bytes[offset + 2]! << 8) | bytes[offset + 3]!) >>> 0; +} + +function bytesToHex(bytes: Uint8Array): string { + let out = ""; + for (const byte of bytes) out += byte.toString(16).padStart(2, "0"); + return out; +} + +function toBufferSource(bytes: Uint8Array): ArrayBuffer { + // Copy into an exact-length ArrayBuffer: WebCrypto rejects views whose + // byteOffset/length do not describe the whole backing buffer on some engines. + const copy = new ArrayBuffer(bytes.byteLength); + new Uint8Array(copy).set(bytes); + return copy; +} + +async function aesGcmEncrypt(key: CryptoKey, iv: Uint8Array, plaintext: Uint8Array): Promise { + return new Uint8Array(await crypto.subtle.encrypt({ name: "AES-GCM", iv: toBufferSource(iv) }, key, toBufferSource(plaintext))); +} + +async function aesGcmDecrypt(key: CryptoKey, iv: Uint8Array, sealed: Uint8Array): Promise { + return new Uint8Array(await crypto.subtle.decrypt({ name: "AES-GCM", iv: toBufferSource(iv) }, key, toBufferSource(sealed))); +} + +// --- SPENC2 header offsets (kept numeric: the format is fixed by contract) --- + +const EPHEMERAL_PUBLIC_OFFSET = 6; +const WRAP_NONCE_OFFSET = 71; +const WRAPPED_CEK_OFFSET = 83; +const SEGMENT_COUNT_OFFSET = 131; +const SEGMENTS_OFFSET = 135; +const WRAPPED_CEK_BYTES = 48; // 32-byte CEK + 16-byte GCM tag + +/** CEK-wrap key: ECDH(ephPriv, serverPub) → HKDF-SHA256(salt ∅, info, L=32). */ +async function directWrapKey(ephemeralPrivate: CryptoKey, serverPublic: CryptoKey): Promise { + const ikm = new Uint8Array(await crypto.subtle.deriveBits({ name: "ECDH", public: serverPublic }, ephemeralPrivate, 256)); + const hkdf = await crypto.subtle.importKey("raw", toBufferSource(ikm), "HKDF", false, ["deriveBits"]); + // WebCrypto HKDF treats an empty salt as HashLen zero bytes (RFC 5869 §2.2). + return new Uint8Array( + await crypto.subtle.deriveBits( + { name: "HKDF", hash: "SHA-256", salt: new Uint8Array(0), info: toBufferSource(WRAP_INFO) }, + hkdf, + 256, + ), + ); +} + +/** + * Seal plaintext for direct upload. Returns the SPENC2 frame and the + * plaintext sha256 (hex) the complete call attests. + */ +export async function sealDirect(bytes: Uint8Array, publicJwk: DirectPublicJwk): Promise<{ frame: Uint8Array; sha256: string }> { + if (bytes.byteLength === 0) throw new Error("direct-storage: nothing to seal"); + const serverPublic = await crypto.subtle.importKey( + "jwk", + { ...publicJwk, ext: true }, + { name: "ECDH", namedCurve: "P-256" }, + false, + [], + ); + // A fresh CEK + ephemeral key per frame: the server can unwrap, but the + // stored object alone (or two uploads of the same plaintext) never lines up. + const cek = crypto.getRandomValues(new Uint8Array(32)); + const ephemeral = await crypto.subtle.generateKey({ name: "ECDH", namedCurve: "P-256" }, true, ["deriveBits"]); + const wrapKey = await directWrapKey(ephemeral.privateKey, serverPublic); + const wrapNonce = crypto.getRandomValues(new Uint8Array(12)); + const wrapAes = await crypto.subtle.importKey("raw", toBufferSource(wrapKey), { name: "AES-GCM" }, false, ["encrypt"]); + const wrappedCek = await aesGcmEncrypt(wrapAes, wrapNonce, cek); + // exportKey("raw") on an EC public key yields 0x04 || X || Y — exactly the frame field. + const ephemeralPoint = new Uint8Array(await crypto.subtle.exportKey("raw", ephemeral.publicKey)); + + const cekAes = await crypto.subtle.importKey("raw", toBufferSource(cek), { name: "AES-GCM" }, false, ["encrypt"]); + const segments: Uint8Array[] = []; + for (let offset = 0; offset < bytes.byteLength; offset += SEGMENT_PLAINTEXT_BYTES) { + const slice = bytes.subarray(offset, Math.min(offset + SEGMENT_PLAINTEXT_BYTES, bytes.byteLength)); + const iv = crypto.getRandomValues(new Uint8Array(12)); + const ciphertext = await aesGcmEncrypt(cekAes, iv, slice); + segments.push(concat([u32be(ciphertext.byteLength), iv, ciphertext])); + } + + const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", toBufferSource(bytes))); + const frame = concat([ + FRAME_MAGIC, + ephemeralPoint, + wrapNonce, + wrappedCek, + u32be(segments.length), + ...segments, + ]); + return { frame, sha256: bytesToHex(digest) }; +} + +/** + * Open a downloaded SPENC2 frame. `cekRaw` is the unwrapped CEK the API + * returned (base64 there, decoded by the caller). Throws on any tampering — + * every segment's GCM tag is verified. + */ +export async function openDirectFrameBytes(blob: Uint8Array, cekRaw: Uint8Array): Promise { + if (blob.byteLength < SEGMENTS_OFFSET) throw new Error("direct-storage: frame is truncated"); + if (blob.byteLength < WRAPPED_CEK_OFFSET + WRAPPED_CEK_BYTES) throw new Error("direct-storage: frame is truncated"); + const magic = blob.subarray(0, 6); + for (let i = 0; i < FRAME_MAGIC.length; i += 1) { + if (magic[i] !== FRAME_MAGIC[i]) throw new Error("direct-storage: not a SPENC2 frame"); + } + if (cekRaw.byteLength !== 32) throw new Error("direct-storage: CEK must be 32 bytes"); + const cek = await crypto.subtle.importKey("raw", toBufferSource(cekRaw), { name: "AES-GCM" }, false, ["decrypt"]); + const segmentCount = readU32be(blob, SEGMENT_COUNT_OFFSET); + if (segmentCount === 0) throw new Error("direct-storage: frame has no segments"); + + const parts: Uint8Array[] = []; + let offset = SEGMENTS_OFFSET; + for (let index = 0; index < segmentCount; index += 1) { + if (offset + 4 > blob.byteLength) throw new Error("direct-storage: segment table is truncated"); + const ctLen = readU32be(blob, offset); + offset += 4; + if (ctLen < 16 || offset + 12 + ctLen > blob.byteLength) throw new Error("direct-storage: segment length is out of range"); + const iv = blob.subarray(offset, offset + 12); + offset += 12; + const ciphertext = blob.subarray(offset, offset + ctLen); + offset += ctLen; + parts.push(await aesGcmDecrypt(cek, iv, ciphertext)); + } + if (offset !== blob.byteLength) throw new Error("direct-storage: frame has trailing bytes"); + const plaintext = concat(parts); + if (plaintext.byteLength === 0) throw new Error("direct-storage: frame decrypts to zero bytes"); + return plaintext; +} + +/** Bytes -> base64 (used for the CEK the download endpoint hands back). */ +export function bytesToBase64(bytes: Uint8Array): string { + let binary = ""; + for (const byte of bytes) binary += String.fromCharCode(byte); + return btoa(binary); +} + +/** base64 -> bytes (browser `atob`; the API returns standard base64). */ +export function base64ToBytes(value: string): Uint8Array { + const binary = atob(value); + const out = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i += 1) out[i] = binary.charCodeAt(i); + return out; +} diff --git a/apps/web/src/lib/files-direct.ts b/apps/web/src/lib/files-direct.ts new file mode 100644 index 0000000..3513e70 --- /dev/null +++ b/apps/web/src/lib/files-direct.ts @@ -0,0 +1,111 @@ +/** + * Direct-to-R2 flows for the Files tab: the api mints presigned URLs and + * unwraps CEKs; payload bytes go browser → R2 → browser, sealed client-side + * (SPENC2 — see ./direct-storage.ts). The server never sees plaintext. + */ +import { api } from "./api-client"; +import { sealDirect, openDirectFrameBytes, base64ToBytes, type DirectPublicJwk } from "./direct-storage"; +import type { GroupFile } from "../components/files/types"; + +export interface DirectUploadConfig { + enabled: boolean; + frameVersion: 2; + publicKeyJwk: DirectPublicJwk | null; + maxBytes: number; +} + +/** + * Browser-side ceiling for the direct path: sealing holds the plaintext and + * the frame in memory at once. Above this the UI falls back to the server + * path (which is itself Cloudflare-capped around 100 MB for now). + */ +export const DIRECT_MEMORY_CAP_BYTES = 512 * 1024 * 1024; + +export async function fetchDirectConfig(): Promise { + return api.get("/files/direct/config"); +} + +/** POST /files/direct/complete response is the created GroupFile. */ +export async function uploadFileDirect( + groupId: string, + file: File, + config: DirectUploadConfig, + onProgress: (percent: number) => void, +): Promise { + if (config.publicKeyJwk === null) throw new Error("direct upload is not enabled"); + const bytes = new Uint8Array(await file.arrayBuffer()); + const { frame, sha256 } = await sealDirect(bytes, config.publicKeyJwk); + const init = await api.post<{ uploadId: string; objectKey: string; putUrl: string }>("/files/direct/init", { + groupId, + filename: file.name, + mimeType: file.type === "" ? "application/octet-stream" : file.type, + sizeBytes: frame.byteLength, + }); + await putToR2(init.putUrl, frame, onProgress); + return api.post("/files/direct/complete", { + uploadId: init.uploadId, + objectKey: init.objectKey, + groupId, + filename: file.name, + mimeType: file.type === "" ? "application/octet-stream" : file.type, + sizeBytes: frame.byteLength, + sha256, + }); +} + +/** + * Presigned R2 PUT — same-origin rules do not apply (different host), the + * signature lives in the query string, and the ONLY header the signature + * covers is content-type, which must match the presign exactly. + */ +function putToR2(url: string, frame: Uint8Array, onProgress: (percent: number) => void): Promise { + return new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.open("PUT", url); + xhr.setRequestHeader("Content-Type", "application/octet-stream"); + xhr.upload.onprogress = (event) => { + if (event.lengthComputable) onProgress(Math.floor((event.loaded / event.total) * 100)); + }; + xhr.onload = () => { + if (xhr.status >= 200 && xhr.status < 300) resolve(); + else reject(new Error(`direct upload failed (HTTP ${xhr.status})`)); + }; + xhr.onerror = () => reject(new Error("direct upload failed — network error")); + xhr.send(new Blob([frame.slice()], { type: "application/octet-stream" })); + }); +} + +export interface DirectDownloadTicket { + url: string; + cekB64: string; + frame: "SPENC2"; + sizeBytes: number; +} + +/** + * Download a direct object: ticket (presigned GET + CEK) → fetch from R2 → + * decrypt in the browser → hand back the plaintext. The api never streams + * the bytes. + */ +export async function downloadFileDirect(itemId: string): Promise { + const ticket = await api.post(`/files/${encodeURIComponent(itemId)}/direct-download`, {}); + const response = await fetch(ticket.url); + if (!response.ok) throw new Error(`direct download failed (HTTP ${response.status})`); + const sealed = new Uint8Array(await response.arrayBuffer()); + return openDirectFrameBytes(sealed, base64ToBytes(ticket.cekB64)); +} + +export function isDirectObject(item: GroupFile): boolean { + return item.file.storageKey?.startsWith("direct/") === true; +} + +export function saveBytes(bytes: Uint8Array, fileName: string): void { + const url = URL.createObjectURL(new Blob([bytes.slice()])); + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = fileName; + document.body.append(anchor); + anchor.click(); + anchor.remove(); + URL.revokeObjectURL(url); +} diff --git a/infra/occupymedia/docker-compose.app.yml b/infra/occupymedia/docker-compose.app.yml index 34a5117..5caed5e 100644 --- a/infra/occupymedia/docker-compose.app.yml +++ b/infra/occupymedia/docker-compose.app.yml @@ -82,6 +82,9 @@ services: FILES_OCID_USER: ${FILES_OCID_USER:?} FILES_OCID_PASSWORD: ${FILES_OCID_PASSWORD:?} FILES_PUBLIC_BASE: ${FILES_PUBLIC_BASE:-https://files.occupymedia.cc} + # Direct-to-R2 client-sealed uploads (SPENC2): the api holds the EC private + # key that unwraps client CEKs; value lives in the compose .env. + DIRECT_EC_PRIVATE_KEY: ${DIRECT_EC_PRIVATE_KEY:-} ARCHIVE_PROVISIONER_HOMESERVER_URL: ${ARCHIVE_PROVISIONER_HOMESERVER_URL:-} ARCHIVE_PROVISIONER_USER: ${ARCHIVE_PROVISIONER_USER:-} ARCHIVE_PROVISIONER_PASSWORD: ${ARCHIVE_PROVISIONER_PASSWORD:-}