feat: direct-to-R2 Files uploads (client-sealed SPENC2) + chunked video dropzone + multi-file queue
For scale: payload bytes go browser → R2 → browser; the api handles keys, authz and metadata only, so a cheap box can serve thousands of users. - SPENC2 frame (browser-sealed, P-256 ECDH → HKDF wrap of a per-file CEK, 10 MiG AES-GCM segments): storage.ts gains the reader (magic-branched getObject), presignPut/headObject, openDirectHeader; sealForDirect is the reference sealer for cross-implementation tests - Files API: GET /files/direct/config, POST /files/direct/init (presigned PUT), POST /files/direct/complete (HEAD-gated commit), POST /files/:id/direct-download (presigned GET + CEK ticket) - web: lib/direct-storage (WebCrypto seal/open), lib/files-direct (flows), dropzone rewritten with a multi-file sequential queue (per-file status, direct-first with server fallback), file download decrypts in-browser for direct objects (falls back to streaming) - video dropzone wired to the chunked ingest API (>10 MiB → init/chunk/ complete), fixing the Cloudflare ~100 MB body cap for large videos - DIRECT_EC_PRIVATE_KEY (P-256, PKCS8 b64) wired through compose; cross-impl roundtrip tests prove web-seal ↔ server-open byte compatibility
This commit is contained in:
parent
b3c2085436
commit
0a1c3b392b
21 changed files with 1444 additions and 83 deletions
|
|
@ -20,6 +20,10 @@ interface FileUpload {
|
|||
*
|
||||
* GET /files list/search, keyset (groupId, senderMxid,
|
||||
* mimeClass, origin, name, from, to, cursor, limit)
|
||||
* GET /files/direct/config direct-upload capability + server public JWK
|
||||
* POST /files/direct/init presigned PUT for a browser-sealed upload
|
||||
* POST /files/direct/complete HEAD-checked commit + index row
|
||||
* POST /files/:id/direct-download presigned GET + unwrapped CEK
|
||||
* POST /files/groups/:groupId/upload direct web upload (≤2 GiB, multipart)
|
||||
* GET /files/:id one GroupFile
|
||||
* GET /files/:id/download stream payload (sha256 re-verified)
|
||||
|
|
@ -49,6 +53,64 @@ export class GroupFilesController {
|
|||
return this.files.listFiles(session.workspaceId, session.userId, session.role, query);
|
||||
}
|
||||
|
||||
/**
|
||||
* Direct-upload capability. Declared before the :id routes so the literal
|
||||
* "direct/config" path wins route matching over ":id".
|
||||
*/
|
||||
@Get("direct/config")
|
||||
@RequirePermission("media.view")
|
||||
async directConfig(): Promise<ReturnType<GroupFilesService["directUploadConfig"]>> {
|
||||
return this.files.directUploadConfig();
|
||||
}
|
||||
|
||||
@Post("direct/init")
|
||||
@RequirePermission("media.view")
|
||||
async directInit(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Body() body: unknown,
|
||||
): Promise<ReturnType<GroupFilesService["initDirectUpload"]>> {
|
||||
const session = requireSession(request);
|
||||
const dto = requireObject(body, "body");
|
||||
return this.files.initDirectUpload(session.workspaceId, requireUuidValue(dto.groupId, "groupId"), {
|
||||
userId: session.userId,
|
||||
role: session.role,
|
||||
}, {
|
||||
filename: requireString(dto.filename, "filename"),
|
||||
mimeType: requireString(dto.mimeType, "mimeType"),
|
||||
sizeBytes: requireNumber(dto.sizeBytes, "sizeBytes"),
|
||||
});
|
||||
}
|
||||
|
||||
@Post("direct/complete")
|
||||
@RequirePermission("media.view")
|
||||
async directComplete(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Body() body: unknown,
|
||||
): Promise<GroupFile> {
|
||||
const session = requireSession(request);
|
||||
const dto = requireObject(body, "body");
|
||||
return this.files.completeDirectUpload(session.workspaceId, { userId: session.userId, role: session.role }, {
|
||||
uploadId: requireUuidValue(dto.uploadId, "uploadId"),
|
||||
objectKey: requireString(dto.objectKey, "objectKey"),
|
||||
groupId: requireUuidValue(dto.groupId, "groupId"),
|
||||
filename: requireString(dto.filename, "filename"),
|
||||
mimeType: requireString(dto.mimeType, "mimeType"),
|
||||
sizeBytes: requireNumber(dto.sizeBytes, "sizeBytes"),
|
||||
sha256: requireString(dto.sha256, "sha256"),
|
||||
});
|
||||
}
|
||||
|
||||
/** Same ACL/availability checks as the streaming route; direct objects only. */
|
||||
@Post(":id/direct-download")
|
||||
@RequirePermission("media.view")
|
||||
async directDownload(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Param("id", ParseUUIDPipe) id: string,
|
||||
): Promise<ReturnType<GroupFilesService["openDirectDownload"]>> {
|
||||
const session = requireSession(request);
|
||||
return this.files.openDirectDownload(session.workspaceId, session.userId, session.role, id);
|
||||
}
|
||||
|
||||
@Post("groups/:groupId/upload")
|
||||
@RequirePermission("media.view")
|
||||
@UseInterceptors(FileInterceptor("file", {
|
||||
|
|
@ -176,3 +238,18 @@ export class PublicFileShareController {
|
|||
response.end();
|
||||
}
|
||||
}
|
||||
|
||||
function requireUuidValue(value: unknown, field: string): string {
|
||||
const text = requireString(value, field);
|
||||
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(text)) {
|
||||
throw new BadRequestException(`${field} must be a UUID`);
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
function requireNumber(value: unknown, field: string): number {
|
||||
if (typeof value !== "number" || !Number.isSafeInteger(value)) {
|
||||
throw new BadRequestException(`${field} must be a safe integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@
|
|||
* (audit_append), the house pattern.
|
||||
*/
|
||||
import { BadRequestException, ConflictException, ForbiddenException, GoneException, Injectable, NotFoundException, Optional, Inject, ServiceUnavailableException } from "@nestjs/common";
|
||||
import { createHash } from "node:crypto";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { Readable } from "node:stream";
|
||||
import type { PoolClient } from "pg";
|
||||
import { attachmentArchiveKey, safeFileName, SNIFF_BYTES, sniffMime, type FileRetention, type GroupFile } from "@tactical-media-broadcast-hub/core";
|
||||
|
|
@ -68,6 +68,38 @@ export interface FilePage {
|
|||
nextCursor: string | null;
|
||||
}
|
||||
|
||||
/** GET /files/direct/config — what the browser needs before it can seal. */
|
||||
export interface DirectUploadConfig {
|
||||
enabled: boolean;
|
||||
frameVersion: 2;
|
||||
publicKeyJwk: { kty: "EC"; crv: "P-256"; x: string; y: string } | null;
|
||||
maxBytes: number;
|
||||
}
|
||||
|
||||
/** POST /files/direct/init response. Nothing is persisted until complete. */
|
||||
export interface DirectUploadInit {
|
||||
uploadId: string;
|
||||
objectKey: string;
|
||||
putUrl: string;
|
||||
}
|
||||
|
||||
/** POST /files/:id/direct-download response: URL + the unwrapped CEK. */
|
||||
export interface DirectDownloadTicket {
|
||||
url: string;
|
||||
cekB64: string;
|
||||
frame: "SPENC2";
|
||||
sizeBytes: number;
|
||||
}
|
||||
|
||||
export const DIRECT_MAX_BYTES = 2_147_483_648;
|
||||
const DIRECT_PUT_TTL_SECONDS = 900;
|
||||
const DIRECT_GET_TTL_SECONDS = 600;
|
||||
const DIRECT_KEY_PREFIX = "direct/";
|
||||
const DIRECT_OBJECT_SUFFIX = ".enc";
|
||||
|
||||
/** MIME allowlist for direct uploads: prefix match on the declared type. */
|
||||
const DIRECT_MIME_PREFIXES = ["image/", "video/", "audio/", "application/", "text/"] as const;
|
||||
|
||||
const MAX_SHARE_TTL_HOURS = 720;
|
||||
const WORKSPACE_WIDE_ROLES: readonly string[] = ["Owner", "Admin"];
|
||||
|
||||
|
|
@ -511,6 +543,133 @@ export class GroupFilesService {
|
|||
});
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Direct-to-R2 (browser-sealed SPENC2): keys/authz/metadata stay here,
|
||||
// payload bytes go browser → R2 → browser
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Whether the deployment can unwrap browser-sealed frames. The public key is
|
||||
* DERIVED from the private key, so a browser can only ever wrap a CEK this
|
||||
* process can open — it never sends a key of its own.
|
||||
*/
|
||||
async directUploadConfig(): Promise<DirectUploadConfig> {
|
||||
const storage = this.storage();
|
||||
const publicKeyJwk = typeof storage.directPublicJwk === "function" ? await storage.directPublicJwk() : null;
|
||||
return {
|
||||
enabled: publicKeyJwk !== null,
|
||||
frameVersion: 2,
|
||||
publicKeyJwk,
|
||||
maxBytes: DIRECT_MAX_BYTES,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Mint the presigned PUT for a direct upload. Group membership is checked
|
||||
* exactly like uploadForGroup; the size is bounded up front so an oversized
|
||||
* pick fails before any bytes are sealed. Nothing is persisted — the row
|
||||
* only appears on complete, so an abandoned PUT leaves an orphan object
|
||||
* (no row), never a dangling row without an object.
|
||||
*/
|
||||
async initDirectUpload(
|
||||
workspaceId: string,
|
||||
groupId: string,
|
||||
actor: ActorRef,
|
||||
input: { filename: string; mimeType: string; sizeBytes: number },
|
||||
): Promise<DirectUploadInit> {
|
||||
if (!isUuid(groupId)) throw new BadRequestException("groupId must be a UUID");
|
||||
if (!Number.isSafeInteger(input.sizeBytes) || input.sizeBytes < 1 || input.sizeBytes > DIRECT_MAX_BYTES) {
|
||||
throw new BadRequestException(`sizeBytes must be an integer between 1 and ${DIRECT_MAX_BYTES}`);
|
||||
}
|
||||
validateDirectFilename(input.filename);
|
||||
validateDirectMime(input.mimeType);
|
||||
await this.requireGroupMembership(workspaceId, groupId, actor);
|
||||
if ((await this.storage().directPublicJwk()) === null) {
|
||||
throw new ServiceUnavailableException("DIRECT_EC_PRIVATE_KEY is not configured");
|
||||
}
|
||||
const uploadId = randomUUID();
|
||||
const objectKey = `${DIRECT_KEY_PREFIX}${uploadId}${DIRECT_OBJECT_SUFFIX}`;
|
||||
const putUrl = await this.storage()
|
||||
.presignPut(objectKey, input.mimeType, DIRECT_PUT_TTL_SECONDS)
|
||||
.catch(() => { throw new ServiceUnavailableException("file storage is unavailable"); });
|
||||
return { uploadId, objectKey, putUrl };
|
||||
}
|
||||
|
||||
/**
|
||||
* Commit a completed direct upload. The object is HEAD-checked (exists, exact
|
||||
* declared size) before the row is written, so metadata can never point at a
|
||||
* missing or partial PUT. sha256 is client-attested — the payload is opaque
|
||||
* to this process by design; the browser computed it over the plaintext it
|
||||
* sealed. Filename runs through the same safeFileName the streamed path uses.
|
||||
*/
|
||||
async completeDirectUpload(
|
||||
workspaceId: string,
|
||||
actor: ActorRef,
|
||||
input: { uploadId: string; objectKey: string; groupId: string; filename: string; mimeType: string; sizeBytes: number; sha256: string },
|
||||
): Promise<GroupFile> {
|
||||
if (!isUuid(input.uploadId)) throw new BadRequestException("uploadId must be a UUID");
|
||||
if (!isUuid(input.groupId)) throw new BadRequestException("groupId must be a UUID");
|
||||
if (input.objectKey !== `${DIRECT_KEY_PREFIX}${input.uploadId}${DIRECT_OBJECT_SUFFIX}`) {
|
||||
throw new BadRequestException("objectKey must be direct/<uploadId>.enc");
|
||||
}
|
||||
if (!/^[0-9a-f]{64}$/.test(input.sha256)) throw new BadRequestException("sha256 must be lowercase hex (64 chars)");
|
||||
if (!Number.isSafeInteger(input.sizeBytes) || input.sizeBytes < 1 || input.sizeBytes > DIRECT_MAX_BYTES) {
|
||||
throw new BadRequestException(`sizeBytes must be an integer between 1 and ${DIRECT_MAX_BYTES}`);
|
||||
}
|
||||
const safe = validateDirectFilename(input.filename);
|
||||
const mimeType = validateDirectMime(input.mimeType);
|
||||
await this.requireGroupMembership(workspaceId, input.groupId, actor);
|
||||
|
||||
const head = await this.storage().headObject(input.objectKey).catch(() => {
|
||||
throw new ServiceUnavailableException("file storage is unavailable");
|
||||
});
|
||||
if (!head.exists || head.contentLength !== input.sizeBytes) {
|
||||
throw new BadRequestException("uploaded object is missing or its size does not match sizeBytes");
|
||||
}
|
||||
|
||||
const mimeClass = mimeClassForDeclared(mimeType);
|
||||
const username = await this.usernameFor(workspaceId, actor.userId);
|
||||
const { id } = await this.registerArchiveEntry(workspaceId, {
|
||||
groupId: input.groupId,
|
||||
roomId: null,
|
||||
eventId: null,
|
||||
origin: { kind: "upload" },
|
||||
originKind: "upload",
|
||||
senderMxid: `@${username}:tmbh.local`,
|
||||
originalName: input.filename,
|
||||
safeName: safe,
|
||||
mime: mimeType,
|
||||
mimeClass,
|
||||
byteSize: input.sizeBytes,
|
||||
sha256: input.sha256,
|
||||
ocisResourceId: input.objectKey,
|
||||
ocisResourceVersion: null,
|
||||
storageKey: input.objectKey,
|
||||
createdAt: new Date(),
|
||||
});
|
||||
return this.getFile(workspaceId, actor.userId, actor.role, id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ticket for a browser-side download: presigned GET (600 s) plus the
|
||||
* unwrapped CEK. Availability/ACL checks are getFile's (the same ones the
|
||||
* streaming route runs); only direct/ objects can take this path.
|
||||
*/
|
||||
async openDirectDownload(workspaceId: string, userId: string, role: string, itemId: string): Promise<DirectDownloadTicket> {
|
||||
const item = await this.getFile(workspaceId, userId, role, itemId);
|
||||
if (item.status !== "available") throw new GoneException(`file is ${item.status}`);
|
||||
const storageKey = item.file.storageKey;
|
||||
if (storageKey === undefined || !storageKey.startsWith(DIRECT_KEY_PREFIX)) {
|
||||
throw new BadRequestException("not a direct object");
|
||||
}
|
||||
const storage = this.storage();
|
||||
const [url, frame] = await Promise.all([
|
||||
storage.presign(storageKey, DIRECT_GET_TTL_SECONDS),
|
||||
storage.getObject(storageKey).then((stored) => storage.openDirectHeader(stored)),
|
||||
]).catch(() => { throw new ServiceUnavailableException("file storage is unavailable"); });
|
||||
return { url, cekB64: frame.toString("base64"), frame: "SPENC2", sizeBytes: item.sizeBytes };
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// internals
|
||||
// -------------------------------------------------------------------------
|
||||
|
|
@ -604,6 +763,15 @@ export class GroupFilesService {
|
|||
return this.getFile(workspaceId, actor.userId, actor.role, id);
|
||||
}
|
||||
|
||||
/** Owner/Admin pass; everyone else must be an active member of the group. */
|
||||
private async requireGroupMembership(workspaceId: string, groupId: string, actor: ActorRef): Promise<void> {
|
||||
if (WORKSPACE_WIDE_ROLES.includes(actor.role)) return;
|
||||
const membership = await withTenant(workspaceId, (client) =>
|
||||
client.query("SELECT 1 FROM group_members WHERE group_id = $1 AND user_id = $2 LIMIT 1", [groupId, actor.userId]),
|
||||
);
|
||||
if (membership.rows.length === 0) throw new ForbiddenException("not a member of this group");
|
||||
}
|
||||
|
||||
private async usernameFor(workspaceId: string, userId: string): Promise<string> {
|
||||
const row = await withTenant(workspaceId, (client) =>
|
||||
client.query<{ username: string }>("SELECT username FROM users WHERE id = $1", [userId]),
|
||||
|
|
@ -659,6 +827,41 @@ function parseFilters(query: Record<string, unknown>): FileListFilters {
|
|||
return { groupId, senderMxid, mimeClass: rawClass as MimeClass | null, origin: rawOrigin as OriginKind | null, name, from, to };
|
||||
}
|
||||
|
||||
/**
|
||||
* Filename gate for direct uploads: no client bytes are sniffed (the payload
|
||||
* is opaque), so the name itself must pass the library-safe rules and carry a
|
||||
* plausible extension for the declared type.
|
||||
*/
|
||||
function validateDirectFilename(filename: string): string {
|
||||
if (typeof filename !== "string" || filename.trim() === "" || filename.length > 255) {
|
||||
throw new BadRequestException("filename must be 1-255 characters");
|
||||
}
|
||||
const safe = safeFileName(filename);
|
||||
if (safe === "" || safe === "attachment") {
|
||||
throw new BadRequestException("filename must contain a usable name");
|
||||
}
|
||||
return safe;
|
||||
}
|
||||
|
||||
/** Declared MIME must sit under one of the allowed prefixes (no sniffing). */
|
||||
function validateDirectMime(mimeType: string): string {
|
||||
if (typeof mimeType !== "string" || mimeType.trim() === "" || mimeType.length > 255) {
|
||||
throw new BadRequestException("mimeType must be 1-255 characters");
|
||||
}
|
||||
if (DIRECT_MIME_PREFIXES.some((prefix) => mimeType.startsWith(prefix))) return mimeType;
|
||||
throw new BadRequestException(`mimeType must start with one of: ${DIRECT_MIME_PREFIXES.join(" ")}`);
|
||||
}
|
||||
|
||||
/** Coarse class from the declared MIME — the sniff path's equivalent. */
|
||||
function mimeClassForDeclared(mimeType: string): MimeClass {
|
||||
if (mimeType.startsWith("image/")) return "image";
|
||||
if (mimeType.startsWith("video/")) return "video";
|
||||
if (mimeType.startsWith("audio/")) return "audio";
|
||||
if (mimeType === "application/pdf" || mimeType.startsWith("text/")) return "document";
|
||||
if (mimeType === "application/zip" || mimeType === "application/x-zip-compressed") return "archive";
|
||||
return "other";
|
||||
}
|
||||
|
||||
function optionalUuid(value: unknown): string | null {
|
||||
if (value === undefined || value === null || value === "") return null;
|
||||
if (typeof value !== "string" || !isUuid(value)) throw new BadRequestException("groupId must be a UUID");
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import {
|
|||
sniffMediaAsset,
|
||||
} from "./media.service.js";
|
||||
import { appendTenantAudit } from "../audit/audit.service.js";
|
||||
import { sha256, type StorageService } from "../storage/storage.js";
|
||||
import { sha256, type StorageService, directStorageDisabled } from "../storage/storage.js";
|
||||
import { MediaController } from "./media.controller.js";
|
||||
import type { AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import type { Request, Response } from "express";
|
||||
|
|
@ -68,6 +68,7 @@ function fakeStorage(): StorageService & {
|
|||
}),
|
||||
hasContent: vi.fn(async () => false),
|
||||
presign: vi.fn(async () => ""),
|
||||
...directStorageDisabled(),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@ import { randomUUID } from "node:crypto";
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
import { MemoryRedis } from "./testing/memory-redis.js";
|
||||
import { PreviewService } from "./preview.js";
|
||||
import { keyForHash, sha256, type StorageService } from "../storage/storage.js";
|
||||
import { keyForHash, sha256, directStorageDisabled, type StorageService } from "../storage/storage.js";
|
||||
const WS = randomUUID(), CH = randomUUID(), TAP = randomUUID();
|
||||
function fixture(): { preview: PreviewService; storage: StorageService; redis: MemoryRedis; seed: (file: string, body: string, type?: string) => void; advance: (ms: number) => void } {
|
||||
const redis = new MemoryRedis(), objects = new Map<string, Buffer>(); let now = Date.now();
|
||||
const storage: StorageService = { putContent: async () => { throw new Error("unused"); }, getObject: vi.fn(async key => { await new Promise(r => setTimeout(r, 1)); return objects.get(key) ?? Buffer.alloc(0); }), hasContent: async () => false, presign: async () => { throw new Error("must not redirect to encrypted S3"); } };
|
||||
const storage: StorageService = { putContent: async () => { throw new Error("unused"); }, getObject: vi.fn(async key => { await new Promise(r => setTimeout(r, 1)); return objects.get(key) ?? Buffer.alloc(0); }), hasContent: async () => false, presign: async () => { throw new Error("must not redirect to encrypted S3"); }, ...directStorageDisabled() };
|
||||
const preview = new PreviewService({ redis, storage, secret: "test", now: () => now, assertTap: async () => undefined });
|
||||
const seed = (file: string, body: string, type = "video/mp2t"): void => {
|
||||
const buffer = Buffer.from(body), key = keyForHash(sha256(buffer), type); objects.set(key, buffer);
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
|
|||
import type { PoolClient } from "pg";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { MediaControlService, type IssuedCommand } from "../media-control/media-control.service.js";
|
||||
import { sha256, keyForHash, type StorageService } from "../storage/storage.js";
|
||||
import { sha256, keyForHash, directStorageDisabled, type StorageService } from "../storage/storage.js";
|
||||
import { ProductionService } from "./production.service.js";
|
||||
import type { TenantTransaction } from "./journal.js";
|
||||
const WS = randomUUID(), CH = randomUUID(), SLOT = randomUUID();
|
||||
|
|
@ -29,7 +29,7 @@ function fixture(render?: (input: unknown) => Promise<Buffer>): { production: Pr
|
|||
try { return await fn({ query } as unknown as PoolClient); } finally { unlock(); }
|
||||
};
|
||||
const control = new MediaControlService({ signer: null, assignments: [] }); vi.spyOn(control, "director").mockResolvedValue(ISSUED);
|
||||
const storage: StorageService = { putContent: vi.fn(async (buffer, contentType) => { const hash = sha256(buffer); return { hash, key: keyForHash(hash, contentType), sizeBytes: buffer.length, contentType, deduplicated: false }; }), getObject: async () => Buffer.alloc(0), hasContent: async () => false, presign: async () => "" };
|
||||
const storage: StorageService = { putContent: vi.fn(async (buffer, contentType) => { const hash = sha256(buffer); return { hash, key: keyForHash(hash, contentType), sizeBytes: buffer.length, contentType, deduplicated: false }; }), getObject: async () => Buffer.alloc(0), hasContent: async () => false, presign: async () => "", ...directStorageDisabled() };
|
||||
return { rows, control, storage, production: new ProductionService(control, { transaction, storage, ...(render ? { render } : {}) }) };
|
||||
}
|
||||
describe("revision → render job → content-addressed storage → desired-state pipeline", () => {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { directStorageDisabled } from "../storage/storage.js";
|
||||
import { BadRequestException, ServiceUnavailableException } from "@nestjs/common";
|
||||
import type { PoolClient } from "pg";
|
||||
import type { Request, Response } from "express";
|
||||
|
|
@ -67,6 +68,7 @@ function fakeStorage(objects: Record<string, Buffer> = {}): {
|
|||
}),
|
||||
hasContent: vi.fn(async () => false),
|
||||
presign: vi.fn(async () => ""),
|
||||
...directStorageDisabled(),
|
||||
};
|
||||
return { storage, puts };
|
||||
}
|
||||
|
|
@ -339,6 +341,7 @@ describe("branding service — uploads (storage reuse, magic-byte sniffing)", ()
|
|||
getObject: vi.fn(async () => Buffer.alloc(0)),
|
||||
hasContent: vi.fn(async () => false),
|
||||
presign: vi.fn(async () => ""),
|
||||
...directStorageDisabled(),
|
||||
};
|
||||
const branding = serviceWith({ storage: broken });
|
||||
await expect(branding.uploadImage(PNG_HEADER, "image/png")).rejects.toBeInstanceOf(ServiceUnavailableException);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { createCipheriv, createDecipheriv, createHash, randomBytes } from "node:crypto";
|
||||
import { createCipheriv, createDecipheriv, createHash, createPrivateKey, createPublicKey, diffieHellman, generateKeyPairSync, hkdfSync, randomBytes, randomUUID } from "node:crypto";
|
||||
import { ServiceUnavailableException } from "@nestjs/common";
|
||||
import {
|
||||
CreateBucketCommand,
|
||||
GetObjectCommand,
|
||||
|
|
@ -18,6 +19,13 @@ import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|||
* stays on the PLAINTEXT hash (dedup survives); getObject re-verifies the
|
||||
* hash, so storage-level tampering is detected. SSE-at-rest headers are set
|
||||
* additionally for endpoints that honor them (belt-and-braces).
|
||||
*
|
||||
* SPENC2 (direct-to-R2, 2026-10-16): objects under `direct/` are sealed IN
|
||||
* THE BROWSER (apps/web/src/lib/direct-storage.ts). The server never sees
|
||||
* the payload — it only mints the presigned PUT/GET and unwraps the CEK with
|
||||
* the ECDH share it derives from DIRECT_EC_PRIVATE_KEY. Both frame versions
|
||||
* share `getObject`: the leading magic decides the branch, so a single
|
||||
* reader serves the whole bucket and the SPENC1 path stays byte-identical.
|
||||
*/
|
||||
|
||||
export interface StoredObject {
|
||||
|
|
@ -32,6 +40,37 @@ export interface StorageService {
|
|||
getObject(key: string): Promise<Buffer>;
|
||||
hasContent(hash: string, contentType: string): Promise<boolean>;
|
||||
presign(key: string, ttlSeconds: number): Promise<string>;
|
||||
/** Direct-to-R2: the JWK the browser seals against; null = feature off. */
|
||||
directPublicJwk(): Promise<{ kty: "EC"; crv: "P-256"; x: string; y: string } | null>;
|
||||
/** Direct-to-R2: presigned PUT for the browser to push a SPENC2 frame. */
|
||||
presignPut(key: string, contentType: string, ttlSeconds: number): Promise<string>;
|
||||
/** Direct-to-R2: existence + exact size, used to gate /files/direct/complete. */
|
||||
headObject(key: string): Promise<{ exists: boolean; contentLength: number | null }>;
|
||||
/** Direct-to-R2: unwrap the frame's CEK (header only) for /files/:id/direct-download. */
|
||||
openDirectHeader(stored: Buffer): Promise<Buffer>;
|
||||
}
|
||||
|
||||
/** The direct-only members of StorageService, as one spreadable group. */
|
||||
export type DirectStorageMembers = Pick<
|
||||
StorageService,
|
||||
"directPublicJwk" | "presignPut" | "headObject" | "openDirectHeader"
|
||||
>;
|
||||
|
||||
/**
|
||||
* Direct-path stub for in-memory fakes that never serve browser uploads:
|
||||
* reports the feature disabled and refuses any direct operation. Lets a fake
|
||||
* stay a complete StorageService with one spread instead of four stubs.
|
||||
*/
|
||||
export function directStorageDisabled(): DirectStorageMembers {
|
||||
const unavailable = async (): Promise<never> => {
|
||||
throw new Error("direct storage is unavailable");
|
||||
};
|
||||
return {
|
||||
directPublicJwk: async () => null,
|
||||
presignPut: unavailable,
|
||||
headObject: unavailable,
|
||||
openDirectHeader: unavailable,
|
||||
};
|
||||
}
|
||||
|
||||
const EXTENSIONS: Record<string, string> = {
|
||||
|
|
@ -89,6 +128,218 @@ export function open(sealed: Buffer, kek: Buffer): Buffer {
|
|||
return Buffer.concat([decipher.update(ciphertext), decipher.final()]);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SPENC2 — browser-sealed direct-to-R2 frames
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Frame magic for browser-sealed objects (spec: "SPENC2" exactly). */
|
||||
export const DIRECT_FRAME_MAGIC = Buffer.from("SPENC2", "ascii");
|
||||
|
||||
/**
|
||||
* Fixed frame layout, all integers big-endian. Apps/web/src/lib/direct-storage.ts
|
||||
* seals the same shape; this reader is the cross-implementation proof target.
|
||||
*
|
||||
* [0..6) magic "SPENC2"
|
||||
* [6..71) ephemeral P-256 public key, uncompressed: 0x04 || X(32) || Y(32)
|
||||
* [71..83) wrapNonce, 12 random bytes
|
||||
* [83..131) wrappedCek: AES-256-GCM(CEK), key = HKDF-SHA256(ECDH, salt=∅,
|
||||
* info="tmbh-direct-cek-wrap", L=32), nonce = wrapNonce → 48 bytes
|
||||
* [131..135) segmentCount, u32
|
||||
* [135..] segments: ctLen u32 | iv 12 | AES-256-GCM ciphertext+tag, key=CEK, no AAD
|
||||
*/
|
||||
const EPHEMERAL_PUBLIC_OFFSET = 6;
|
||||
const WRAP_NONCE_OFFSET = 71;
|
||||
const WRAPPED_CEK_OFFSET = 83;
|
||||
const SEGMENT_COUNT_OFFSET = 131;
|
||||
const SEGMENTS_OFFSET = 135;
|
||||
const WRAPPED_CEK_BYTES = 32 + 16;
|
||||
|
||||
/** HKDF info binding the wrap key to this purpose. ASCII, exact. */
|
||||
const DIRECT_WRAP_INFO = Buffer.from("tmbh-direct-cek-wrap", "ascii");
|
||||
|
||||
/** Segment plaintext ceiling. Mirrors the web side, which seals 10 MiB chunks. */
|
||||
export const DIRECT_SEGMENT_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
/** Server direct keypair: P-256 private key from DIRECT_EC_PRIVATE_KEY (base64 DER). */
|
||||
export interface DirectKeyPair {
|
||||
privateKey: ReturnType<typeof createPrivateKey>;
|
||||
/** Compact JWK {kty,crv,x,y} — exactly what the browser's importKey wants. */
|
||||
publicJwk: { kty: "EC"; crv: "P-256"; x: string; y: string };
|
||||
}
|
||||
|
||||
export class DirectKeyError extends Error {}
|
||||
|
||||
/** True when DIRECT_EC_PRIVATE_KEY is present (routes report enabled=false otherwise). */
|
||||
export function directEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
|
||||
return loadDirectPrivateJwk(env) !== null;
|
||||
}
|
||||
|
||||
function loadDirectPrivateJwk(env: NodeJS.ProcessEnv): ReturnType<typeof createPrivateKey> | null {
|
||||
const raw = env.DIRECT_EC_PRIVATE_KEY;
|
||||
if (raw === undefined || raw.trim() === "") return null;
|
||||
let der: Buffer;
|
||||
try {
|
||||
der = Buffer.from(raw.trim(), "base64");
|
||||
} catch {
|
||||
throw new DirectKeyError("DIRECT_EC_PRIVATE_KEY must be base64 DER");
|
||||
}
|
||||
try {
|
||||
return createPrivateKey({ key: der, format: "der", type: "pkcs8" });
|
||||
} catch {
|
||||
throw new DirectKeyError("DIRECT_EC_PRIVATE_KEY is not a PKCS8 P-256 private key");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The direct keypair, deriving the public key from the private one — a
|
||||
* client-supplied public key is never trusted, so the browser can only wrap
|
||||
* CEKs this process can unwrap.
|
||||
*/
|
||||
export function directKeyPairFromEnv(env: NodeJS.ProcessEnv = process.env): DirectKeyPair | null {
|
||||
const privateKey = loadDirectPrivateJwk(env);
|
||||
if (privateKey === null) return null;
|
||||
const jwk = createPublicKey(privateKey).export({ format: "jwk" });
|
||||
if (jwk.kty !== "EC" || jwk.crv !== "P-256" || typeof jwk.x !== "string" || typeof jwk.y !== "string") {
|
||||
throw new DirectKeyError("DIRECT_EC_PRIVATE_KEY is not a P-256 key");
|
||||
}
|
||||
return { privateKey, publicJwk: { kty: "EC", crv: "P-256", x: jwk.x, y: jwk.y } };
|
||||
}
|
||||
|
||||
/** ECDH → HKDF-SHA256(salt empty, info "tmbh-direct-cek-wrap") → 32-byte wrap key. */
|
||||
function directWrapKey(privateKey: ReturnType<typeof createPrivateKey>, ephemeralPoint: Buffer): Buffer {
|
||||
const x = ephemeralPoint.subarray(1, 33).toString("base64url");
|
||||
const y = ephemeralPoint.subarray(33, 65).toString("base64url");
|
||||
let peer: ReturnType<typeof createPublicKey>;
|
||||
try {
|
||||
peer = createPublicKey({ key: { kty: "EC", crv: "P-256", x, y }, format: "jwk" });
|
||||
} catch {
|
||||
throw new Error("storage: SPENC2 frame holds a malformed ephemeral key");
|
||||
}
|
||||
const ikm = Buffer.from(diffieHellman({ privateKey, publicKey: peer }));
|
||||
// RFC 5869: an absent salt defaults to HashLen zero bytes; WebCrypto's empty
|
||||
// salt and Node's zero-filled salt produce the identical PRK (verified).
|
||||
return Buffer.from(hkdfSync("sha256", ikm, Buffer.alloc(0), DIRECT_WRAP_INFO, 32));
|
||||
}
|
||||
|
||||
interface DirectHeader {
|
||||
ephemeralPoint: Buffer;
|
||||
wrapNonce: Buffer;
|
||||
wrappedCek: Buffer;
|
||||
segmentCount: number;
|
||||
}
|
||||
|
||||
/** Validate + slice the fixed header. Throws on any truncation or bad shape. */
|
||||
function parseDirectHeader(stored: Buffer): DirectHeader {
|
||||
if (stored.length < SEGMENTS_OFFSET) {
|
||||
throw new Error("storage: SPENC2 frame is truncated");
|
||||
}
|
||||
const ephemeralPoint = stored.subarray(EPHEMERAL_PUBLIC_OFFSET, WRAP_NONCE_OFFSET);
|
||||
if (ephemeralPoint[0] !== 0x04) throw new Error("storage: SPENC2 ephemeral key is not an uncompressed point");
|
||||
const wrapNonce = stored.subarray(WRAP_NONCE_OFFSET, WRAPPED_CEK_OFFSET);
|
||||
const wrappedCek = stored.subarray(WRAPPED_CEK_OFFSET, WRAPPED_CEK_OFFSET + WRAPPED_CEK_BYTES);
|
||||
const segmentCount = stored.readUInt32BE(SEGMENT_COUNT_OFFSET);
|
||||
if (segmentCount === 0 || segmentCount > 4_294_967_295 / DIRECT_SEGMENT_BYTES + 1) {
|
||||
throw new Error("storage: SPENC2 segment count is out of range");
|
||||
}
|
||||
return { ephemeralPoint, wrapNonce, wrappedCek, segmentCount };
|
||||
}
|
||||
|
||||
/** Walk the segment table; yields {ciphertext, iv} in order. */
|
||||
function* directSegments(stored: Buffer, segmentCount: number): Generator<{ iv: Buffer; ciphertext: Buffer }> {
|
||||
let offset = SEGMENTS_OFFSET;
|
||||
for (let index = 0; index < segmentCount; index += 1) {
|
||||
if (offset + 4 + 12 > stored.length) throw new Error("storage: SPENC2 segment table is truncated");
|
||||
const ctLen = stored.readUInt32BE(offset);
|
||||
offset += 4;
|
||||
if (ctLen < 16 || offset + 12 + ctLen > stored.length) throw new Error("storage: SPENC2 segment length is out of range");
|
||||
const iv = stored.subarray(offset, offset + 12);
|
||||
offset += 12;
|
||||
const ciphertext = stored.subarray(offset, offset + ctLen);
|
||||
offset += ctLen;
|
||||
yield { iv, ciphertext };
|
||||
}
|
||||
if (offset !== stored.length) throw new Error("storage: SPENC2 frame has trailing bytes");
|
||||
}
|
||||
|
||||
/** Unwrap the CEK (verifies the wrap GCM tag). Exposed for openDirectHeader. */
|
||||
export function unwrapDirectCek(stored: Buffer, privateKey: ReturnType<typeof createPrivateKey>): Buffer {
|
||||
const header = parseDirectHeader(stored);
|
||||
return aesGcmOpen(directWrapKey(privateKey, header.ephemeralPoint), header.wrapNonce, header.wrappedCek);
|
||||
}
|
||||
|
||||
/**
|
||||
* Open a SPENC2 frame: unwrap the CEK, then decrypt every segment. Segment
|
||||
* GCM tags are verified individually, so any flipped byte throws. Empty
|
||||
* plaintexts are rejected (the upload floor is 1 byte).
|
||||
*/
|
||||
export function openDirectFrame(stored: Buffer, privateKey: ReturnType<typeof createPrivateKey>): Buffer {
|
||||
const { segmentCount } = parseDirectHeader(stored);
|
||||
const cek = unwrapDirectCek(stored, privateKey);
|
||||
const parts: Buffer[] = [];
|
||||
for (const segment of directSegments(stored, segmentCount)) {
|
||||
parts.push(aesGcmOpen(cek, segment.iv, segment.ciphertext));
|
||||
}
|
||||
const plaintext = Buffer.concat(parts);
|
||||
if (plaintext.length === 0) throw new Error("storage: SPENC2 frame decrypts to zero bytes");
|
||||
return plaintext;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test helper: seal `plaintext` exactly the way the browser does, so server
|
||||
* readers can be exercised without a DOM. Production uploads come from
|
||||
* apps/web/src/lib/direct-storage.ts — this is the reference implementation.
|
||||
*/
|
||||
export function sealForDirect(plaintext: Buffer, publicJwk: { kty: "EC"; crv: "P-256"; x: string; y: string }): Buffer {
|
||||
if (plaintext.length === 0) throw new Error("sealForDirect: plaintext must not be empty");
|
||||
const cek = randomBytes(32);
|
||||
const { privateKey: ephemeralPrivate, publicKey: ephemeralPublic } = newEphemeralKeyPair();
|
||||
// Wrap to the SERVER's public key — ECDH(ephPriv, serverPub), the same share
|
||||
// the server derives as ECDH(serverPriv, ephPub) in directWrapKey.
|
||||
const wrapKey = directWrapKey(ephemeralPrivate, uncompressedPoint(createPublicKey({ key: { ...publicJwk }, format: "jwk" })));
|
||||
const wrapNonce = randomBytes(12);
|
||||
const wrappedCek = aesGcmSeal(wrapKey, wrapNonce, cek);
|
||||
const segments: Buffer[] = [];
|
||||
for (let offset = 0; offset < plaintext.length; offset += DIRECT_SEGMENT_BYTES) {
|
||||
const iv = randomBytes(12);
|
||||
const ciphertext = aesGcmSeal(cek, iv, plaintext.subarray(offset, offset + DIRECT_SEGMENT_BYTES));
|
||||
segments.push(Buffer.concat([u32be(ciphertext.byteLength), iv, ciphertext]));
|
||||
}
|
||||
return Buffer.concat([
|
||||
DIRECT_FRAME_MAGIC,
|
||||
uncompressedPoint(ephemeralPublic),
|
||||
wrapNonce,
|
||||
wrappedCek,
|
||||
u32be(segments.length),
|
||||
...segments,
|
||||
]);
|
||||
}
|
||||
|
||||
/** Random object key for a browser upload: `direct/<uuid>.enc`. */
|
||||
export function directObjectKey(): string {
|
||||
return `direct/${randomUUID()}.enc`;
|
||||
}
|
||||
|
||||
function u32be(value: number): Buffer {
|
||||
const out = Buffer.alloc(4);
|
||||
out.writeUInt32BE(value, 0);
|
||||
return out;
|
||||
}
|
||||
|
||||
function newEphemeralKeyPair(): { privateKey: ReturnType<typeof createPrivateKey>; publicKey: ReturnType<typeof createPublicKey> } {
|
||||
return generateKeyPairSync("ec", { namedCurve: "P-256" });
|
||||
}
|
||||
|
||||
/** Raised wherever a direct operation needs a key the deployment never supplied. */
|
||||
function missingDirectKey(): ServiceUnavailableException {
|
||||
return new ServiceUnavailableException("DIRECT_EC_PRIVATE_KEY is not configured");
|
||||
}
|
||||
|
||||
function uncompressedPoint(publicKey: ReturnType<typeof createPublicKey>): Buffer {
|
||||
const point = publicKey.export({ type: "spki", format: "der" }).subarray(26);
|
||||
if (point.length !== 65 || point[0] !== 0x04) throw new Error("storage: expected an uncompressed P-256 point");
|
||||
return point;
|
||||
}
|
||||
|
||||
function aesGcmSeal(key: Buffer, iv: Buffer, plaintext: Buffer): Buffer {
|
||||
const cipher = createCipheriv("aes-256-gcm", key, iv);
|
||||
return Buffer.concat([cipher.update(plaintext), cipher.final(), cipher.getAuthTag()]);
|
||||
|
|
@ -122,6 +373,8 @@ export interface S3StorageConfig {
|
|||
bucket?: string;
|
||||
forcePathStyle?: boolean;
|
||||
mediaKek?: Buffer;
|
||||
/** Overrides DIRECT_EC_PRIVATE_KEY (tests). */
|
||||
directPrivateKey?: ReturnType<typeof createPrivateKey>;
|
||||
}
|
||||
|
||||
export function s3ConfigFromEnv(): S3StorageConfig {
|
||||
|
|
@ -139,11 +392,13 @@ export class S3Storage implements StorageService {
|
|||
private readonly client: S3Client;
|
||||
private readonly bucket: string;
|
||||
private readonly kek: Buffer;
|
||||
private readonly directKey: DirectKeyPair | null;
|
||||
private bucketReady = false;
|
||||
|
||||
constructor(config: S3StorageConfig) {
|
||||
this.bucket = config.bucket ?? "tactical-media-broadcast-hub-media";
|
||||
this.kek = config.mediaKek ?? loadMediaKek();
|
||||
this.directKey = config.directPrivateKey !== undefined ? directKeyPairFromPrivateKey(config.directPrivateKey) : directKeyPairFromEnv();
|
||||
this.client = new S3Client({
|
||||
endpoint: config.endpoint,
|
||||
region: config.region ?? "us-east-1",
|
||||
|
|
@ -155,6 +410,11 @@ export class S3Storage implements StorageService {
|
|||
});
|
||||
}
|
||||
|
||||
/** The JWK the browser seals against; null when direct upload is disabled. */
|
||||
async directPublicJwk(): Promise<{ kty: "EC"; crv: "P-256"; x: string; y: string } | null> {
|
||||
return this.directKey === null ? null : { ...this.directKey.publicJwk };
|
||||
}
|
||||
|
||||
private async ensureBucket(): Promise<void> {
|
||||
if (this.bucketReady) return;
|
||||
try {
|
||||
|
|
@ -205,9 +465,21 @@ export class S3Storage implements StorageService {
|
|||
const res = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: key }));
|
||||
if (res.Body === undefined) throw new Error(`empty body for ${key}`);
|
||||
const stored = Buffer.from(await res.Body.transformToByteArray());
|
||||
const plaintext = open(stored, this.kek);
|
||||
const leading = stored.subarray(0, 6).toString("latin1");
|
||||
let plaintext: Buffer;
|
||||
if (leading === "SPENC1") {
|
||||
plaintext = open(stored, this.kek);
|
||||
} else if (leading === "SPENC2") {
|
||||
// Browser-sealed frame: only this process can unwrap the CEK.
|
||||
if (this.directKey === null) throw missingDirectKey();
|
||||
plaintext = openDirectFrame(stored, this.directKey.privateKey);
|
||||
} else {
|
||||
plaintext = open(stored, this.kek); // raises the magic-mismatch error
|
||||
}
|
||||
// Content addressing is on the PLAINTEXT hash, so tampering (or a
|
||||
// mismatched key) is detected here for both frame versions.
|
||||
const hashFromKey = key.split("/")[1]?.split(".")[0];
|
||||
if (hashFromKey !== undefined && sha256(plaintext) !== hashFromKey) {
|
||||
if (hashFromKey !== undefined && hashFromKey.length === 64 && sha256(plaintext) !== hashFromKey) {
|
||||
throw new Error(`storage: integrity failure for ${key} (plaintext hash mismatch)`);
|
||||
}
|
||||
return plaintext;
|
||||
|
|
@ -221,4 +493,34 @@ export class S3Storage implements StorageService {
|
|||
async presign(key: string, ttlSeconds: number): Promise<string> {
|
||||
return getSignedUrl(this.client, new GetObjectCommand({ Bucket: this.bucket, Key: key }), { expiresIn: ttlSeconds });
|
||||
}
|
||||
|
||||
async presignPut(key: string, contentType: string, ttlSeconds: number): Promise<string> {
|
||||
return getSignedUrl(this.client, new PutObjectCommand({ Bucket: this.bucket, Key: key, ContentType: contentType }), { expiresIn: ttlSeconds });
|
||||
}
|
||||
|
||||
async headObject(key: string): Promise<{ exists: boolean; contentLength: number | null }> {
|
||||
try {
|
||||
const res = await this.client.send(new HeadObjectCommand({ Bucket: this.bucket, Key: key }));
|
||||
return { exists: true, contentLength: res.ContentLength ?? null };
|
||||
} catch (err) {
|
||||
const status = err instanceof S3ServiceException ? err.$metadata.httpStatusCode : undefined;
|
||||
const name = err instanceof S3ServiceException ? err.name : "";
|
||||
if (status === 404 || name === "NotFound") return { exists: false, contentLength: null };
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async openDirectHeader(stored: Buffer): Promise<Buffer> {
|
||||
if (this.directKey === null) throw missingDirectKey();
|
||||
return unwrapDirectCek(stored, this.directKey.privateKey);
|
||||
}
|
||||
}
|
||||
|
||||
/** Build a DirectKeyPair from an already-loaded private key (config + tests). */
|
||||
function directKeyPairFromPrivateKey(privateKey: ReturnType<typeof createPrivateKey>): DirectKeyPair {
|
||||
const jwk = createPublicKey(privateKey).export({ format: "jwk" });
|
||||
if (jwk.kty !== "EC" || jwk.crv !== "P-256" || typeof jwk.x !== "string" || typeof jwk.y !== "string") {
|
||||
throw new DirectKeyError("direct key is not a P-256 key");
|
||||
}
|
||||
return { privateKey, publicJwk: { kty: "EC", crv: "P-256", x: jwk.x, y: jwk.y } };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { keyForHash, sha256, type StorageService } from "../storage/storage.js";
|
||||
import { keyForHash, sha256, type StorageService, directStorageDisabled } from "../storage/storage.js";
|
||||
import type { PoolClient, QueryResult } from "pg";
|
||||
import { vi } from "vitest";
|
||||
import {
|
||||
|
|
@ -43,6 +43,7 @@ export function serviceWithDatabase(
|
|||
}),
|
||||
hasContent: async (hash, type) => objects.has(keyForHash(hash, type)),
|
||||
presign: async () => { throw new Error("Encrypted source must be decrypted before probing"); },
|
||||
...directStorageDisabled(),
|
||||
};
|
||||
database.transitions = [];
|
||||
database.statements = [];
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { join } from "node:path";
|
|||
import { Pool, type PoolClient } from "pg";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { runMigrations } from "../db/migrate.js";
|
||||
import { sha256, keyForHash, type StorageService } from "../storage/storage.js";
|
||||
import { sha256, keyForHash, type StorageService, directStorageDisabled } from "../storage/storage.js";
|
||||
import { VideoService, type VideoServiceDependencies } from "./video.service.js";
|
||||
|
||||
const dbAvailable = process.env.DATABASE_URL !== undefined;
|
||||
|
|
@ -45,6 +45,7 @@ describe.skipIf(!dbAvailable)("video library (live DB)", () => {
|
|||
},
|
||||
hasContent: async (hash, type) => objects.has(keyForHash(hash, type)),
|
||||
presign: async () => "unused",
|
||||
...directStorageDisabled(),
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { readFile } from "node:fs/promises";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { keyForHash, sha256, seal, open, type StorageService } from "../storage/storage.js";
|
||||
import { keyForHash, sha256, seal, open, type StorageService, directStorageDisabled } from "../storage/storage.js";
|
||||
import { probeVideoFile } from "./ffprobe.js";
|
||||
import { serviceWithDatabase, workspaceId } from "./test-fixtures.js";
|
||||
|
||||
|
|
@ -30,6 +30,7 @@ describe.skipIf(fixture === undefined)("real ffprobe roundtrip (encrypted in-mem
|
|||
},
|
||||
hasContent: async (hash, contentType) => objects.has(keyForHash(hash, contentType)),
|
||||
presign: async () => { throw new Error("must decrypt, not presign"); },
|
||||
...directStorageDisabled(),
|
||||
};
|
||||
const { service } = serviceWithDatabase({}, probeVideoFile, storage);
|
||||
const uploaded = await service.uploadVideoBytes(workspaceId, bytes, "video/mp4");
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import { UploadDropzone } from "../../../components/files/upload-dropzone";
|
|||
import { ShareDialog } from "../../../components/files/share-dialog";
|
||||
import { RetentionDialog } from "../../../components/files/retention-dialog";
|
||||
import { downloadGroupFile } from "../../../components/files/files-client";
|
||||
import { downloadFileDirect, isDirectObject, saveBytes } from "../../../lib/files-direct";
|
||||
import { useFilesLang } from "../../../components/files/i18n";
|
||||
import { MIME_CLASS_LABEL } from "../../../components/files/labels";
|
||||
import type {
|
||||
|
|
@ -171,8 +172,23 @@ export default function FilesPage(): React.ReactElement {
|
|||
async function handleDownload(item: GroupFile): Promise<void> {
|
||||
setActionError(null);
|
||||
try {
|
||||
if (isDirectObject(item)) {
|
||||
const bytes = await downloadFileDirect(item.id);
|
||||
saveBytes(bytes, item.name);
|
||||
return;
|
||||
}
|
||||
await downloadGroupFile(item.id, item.name);
|
||||
} catch (err) {
|
||||
// Direct path is best-effort: fall back to streaming via the api once.
|
||||
if (isDirectObject(item)) {
|
||||
try {
|
||||
await downloadGroupFile(item.id, item.name);
|
||||
return;
|
||||
} catch (fallbackErr) {
|
||||
setActionError(errorMessage(fallbackErr));
|
||||
return;
|
||||
}
|
||||
}
|
||||
setActionError(errorMessage(err));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ export interface GroupFile {
|
|||
checksum: { value: string };
|
||||
createdAt: string;
|
||||
indexedAt: string;
|
||||
file: { resourceId: string };
|
||||
file: { resourceId: string; storageKey?: string };
|
||||
shareLink: { url: string; access: "members" | "public"; expiresAt: string | null };
|
||||
retention: FileRetention;
|
||||
status: FileStatus;
|
||||
|
|
|
|||
|
|
@ -4,19 +4,40 @@ import { useCallback, useRef, useState } from "react";
|
|||
import { Button } from "../ui/primitives";
|
||||
import { Loading } from "../brand/loading";
|
||||
import { apiUploadFile, errorMessage } from "../../lib/api-client";
|
||||
import { fetchDirectConfig, uploadFileDirect, DIRECT_MEMORY_CAP_BYTES, type DirectUploadConfig } from "../../lib/files-direct";
|
||||
import { formatBytes } from "../video/format";
|
||||
import { useFilesLang } from "./i18n";
|
||||
import type { GroupFile } from "./types";
|
||||
|
||||
/**
|
||||
* Drag-drop + picker for POST /files/groups/:groupId/upload. The API accepts a
|
||||
* single `file` field capped at 2 GiB; the cap is enforced here too so an
|
||||
* oversized pick fails fast with a friendly message instead of a 413.
|
||||
* Drag-drop + picker for group file uploads — MULTIPLE files at once.
|
||||
*
|
||||
* Two transports per file (chosen per file, sequential queue):
|
||||
* direct — browser seals (SPENC2) and PUTs straight to R2 via a presigned
|
||||
* URL; the api handles keys/metadata only. Chosen when the
|
||||
* deployment enables it and the file fits the memory cap
|
||||
* (sealing holds plaintext + frame in memory at once).
|
||||
* server — multipart POST /files/groups/:groupId/upload (≤2 GiB api-side).
|
||||
*
|
||||
* A file that fails on the direct path falls back to the server path before
|
||||
* being marked failed, and one file's failure never aborts the queue.
|
||||
*/
|
||||
|
||||
const MAX_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024;
|
||||
const MAX_LABEL = "2 GiB";
|
||||
|
||||
type QueueState = "queued" | "uploading" | "done" | "failed";
|
||||
|
||||
interface QueueItem {
|
||||
id: string;
|
||||
name: string;
|
||||
size: number;
|
||||
state: QueueState;
|
||||
progress: number;
|
||||
error: string | null;
|
||||
file: File;
|
||||
}
|
||||
|
||||
export function UploadDropzone({ groupId, live, onUploaded }: {
|
||||
groupId: string;
|
||||
live: boolean;
|
||||
|
|
@ -25,57 +46,92 @@ export function UploadDropzone({ groupId, live, onUploaded }: {
|
|||
const { t } = useFilesLang();
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
const busyRef = useRef(false);
|
||||
const configRef = useRef<DirectUploadConfig | null | undefined>(undefined);
|
||||
const [dragging, setDragging] = useState(false);
|
||||
const [uploading, setUploading] = useState<string | null>(null);
|
||||
const [progress, setProgress] = useState(0);
|
||||
const [queue, setQueue] = useState<QueueItem[]>([]);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const disabled = !live || groupId === "" || uploading !== null;
|
||||
const uploading = queue.some((item) => item.state === "uploading" || item.state === "queued");
|
||||
const disabled = !live || groupId === "" || uploading;
|
||||
|
||||
const register = useCallback(
|
||||
async (file: File): Promise<void> => {
|
||||
if (disabled || busyRef.current) return;
|
||||
if (file.size === 0) {
|
||||
setError(t({ en: `${file.name}: the file is empty.`, es: `${file.name}: el archivo está vacío.` }));
|
||||
return;
|
||||
}
|
||||
if (file.size > MAX_UPLOAD_BYTES) {
|
||||
setError(
|
||||
t({
|
||||
en: `${file.name} is ${formatBytes(file.size)} — the limit is ${MAX_LABEL}.`,
|
||||
es: `${file.name} pesa ${formatBytes(file.size)} — el límite es ${MAX_LABEL}.`,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
setError(null);
|
||||
busyRef.current = true;
|
||||
setProgress(0);
|
||||
setUploading(`${file.name} (${formatBytes(file.size)})`);
|
||||
const patch = useCallback((id: string, changes: Partial<QueueItem>): void => {
|
||||
setQueue((current) => current.map((item) => (item.id === id ? { ...item, ...changes } : item)));
|
||||
}, []);
|
||||
|
||||
/** One file through whichever transport applies; direct failure falls back. */
|
||||
const uploadOne = useCallback(async (item: QueueItem, onProgress: (percent: number) => void): Promise<GroupFile> => {
|
||||
let config = configRef.current;
|
||||
if (config === undefined) {
|
||||
config = await fetchDirectConfig().catch(() => null);
|
||||
configRef.current = config ?? null;
|
||||
}
|
||||
const directPossible =
|
||||
config !== null && config.enabled && config.publicKeyJwk !== null &&
|
||||
item.size <= config.maxBytes && item.size <= DIRECT_MEMORY_CAP_BYTES;
|
||||
if (directPossible && config !== null && config.publicKeyJwk !== null) {
|
||||
try {
|
||||
const created = await apiUploadFile<GroupFile>(
|
||||
`/files/groups/${encodeURIComponent(groupId)}/upload`,
|
||||
file,
|
||||
setProgress,
|
||||
);
|
||||
return await uploadFileDirect(groupId, item.file, config, onProgress);
|
||||
} catch (directError) {
|
||||
// Direct path is best-effort: fall back to the server upload, but say why.
|
||||
setError(t({
|
||||
en: `${item.name}: direct upload failed (${errorMessage(directError)}) — uploading via server instead.`,
|
||||
es: `${item.name}: la subida directa falló (${errorMessage(directError)}) — subiendo por el servidor.`,
|
||||
}));
|
||||
}
|
||||
}
|
||||
return apiUploadFile<GroupFile>(`/files/groups/${encodeURIComponent(groupId)}/upload`, item.file, onProgress);
|
||||
}, [groupId, t]);
|
||||
|
||||
const runQueue = useCallback(async (items: QueueItem[]): Promise<void> => {
|
||||
busyRef.current = true;
|
||||
for (const item of items) {
|
||||
patch(item.id, { state: "uploading", progress: 0 });
|
||||
try {
|
||||
const created = await uploadOne(item, (progress) => patch(item.id, { progress }));
|
||||
patch(item.id, { state: "done", progress: 100 });
|
||||
onUploaded(created);
|
||||
} catch (err) {
|
||||
setError(errorMessage(err));
|
||||
} finally {
|
||||
setUploading(null);
|
||||
busyRef.current = false;
|
||||
patch(item.id, { state: "failed", error: errorMessage(err) });
|
||||
}
|
||||
},
|
||||
[disabled, groupId, onUploaded, t],
|
||||
);
|
||||
}
|
||||
busyRef.current = false;
|
||||
// Let the results stay visible briefly, then clear finished entries.
|
||||
window.setTimeout(() => {
|
||||
setQueue((current) => current.filter((entry) => entry.state === "uploading" || entry.state === "queued"));
|
||||
}, 8000);
|
||||
}, [onUploaded, patch, uploadOne]);
|
||||
|
||||
const register = useCallback((files: File[]): void => {
|
||||
if (disabled || files.length === 0) return;
|
||||
setError(null);
|
||||
const accepted: QueueItem[] = [];
|
||||
for (const file of files) {
|
||||
const id = `${file.name}:${file.size}:${Math.random().toString(36).slice(2, 8)}`;
|
||||
if (file.size === 0) {
|
||||
accepted.push({ id, name: file.name, size: 0, state: "failed", progress: 0, error: t({ en: "the file is empty.", es: "el archivo está vacío." }), file });
|
||||
continue;
|
||||
}
|
||||
if (file.size > MAX_UPLOAD_BYTES) {
|
||||
accepted.push({
|
||||
id, name: file.name, size: file.size, state: "failed", progress: 0,
|
||||
error: t({ en: `is ${formatBytes(file.size)} — the limit is ${MAX_LABEL}.`, es: `pesa ${formatBytes(file.size)} — el límite es ${MAX_LABEL}.` }),
|
||||
file,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
accepted.push({ id, name: file.name, size: file.size, state: "queued", progress: 0, error: null, file });
|
||||
}
|
||||
setQueue((current) => [...current, ...accepted]);
|
||||
if (accepted.some((entry) => entry.state === "queued")) void runQueue(accepted);
|
||||
}, [disabled, runQueue, t]);
|
||||
|
||||
const hint = !live
|
||||
? t({ en: "Offline — connect to the API to upload.", es: "Sin conexión — conéctate a la API para subir." })
|
||||
: groupId === ""
|
||||
? t({ en: "Choose a group first.", es: "Elige un grupo primero." })
|
||||
: t({
|
||||
en: `Drop a file, paste one, or browse. Up to ${MAX_LABEL}; stored with re-verified bytes.`,
|
||||
es: `Arrastra un archivo, pega uno o búscalo. Hasta ${MAX_LABEL}; guardado con bytes reverificados.`,
|
||||
en: `Drop files, paste one, or browse. Multiple files welcome — up to ${MAX_LABEL} each; encrypted, stored in the hub's object storage.`,
|
||||
es: `Arrastra archivos, pega uno o búscalos. Varios archivos a la vez — hasta ${MAX_LABEL} cada uno; cifrados en el almacenamiento del hub.`,
|
||||
});
|
||||
|
||||
return (
|
||||
|
|
@ -84,7 +140,7 @@ export function UploadDropzone({ groupId, live, onUploaded }: {
|
|||
role="button"
|
||||
tabIndex={disabled ? -1 : 0}
|
||||
aria-disabled={disabled}
|
||||
aria-label={t({ en: "Upload a file", es: "Subir un archivo" })}
|
||||
aria-label={t({ en: "Upload files", es: "Subir archivos" })}
|
||||
onClick={() => {
|
||||
if (!disabled) inputRef.current?.click();
|
||||
}}
|
||||
|
|
@ -104,8 +160,8 @@ export function UploadDropzone({ groupId, live, onUploaded }: {
|
|||
e.preventDefault();
|
||||
setDragging(false);
|
||||
if (disabled) return;
|
||||
const file = e.dataTransfer.files[0];
|
||||
if (file !== undefined) void register(file);
|
||||
const files = Array.from(e.dataTransfer.files);
|
||||
if (files.length > 0) register(files);
|
||||
}}
|
||||
className={`flex flex-col items-center justify-center gap-2 rounded-lg border border-dashed px-4 py-6 text-center transition-colors duration-150 ${
|
||||
disabled
|
||||
|
|
@ -115,20 +171,33 @@ export function UploadDropzone({ groupId, live, onUploaded }: {
|
|||
: "cursor-pointer border-border hover:border-accent hover:bg-surface-2"
|
||||
}`}
|
||||
>
|
||||
{uploading !== null ? (
|
||||
<div className="w-full max-w-md space-y-2">
|
||||
<span className="inline-flex items-center gap-2 text-xs text-text-dim">
|
||||
<Loading size={14} label={t({ en: "Uploading", es: "Subiendo" })} />
|
||||
{progress < 100
|
||||
? t({ en: `Uploading ${progress}%`, es: `Subiendo ${progress}%` })
|
||||
: t({ en: "Upload sent — indexing…", es: "Subida enviada — indexando…" })}{" "}
|
||||
{uploading}
|
||||
</span>
|
||||
<progress className="h-2 w-full" aria-label={t({ en: "File upload progress", es: "Progreso de subida" })} value={progress} max={100} />
|
||||
{queue.length > 0 ? (
|
||||
<div className="w-full max-w-md space-y-1">
|
||||
{queue.map((item) => (
|
||||
<div key={item.id} className="flex items-center gap-2 text-xs">
|
||||
{item.state === "uploading" ? (
|
||||
<Loading size={12} label={t({ en: "Uploading", es: "Subiendo" })} />
|
||||
) : item.state === "done" ? (
|
||||
<span aria-label={t({ en: "Done", es: "Listo" })}>✓</span>
|
||||
) : item.state === "failed" ? (
|
||||
<span aria-label={t({ en: "Failed", es: "Falló" })}>✗</span>
|
||||
) : (
|
||||
<span aria-label={t({ en: "Queued", es: "En cola" })}>…</span>
|
||||
)}
|
||||
<span className={item.state === "failed" ? "text-danger" : "text-text-dim"}>
|
||||
{item.name} ({formatBytes(item.size)})
|
||||
{item.state === "uploading" ? ` — ${item.progress}%` : ""}
|
||||
{item.state === "failed" && item.error !== null ? ` — ${item.error}` : ""}
|
||||
</span>
|
||||
{item.state === "uploading" && (
|
||||
<progress className="h-1.5 w-20" aria-label={t({ en: "Upload progress", es: "Progreso de subida" })} value={item.progress} max={100} />
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<span className="text-sm text-text">{t({ en: "Drop a file here", es: "Suelta un archivo aquí" })}</span>
|
||||
<span className="text-sm text-text">{t({ en: "Drop files here", es: "Suelta archivos aquí" })}</span>
|
||||
<span className="text-2xs leading-relaxed text-text-faint">{hint}</span>
|
||||
{!disabled && (
|
||||
<Button
|
||||
|
|
@ -138,7 +207,7 @@ export function UploadDropzone({ groupId, live, onUploaded }: {
|
|||
inputRef.current?.click();
|
||||
}}
|
||||
>
|
||||
＋ {t({ en: "Upload a file", es: "Subir un archivo" })}
|
||||
＋ {t({ en: "Upload files", es: "Subir archivos" })}
|
||||
</Button>
|
||||
)}
|
||||
</>
|
||||
|
|
@ -146,11 +215,12 @@ export function UploadDropzone({ groupId, live, onUploaded }: {
|
|||
<input
|
||||
ref={inputRef}
|
||||
type="file"
|
||||
multiple
|
||||
className="hidden"
|
||||
onChange={(e) => {
|
||||
const file = e.target.files?.[0];
|
||||
const files = Array.from(e.target.files ?? []);
|
||||
e.target.value = "";
|
||||
if (file !== undefined) void register(file);
|
||||
if (files.length > 0) register(files);
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -4,13 +4,17 @@ import { useCallback, useEffect, useRef, useState } from "react";
|
|||
import { Button } from "../ui/primitives";
|
||||
import { Loading } from "../brand/loading";
|
||||
import { apiUploadFile, errorMessage } from "../../lib/api-client";
|
||||
import { CHUNK_SIZE_BYTES, uploadVideoChunked } from "../../lib/chunked-upload";
|
||||
import type { VideoStatusDto } from "../../lib/video-types";
|
||||
import { formatBytes } from "./format";
|
||||
|
||||
/**
|
||||
* Drag-drop + file-picker target for POST /video/upload.
|
||||
* The API stores encrypted bytes, mints the content-addressed key, and probes
|
||||
* the stored object before responding. Progress measures the actual transfer.
|
||||
* Drag-drop + file-picker target for video ingest. Files at or under one chunk
|
||||
* keep the single multipart POST /video/upload; larger ones go through the
|
||||
* chunked API (init → PUT slices → complete) so no single request approaches
|
||||
* Cloudflare's ~100 MB proxied-body cap. The API stores encrypted bytes, mints
|
||||
* the content-addressed key, and probes the stored object before responding.
|
||||
* Progress measures the actual transfer.
|
||||
*/
|
||||
|
||||
const MAX_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024;
|
||||
|
|
@ -46,7 +50,7 @@ export function UploadDropzone({ live, canUpload, onStarted, onSettled, onUpload
|
|||
return;
|
||||
}
|
||||
if (file.size === 0 || file.size > MAX_UPLOAD_BYTES) {
|
||||
setError("Choose a non-empty video up to 100 MiB.");
|
||||
setError("Choose a non-empty video up to 2 GiB.");
|
||||
return;
|
||||
}
|
||||
setError(null);
|
||||
|
|
@ -55,7 +59,10 @@ export function UploadDropzone({ live, canUpload, onStarted, onSettled, onUpload
|
|||
setUploading(`${file.name} (${formatBytes(file.size)})`);
|
||||
onStarted(draft);
|
||||
try {
|
||||
const result = await apiUploadFile<VideoStatusDto>("/video/upload", file, setProgress);
|
||||
// Above one chunk the whole-body POST would trip Cloudflare's body cap.
|
||||
const result = file.size > CHUNK_SIZE_BYTES
|
||||
? await uploadVideoChunked(file, setProgress)
|
||||
: await apiUploadFile<VideoStatusDto>("/video/upload", file, setProgress);
|
||||
onUploaded(result);
|
||||
if (result.processingJob.state === "failed") setError(result.processingJob.error ?? "Video probing failed.");
|
||||
} catch (err) {
|
||||
|
|
@ -85,7 +92,7 @@ export function UploadDropzone({ live, canUpload, onStarted, onSettled, onUpload
|
|||
? "Offline — connect to the API to upload."
|
||||
: !canUpload
|
||||
? "Uploading needs the media.manage permission."
|
||||
: "Drop a video file, paste one, or browse. Up to 100 MiB; stored encrypted and checked before it is ready.";
|
||||
: "Drop a video file, paste one, or browse. Up to 2 GiB — larger videos upload in 10 MiB chunks; stored encrypted and checked before it is ready.";
|
||||
|
||||
return (
|
||||
<div className="mb-4">
|
||||
|
|
|
|||
|
|
@ -58,15 +58,34 @@ function messageFrom(body: unknown, fallback: string): string {
|
|||
return fallback;
|
||||
}
|
||||
|
||||
/** Multipart bytes with browser upload progress; shares the API's auth/error rules. */
|
||||
export async function apiUploadFile<T>(path: string, file: File, onProgress: (percent: number) => void): Promise<T> {
|
||||
/** Byte payloads XHR can stream with upload progress. */
|
||||
type ByteBody = FormData | Blob;
|
||||
|
||||
interface ByteRequest {
|
||||
method: "POST" | "PUT";
|
||||
body: ByteBody;
|
||||
/**
|
||||
* Omitted for multipart, where the browser must own the boundary; set for
|
||||
* raw bodies (chunked ingest slices are application/octet-stream).
|
||||
*/
|
||||
contentType?: string;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* XHR transport shared by every byte upload (multipart and chunked slices) so
|
||||
* credentials, the bearer header, upload progress, and the epoch-stale retry
|
||||
* behave identically no matter which endpoint carries the bytes.
|
||||
*/
|
||||
async function apiSendBytes<T>(path: string, request: ByteRequest, onProgress: (percent: number) => void): Promise<T> {
|
||||
const send = async (retried: boolean): Promise<T> => {
|
||||
const token = getSessionToken();
|
||||
const response = await new Promise<{ status: number; text: string }>((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open("POST", buildUrl(path, undefined));
|
||||
xhr.open(request.method, buildUrl(path, undefined));
|
||||
xhr.withCredentials = true;
|
||||
xhr.timeout = 120_000;
|
||||
xhr.timeout = request.timeoutMs ?? 120_000;
|
||||
if (request.contentType !== undefined) xhr.setRequestHeader("Content-Type", request.contentType);
|
||||
if (token !== null) xhr.setRequestHeader("Authorization", `Bearer ${token}`);
|
||||
xhr.upload.onprogress = (event) => {
|
||||
if (event.lengthComputable) onProgress(Math.floor((event.loaded / event.total) * 100));
|
||||
|
|
@ -76,9 +95,7 @@ export async function apiUploadFile<T>(path: string, file: File, onProgress: (pe
|
|||
xhr.onerror = () => reject(new ApiUnreachableError(new Error("Upload network error")));
|
||||
xhr.ontimeout = () => reject(new Error("Upload timed out; check the video library before retrying."));
|
||||
xhr.onabort = () => reject(new Error("Upload cancelled"));
|
||||
const form = new FormData();
|
||||
form.append("file", file);
|
||||
xhr.send(form);
|
||||
xhr.send(request.body);
|
||||
});
|
||||
let body: unknown = null;
|
||||
if (response.text !== "") {
|
||||
|
|
@ -99,6 +116,21 @@ export async function apiUploadFile<T>(path: string, file: File, onProgress: (pe
|
|||
return send(false);
|
||||
}
|
||||
|
||||
/** Multipart bytes with browser upload progress; shares the API's auth/error rules. */
|
||||
export async function apiUploadFile<T>(path: string, file: File, onProgress: (percent: number) => void): Promise<T> {
|
||||
const form = new FormData();
|
||||
form.append("file", file);
|
||||
return apiSendBytes<T>(path, { method: "POST", body: form }, onProgress);
|
||||
}
|
||||
|
||||
/**
|
||||
* Raw application/octet-stream PUT with browser upload progress — one slice of
|
||||
* a chunked upload. Same auth, error, and retry rules as the multipart path.
|
||||
*/
|
||||
export function apiPutBytes<T>(path: string, bytes: Blob, onProgress: (percent: number) => void): Promise<T> {
|
||||
return apiSendBytes<T>(path, { method: "PUT", body: bytes, contentType: "application/octet-stream" }, onProgress);
|
||||
}
|
||||
|
||||
export function reissueSession(): Promise<string> {
|
||||
if (reissuePromise === null) {
|
||||
reissuePromise = apiFetch<{ token: string }>("/auth/reissue", { method: "POST", redirectOn401: false })
|
||||
|
|
|
|||
218
apps/web/src/lib/chunked-upload.test.ts
Normal file
218
apps/web/src/lib/chunked-upload.test.ts
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
import { strict as assert } from "node:assert";
|
||||
import { test } from "node:test";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { API_URL, ApiError } from "./api-client";
|
||||
import { setSessionToken } from "./session-token";
|
||||
import { CHUNK_SIZE_BYTES, chunkBounds, uploadVideoChunked, wholeFilePercent } from "./chunked-upload";
|
||||
import type { VideoStatusDto } from "./video-types";
|
||||
|
||||
/**
|
||||
* Chunked ingest wiring: init → sequential octet-stream PUTs → complete.
|
||||
* Transport is stubbed at the same seams the existing control-room test uses
|
||||
* (globalThis.fetch and a fake XHR), so the assertions cover the REAL
|
||||
* api-client wrappers: bearer header, credentials, method, content-type, and
|
||||
* the whole-file progress math.
|
||||
*/
|
||||
|
||||
type FetchCall = { url: string; init: RequestInit };
|
||||
|
||||
/** The XMLHttpRequest global is typed as the browser constructor; tests install doubles. */
|
||||
function setGlobalXHR(ctor: unknown): void {
|
||||
(globalThis as { XMLHttpRequest: unknown }).XMLHttpRequest = ctor;
|
||||
}
|
||||
|
||||
|
||||
class FakeXHR {
|
||||
static instances: FakeXHR[] = [];
|
||||
method = "";
|
||||
url = "";
|
||||
body: Blob | FormData | null = null;
|
||||
withCredentials = false;
|
||||
timeout = 0;
|
||||
headers: Record<string, string> = {};
|
||||
status = 200;
|
||||
responseText = "{}";
|
||||
upload: { onprogress: ((event: { lengthComputable: boolean; loaded: number; total: number }) => void) | null; onload: (() => void) | null } = { onprogress: null, onload: null };
|
||||
onload: (() => void) | null = null;
|
||||
onerror: (() => void) | null = null;
|
||||
ontimeout: (() => void) | null = null;
|
||||
onabort: (() => void) | null = null;
|
||||
|
||||
constructor() { FakeXHR.instances.push(this); }
|
||||
|
||||
open(method: string, url: string): void { this.method = method; this.url = url; }
|
||||
setRequestHeader(key: string, value: string): void { this.headers[key.toLowerCase()] = value; }
|
||||
send(body: Blob | FormData): void {
|
||||
this.body = body;
|
||||
// Half of the slice, then the completed slice — mirrors real xhr.upload events.
|
||||
queueMicrotask(() => {
|
||||
const size = body instanceof Blob ? body.size : 0;
|
||||
this.upload.onprogress?.({ lengthComputable: true, loaded: Math.floor(size / 2), total: size });
|
||||
this.upload.onload?.();
|
||||
this.onload?.();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const videoStatus: VideoStatusDto = {
|
||||
videoAsset: { id: "va-1", workspaceId: "ws", storageKey: "spenc1/key", mimeType: "video/mp4", sizeBytes: 2500 },
|
||||
processingJob: { id: "job-1", workspaceId: "ws", videoAssetId: "va-1", state: "complete" },
|
||||
};
|
||||
|
||||
/** Server-driven plan: 2500 bytes in 1000-byte chunks → 1000 + 1000 + 500. */
|
||||
async function stubHappyPath(events: string[]): Promise<() => void> {
|
||||
const originalFetch = globalThis.fetch;
|
||||
globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const url = String(input);
|
||||
events.push(`fetch:${url.replace(API_URL ?? "", "")}`);
|
||||
if (url.endsWith("/video/upload/init")) {
|
||||
const body = JSON.parse(String(init?.body)) as { filename: string; mimeType: string; totalBytes: number };
|
||||
events.push(`init:${body.filename}:${body.mimeType}:${body.totalBytes}`);
|
||||
return Response.json({ uploadId: "upl-1", chunkSizeBytes: 1000, totalChunks: 3 });
|
||||
}
|
||||
if (url.endsWith("/video/upload/upl-1/complete")) return Response.json(videoStatus);
|
||||
return Response.json({ message: `unexpected fetch ${url}` }, { status: 500 });
|
||||
}) as typeof fetch;
|
||||
return () => { globalThis.fetch = originalFetch; };
|
||||
}
|
||||
|
||||
test("chunk bounds cover the file exactly; every chunk is full except the last", () => {
|
||||
assert.deepEqual(chunkBounds(2500, 0, 1000), { start: 0, end: 1000 });
|
||||
assert.deepEqual(chunkBounds(2500, 1, 1000), { start: 1000, end: 2000 });
|
||||
assert.deepEqual(chunkBounds(2500, 2, 1000), { start: 2000, end: 2500 });
|
||||
assert.deepEqual(chunkBounds(1000, 0, 1000), { start: 0, end: 1000 }); // exactly one chunk
|
||||
// whole-file percent accumulates across chunks and never exceeds 100
|
||||
assert.equal(wholeFilePercent(0, 500, 2500), 20);
|
||||
assert.equal(wholeFilePercent(1000, 500, 2500), 60);
|
||||
assert.equal(wholeFilePercent(2500, 0, 2500), 100);
|
||||
assert.equal(wholeFilePercent(2500, 999, 2500), 100); // clamped
|
||||
assert.equal(wholeFilePercent(0, 0, 0), 0); // degenerate file never divides by zero
|
||||
});
|
||||
|
||||
test("client CHUNK_SIZE_BYTES mirrors apps/api upload-limits.ts", () => {
|
||||
assert.equal(CHUNK_SIZE_BYTES, 10 * 1024 * 1024);
|
||||
const candidates = ["../api/src/video/upload-limits.ts", "../../api/src/video/upload-limits.ts"];
|
||||
const source = candidates.map((p) => { try { return readFileSync(p, "utf8"); } catch { return null; } }).find((s) => s !== null);
|
||||
if (typeof source === "string") {
|
||||
const match = /export const CHUNK_SIZE_BYTES = ([^;]+);/.exec(source);
|
||||
assert.ok(match, "server CHUNK_SIZE_BYTES not found");
|
||||
// eslint-disable-next-line no-new-func
|
||||
assert.equal(Function(`"use strict";return (${match[1]!});`)(), CHUNK_SIZE_BYTES);
|
||||
}
|
||||
});
|
||||
|
||||
test("upload runs init → sequential chunk PUTs → complete with the same auth as the multipart path", async () => {
|
||||
setSessionToken("tok-1");
|
||||
const events: string[] = [];
|
||||
const restore = await stubHappyPath(events);
|
||||
const originalXHR = globalThis.XMLHttpRequest;
|
||||
FakeXHR.instances = [];
|
||||
setGlobalXHR(FakeXHR);
|
||||
const progress: number[] = [];
|
||||
const bytes = new Uint8Array(2500).map((_, i) => i % 251);
|
||||
const file = new File([bytes], "clip.mp4", { type: "video/mp4" });
|
||||
try {
|
||||
const result = await uploadVideoChunked(file, (percent) => progress.push(percent));
|
||||
assert.deepEqual(result, videoStatus);
|
||||
|
||||
// init + complete went through fetch with JSON bodies, bearer auth, cookies
|
||||
assert.deepEqual(events, [
|
||||
"fetch:/video/upload/init", "init:clip.mp4:video/mp4:2500",
|
||||
"fetch:/video/upload/upl-1/complete",
|
||||
]);
|
||||
|
||||
// strictly sequential PUTs, index order preserved
|
||||
const puts = FakeXHR.instances;
|
||||
assert.equal(puts.length, 3);
|
||||
assert.deepEqual(puts.map((x) => x.method), ["PUT", "PUT", "PUT"]);
|
||||
assert.deepEqual(puts.map((x) => x.url.replace(API_URL ?? "", "")), [
|
||||
"/video/upload/upl-1/chunk/0",
|
||||
"/video/upload/upl-1/chunk/1",
|
||||
"/video/upload/upl-1/chunk/2",
|
||||
]);
|
||||
for (const put of puts) {
|
||||
assert.equal(put.headers["content-type"], "application/octet-stream");
|
||||
assert.equal(put.headers.authorization, "Bearer tok-1");
|
||||
assert.equal(put.withCredentials, true);
|
||||
assert.ok(put.body instanceof Blob);
|
||||
}
|
||||
assert.deepEqual(puts.map((put) => (put.body as Blob).size), [1000, 1000, 500]);
|
||||
|
||||
// slices reassemble to the original file, byte for byte
|
||||
const sent = new Uint8Array(await Promise.all(puts.map(async (put) => new Uint8Array(await (put.body as Blob).arrayBuffer()))).then((parts) => parts.flatMap((p) => Array.from(p))));
|
||||
assert.deepEqual(Array.from(sent), Array.from(bytes));
|
||||
|
||||
// progress is monotonic, spans chunk boundaries, and lands on 100
|
||||
assert.ok(progress.length > 0);
|
||||
assert.equal(progress.at(-1)!, 100);
|
||||
assert.equal(progress[0]!, 20); // half of chunk 0 of 3
|
||||
assert.ok(progress.includes(60)); // half of chunk 1: 1000 done + 500 in flight
|
||||
assert.ok(progress.includes(90));
|
||||
for (let i = 1; i < progress.length; i += 1) assert.ok(progress[i]! >= progress[i - 1]!, `progress went backwards: ${progress.join(",")}`);
|
||||
} finally {
|
||||
setGlobalXHR(originalXHR);
|
||||
await restore();
|
||||
setSessionToken(null);
|
||||
}
|
||||
});
|
||||
|
||||
test("a failed chunk aborts the upload: later chunks and complete are never sent", async () => {
|
||||
setSessionToken("tok-1");
|
||||
const events: string[] = [];
|
||||
const restore = await stubHappyPath(events);
|
||||
const originalXHR = globalThis.XMLHttpRequest;
|
||||
class FailingXHR extends FakeXHR {
|
||||
override send(body: Blob | FormData): void {
|
||||
this.body = body;
|
||||
queueMicrotask(() => {
|
||||
if (this.url.endsWith("/chunk/1")) {
|
||||
this.status = 500;
|
||||
this.responseText = JSON.stringify({ message: "disk full" });
|
||||
}
|
||||
this.upload.onload?.();
|
||||
this.onload?.();
|
||||
});
|
||||
}
|
||||
}
|
||||
FakeXHR.instances = [];
|
||||
setGlobalXHR(FailingXHR);
|
||||
try {
|
||||
const file = new File([new Uint8Array(2500)], "clip.mp4", { type: "video/mp4" });
|
||||
await assert.rejects(
|
||||
uploadVideoChunked(file, () => undefined),
|
||||
(err: unknown) => {
|
||||
assert.ok(err instanceof ApiError);
|
||||
assert.equal(err.status, 500);
|
||||
assert.equal(err.message, "disk full");
|
||||
return true;
|
||||
},
|
||||
);
|
||||
assert.equal(FakeXHR.instances.length, 2); // chunk 2 never left the browser
|
||||
assert.ok(!events.some((e) => e.includes("complete")));
|
||||
} finally {
|
||||
setGlobalXHR(originalXHR);
|
||||
await restore();
|
||||
setSessionToken(null);
|
||||
}
|
||||
});
|
||||
|
||||
test("an init plan that disagrees with the client is rejected before any bytes move", async () => {
|
||||
setSessionToken("tok-1");
|
||||
const originalFetch = globalThis.fetch;
|
||||
globalThis.fetch = (async () => Response.json({ uploadId: "upl-2", chunkSizeBytes: 1000, totalChunks: 4 })) as typeof fetch;
|
||||
const originalXHR = globalThis.XMLHttpRequest;
|
||||
FakeXHR.instances = [];
|
||||
setGlobalXHR(FakeXHR);
|
||||
try {
|
||||
const file = new File([new Uint8Array(2500)], "clip.mp4", { type: "video/mp4" });
|
||||
await assert.rejects(
|
||||
uploadVideoChunked(file, () => undefined),
|
||||
/server expects 4 chunks, client computed 3/,
|
||||
);
|
||||
assert.equal(FakeXHR.instances.length, 0);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
setGlobalXHR(originalXHR);
|
||||
setSessionToken(null);
|
||||
}
|
||||
});
|
||||
71
apps/web/src/lib/chunked-upload.ts
Normal file
71
apps/web/src/lib/chunked-upload.ts
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
import { api, apiPutBytes } from "./api-client";
|
||||
import type { VideoStatusDto } from "./video-types";
|
||||
|
||||
/**
|
||||
* Client half of the chunked ingest protocol — see apps/api/src/video/upload-limits.ts
|
||||
* and the /video/upload/init | chunk | complete routes in VideoController.
|
||||
*
|
||||
* Files at or under one chunk stay on the single multipart POST /video/upload;
|
||||
* anything larger is sliced here so no single request approaches Cloudflare's
|
||||
* ~100 MB proxied-body cap. Kept as a pure lib module (no React) so the
|
||||
* sequencing and progress math are unit-testable with node:test.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Client mirror of CHUNK_SIZE_BYTES in apps/api/src/video/upload-limits.ts.
|
||||
* It only picks the path (whole-body vs chunked); the server's chunkSizeBytes
|
||||
* from /video/upload/init drives the actual slicing.
|
||||
*/
|
||||
export const CHUNK_SIZE_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
/** Response of POST /video/upload/init. */
|
||||
export interface ChunkedUploadInit {
|
||||
uploadId: string;
|
||||
chunkSizeBytes: number;
|
||||
totalChunks: number;
|
||||
}
|
||||
|
||||
/** Byte offset → slice bounds for one chunk; every chunk is full except the last. */
|
||||
export function chunkBounds(totalBytes: number, index: number, chunkSizeBytes: number): { start: number; end: number } {
|
||||
return { start: index * chunkSizeBytes, end: Math.min((index + 1) * chunkSizeBytes, totalBytes) };
|
||||
}
|
||||
|
||||
/** Upload progress as a whole-file percentage: sent bytes over total bytes, clamped to 100. */
|
||||
export function wholeFilePercent(completedBytes: number, inFlightBytes: number, totalBytes: number): number {
|
||||
if (totalBytes <= 0) return 0;
|
||||
return Math.min(100, Math.floor(((completedBytes + inFlightBytes) / totalBytes) * 100));
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunked ingest for files too large for one proxied request: init, then one
|
||||
* application/octet-stream PUT per slice — strictly sequential, so chunk order
|
||||
* is guaranteed — then complete, which runs the same seal/probe path as
|
||||
* POST /video/upload and returns the same VideoStatusDto shape.
|
||||
*
|
||||
* Any failure rejects and abandons the upload (no resume in v1); server-side
|
||||
* TTL sweeping reclaims the partial file. Progress is reported as
|
||||
* (completedBytes + bytes of the slice in flight) / totalBytes.
|
||||
*/
|
||||
export async function uploadVideoChunked(file: File, onProgress: (percent: number) => void): Promise<VideoStatusDto> {
|
||||
const init = await api.post<ChunkedUploadInit>("/video/upload/init", {
|
||||
filename: file.name,
|
||||
mimeType: file.type,
|
||||
totalBytes: file.size,
|
||||
});
|
||||
const chunkSizeBytes = init.chunkSizeBytes > 0 ? init.chunkSizeBytes : CHUNK_SIZE_BYTES;
|
||||
const totalChunks = Math.ceil(file.size / chunkSizeBytes);
|
||||
if (init.totalChunks !== totalChunks) {
|
||||
throw new Error(`Upload rejected: server expects ${init.totalChunks} chunks, client computed ${totalChunks}.`);
|
||||
}
|
||||
let completedBytes = 0;
|
||||
for (let index = 0; index < totalChunks; index += 1) {
|
||||
const { start, end } = chunkBounds(file.size, index, chunkSizeBytes);
|
||||
const slice = file.slice(start, end);
|
||||
await apiPutBytes(`/video/upload/${init.uploadId}/chunk/${index}`, slice, (percent) => {
|
||||
onProgress(wholeFilePercent(completedBytes, (slice.size * percent) / 100, file.size));
|
||||
});
|
||||
completedBytes += slice.size;
|
||||
onProgress(wholeFilePercent(completedBytes, 0, file.size));
|
||||
}
|
||||
return api.post<VideoStatusDto>(`/video/upload/${init.uploadId}/complete`, {});
|
||||
}
|
||||
44
apps/web/src/lib/direct-storage.test.ts
Normal file
44
apps/web/src/lib/direct-storage.test.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
/**
|
||||
* Cross-implementation proof for SPENC2: the web seal (WebCrypto) must open
|
||||
* with the server reader (node:crypto) and vice versa. Run under tsx/node
|
||||
* (WebCrypto is global in node 22).
|
||||
*/
|
||||
import { strict as assert } from "node:assert";
|
||||
import { test } from "node:test";
|
||||
import { generateKeyPairSync } from "node:crypto";
|
||||
import { sealForDirect, openDirectFrame, unwrapDirectCek } from "../../../api/src/storage/storage.js";
|
||||
import { sealDirect, openDirectFrameBytes, bytesToBase64 } from "./direct-storage.js";
|
||||
|
||||
const { publicKey, privateKey } = generateKeyPairSync("ec", { namedCurve: "prime256v1" });
|
||||
const jwk = publicKey.export({ format: "jwk" }) as { kty: "EC"; crv: "P-256"; x: string; y: string };
|
||||
|
||||
test("web sealDirect → server openDirectFrame roundtrip (25 MiB spans 3 segments)", async () => {
|
||||
const plaintext = new Uint8Array(25 * 1024 * 1024 + 7);
|
||||
for (let i = 0; i < plaintext.length; i += 4096) plaintext[i] = i % 251;
|
||||
const { frame, sha256 } = await sealDirect(plaintext, jwk);
|
||||
const opened = openDirectFrame(Buffer.from(frame), privateKey);
|
||||
assert.equal(opened.length, plaintext.length);
|
||||
assert.equal(Buffer.from(opened).equals(Buffer.from(plaintext)), true);
|
||||
assert.equal(sha256.length, 64);
|
||||
});
|
||||
|
||||
test("server sealForDirect → web openDirectFrameBytes roundtrip", async () => {
|
||||
const plaintext = Buffer.from("server-sealed bytes for the browser to open");
|
||||
const frame = sealForDirect(plaintext, jwk);
|
||||
const cek = unwrapDirectCek(frame, privateKey);
|
||||
const opened = await openDirectFrameBytes(new Uint8Array(frame), new Uint8Array(cek));
|
||||
assert.equal(Buffer.from(opened).equals(plaintext), true);
|
||||
});
|
||||
|
||||
test("tampered segment ciphertext throws on open", async () => {
|
||||
const plaintext = new Uint8Array(1024);
|
||||
const { frame } = await sealDirect(plaintext, jwk);
|
||||
const copy = Buffer.from(frame);
|
||||
copy[copy.length - 20] ^= 0xff;
|
||||
assert.throws(() => openDirectFrame(copy, privateKey));
|
||||
});
|
||||
|
||||
test("bytesToBase64 matches node's base64", () => {
|
||||
const bytes = new Uint8Array([0, 1, 2, 250, 251, 252, 253, 254, 255]);
|
||||
assert.equal(bytesToBase64(bytes), Buffer.from(bytes).toString("base64"));
|
||||
});
|
||||
200
apps/web/src/lib/direct-storage.ts
Normal file
200
apps/web/src/lib/direct-storage.ts
Normal file
|
|
@ -0,0 +1,200 @@
|
|||
/**
|
||||
* Direct-to-R2 client-side encryption (WebCrypto ONLY — no dependencies).
|
||||
*
|
||||
* Frame format "SPENC2", all integers big-endian, mirrors the server reader
|
||||
* in apps/api/src/storage/storage.ts byte for byte:
|
||||
*
|
||||
* [0..6) magic ASCII "SPENC2"
|
||||
* [6..71) ephemeral P-256 public key, uncompressed: 0x04 || X(32) || Y(32)
|
||||
* [71..83) wrapNonce, 12 random bytes
|
||||
* [83..131) wrappedCek: AES-256-GCM over the 32-byte CEK;
|
||||
* key = HKDF-SHA256(ikm=ECDH(ephPriv, serverPub), salt empty,
|
||||
* info ASCII "tmbh-direct-cek-wrap", L=32); nonce = wrapNonce
|
||||
* → 48 bytes (32 ct + 16 tag)
|
||||
* [131..135) segmentCount, u32
|
||||
* [135..] segments, in order: ctLen u32 | iv 12 | AES-256-GCM ciphertext
|
||||
* WITH the 16-byte tag appended (WebCrypto default), key = CEK, no AAD
|
||||
*
|
||||
* Segment plaintext size: 10 MiB except the last.
|
||||
*
|
||||
* The API never sees payload bytes: it mints a presigned PUT for the sealed
|
||||
* frame, and later a presigned GET plus the unwrapped CEK for download.
|
||||
*/
|
||||
|
||||
export const DIRECT_FRAME_VERSION = 2;
|
||||
const SEGMENT_PLAINTEXT_BYTES = 10 * 1024 * 1024;
|
||||
const WRAP_INFO = new TextEncoder().encode("tmbh-direct-cek-wrap");
|
||||
const FRAME_MAGIC = new TextEncoder().encode("SPENC2");
|
||||
|
||||
/** EC public key JWK as served by GET /files/direct/config. */
|
||||
export interface DirectPublicJwk {
|
||||
kty: "EC";
|
||||
crv: "P-256";
|
||||
x: string;
|
||||
y: string;
|
||||
}
|
||||
|
||||
/** --- small typed helpers over Buffer-free Uint8Array math ----------------- */
|
||||
|
||||
|
||||
function concat(parts: readonly Uint8Array[]): Uint8Array {
|
||||
const total = parts.reduce((sum, part) => sum + part.byteLength, 0);
|
||||
const out = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const part of parts) {
|
||||
out.set(part, offset);
|
||||
offset += part.byteLength;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function u32be(value: number): Uint8Array {
|
||||
if (!Number.isSafeInteger(value) || value < 0 || value > 0xffff_ffff) throw new Error("direct-storage: u32 out of range");
|
||||
return new Uint8Array([(value >>> 24) & 0xff, (value >>> 16) & 0xff, (value >>> 8) & 0xff, value & 0xff]);
|
||||
}
|
||||
|
||||
function readU32be(bytes: Uint8Array, offset: number): number {
|
||||
if (offset + 4 > bytes.length) throw new Error("direct-storage: truncated u32");
|
||||
return ((bytes[offset]! << 24) | (bytes[offset + 1]! << 16) | (bytes[offset + 2]! << 8) | bytes[offset + 3]!) >>> 0;
|
||||
}
|
||||
|
||||
function bytesToHex(bytes: Uint8Array): string {
|
||||
let out = "";
|
||||
for (const byte of bytes) out += byte.toString(16).padStart(2, "0");
|
||||
return out;
|
||||
}
|
||||
|
||||
function toBufferSource(bytes: Uint8Array): ArrayBuffer {
|
||||
// Copy into an exact-length ArrayBuffer: WebCrypto rejects views whose
|
||||
// byteOffset/length do not describe the whole backing buffer on some engines.
|
||||
const copy = new ArrayBuffer(bytes.byteLength);
|
||||
new Uint8Array(copy).set(bytes);
|
||||
return copy;
|
||||
}
|
||||
|
||||
async function aesGcmEncrypt(key: CryptoKey, iv: Uint8Array, plaintext: Uint8Array): Promise<Uint8Array> {
|
||||
return new Uint8Array(await crypto.subtle.encrypt({ name: "AES-GCM", iv: toBufferSource(iv) }, key, toBufferSource(plaintext)));
|
||||
}
|
||||
|
||||
async function aesGcmDecrypt(key: CryptoKey, iv: Uint8Array, sealed: Uint8Array): Promise<Uint8Array> {
|
||||
return new Uint8Array(await crypto.subtle.decrypt({ name: "AES-GCM", iv: toBufferSource(iv) }, key, toBufferSource(sealed)));
|
||||
}
|
||||
|
||||
// --- SPENC2 header offsets (kept numeric: the format is fixed by contract) ---
|
||||
|
||||
const EPHEMERAL_PUBLIC_OFFSET = 6;
|
||||
const WRAP_NONCE_OFFSET = 71;
|
||||
const WRAPPED_CEK_OFFSET = 83;
|
||||
const SEGMENT_COUNT_OFFSET = 131;
|
||||
const SEGMENTS_OFFSET = 135;
|
||||
const WRAPPED_CEK_BYTES = 48; // 32-byte CEK + 16-byte GCM tag
|
||||
|
||||
/** CEK-wrap key: ECDH(ephPriv, serverPub) → HKDF-SHA256(salt ∅, info, L=32). */
|
||||
async function directWrapKey(ephemeralPrivate: CryptoKey, serverPublic: CryptoKey): Promise<Uint8Array> {
|
||||
const ikm = new Uint8Array(await crypto.subtle.deriveBits({ name: "ECDH", public: serverPublic }, ephemeralPrivate, 256));
|
||||
const hkdf = await crypto.subtle.importKey("raw", toBufferSource(ikm), "HKDF", false, ["deriveBits"]);
|
||||
// WebCrypto HKDF treats an empty salt as HashLen zero bytes (RFC 5869 §2.2).
|
||||
return new Uint8Array(
|
||||
await crypto.subtle.deriveBits(
|
||||
{ name: "HKDF", hash: "SHA-256", salt: new Uint8Array(0), info: toBufferSource(WRAP_INFO) },
|
||||
hkdf,
|
||||
256,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Seal plaintext for direct upload. Returns the SPENC2 frame and the
|
||||
* plaintext sha256 (hex) the complete call attests.
|
||||
*/
|
||||
export async function sealDirect(bytes: Uint8Array, publicJwk: DirectPublicJwk): Promise<{ frame: Uint8Array; sha256: string }> {
|
||||
if (bytes.byteLength === 0) throw new Error("direct-storage: nothing to seal");
|
||||
const serverPublic = await crypto.subtle.importKey(
|
||||
"jwk",
|
||||
{ ...publicJwk, ext: true },
|
||||
{ name: "ECDH", namedCurve: "P-256" },
|
||||
false,
|
||||
[],
|
||||
);
|
||||
// A fresh CEK + ephemeral key per frame: the server can unwrap, but the
|
||||
// stored object alone (or two uploads of the same plaintext) never lines up.
|
||||
const cek = crypto.getRandomValues(new Uint8Array(32));
|
||||
const ephemeral = await crypto.subtle.generateKey({ name: "ECDH", namedCurve: "P-256" }, true, ["deriveBits"]);
|
||||
const wrapKey = await directWrapKey(ephemeral.privateKey, serverPublic);
|
||||
const wrapNonce = crypto.getRandomValues(new Uint8Array(12));
|
||||
const wrapAes = await crypto.subtle.importKey("raw", toBufferSource(wrapKey), { name: "AES-GCM" }, false, ["encrypt"]);
|
||||
const wrappedCek = await aesGcmEncrypt(wrapAes, wrapNonce, cek);
|
||||
// exportKey("raw") on an EC public key yields 0x04 || X || Y — exactly the frame field.
|
||||
const ephemeralPoint = new Uint8Array(await crypto.subtle.exportKey("raw", ephemeral.publicKey));
|
||||
|
||||
const cekAes = await crypto.subtle.importKey("raw", toBufferSource(cek), { name: "AES-GCM" }, false, ["encrypt"]);
|
||||
const segments: Uint8Array[] = [];
|
||||
for (let offset = 0; offset < bytes.byteLength; offset += SEGMENT_PLAINTEXT_BYTES) {
|
||||
const slice = bytes.subarray(offset, Math.min(offset + SEGMENT_PLAINTEXT_BYTES, bytes.byteLength));
|
||||
const iv = crypto.getRandomValues(new Uint8Array(12));
|
||||
const ciphertext = await aesGcmEncrypt(cekAes, iv, slice);
|
||||
segments.push(concat([u32be(ciphertext.byteLength), iv, ciphertext]));
|
||||
}
|
||||
|
||||
const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", toBufferSource(bytes)));
|
||||
const frame = concat([
|
||||
FRAME_MAGIC,
|
||||
ephemeralPoint,
|
||||
wrapNonce,
|
||||
wrappedCek,
|
||||
u32be(segments.length),
|
||||
...segments,
|
||||
]);
|
||||
return { frame, sha256: bytesToHex(digest) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Open a downloaded SPENC2 frame. `cekRaw` is the unwrapped CEK the API
|
||||
* returned (base64 there, decoded by the caller). Throws on any tampering —
|
||||
* every segment's GCM tag is verified.
|
||||
*/
|
||||
export async function openDirectFrameBytes(blob: Uint8Array, cekRaw: Uint8Array): Promise<Uint8Array> {
|
||||
if (blob.byteLength < SEGMENTS_OFFSET) throw new Error("direct-storage: frame is truncated");
|
||||
if (blob.byteLength < WRAPPED_CEK_OFFSET + WRAPPED_CEK_BYTES) throw new Error("direct-storage: frame is truncated");
|
||||
const magic = blob.subarray(0, 6);
|
||||
for (let i = 0; i < FRAME_MAGIC.length; i += 1) {
|
||||
if (magic[i] !== FRAME_MAGIC[i]) throw new Error("direct-storage: not a SPENC2 frame");
|
||||
}
|
||||
if (cekRaw.byteLength !== 32) throw new Error("direct-storage: CEK must be 32 bytes");
|
||||
const cek = await crypto.subtle.importKey("raw", toBufferSource(cekRaw), { name: "AES-GCM" }, false, ["decrypt"]);
|
||||
const segmentCount = readU32be(blob, SEGMENT_COUNT_OFFSET);
|
||||
if (segmentCount === 0) throw new Error("direct-storage: frame has no segments");
|
||||
|
||||
const parts: Uint8Array[] = [];
|
||||
let offset = SEGMENTS_OFFSET;
|
||||
for (let index = 0; index < segmentCount; index += 1) {
|
||||
if (offset + 4 > blob.byteLength) throw new Error("direct-storage: segment table is truncated");
|
||||
const ctLen = readU32be(blob, offset);
|
||||
offset += 4;
|
||||
if (ctLen < 16 || offset + 12 + ctLen > blob.byteLength) throw new Error("direct-storage: segment length is out of range");
|
||||
const iv = blob.subarray(offset, offset + 12);
|
||||
offset += 12;
|
||||
const ciphertext = blob.subarray(offset, offset + ctLen);
|
||||
offset += ctLen;
|
||||
parts.push(await aesGcmDecrypt(cek, iv, ciphertext));
|
||||
}
|
||||
if (offset !== blob.byteLength) throw new Error("direct-storage: frame has trailing bytes");
|
||||
const plaintext = concat(parts);
|
||||
if (plaintext.byteLength === 0) throw new Error("direct-storage: frame decrypts to zero bytes");
|
||||
return plaintext;
|
||||
}
|
||||
|
||||
/** Bytes -> base64 (used for the CEK the download endpoint hands back). */
|
||||
export function bytesToBase64(bytes: Uint8Array): string {
|
||||
let binary = "";
|
||||
for (const byte of bytes) binary += String.fromCharCode(byte);
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
/** base64 -> bytes (browser `atob`; the API returns standard base64). */
|
||||
export function base64ToBytes(value: string): Uint8Array {
|
||||
const binary = atob(value);
|
||||
const out = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i += 1) out[i] = binary.charCodeAt(i);
|
||||
return out;
|
||||
}
|
||||
111
apps/web/src/lib/files-direct.ts
Normal file
111
apps/web/src/lib/files-direct.ts
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
/**
|
||||
* Direct-to-R2 flows for the Files tab: the api mints presigned URLs and
|
||||
* unwraps CEKs; payload bytes go browser → R2 → browser, sealed client-side
|
||||
* (SPENC2 — see ./direct-storage.ts). The server never sees plaintext.
|
||||
*/
|
||||
import { api } from "./api-client";
|
||||
import { sealDirect, openDirectFrameBytes, base64ToBytes, type DirectPublicJwk } from "./direct-storage";
|
||||
import type { GroupFile } from "../components/files/types";
|
||||
|
||||
export interface DirectUploadConfig {
|
||||
enabled: boolean;
|
||||
frameVersion: 2;
|
||||
publicKeyJwk: DirectPublicJwk | null;
|
||||
maxBytes: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Browser-side ceiling for the direct path: sealing holds the plaintext and
|
||||
* the frame in memory at once. Above this the UI falls back to the server
|
||||
* path (which is itself Cloudflare-capped around 100 MB for now).
|
||||
*/
|
||||
export const DIRECT_MEMORY_CAP_BYTES = 512 * 1024 * 1024;
|
||||
|
||||
export async function fetchDirectConfig(): Promise<DirectUploadConfig> {
|
||||
return api.get<DirectUploadConfig>("/files/direct/config");
|
||||
}
|
||||
|
||||
/** POST /files/direct/complete response is the created GroupFile. */
|
||||
export async function uploadFileDirect(
|
||||
groupId: string,
|
||||
file: File,
|
||||
config: DirectUploadConfig,
|
||||
onProgress: (percent: number) => void,
|
||||
): Promise<GroupFile> {
|
||||
if (config.publicKeyJwk === null) throw new Error("direct upload is not enabled");
|
||||
const bytes = new Uint8Array(await file.arrayBuffer());
|
||||
const { frame, sha256 } = await sealDirect(bytes, config.publicKeyJwk);
|
||||
const init = await api.post<{ uploadId: string; objectKey: string; putUrl: string }>("/files/direct/init", {
|
||||
groupId,
|
||||
filename: file.name,
|
||||
mimeType: file.type === "" ? "application/octet-stream" : file.type,
|
||||
sizeBytes: frame.byteLength,
|
||||
});
|
||||
await putToR2(init.putUrl, frame, onProgress);
|
||||
return api.post<GroupFile>("/files/direct/complete", {
|
||||
uploadId: init.uploadId,
|
||||
objectKey: init.objectKey,
|
||||
groupId,
|
||||
filename: file.name,
|
||||
mimeType: file.type === "" ? "application/octet-stream" : file.type,
|
||||
sizeBytes: frame.byteLength,
|
||||
sha256,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Presigned R2 PUT — same-origin rules do not apply (different host), the
|
||||
* signature lives in the query string, and the ONLY header the signature
|
||||
* covers is content-type, which must match the presign exactly.
|
||||
*/
|
||||
function putToR2(url: string, frame: Uint8Array, onProgress: (percent: number) => void): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open("PUT", url);
|
||||
xhr.setRequestHeader("Content-Type", "application/octet-stream");
|
||||
xhr.upload.onprogress = (event) => {
|
||||
if (event.lengthComputable) onProgress(Math.floor((event.loaded / event.total) * 100));
|
||||
};
|
||||
xhr.onload = () => {
|
||||
if (xhr.status >= 200 && xhr.status < 300) resolve();
|
||||
else reject(new Error(`direct upload failed (HTTP ${xhr.status})`));
|
||||
};
|
||||
xhr.onerror = () => reject(new Error("direct upload failed — network error"));
|
||||
xhr.send(new Blob([frame.slice()], { type: "application/octet-stream" }));
|
||||
});
|
||||
}
|
||||
|
||||
export interface DirectDownloadTicket {
|
||||
url: string;
|
||||
cekB64: string;
|
||||
frame: "SPENC2";
|
||||
sizeBytes: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a direct object: ticket (presigned GET + CEK) → fetch from R2 →
|
||||
* decrypt in the browser → hand back the plaintext. The api never streams
|
||||
* the bytes.
|
||||
*/
|
||||
export async function downloadFileDirect(itemId: string): Promise<Uint8Array> {
|
||||
const ticket = await api.post<DirectDownloadTicket>(`/files/${encodeURIComponent(itemId)}/direct-download`, {});
|
||||
const response = await fetch(ticket.url);
|
||||
if (!response.ok) throw new Error(`direct download failed (HTTP ${response.status})`);
|
||||
const sealed = new Uint8Array(await response.arrayBuffer());
|
||||
return openDirectFrameBytes(sealed, base64ToBytes(ticket.cekB64));
|
||||
}
|
||||
|
||||
export function isDirectObject(item: GroupFile): boolean {
|
||||
return item.file.storageKey?.startsWith("direct/") === true;
|
||||
}
|
||||
|
||||
export function saveBytes(bytes: Uint8Array, fileName: string): void {
|
||||
const url = URL.createObjectURL(new Blob([bytes.slice()]));
|
||||
const anchor = document.createElement("a");
|
||||
anchor.href = url;
|
||||
anchor.download = fileName;
|
||||
document.body.append(anchor);
|
||||
anchor.click();
|
||||
anchor.remove();
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
|
|
@ -82,6 +82,9 @@ services:
|
|||
FILES_OCID_USER: ${FILES_OCID_USER:?}
|
||||
FILES_OCID_PASSWORD: ${FILES_OCID_PASSWORD:?}
|
||||
FILES_PUBLIC_BASE: ${FILES_PUBLIC_BASE:-https://files.occupymedia.cc}
|
||||
# Direct-to-R2 client-sealed uploads (SPENC2): the api holds the EC private
|
||||
# key that unwraps client CEKs; value lives in the compose .env.
|
||||
DIRECT_EC_PRIVATE_KEY: ${DIRECT_EC_PRIVATE_KEY:-}
|
||||
ARCHIVE_PROVISIONER_HOMESERVER_URL: ${ARCHIVE_PROVISIONER_HOMESERVER_URL:-}
|
||||
ARCHIVE_PROVISIONER_USER: ${ARCHIVE_PROVISIONER_USER:-}
|
||||
ARCHIVE_PROVISIONER_PASSWORD: ${ARCHIVE_PROVISIONER_PASSWORD:-}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue