feat: video direct-to-R2 ingest — browser seals, worker probes off R2
POST /video/direct/init|complete: the browser seals (SPENC2) and PUTs straight to object storage; the api HEAD-checks, registers the unprobed asset and queues a probe job — video bytes never cross the api box. Worker side: produce_job_claim widened to probe jobs (042), the runner gains a probe lane (sealed fetch → SPENC1/SPENC2 open by magic → scratch → ffprobe → metadata write-back → complete 100), worker storage reads SPENC2 frames via DIRECT_EC_PRIVATE_KEY (inherited through compose appenv), and a ported ffprobe util. Probe rows carrying input_json fail instead of stranding; the claim guard no longer strands probe rows as invalid.
This commit is contained in:
parent
a7c3f26a4f
commit
1429e4e9af
15 changed files with 1325 additions and 90 deletions
|
|
@ -21,6 +21,9 @@ import { parseId, parseLimit, requireObject, requireSession, requireString } fro
|
|||
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import {
|
||||
VideoService,
|
||||
type DirectUploadCompleteInput,
|
||||
type DirectUploadConfig,
|
||||
type DirectUploadInit,
|
||||
type ProduceRequestResult,
|
||||
type VideoLibraryPage,
|
||||
type VideoStatus,
|
||||
|
|
@ -39,6 +42,7 @@ import {
|
|||
type InitChunkedUploadRequest,
|
||||
} from "./chunked-upload.js";
|
||||
import { loadMatrixBytes, parseMxcUri } from "./import-sources.js";
|
||||
import { isUuid } from "../http/validation.js";
|
||||
|
||||
interface VideoFile {
|
||||
buffer: Buffer;
|
||||
|
|
@ -68,6 +72,42 @@ export class VideoController {
|
|||
* MEMORY: reading the assembled file for the seal peaks around 2× file size of
|
||||
* RAM (see chunked-upload.ts) — a 20 GB upload needs a ≥ 48 GB host.
|
||||
*/
|
||||
/**
|
||||
* Direct-to-R2 ingest: the BROWSER seals (SPENC2) and PUTs straight to object
|
||||
* storage, so this box never touches video bytes. The api only mints the
|
||||
* presigned PUT (init), HEAD-checks + registers + queues a probe job for the
|
||||
* media-worker (complete). Same ceiling as the chunked path, without the RAM.
|
||||
*
|
||||
* GET /video/direct/config capability + the JWK the browser seals against
|
||||
* POST /video/direct/init presigned PUT for a browser-sealed frame
|
||||
* POST /video/direct/complete HEAD-checked register + queued probe job
|
||||
*/
|
||||
@Get("direct/config")
|
||||
@RequirePermission("media.view")
|
||||
async directConfig(): Promise<DirectUploadConfig> {
|
||||
return this.video.directUploadConfig();
|
||||
}
|
||||
|
||||
@Post("direct/init")
|
||||
@RequirePermission("media.manage")
|
||||
async initDirectUpload(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Body() body: unknown,
|
||||
): Promise<DirectUploadInit> {
|
||||
const session = requireSession(request);
|
||||
return this.video.initDirectUpload(session.workspaceId, requireDirectInit(body));
|
||||
}
|
||||
|
||||
@Post("direct/complete")
|
||||
@RequirePermission("media.manage")
|
||||
async completeDirectUpload(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Body() body: unknown,
|
||||
): Promise<VideoStatus> {
|
||||
const session = requireSession(request);
|
||||
return this.video.completeDirectUpload(session.workspaceId, parseDirectComplete(body));
|
||||
}
|
||||
|
||||
@Post("upload/init")
|
||||
@RequirePermission("media.manage")
|
||||
async initChunkedUpload(
|
||||
|
|
@ -201,6 +241,41 @@ export class VideoController {
|
|||
}
|
||||
}
|
||||
|
||||
/** POST /video/direct/init body: the browser's pick, bounded before any bytes are sealed. */
|
||||
function requireDirectInit(value: unknown): { filename: string; mimeType: string; sizeBytes: number } {
|
||||
const body = requireObject(value, "body");
|
||||
const sizeBytes = body.sizeBytes;
|
||||
if (typeof sizeBytes !== "number" || !Number.isSafeInteger(sizeBytes) || sizeBytes < 1) {
|
||||
throw new BadRequestException("sizeBytes must be a positive safe integer");
|
||||
}
|
||||
if (sizeBytes > MAX_CHUNKED_UPLOAD_BYTES) {
|
||||
throw new BadRequestException("sizeBytes exceeds the 100 GiB chunked upload limit");
|
||||
}
|
||||
return {
|
||||
filename: requireString(body.filename, "filename"),
|
||||
mimeType: requireString(body.mimeType, "mimeType"),
|
||||
sizeBytes,
|
||||
};
|
||||
}
|
||||
|
||||
/** POST /video/direct/complete body: the commit half of the direct handshake. */
|
||||
function parseDirectComplete(value: unknown): DirectUploadCompleteInput {
|
||||
const body = requireObject(value, "body");
|
||||
const uploadId = requireString(body.uploadId, "uploadId");
|
||||
if (!isUuid(uploadId)) throw new BadRequestException("uploadId must be a UUID");
|
||||
const sizeBytes = body.sizeBytes;
|
||||
if (typeof sizeBytes !== "number" || !Number.isSafeInteger(sizeBytes) || sizeBytes < 1) {
|
||||
throw new BadRequestException("sizeBytes must be a positive safe integer");
|
||||
}
|
||||
return {
|
||||
uploadId,
|
||||
objectKey: requireString(body.objectKey, "objectKey"),
|
||||
sizeBytes,
|
||||
mimeType: requireString(body.mimeType, "mimeType"),
|
||||
filename: requireString(body.filename, "filename"),
|
||||
};
|
||||
}
|
||||
|
||||
function parseUploadVideo(value: unknown): UploadVideoDto {
|
||||
const body = requireObject(value, "body");
|
||||
const sizeBytes = body.sizeBytes;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
|
@ -6,7 +7,10 @@ import type { PoolClient } from "pg";
|
|||
import { appendTenantAudit, type AuditEvent } from "../audit/audit.service.js";
|
||||
import { withTenant } from "../db/client.js";
|
||||
import { probeVideoFile, type VideoProbeResult } from "./ffprobe.js";
|
||||
import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js";
|
||||
import {
|
||||
S3Storage, s3ConfigFromEnv, type StorageService,
|
||||
directEnabled, directObjectKey,
|
||||
} from "../storage/storage.js";
|
||||
import { type ProduceRequest } from "./produce-request.js";
|
||||
import { MAX_CHUNKED_UPLOAD_BYTES, MAX_VIDEO_UPLOAD_BYTES } from "./upload-limits.js";
|
||||
|
||||
|
|
@ -83,6 +87,34 @@ export interface VideoStatus {
|
|||
|
||||
export type VideoUploadResult = VideoStatus;
|
||||
|
||||
/** GET /video/direct/config — the browser's capability check before it seals. */
|
||||
export interface DirectUploadConfig {
|
||||
enabled: boolean;
|
||||
publicKeyJwk: { kty: "EC"; crv: "P-256"; x: string; y: string } | null;
|
||||
maxBytes: number;
|
||||
}
|
||||
|
||||
/** POST /video/direct/init response. Nothing is persisted until complete. */
|
||||
export interface DirectUploadInit {
|
||||
uploadId: string;
|
||||
objectKey: string;
|
||||
putUrl: string;
|
||||
}
|
||||
|
||||
/** POST /video/direct/complete request (HEAD-checked, then registered + queued). */
|
||||
export interface DirectUploadCompleteInput {
|
||||
uploadId: string;
|
||||
objectKey: string;
|
||||
sizeBytes: number;
|
||||
mimeType: string;
|
||||
filename: string;
|
||||
}
|
||||
|
||||
/** Presigned PUT lifetime for a browser upload (spec: 900 s). */
|
||||
export const DIRECT_PUT_TTL_SECONDS = 900;
|
||||
const DIRECT_KEY_PREFIX = "direct/";
|
||||
const DIRECT_OBJECT_SUFFIX = ".enc";
|
||||
|
||||
type TenantTransaction = <T>(workspaceId: string, fn: (client: PoolClient) => Promise<T>) => Promise<T>;
|
||||
type AuditAppender = (workspaceId: string, event: AuditEvent) => Promise<string>;
|
||||
|
||||
|
|
@ -177,6 +209,96 @@ export class VideoService {
|
|||
return this.registerAndProbe(workspaceId, stored.key, stored.contentType, stored.sizeBytes);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Direct-to-R2 (browser-sealed SPENC2): the api mints the presign and the
|
||||
// rows; the payload goes browser → R2 → media-worker. No video bytes ever
|
||||
// touch this process, so the chunked/whole-body memory ceilings do not apply.
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
/** Whether this deployment can unwrap browser-sealed frames (DIRECT_EC_PRIVATE_KEY). */
|
||||
async directUploadConfig(): Promise<DirectUploadConfig> {
|
||||
const storage = this.requireStorage();
|
||||
const publicKeyJwk = await storage.directPublicJwk();
|
||||
return {
|
||||
enabled: directEnabled(),
|
||||
publicKeyJwk,
|
||||
maxBytes: MAX_CHUNKED_UPLOAD_BYTES,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Mint the presigned PUT for a browser upload. The size is bounded up front
|
||||
* so an oversized pick fails before any bytes are sealed; nothing is
|
||||
* persisted — the asset row only appears on complete, so an abandoned PUT
|
||||
* leaves an orphan object (no row), never a dangling row without an object.
|
||||
*/
|
||||
async initDirectUpload(
|
||||
workspaceId: string,
|
||||
input: { filename: string; mimeType: string; sizeBytes: number },
|
||||
): Promise<DirectUploadInit> {
|
||||
validateDirectUpload(input);
|
||||
if ((await this.requireStorage().directPublicJwk()) === null) {
|
||||
throw new ServiceUnavailableException("DIRECT_EC_PRIVATE_KEY is not configured");
|
||||
}
|
||||
const uploadId = randomUUID();
|
||||
const objectKey = directObjectKey();
|
||||
const putUrl = await this.requireStorage()
|
||||
.presignPut(objectKey, input.mimeType, DIRECT_PUT_TTL_SECONDS)
|
||||
.catch(() => { throw new ServiceUnavailableException("Video storage is unavailable: could not presign the direct PUT"); });
|
||||
return { uploadId, objectKey, putUrl };
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a completed direct upload. The object is HEAD-checked (exists,
|
||||
* exact declared size) BEFORE any row is written, so metadata can never point
|
||||
* at a missing or partial PUT — the same gate /files/direct/complete uses.
|
||||
*
|
||||
* The probe deliberately does NOT run here: the insert queues kind 'probe'
|
||||
* for the media-worker, which downloads the sealed object off R2, decrypts it
|
||||
* and runs ffprobe itself. This box never touches video bytes on this path.
|
||||
*/
|
||||
async completeDirectUpload(workspaceId: string, input: DirectUploadCompleteInput): Promise<VideoStatus> {
|
||||
if (!isUuid(input.uploadId)) throw new BadRequestException("uploadId must be a UUID");
|
||||
if (input.objectKey !== `${DIRECT_KEY_PREFIX}${input.uploadId}${DIRECT_OBJECT_SUFFIX}`) {
|
||||
throw new BadRequestException("objectKey must be direct/<uploadId>.enc");
|
||||
}
|
||||
validateDirectUpload(input);
|
||||
|
||||
const head = await this.requireStorage().headObject(input.objectKey).catch(() => {
|
||||
throw new ServiceUnavailableException("Video storage is unavailable: could not verify the uploaded object");
|
||||
});
|
||||
if (!head.exists || head.contentLength !== input.sizeBytes) {
|
||||
throw new BadRequestException("uploaded object is missing or its size does not match sizeBytes");
|
||||
}
|
||||
|
||||
const registered = await this.dependencies.tenantTransaction(workspaceId, async (client) => {
|
||||
const assetResult = await client.query(
|
||||
`INSERT INTO video_assets (workspace_id, storage_key, mime_type, size_bytes)
|
||||
VALUES ($1, $2, $3, $4) RETURNING *`,
|
||||
[workspaceId, input.objectKey, input.mimeType, input.sizeBytes],
|
||||
);
|
||||
const videoAsset = assetFromRow(requiredRow(assetResult.rows[0], "created video asset"));
|
||||
// kind 'probe' is 039's default: the media-worker claims it and runs ffprobe.
|
||||
const jobResult = await client.query(
|
||||
`INSERT INTO video_processing_jobs (workspace_id, video_asset_id, state, kind)
|
||||
VALUES ($1, $2, 'queued', 'probe') RETURNING *`,
|
||||
[workspaceId, videoAsset.id],
|
||||
);
|
||||
return {
|
||||
videoAsset,
|
||||
processingJob: jobFromRow(requiredRow(jobResult.rows[0], "created video processing job")),
|
||||
};
|
||||
});
|
||||
|
||||
await this.audit(workspaceId, "video.uploaded", registered.videoAsset.id, {
|
||||
mimeType: input.mimeType,
|
||||
sizeBytes: input.sizeBytes,
|
||||
uploadPath: "direct",
|
||||
});
|
||||
// The worker probes; status reports the queued job the drawer polls on.
|
||||
return this.getVideoStatus(registered.videoAsset.id, workspaceId);
|
||||
}
|
||||
|
||||
async uploadVideo(
|
||||
workspaceId: string,
|
||||
storageKey: string,
|
||||
|
|
@ -483,6 +605,22 @@ export class VideoService {
|
|||
}
|
||||
}
|
||||
|
||||
/** Shared validation for the direct init/complete payloads (no sniffing: the declared type stands). */
|
||||
function validateDirectUpload(input: { filename?: string; mimeType: string; sizeBytes: number }): void {
|
||||
if (input.filename !== undefined && (input.filename.trim() === "" || input.filename.length > 255)) {
|
||||
throw new BadRequestException("filename must be 1-255 characters");
|
||||
}
|
||||
if (!input.mimeType.toLowerCase().startsWith("video/")) {
|
||||
throw new BadRequestException("mimeType must be a video media type");
|
||||
}
|
||||
if (!Number.isSafeInteger(input.sizeBytes) || input.sizeBytes < 1) {
|
||||
throw new BadRequestException("sizeBytes must be a positive safe integer");
|
||||
}
|
||||
if (input.sizeBytes > MAX_CHUNKED_UPLOAD_BYTES) {
|
||||
throw new BadRequestException("video exceeds the 100 GiB chunked upload limit");
|
||||
}
|
||||
}
|
||||
|
||||
function validateUpload(storageKey: string, mimeType: string, sizeBytes: number): void {
|
||||
if (storageKey.trim() === "") throw new BadRequestException("storageKey must be a non-empty string");
|
||||
if (!mimeType.toLowerCase().startsWith("video/")) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue