feat: chunked video ingest API (init / chunk / complete)

Cloudflare's proxy plan rejects bodies above ~100 MB, so whole-body video
uploads cannot exceed it no matter the origin limits. Clients now slice into
10 MB chunks: POST /video/upload/init, PUT /video/upload/:id/chunk/:index,
POST /video/upload/:id/complete reassembles to disk, seals, stores to R2 and
runs the normal probe + job pipeline. Max chunked upload: 100 GiB. Tests 17/17.
This commit is contained in:
glass beadian 2026-10-15 22:28:45 +00:00
parent 18afff5e49
commit 93edeab5ca
6 changed files with 843 additions and 5 deletions

View file

@ -1,4 +1,4 @@
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { BadRequestException, Inject, Injectable, NotFoundException, Optional, ServiceUnavailableException } from "@nestjs/common";
@ -8,7 +8,7 @@ import { withTenant } from "../db/client.js";
import { probeVideoFile, type VideoProbeResult } from "./ffprobe.js";
import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js";
import { type ProduceRequest } from "./produce-request.js";
import { MAX_VIDEO_UPLOAD_BYTES } from "./upload-limits.js";
import { MAX_CHUNKED_UPLOAD_BYTES, MAX_VIDEO_UPLOAD_BYTES } from "./upload-limits.js";
const NO_OP_TRANSCODE_NOTE = "Transcode skipped: original stored object retained; rendering belongs to the media-worker compositor.";
@ -139,6 +139,44 @@ export class VideoService {
return result;
}
/**
* Chunked-ingest tail end: seal + store + probe an already-assembled file on
* local disk (see chunked-upload.ts). The bytes never sat in a request body,
* so the 2 GiB whole-body cap does not apply — MAX_CHUNKED_UPLOAD_BYTES does.
*
* MEMORY: reading the assembled file into one Buffer peaks around 2× file
* size of RAM (seal takes a Buffer and S3 takes the sealed copy). A 20 GB
* upload therefore needs a host with ≥ 48 GB RAM; /tmp must also hold the
* file itself.
*/
async uploadVideoFromPath(
workspaceId: string,
inputPath: string,
mimeType: string,
sizeBytes: number,
): Promise<VideoUploadResult> {
validateUpload("pending", mimeType, sizeBytes);
if (sizeBytes > MAX_CHUNKED_UPLOAD_BYTES) {
throw new BadRequestException("video exceeds the 100 GiB chunked upload limit");
}
const onDisk = await stat(inputPath).catch(() => {
throw new BadRequestException("assembled upload file is missing on disk");
});
if (onDisk.size !== sizeBytes) {
throw new BadRequestException(
`assembled upload is ${onDisk.size} bytes on disk but declared ${sizeBytes}`,
);
}
const storage = this.requireStorage();
const bytes = await readFile(inputPath).catch(() => {
throw new ServiceUnavailableException("could not read the assembled upload from disk");
});
const stored = await storage.putContent(bytes, mimeType).catch(() => {
throw new ServiceUnavailableException("Video storage unavailable: could not store encrypted upload in S3/MinIO.");
});
return this.registerAndProbe(workspaceId, stored.key, stored.contentType, stored.sizeBytes);
}
async uploadVideo(
workspaceId: string,
storageKey: string,