feat: chunked video ingest API (init / chunk / complete)
Cloudflare's proxy plan rejects bodies above ~100 MB, so whole-body video uploads cannot exceed it no matter the origin limits. Clients now slice into 10 MB chunks: POST /video/upload/init, PUT /video/upload/:id/chunk/:index, POST /video/upload/:id/complete reassembles to disk, seals, stores to R2 and runs the normal probe + job pipeline. Max chunked upload: 100 GiB. Tests 17/17.
This commit is contained in:
parent
18afff5e49
commit
93edeab5ca
6 changed files with 843 additions and 5 deletions
|
|
@ -1,4 +1,4 @@
|
|||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { BadRequestException, Inject, Injectable, NotFoundException, Optional, ServiceUnavailableException } from "@nestjs/common";
|
||||
|
|
@ -8,7 +8,7 @@ import { withTenant } from "../db/client.js";
|
|||
import { probeVideoFile, type VideoProbeResult } from "./ffprobe.js";
|
||||
import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js";
|
||||
import { type ProduceRequest } from "./produce-request.js";
|
||||
import { MAX_VIDEO_UPLOAD_BYTES } from "./upload-limits.js";
|
||||
import { MAX_CHUNKED_UPLOAD_BYTES, MAX_VIDEO_UPLOAD_BYTES } from "./upload-limits.js";
|
||||
|
||||
const NO_OP_TRANSCODE_NOTE = "Transcode skipped: original stored object retained; rendering belongs to the media-worker compositor.";
|
||||
|
||||
|
|
@ -139,6 +139,44 @@ export class VideoService {
|
|||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunked-ingest tail end: seal + store + probe an already-assembled file on
|
||||
* local disk (see chunked-upload.ts). The bytes never sat in a request body,
|
||||
* so the 2 GiB whole-body cap does not apply — MAX_CHUNKED_UPLOAD_BYTES does.
|
||||
*
|
||||
* MEMORY: reading the assembled file into one Buffer peaks around 2× file
|
||||
* size of RAM (seal takes a Buffer and S3 takes the sealed copy). A 20 GB
|
||||
* upload therefore needs a host with ≥ 48 GB RAM; /tmp must also hold the
|
||||
* file itself.
|
||||
*/
|
||||
async uploadVideoFromPath(
|
||||
workspaceId: string,
|
||||
inputPath: string,
|
||||
mimeType: string,
|
||||
sizeBytes: number,
|
||||
): Promise<VideoUploadResult> {
|
||||
validateUpload("pending", mimeType, sizeBytes);
|
||||
if (sizeBytes > MAX_CHUNKED_UPLOAD_BYTES) {
|
||||
throw new BadRequestException("video exceeds the 100 GiB chunked upload limit");
|
||||
}
|
||||
const onDisk = await stat(inputPath).catch(() => {
|
||||
throw new BadRequestException("assembled upload file is missing on disk");
|
||||
});
|
||||
if (onDisk.size !== sizeBytes) {
|
||||
throw new BadRequestException(
|
||||
`assembled upload is ${onDisk.size} bytes on disk but declared ${sizeBytes}`,
|
||||
);
|
||||
}
|
||||
const storage = this.requireStorage();
|
||||
const bytes = await readFile(inputPath).catch(() => {
|
||||
throw new ServiceUnavailableException("could not read the assembled upload from disk");
|
||||
});
|
||||
const stored = await storage.putContent(bytes, mimeType).catch(() => {
|
||||
throw new ServiceUnavailableException("Video storage unavailable: could not store encrypted upload in S3/MinIO.");
|
||||
});
|
||||
return this.registerAndProbe(workspaceId, stored.key, stored.contentType, stored.sizeBytes);
|
||||
}
|
||||
|
||||
async uploadVideo(
|
||||
workspaceId: string,
|
||||
storageKey: string,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue