glass beadian
8401aa3d37
Card and audio-less-clip anullsrc sources are infinite; concat never terminates an infinite segment, so the encode hung forever at 30%. atrim pins each bed to its item's span (card duration / clip trim).
383 lines
18 KiB
TypeScript
383 lines
18 KiB
TypeScript
/**
|
|
* Assemble command builder (editor-v2 L1; adopted plan §2 graph, BINDING).
|
|
*
|
|
* Typed plan in, argv out — the same discipline as produce/command-builder.ts:
|
|
* nothing here takes a free-form filter string, and NO operator text ever
|
|
* reaches a filter. That is the plan's injection stance made concrete:
|
|
*
|
|
* clips → the asset's scratch path (named by content hash) + numeric -ss/-t
|
|
* cards → a PNG whose pixels are the api-side renderer's (rasterized off the
|
|
* filtergraph entirely — "no drawtext at all" per the verdict)
|
|
* overlays → PNGs too: an EdlOverlay names the api's overlay templates, and the
|
|
* SAME mechanism the compose path uses for card/overlay pixels —
|
|
* production/render.ts's overlaySvg → sharp worker → PNG — produces
|
|
* them (see overlay-png.ts); ffmpeg only ever composites the PNG
|
|
* text → never a filter argument anywhere in this module
|
|
*
|
|
* Graph (one process, one filter_complex, one encode):
|
|
* per clip i [i:v] transpose → scale/crop-or-pad (fit) → setsar=1 → fps=30 → format [vK]
|
|
* [i:a] aformat… , volume=dB [aK]
|
|
* per card k (-loop 1 -t dur PNG) [c:v] fps=30,format [vK] + anullsrc → [aK]
|
|
* concat n=N:v=1:a=1 → [vcat][acat]
|
|
* per overlay [vcat][oJ] overlay=enable='between(t,s,e)' [vJ] (absolute timeline ms)
|
|
* voiceover adelay=offsetMs → volume=gain; duckOriginal → sidechaincompress
|
|
* keyed by the VO (the produce path's duck chain), else amix duration=first
|
|
* out libx264 preset medium crf 20, aac 192k, +faststart
|
|
*/
|
|
import { EDL_ASPECTS, type AssembleInput, type EdlClip, type EdlItem, type EdlOverlay, type EdlRotate } from "./edl-plan";
|
|
import { PRODUCE_PRESET } from "./command-builder";
|
|
|
|
export class AssembleBuildError extends Error {
|
|
constructor(message: string) { super(message); this.name = "AssembleBuildError"; }
|
|
}
|
|
function fail(message: string): never { throw new AssembleBuildError(message); }
|
|
|
|
/** Output encoder settings: plan §2 (medium/CRF 20/aac 192k), sample rate from the produce preset. */
|
|
export const ASSEMBLE_PRESET = {
|
|
videoCodec: "libx264", preset: "medium", crf: 20,
|
|
audioCodec: "aac", audioBitrateKbps: 192,
|
|
sampleRate: PRODUCE_PRESET.sampleRate, fps: PRODUCE_PRESET.fps,
|
|
/** 15-min hard cap on one assemble (the gate the runner enforces too). */
|
|
maxDurationMs: 15 * 60_000,
|
|
} as const;
|
|
|
|
/** Scratch paths are absolute, depth-bounded, and every component is name-safe. */
|
|
const SAFE_NAME = /^[0-9a-zA-Z][0-9a-zA-Z._-]{0,127}$/;
|
|
function scratchPath(path: string, field: string): string {
|
|
if (!path.startsWith("/") || path.includes("..")) fail(`${field} must be an absolute scratch path`);
|
|
const parts = path.slice(1).split("/");
|
|
if (parts.length > 8) fail(`${field} is too deep`);
|
|
for (const part of parts) if (!SAFE_NAME.test(part)) fail(`${field} has an unsafe component`);
|
|
return path;
|
|
}
|
|
|
|
/** A downloaded source/overlay/VO file: a bare 64-hex name + known extension. */
|
|
const SCRATCH_FILE = /^[0-9a-f]{64}\.(mp4|png|webm|m4a|aac|mp3|wav|bin)$/;;
|
|
|
|
export interface AssembleClipSource {
|
|
/** The EdlClip this input renders (its trim/rotate/volume ride along). */
|
|
clip: EdlClip;
|
|
/** Scratch path of the downloaded asset bytes. */
|
|
path: string;
|
|
/** The probe row's durationMs — the pre-spawn trim gate is checked against it. */
|
|
durationMs: number;
|
|
/** hasAudio===false → the clip's audio is a matching anullsrc segment, never [i:a]. */
|
|
hasAudio: boolean;
|
|
}
|
|
|
|
export interface AssembleCardSource {
|
|
itemIndex: number;
|
|
/** Scratch path of the card's full-frame PNG (renderer output, the target canvas size). */
|
|
path: string;
|
|
durationMs: number;
|
|
}
|
|
|
|
export interface AssembleOverlaySource {
|
|
overlay: EdlOverlay;
|
|
/** Scratch path of the rendered overlay PNG. */
|
|
path: string;
|
|
}
|
|
|
|
export interface AssembleVoiceoverSource {
|
|
path: string;
|
|
gainDb: number;
|
|
duckOriginal: boolean;
|
|
offsetMs: number;
|
|
}
|
|
|
|
/**
|
|
* The validated, downloaded, pre-gated plan. Built by the runner once every
|
|
* input exists on disk and every trim has been checked against its probe
|
|
* duration; the builder only re-validates shapes from here on.
|
|
*/
|
|
export interface AssemblePlan {
|
|
input: AssembleInput;
|
|
/** Timeline inputs in ORDER (concat order), each already carrying its scratch path. */
|
|
clips: AssembleClipSource[];
|
|
cards: AssembleCardSource[];
|
|
overlays: AssembleOverlaySource[];
|
|
voiceover: AssembleVoiceoverSource | null;
|
|
/** Scratch path of the mp4 written. */
|
|
outputPath: string;
|
|
}
|
|
|
|
/** Where each downloaded file landed in the argv input order (0-based `-i` indexes). */
|
|
export interface AssembleInputIndex {
|
|
/** One index per timeline item, in timeline order. */
|
|
timeline: number[];
|
|
/** One index per overlay PNG, in overlay order. */
|
|
overlays: number[];
|
|
voiceover: number | null;
|
|
}
|
|
|
|
/** Transpose filter per closed rotation enum (0 → passthrough keeps the chain uniform). */
|
|
export function transposeFor(rotate: EdlRotate): string {
|
|
switch (rotate) {
|
|
case 0: return "transpose=nosuch"; // replaced by the caller; unreachable
|
|
case 90: return "transpose=1";
|
|
case 180: return "transpose=1,transpose=1";
|
|
case 270: return "transpose=2";
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The per-clip video fit chain, EXPORTED for the unit tests and for reasoning
|
|
* about the two fit modes:
|
|
*
|
|
* cover = center-crop: scale so the canvas is fully covered, then crop the excess
|
|
* contain = letterbox: scale to fit INSIDE the canvas, then pad the remainder with
|
|
* the identity black (#060403 lives in the renderer's pixels, not ffmpeg's
|
|
* pad color, so ffmpeg pads plain black)
|
|
*
|
|
* Rotation is applied BEFORE fitting (a 90° phone clip of a vertical action shot
|
|
* must fit the portrait canvas, not the pre-rotation rectangle).
|
|
*/
|
|
export function clipVideoChain(rotate: EdlRotate, fit: "cover" | "contain", aspect: keyof typeof EDL_ASPECTS): string {
|
|
const { w: W, h: H } = EDL_ASPECTS[aspect];
|
|
const rotated = rotate !== 0 ? `${transposeFor(rotate)},` : "";
|
|
if (fit === "cover") {
|
|
return `${rotated}scale=${W}:${H}:force_original_aspect_ratio=increase:force_divisible_by=2,crop=${W}:${H},setsar=1,fps=${ASSEMBLE_PRESET.fps},format=yuv420p`;
|
|
}
|
|
return `${rotated}scale=${W}:${H}:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=${W}:${H}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=${ASSEMBLE_PRESET.fps},format=yuv420p`;
|
|
}
|
|
|
|
function clipAudioChain(volumeDb: number, hasAudio: boolean, inputIndex: number, trimSpanSec: string): string {
|
|
const aformat = `aformat=sample_fmts=fltp:sample_rates=${ASSEMBLE_PRESET.sampleRate}:channel_layouts=stereo`;
|
|
if (!hasAudio) return `anullsrc=r=${ASSEMBLE_PRESET.sampleRate}:cl=stereo,aformat=sample_fmts=fltp:sample_rates=${ASSEMBLE_PRESET.sampleRate}:channel_layouts=stereo`;
|
|
return `[${inputIndex}:a]${aformat}${volumeDb === 0 ? "" : `,volume=${volumeDb}dB`}`;
|
|
}
|
|
|
|
/** Absolute timeline ms → seconds for `between(t,…)`; 3 decimals keeps ms honest. */
|
|
export function secondsOf(ms: number): string { return (ms / 1000).toFixed(3); }
|
|
|
|
/** An item's timeline length in ms: a clip's trim span, a card's duration. */
|
|
export function itemDurationMs(item: EdlItem): number {
|
|
return item.kind === "clip" ? item.trimEndMs - item.trimStartMs : item.durationMs;
|
|
}
|
|
|
|
/** Timeline-relative start of item i (item durations accumulate, in order). */
|
|
export function itemStartMs(items: readonly EdlItem[], index: number): number {
|
|
let start = 0;
|
|
for (let i = 0; i < index; i += 1) start += itemDurationMs(items[i]!);
|
|
return start;
|
|
}
|
|
|
|
/** Timeline length in ms (concat order is items order). */
|
|
export function timelineTotalMs(items: readonly EdlItem[]): number {
|
|
return items.reduce((total, item) => total + itemDurationMs(item), 0);
|
|
}
|
|
|
|
/**
|
|
* The whole `-filter_complex` string. Ordering is load-bearing:
|
|
* 1. per-item chains (timeline order) — clip chains consume their input pads,
|
|
* card video chains consume the looped PNG pads, card audio is anullsrc
|
|
* 2. concat v=1:a=1 over every item in timeline order
|
|
* 3. overlay chain over [vcat], each gated by enable='between(t,s,e)' with
|
|
* ABSOLUTE timeline seconds (the overlays track is timeline-absolute per plan d3)
|
|
* 4. audio: [acat] alone, + adelay'd VO mixed with amix duration=first, or
|
|
* ducked through sidechaincompress keyed by the (delayed) VO
|
|
*/
|
|
export function buildAssembleFilter(plan: AssemblePlan, index: AssembleInputIndex): string {
|
|
const project = plan.input.project;
|
|
const items = project.items;
|
|
if (index.timeline.length !== items.length) fail("one input index per timeline item is required");
|
|
if (index.overlays.length !== plan.overlays.length) fail("one input index per overlay is required");
|
|
if (plan.clips.length + plan.cards.length !== items.length) fail("clips + cards must cover every timeline item");
|
|
if (project.voiceover !== null && plan.voiceover === null) fail("the project names a voiceover the plan did not download");
|
|
if (project.voiceover === null && plan.voiceover !== null) fail("the plan carries a voiceover the project does not name");
|
|
|
|
const chains: string[] = [];
|
|
let clipCursor = 0;
|
|
let cardCursor = 0;
|
|
|
|
items.forEach((entry, i) => {
|
|
const v = `v${i}`;
|
|
const a = `a${i}`;
|
|
if (entry.kind === "clip") {
|
|
// Match by VALUE (assetId + trim + rotate + volume): the runner builds the
|
|
// plan from parsed items, and a re-parse produces equal-but-distinct objects.
|
|
const source = plan.clips[clipCursor];
|
|
if (source === undefined || clipEquals(source.clip, entry) === false) fail(`clip ${i} is missing its downloaded source`);
|
|
const at = index.timeline[i];
|
|
if (at === undefined) fail(`clip ${i} has no input index`);
|
|
chains.push(`[${at}:v]${clipVideoChain(entry.rotate, project.fit, project.aspect)}[${v}]`);
|
|
chains.push(`${clipAudioChain(entry.volumeDb, source.hasAudio, at, ((entry.trimEndMs - entry.trimStartMs) / 1000).toFixed(3))}[${a}]`);
|
|
clipCursor += 1;
|
|
} else {
|
|
const source = plan.cards[cardCursor];
|
|
if (source === undefined || source.itemIndex !== i) fail(`card ${i} is missing its rendered PNG`);
|
|
const at = index.timeline[i];
|
|
if (at === undefined) fail(`card ${i} has no input index`);
|
|
// -loop 1 -t {dur} supplies the frames; the chain only normalizes to the canvas.
|
|
chains.push(`[${at}:v]scale=${EDL_ASPECTS[project.aspect].w}:${EDL_ASPECTS[project.aspect].h},setsar=1,fps=${ASSEMBLE_PRESET.fps},format=yuv420p[${v}]`);
|
|
chains.push(`anullsrc=r=${ASSEMBLE_PRESET.sampleRate}:cl=stereo,atrim=0:${(source.durationMs / 1000).toFixed(3)},asetpts=PTS-STARTPTS,aformat=sample_fmts=fltp:sample_rates=${ASSEMBLE_PRESET.sampleRate}:channel_layouts=stereo[${a}]`);
|
|
cardCursor += 1;
|
|
}
|
|
});
|
|
if (clipCursor !== plan.clips.length || cardCursor !== plan.cards.length) fail("clips/cards disagree with the timeline");
|
|
|
|
// concat v=1:a=1 consumes SEGMENT-INTERLEAVED pairs ([v0][a0][v1][a1]…) —
|
|
// grouped v-then-a order is a runtime ffmpeg failure.
|
|
const concatIn = items.map((_, i) => `[v${i}][a${i}]`).join("");
|
|
chains.push(`${concatIn}concat=n=${items.length}:v=1:a=1[vcat][acat]`);
|
|
|
|
// Overlays: absolute-timeline windows over the concatenated base. PNG inputs are
|
|
// single-frame loops held by eof_action=repeat (the compositor's asset rule).
|
|
let cur = "vcat";
|
|
plan.overlays.forEach((source, j) => {
|
|
const next = j === plan.overlays.length - 1 ? "vout" : `ov${j}`;
|
|
const at = index.overlays[j];
|
|
if (at === undefined) fail(`overlay ${j} has no input index`);
|
|
const s = secondsOf(source.overlay.startMs);
|
|
const e = secondsOf(source.overlay.endMs);
|
|
chains.push(`[${cur}][${at}:v]overlay=x=0:y=0:eof_action=repeat:enable='between(t,${s},${e})'[${next}]`);
|
|
cur = next;
|
|
});
|
|
// With no overlays the concatenated base IS the video out; a null passthrough
|
|
// keeps [vout] a real, mapped label.
|
|
if (plan.overlays.length === 0) chains.push(`[vcat]null[vout]`);
|
|
|
|
// Audio: the concatenated bed, plus the voiceover.
|
|
const vo = plan.voiceover;
|
|
if (vo === null) {
|
|
chains.push(`[acat]anull[aout]`);
|
|
return chains.join(";");
|
|
}
|
|
const voIndex = index.voiceover ?? fail("voiceover input index required");
|
|
const aformat = `aformat=sample_fmts=fltp:sample_rates=${ASSEMBLE_PRESET.sampleRate}:channel_layouts=stereo`;
|
|
const delay = vo.offsetMs > 0 ? `adelay=${vo.offsetMs}:all=1` : "";
|
|
const gain = vo.gainDb !== 0 ? `volume=${vo.gainDb}dB` : "";
|
|
const voChain = `[${voIndex}:a]${aformat}${delay ? `,${delay}` : ""}${gain ? `,${gain}` : ""}`;
|
|
if (vo.duckOriginal) {
|
|
// The duck path: the delayed VO keys a compressor on the bed, and the same
|
|
// delayed VO is what gets mixed in — the sidechain "hears" the VO exactly
|
|
// where the audience does, so the duck opens in time with the offset.
|
|
chains.push(`${voChain},asplit=2[voMix][voKey]`);
|
|
chains.push(`[acat][voKey]sidechaincompress=threshold=0.03:ratio=8:attack=20:release=300:makeup=1:link=average[duck]`);
|
|
chains.push(`[duck][voMix]amix=inputs=2:duration=first:dropout_transition=2:normalize=0[aout]`);
|
|
} else {
|
|
chains.push(`${voChain}[voGain]`);
|
|
chains.push(`[acat][voGain]amix=inputs=2:duration=first:dropout_transition=2:normalize=0[aout]`);
|
|
}
|
|
return chains.join(";");
|
|
}
|
|
|
|
/** Value equality for a clip (parse identity is not stable across a re-parse). */
|
|
function clipEquals(a: EdlClip, b: EdlClip): boolean {
|
|
return a.assetId === b.assetId && a.trimStartMs === b.trimStartMs && a.trimEndMs === b.trimEndMs
|
|
&& a.rotate === b.rotate && a.volumeDb === b.volumeDb;
|
|
}
|
|
|
|
/** Pre-spawn trim gate: every clip's out-point must sit inside the probed source. */
|
|
export function validateTrims(items: readonly EdlItem[], sources: readonly AssembleClipSource[]): void {
|
|
items.forEach((entry, i) => {
|
|
if (entry.kind !== "clip") return;
|
|
const source = sources.find((candidate) => clipEquals(candidate.clip, entry));
|
|
if (source === undefined) fail(`items[${i}]: no downloaded source for the clip`);
|
|
if (entry.trimStartMs >= entry.trimEndMs) fail(`items[${i}]: trimStartMs must be before trimEndMs`);
|
|
if (entry.trimEndMs > source.durationMs) {
|
|
// The plan's late-failure guard: an out-point past the real duration only
|
|
// surfaces as ffmpeg's "Invalid data found while processing input" — fail
|
|
// BEFORE the spawn, with a message a crew can act on.
|
|
fail(`items[${i}]: trim ${entry.trimEndMs}ms exceeds the source's probed ${source.durationMs}ms`);
|
|
}
|
|
});
|
|
}
|
|
|
|
function validatePlan(plan: AssemblePlan): void {
|
|
scratchPath(plan.outputPath, "outputPath");
|
|
for (const source of plan.clips) {
|
|
scratchPath(source.path, "clip.path");
|
|
if (!SCRATCH_FILE.test(source.path.split("/").at(-1) ?? "")) fail("clip.path must be a downloaded <sha256>.<ext> scratch file");
|
|
if (!(source.durationMs > 0)) fail("clip.durationMs must be positive");
|
|
}
|
|
for (const card of plan.cards) {
|
|
scratchPath(card.path, "card.path");
|
|
if (!SCRATCH_FILE.test(card.path.split("/").at(-1) ?? "")) fail("card.path must be a rendered <sha256>.png scratch file");
|
|
}
|
|
for (const source of plan.overlays) {
|
|
scratchPath(source.path, "overlay.path");
|
|
if (!SCRATCH_FILE.test(source.path.split("/").at(-1) ?? "")) fail("overlay.path must be a rendered <sha256>.png scratch file");
|
|
}
|
|
if (plan.voiceover !== null) {
|
|
scratchPath(plan.voiceover.path, "voiceover.path");
|
|
if (!SCRATCH_FILE.test(plan.voiceover.path.split("/").at(-1) ?? "")) fail("voiceover.path must be a downloaded scratch file");
|
|
}
|
|
validateTrims(plan.input.project.items, plan.clips);
|
|
}
|
|
|
|
/**
|
|
* Full assemble argv. Inputs in strict order: one per timeline item, then one
|
|
* per overlay PNG, then the voiceover; every timeline clip carries its own
|
|
* `-ss/-t` BEFORE its `-i` (input seek: accurate under re-encode), cards carry
|
|
* `-loop 1 -t dur`. A global `-t` bounds the encode at the timeline length so a
|
|
* looping card input can never run away.
|
|
*/
|
|
export function buildAssembleArgs(plan: AssemblePlan): string[] {
|
|
validatePlan(plan);
|
|
const project = plan.input.project;
|
|
const { w: W, h: H } = EDL_ASPECTS[project.aspect];
|
|
const args = ["-hide_banner", "-nostdin", "-loglevel", "error", "-y"];
|
|
const index: AssembleInputIndex = { timeline: [], overlays: [], voiceover: null };
|
|
let next = 0;
|
|
|
|
project.items.forEach((entry) => {
|
|
if (entry.kind === "clip") {
|
|
const source = plan.clips.find((source) => clipEquals(source.clip, entry));
|
|
if (source === undefined) fail("clip source missing");
|
|
const durationSec = (entry.trimEndMs - entry.trimStartMs) / 1000;
|
|
args.push("-ss", (entry.trimStartMs / 1000).toFixed(3), "-t", durationSec.toFixed(3), "-i", `file:${source.path}`);
|
|
} else {
|
|
const source = plan.cards.find((card) => card.itemIndex === project.items.indexOf(entry));
|
|
if (source === undefined) fail("card source missing");
|
|
args.push("-loop", "1", "-t", (entry.durationMs / 1000).toFixed(3), "-i", `file:${source.path}`);
|
|
}
|
|
index.timeline.push(next);
|
|
next += 1;
|
|
});
|
|
|
|
for (const source of plan.overlays) {
|
|
args.push("-i", `file:${source.path}`);
|
|
index.overlays.push(next);
|
|
next += 1;
|
|
}
|
|
|
|
if (plan.voiceover !== null) {
|
|
args.push("-i", `file:${plan.voiceover.path}`);
|
|
index.voiceover = next;
|
|
next += 1;
|
|
}
|
|
|
|
args.push("-filter_complex", buildAssembleFilter(plan, index));
|
|
const totalSec = timelineTotalMs(project.items) / 1000;
|
|
args.push("-t", totalSec.toFixed(3));
|
|
args.push(
|
|
"-map", "[vout]", "-map", "[aout]",
|
|
"-c:v", ASSEMBLE_PRESET.videoCodec, "-preset", ASSEMBLE_PRESET.preset, "-crf", String(ASSEMBLE_PRESET.crf),
|
|
"-pix_fmt", "yuv420p",
|
|
"-g", String(ASSEMBLE_PRESET.fps * 2), "-keyint_min", String(ASSEMBLE_PRESET.fps * 2), "-sc_threshold", "0",
|
|
"-c:a", ASSEMBLE_PRESET.audioCodec, "-b:a", `${ASSEMBLE_PRESET.audioBitrateKbps}k`,
|
|
"-ar", String(ASSEMBLE_PRESET.sampleRate),
|
|
"-movflags", "+faststart",
|
|
"-metadata", `width=${W}`, "-metadata", `height=${H}`,
|
|
"-f", "mp4", `file:${plan.outputPath}`,
|
|
);
|
|
return args;
|
|
}
|
|
|
|
/** Failure reasons the runner maps onto the job row; none carries operator text or a path. */
|
|
export type AssembleFailure =
|
|
| "invalid request"
|
|
| "asset unavailable"
|
|
| "render unavailable"
|
|
| "download failed"
|
|
| "encode timed out"
|
|
| "encode failed"
|
|
| "publish failed";
|
|
|
|
export function assembleError(failure: AssembleFailure, detail?: string): Error {
|
|
const error = new Error(`assemble: ${failure}`);
|
|
error.name = failure;
|
|
if (detail !== undefined) (error as Error & { detail?: string }).detail = detail;
|
|
return error;
|
|
}
|