Tactical Media Broadcast Hub — initial release
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
This commit is contained in:
commit
aa5e262f3a
880 changed files with 111042 additions and 0 deletions
203
apps/media-worker/src/compositor/command-builder.test.ts
Normal file
203
apps/media-worker/src/compositor/command-builder.test.ts
Normal file
|
|
@ -0,0 +1,203 @@
|
|||
/**
|
||||
* MediaCommandBuilder: argv shape, typed-param rejection, and golden-frame
|
||||
* hashes per layout (deterministic synthetic inputs: seeded noise over test
|
||||
* patterns; framemd5 of the composited output, video + program audio).
|
||||
*
|
||||
* Hashes are pinned to the system FFmpeg (5.1.x here). UPDATE_GOLDEN=1 rewrites
|
||||
* src/compositor/golden-frames.json after an intentional graph change or an
|
||||
* FFmpeg upgrade.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import {
|
||||
buildCompositorArgs, buildEncoderArgs, buildFilterGraph, buildIngestArgs, CommandBuildError, layoutRects, rectFromBasisPoints,
|
||||
SYNTHETIC_PATTERNS, type Canvas, type CompositorPlan, type PxRect,
|
||||
} from "./command-builder";
|
||||
import type { SceneLayout } from "../production";
|
||||
import { makeAssets } from "./testing";
|
||||
|
||||
const canvas: Canvas = { width: 320, height: 180, fps: 30 };
|
||||
const GOLDEN = join(__dirname, "golden-frames.json");
|
||||
|
||||
function synthetic(i: number) {
|
||||
return { video: { kind: "synthetic" as const, pattern: SYNTHETIC_PATTERNS[i % SYNTHETIC_PATTERNS.length]!, seed: 1000 + i }, audio: { kind: "synthetic" as const, toneHz: 220 + 110 * i } };
|
||||
}
|
||||
|
||||
const CUSTOM: PxRect[] = [{ x: 0, y: 0, w: 160, h: 90 }, { x: 160, y: 0, w: 160, h: 90 }, { x: 80, y: 90, w: 160, h: 90 }];
|
||||
const CUSTOM_OVERLAP: PxRect[] = [{ x: 0, y: 0, w: 200, h: 112 }, { x: 120, y: 68, w: 200, h: 112 }];
|
||||
|
||||
interface Case { name: string; layout: SceneLayout; rects: PxRect[]; empty?: number[]; overlays: "none" | "lower" | "chyron" | "both"; mix: Array<[number, number]>; master: number }
|
||||
|
||||
const CASES: Case[] = [
|
||||
{ name: "single", layout: "single", rects: layoutRects("single", canvas), overlays: "none", mix: [[0, 0]], master: 0 },
|
||||
{ name: "pip", layout: "pip", rects: [{ x: 0, y: 0, w: 320, h: 180 }, rectFromBasisPoints({ x: 6000, y: 6000, w: 3500, h: 3500 }, canvas)], overlays: "lower", mix: [[0, 0], [1, -12]], master: -3 },
|
||||
{ name: "2x1", layout: "2x1", rects: layoutRects("2x1", canvas), overlays: "none", mix: [[0, -3], [1, -3]], master: 0 },
|
||||
{ name: "2x2", layout: "2x2", rects: layoutRects("2x2", canvas), empty: [3], overlays: "chyron", mix: [[0, 0]], master: -6 },
|
||||
{ name: "3x3", layout: "3x3", rects: layoutRects("3x3", canvas), empty: [4, 8], overlays: "both", mix: [[0, 0], [1, -6], [2, -24]], master: 0 },
|
||||
{ name: "4x4", layout: "4x4", rects: layoutRects("4x4", canvas), empty: [5, 10, 15], overlays: "none", mix: [], master: 0 },
|
||||
{ name: "custom", layout: "custom", rects: CUSTOM, overlays: "lower", mix: [[2, 0]], master: 0 },
|
||||
{ name: "custom-overlap", layout: "custom", rects: CUSTOM_OVERLAP, overlays: "both", mix: [[0, 0], [1, 0]], master: -9 },
|
||||
];
|
||||
|
||||
function plan(c: Case, assets: string[]): CompositorPlan {
|
||||
const tiles = c.rects.map((rect, i) => ({ source: c.empty?.includes(i) ? null : i % 4, rect, fit: (["contain", "cover", "crop"] as const)[i % 3]! }));
|
||||
const overlays = [
|
||||
...(c.overlays === "lower" || c.overlays === "both" ? [{ kind: "image" as const, asset: 0, x: 12, y: 140 }] : []),
|
||||
...(c.overlays === "chyron" || c.overlays === "both" ? [{ kind: "chyron" as const, asset: 1, y: 164, speedPxPerSec: 120 }] : []),
|
||||
];
|
||||
return {
|
||||
sources: [0, 1, 2, 3].map(synthetic), assets,
|
||||
graph: { canvas, layout: c.layout, tiles, overlays, audio: { inputs: c.mix.map(([source, gainDb]) => ({ source, gainDb })), masterGainDb: c.master } },
|
||||
output: { kind: "framemd5", frames: 4 },
|
||||
};
|
||||
}
|
||||
|
||||
function hashes(p: CompositorPlan): string[] {
|
||||
const r = spawnSync("ffmpeg", buildCompositorArgs(p), { encoding: "utf8", timeout: 60_000 });
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
// framemd5 lines: stream, dts, pts, duration, size, hash — keep stream/pts/size/hash.
|
||||
return r.stdout.split("\n").filter((l) => l && !l.startsWith("#")).map((l) => l.split(",").map((x) => x.trim())).map((f) => `${f[0]}:${f[2]}:${f[4]}:${f[5]}`);
|
||||
}
|
||||
|
||||
test("golden-frame hashes per layout (deterministic synthetic inputs)", { timeout: 300_000 }, () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "golden-"));
|
||||
try {
|
||||
const assets = makeAssets(dir);
|
||||
const got: Record<string, string[]> = {};
|
||||
for (const c of CASES) {
|
||||
const p = plan(c, assets);
|
||||
const h = hashes(p);
|
||||
assert.ok(h.some((l) => l.startsWith("0:")) && h.some((l) => l.startsWith("1:")), `${c.name}: video and audio frames`);
|
||||
assert.deepEqual(hashes(p), h, `${c.name}: two runs hash identically`);
|
||||
got[c.name] = h;
|
||||
}
|
||||
if (process.env.UPDATE_GOLDEN || !existsSync(GOLDEN)) {
|
||||
writeFileSync(GOLDEN, `${JSON.stringify({ ffmpeg: ffmpegVersion(), canvas, frames: 4, layouts: got }, null, 2)}\n`);
|
||||
}
|
||||
const golden = JSON.parse(readFileSync(GOLDEN, "utf8")) as { ffmpeg: string; layouts: Record<string, string[]> };
|
||||
for (const c of CASES) assert.deepEqual(got[c.name], golden.layouts[c.name], `${c.name} matches golden (pinned to ${golden.ffmpeg}; UPDATE_GOLDEN=1 after an intentional change)`);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function ffmpegVersion(): string {
|
||||
return spawnSync("ffmpeg", ["-version"], { encoding: "utf8" }).stdout.split("\n")[0] ?? "unknown";
|
||||
}
|
||||
|
||||
test("layout cells are even, cover the canvas, and xstack/overlay is chosen by geometry", () => {
|
||||
for (const layout of ["2x1", "2x2", "3x3", "4x4"] as const) {
|
||||
for (const c of [canvas, { width: 1920, height: 1080, fps: 30 }, { width: 1280, height: 720, fps: 30 }]) {
|
||||
const rects = layoutRects(layout, c);
|
||||
assert.equal(rects.reduce((a, r) => a + r.w * r.h, 0), c.width * c.height, `${layout} ${c.width}x${c.height} tiles cover the canvas`);
|
||||
for (const r of rects) for (const v of [r.x, r.y, r.w, r.h]) assert.equal(v % 2, 0);
|
||||
}
|
||||
}
|
||||
const idx = { video: [0, 1, 2, 3], audio: [null, null, null, null], assets: [] };
|
||||
const base = { sources: [0, 1, 2, 3].map(synthetic), assets: [], output: { kind: "framemd5" as const, frames: 1 } };
|
||||
const audio = { inputs: [], masterGainDb: 0 };
|
||||
const grid = buildFilterGraph({ ...base, graph: { canvas, layout: "2x2", tiles: layoutRects("2x2", canvas).map((rect, i) => ({ source: i, rect, fit: "contain" as const })), overlays: [], audio } }, idx);
|
||||
assert.match(grid, /xstack=inputs=4:layout=0_0\|160_0\|0_90\|160_90:fill=black/);
|
||||
assert.match(grid, /anullsrc=r=48000:cl=stereo/);
|
||||
const pip = buildFilterGraph({ ...base, graph: { canvas, layout: "pip", tiles: [{ source: 0, rect: { x: 0, y: 0, w: 320, h: 180 }, fit: "cover" }, { source: 1, rect: { x: 200, y: 100, w: 100, h: 56 }, fit: "contain" }], overlays: [], audio } }, idx);
|
||||
assert.match(pip, /overlay=x=200:y=100/);
|
||||
assert.doesNotMatch(pip, /xstack/);
|
||||
// Overlapping custom rects: the base is the first tile padded (paced by a live input), not a lavfi color source.
|
||||
const overlap = buildFilterGraph({ ...base, graph: { canvas, layout: "custom", tiles: CUSTOM_OVERLAP.map((rect, i) => ({ source: i, rect, fit: "cover" as const })), overlays: [], audio } }, idx);
|
||||
assert.match(overlap, /\[t0\]pad=320:180:0:0:color=black\[bg\];\[bg\]\[t1\]overlay=x=120:y=68/);
|
||||
assert.doesNotMatch(overlap, /color=c=black/);
|
||||
const shared = buildFilterGraph({ ...base, graph: { canvas, layout: "2x1", tiles: layoutRects("2x1", canvas).map((rect) => ({ source: 0, rect, fit: "crop" as const })), overlays: [], audio } }, idx);
|
||||
assert.match(shared, /\[0:v\]split=2\[s0_0\]\[s0_1\]/);
|
||||
});
|
||||
|
||||
test("program mix follows §2.3: volume per input → amix → master gain, 3 dB steps only", () => {
|
||||
const idx = { video: [0, 2, 4], audio: [1, 3, 5], assets: [] };
|
||||
const p = (inputs: Array<{ source: number; gainDb: number }>, masterGainDb = 0): CompositorPlan => ({
|
||||
sources: [0, 1, 2].map(synthetic), assets: [], output: { kind: "framemd5", frames: 1 },
|
||||
graph: { canvas, layout: "single", tiles: [{ source: 0, rect: { x: 0, y: 0, w: 320, h: 180 }, fit: "contain" }], overlays: [], audio: { inputs, masterGainDb } },
|
||||
});
|
||||
const g = buildFilterGraph(p([{ source: 0, gainDb: -6 }, { source: 2, gainDb: 0 }], -3), idx);
|
||||
assert.match(g, /\[1:a\]aformat=[^;]*,volume=-6dB\[m0\]/);
|
||||
assert.match(g, /\[5:a\]aformat=[^;]*,volume=0dB\[m1\]/);
|
||||
assert.match(g, /\[m0\]\[m1\]amix=inputs=2:duration=longest:dropout_transition=2:normalize=1,volume=-3dB/);
|
||||
assert.doesNotMatch(buildFilterGraph(p([{ source: 0, gainDb: 0 }]), idx), /amix/);
|
||||
for (const bad of [-1, -25, 3, 1.5]) assert.throws(() => buildCompositorArgs(p([{ source: 0, gainDb: bad }])), /gainDb/);
|
||||
assert.throws(() => buildCompositorArgs(p([{ source: 0, gainDb: 0 }, { source: 0, gainDb: -3 }])), /repeats a source/);
|
||||
assert.throws(() => buildCompositorArgs(p([{ source: 0, gainDb: 0 }], -2)), /masterGainDb/);
|
||||
});
|
||||
|
||||
test("typed params only: hostile values never reach argv", () => {
|
||||
const good: CompositorPlan = {
|
||||
sources: [{ video: { kind: "rawPipe", fd: 4 }, audio: { kind: "rawPipe", fd: 5 } }],
|
||||
assets: [`/scratch/assets/${"a".repeat(64)}.png`],
|
||||
graph: { canvas, layout: "single", tiles: [{ source: 0, rect: { x: 0, y: 0, w: 320, h: 180 }, fit: "contain" }], overlays: [{ kind: "chyron", asset: 0, y: 160, speedPxPerSec: 100 }], audio: { inputs: [{ source: 0, gainDb: 0 }], masterGainDb: 0 } },
|
||||
output: { kind: "gate", videoFd: 1, audioFd: 3 },
|
||||
};
|
||||
const args = buildCompositorArgs(good);
|
||||
assert.ok(args.includes("pipe:4") && args.includes("pipe:5") && args.includes(`file:/scratch/assets/${"a".repeat(64)}.png`));
|
||||
assert.deepEqual(args.slice(-1), ["pipe:3"]);
|
||||
const hostileAssets = ["/scratch/../etc/passwd.png", "/scratch/x;rm.png", "relative/" + "a".repeat(64) + ".png", `/scratch/${"a".repeat(64)}.png'`, `/s/movie=${"a".repeat(58)}.png`, "/scratch/assets/notahash.png"];
|
||||
for (const asset of hostileAssets) assert.throws(() => buildCompositorArgs({ ...good, assets: [asset] }), CommandBuildError, asset);
|
||||
const g = good.graph;
|
||||
assert.throws(() => buildCompositorArgs({ ...good, graph: { ...g, tiles: [{ ...g.tiles[0]!, rect: { x: 1, y: 0, w: 320, h: 180 } }] } }), /even/);
|
||||
assert.throws(() => buildCompositorArgs({ ...good, graph: { ...g, tiles: [{ ...g.tiles[0]!, rect: { x: 0, y: 0, w: 322, h: 180 } }] } }), CommandBuildError);
|
||||
assert.throws(() => buildCompositorArgs({ ...good, graph: { ...g, overlays: [{ kind: "chyron", asset: 0, y: 160, speedPxPerSec: 39 }] } }), /speedPxPerSec/);
|
||||
assert.throws(() => buildCompositorArgs({ ...good, graph: { ...g, overlays: [{ kind: "image", asset: 1, x: 0, y: 0 }] } }), /asset/);
|
||||
assert.throws(() => buildCompositorArgs({ ...good, sources: [{ video: { kind: "rawPipe", fd: 1.5 }, audio: null }] }), /fd/);
|
||||
assert.throws(() => buildCompositorArgs({ ...good, graph: { ...g, canvas: { width: 321, height: 180, fps: 30 } } }), /even/);
|
||||
assert.throws(() => buildCompositorArgs({ ...good, graph: { ...g, canvas: { width: 320, height: 180, fps: 7 } } }), /fps/);
|
||||
// Nothing in argv is a shell: no metacharacters outside the filtergraph and the expression quotes.
|
||||
for (const a of args) if (!a.includes("[")) assert.doesNotMatch(a, /[;&|`$<>]/, a);
|
||||
});
|
||||
|
||||
test("ingest normalize and persistent encoder argv", () => {
|
||||
const ingest = buildIngestArgs({ input: { kind: "mpegtsPipe" }, canvas: { width: 1920, height: 1080, fps: 30 }, audio: "source" });
|
||||
assert.deepEqual(ingest.slice(ingest.indexOf("-f"), ingest.indexOf("-f") + 4), ["-f", "mpegts", "-i", "pipe:0"]);
|
||||
assert.ok(ingest.includes("pipe:1") && ingest.includes("pipe:3"));
|
||||
assert.ok(ingest.some((a) => a.startsWith("scale=1920:1080:force_original_aspect_ratio=decrease") && a.includes("fps=30")));
|
||||
const silent = buildIngestArgs({ input: { kind: "mpegtsPipe" }, canvas, audio: "none" });
|
||||
assert.ok(silent.includes("anullsrc=r=48000:cl=stereo") && silent.includes("1:a:0"));
|
||||
const enc = buildEncoderArgs({ canvas: { width: 1920, height: 1080, fps: 30 }, videoBitrateKbps: 6000, audioBitrateKbps: 128, output: { kind: "flvPipe", fd: 4 } });
|
||||
assert.deepEqual(enc.slice(-3), ["-f", "flv", "pipe:4"]);
|
||||
assert.ok(enc.includes("pipe:0") && enc.includes("pipe:3") && enc.includes("6000k") && enc.includes("60"));
|
||||
assert.throws(() => buildEncoderArgs({ canvas, videoBitrateKbps: 1e9, audioBitrateKbps: 128, output: { kind: "null" } }));
|
||||
});
|
||||
|
||||
test("ingest ×N normalize stage end to end: MPEG-TS pipe (4:3, no audio) and synthetic sources fill their buses", { timeout: 60_000 }, async () => {
|
||||
const { IngestProcess } = await import("./ingest");
|
||||
const { ensureGateBinary, passthroughRedactor } = await import("./testing");
|
||||
assert.ok(ensureGateBinary(), "native helpers (source bus) build");
|
||||
// 4:3 MPEG-TS without audio on the ingest's stdin, as the egress relay would deliver it.
|
||||
const ts = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=s=640x480:r=25:d=2", "-c:v", "mpeg2video", "-f", "mpegts", "-"], { maxBuffer: 64 << 20 });
|
||||
assert.equal(ts.status, 0);
|
||||
const dir = mkdtempSync(join(tmpdir(), "ingest-"));
|
||||
const tsPath = join(dir, "in.ts");
|
||||
writeFileSync(tsPath, ts.stdout);
|
||||
const { openSync, closeSync } = await import("node:fs");
|
||||
const fd = openSync(tsPath, "r");
|
||||
const exits: string[] = [];
|
||||
const pipeIngest = new IngestProcess("a", { input: { kind: "mpegtsPipe" }, canvas, audio: "none" }, fd, join(dir, "scratch"), passthroughRedactor, () => undefined, (_id, r) => exits.push(r));
|
||||
const synth = new IngestProcess("b", { input: { kind: "synthetic", pattern: "smptehdbars", seed: 7, toneHz: 440 }, canvas, audio: "source" }, null, join(dir, "scratch"), passthroughRedactor, () => undefined, (_id, r) => exits.push(r));
|
||||
try {
|
||||
await pipeIngest.start();
|
||||
closeSync(fd);
|
||||
await synth.start();
|
||||
await new Promise((r) => setTimeout(r, 1_500));
|
||||
// 2 s of 25 fps input → 30 fps CFR on the canvas (decoded faster than real time).
|
||||
const piped = await pipeIngest.bus.stats();
|
||||
assert.ok(piped.pairs >= 50, `mpegts pairs ${piped.pairs}`);
|
||||
assert.equal(piped.silenceFilled, 0, "anullsrc supplies audio for a silent source");
|
||||
// -re synthetic: real-time pace.
|
||||
const synthetic = await synth.bus.stats();
|
||||
assert.ok(synthetic.pairs >= 30 && synthetic.pairs <= 60, `synthetic pairs ${synthetic.pairs}`);
|
||||
assert.deepEqual(exits, []);
|
||||
} finally {
|
||||
await pipeIngest.stop();
|
||||
await synth.stop();
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
486
apps/media-worker/src/compositor/command-builder.ts
Normal file
486
apps/media-worker/src/compositor/command-builder.ts
Normal file
|
|
@ -0,0 +1,486 @@
|
|||
/**
|
||||
* MediaCommandBuilder (docs/design/streaming-architecture.md §1.4, §2.3, §4, §6 S2).
|
||||
*
|
||||
* Typed params in, argv out. Nothing here takes a free-form string: sources are
|
||||
* fds or closed synthetic patterns, assets are validated scratch paths to PNGs
|
||||
* named by content id, geometry is integers, gains are 3 dB steps. Every
|
||||
* filtergraph token is produced from those values, so no operator text can
|
||||
* reach a graph or argv (§4.3 "Injection").
|
||||
*/
|
||||
import { CHYRON_SPEED_MAX, CHYRON_SPEED_MIN, GAIN_DB_MAX, GAIN_DB_MIN, gainDb, type SceneLayout, type SlotFit } from "../production";
|
||||
import { assertGeometry, GATE_CHANNELS, GATE_SAMPLE_RATE, type FrameGeometry } from "../gate/protocol";
|
||||
|
||||
export type Canvas = FrameGeometry;
|
||||
|
||||
export interface PxRect { x: number; y: number; w: number; h: number }
|
||||
|
||||
/** Closed set of deterministic test patterns (golden frames, spike, CPU smoke). */
|
||||
export type SyntheticPattern = "testsrc2" | "smptehdbars" | "mandelbrot" | "rgbtestsrc";
|
||||
export const SYNTHETIC_PATTERNS: readonly SyntheticPattern[] = ["testsrc2", "smptehdbars", "mandelbrot", "rgbtestsrc"];
|
||||
|
||||
export type VideoInput = { kind: "rawPipe"; fd: number } | { kind: "synthetic"; pattern: SyntheticPattern; seed: number };
|
||||
export type AudioInput = { kind: "rawPipe"; fd: number } | { kind: "synthetic"; toneHz: number };
|
||||
|
||||
export interface CompositorSource { video: VideoInput; audio: AudioInput | null }
|
||||
|
||||
export interface TileSpec {
|
||||
/** Index into `sources`; null renders black. */
|
||||
source: number | null;
|
||||
/** Pixel rect inside the canvas; computed by `layoutRects` for grid layouts. */
|
||||
rect: PxRect;
|
||||
fit: SlotFit;
|
||||
}
|
||||
|
||||
export type OverlaySpec =
|
||||
| { kind: "image"; asset: number; x: number; y: number }
|
||||
| { kind: "chyron"; asset: number; y: number; speedPxPerSec: number };
|
||||
|
||||
export interface MixInput { source: number; gainDb: number }
|
||||
|
||||
export interface CompositorGraph {
|
||||
canvas: Canvas;
|
||||
layout: SceneLayout;
|
||||
tiles: TileSpec[];
|
||||
/** Drawn in order (lowest layer first). */
|
||||
overlays: OverlaySpec[];
|
||||
audio: { inputs: MixInput[]; masterGainDb: number };
|
||||
}
|
||||
|
||||
export type CompositorOutput =
|
||||
| { kind: "gate"; videoFd: 1; audioFd: 3 }
|
||||
/** Golden-frame hashing: framemd5 of the first `frames` video frames (and the audio in that span) on stdout. */
|
||||
| { kind: "framemd5"; frames: number };
|
||||
|
||||
export interface CompositorPlan {
|
||||
sources: CompositorSource[];
|
||||
/** Absolute scratch paths of rendered overlay PNGs, by content id. */
|
||||
assets: string[];
|
||||
graph: CompositorGraph;
|
||||
output: CompositorOutput;
|
||||
}
|
||||
|
||||
export class CommandBuildError extends Error {
|
||||
constructor(message: string) { super(message); this.name = "CommandBuildError"; }
|
||||
}
|
||||
function fail(message: string): never { throw new CommandBuildError(message); }
|
||||
|
||||
function int(value: number, field: string, min: number, max: number): number {
|
||||
if (!Number.isSafeInteger(value) || value < min || value > max) fail(`${field} must be an integer in ${min}..${max}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function fd(value: number, field: string): number { return int(value, field, 0, 1023); }
|
||||
|
||||
/** Rendered assets live in the worker's scratch dir and are named by sha256 (§4.3). */
|
||||
const ASSET_PATH = /^\/(?:[A-Za-z0-9._-]+\/)*[0-9a-f]{64}\.png$/;
|
||||
function assetPath(path: string, field: string): string {
|
||||
if (!ASSET_PATH.test(path) || path.split("/").includes("..")) fail(`${field} must be an absolute scratch path to <sha256>.png`);
|
||||
return path;
|
||||
}
|
||||
|
||||
function even(n: number): number { return Math.round(n / 2) * 2; }
|
||||
|
||||
/** Even cell boundaries so every tile has even size and offset (yuv420p chroma alignment). */
|
||||
function boundaries(total: number, parts: number): number[] {
|
||||
return Array.from({ length: parts + 1 }, (_, i) => (i === parts ? total : even((i * total) / parts)));
|
||||
}
|
||||
|
||||
const GRID: Partial<Record<SceneLayout, [cols: number, rows: number]>> = { "2x1": [2, 1], "2x2": [2, 2], "3x3": [3, 3], "4x4": [4, 4] };
|
||||
|
||||
/** Cell rects for grid layouts, in slot-index order (row-major). `single`/`pip` slot 0 is the full canvas. */
|
||||
export function layoutRects(layout: SceneLayout, canvas: Canvas): PxRect[] {
|
||||
if (layout === "single" || layout === "pip") return [{ x: 0, y: 0, w: canvas.width, h: canvas.height }];
|
||||
const grid = GRID[layout];
|
||||
if (!grid) return [];
|
||||
const xs = boundaries(canvas.width, grid[0]);
|
||||
const ys = boundaries(canvas.height, grid[1]);
|
||||
const rects: PxRect[] = [];
|
||||
for (let r = 0; r < grid[1]; r += 1) {
|
||||
for (let c = 0; c < grid[0]; c += 1) rects.push({ x: xs[c] ?? 0, y: ys[r] ?? 0, w: (xs[c + 1] ?? 0) - (xs[c] ?? 0), h: (ys[r + 1] ?? 0) - (ys[r] ?? 0) });
|
||||
}
|
||||
return rects;
|
||||
}
|
||||
|
||||
/** Basis points (0..10000) of the canvas to an even pixel rect. */
|
||||
export function rectFromBasisPoints(r: { x: number; y: number; w: number; h: number }, canvas: Canvas): PxRect {
|
||||
const x = even((r.x * canvas.width) / 10_000);
|
||||
const y = even((r.y * canvas.height) / 10_000);
|
||||
const w = Math.max(2, Math.min(canvas.width - x, even((r.w * canvas.width) / 10_000)));
|
||||
const h = Math.max(2, Math.min(canvas.height - y, even((r.h * canvas.height) / 10_000)));
|
||||
return { x, y, w, h };
|
||||
}
|
||||
|
||||
function checkRect(r: PxRect, canvas: Canvas, field: string): void {
|
||||
int(r.x, `${field}.x`, 0, canvas.width - 2);
|
||||
int(r.y, `${field}.y`, 0, canvas.height - 2);
|
||||
int(r.w, `${field}.w`, 2, canvas.width);
|
||||
int(r.h, `${field}.h`, 2, canvas.height);
|
||||
if (r.x % 2 || r.y % 2 || r.w % 2 || r.h % 2) fail(`${field} must be even-aligned`);
|
||||
if (r.x + r.w > canvas.width || r.y + r.h > canvas.height) fail(`${field} must lie inside the canvas`);
|
||||
}
|
||||
|
||||
function overlaps(a: PxRect, b: PxRect): boolean {
|
||||
return a.x < b.x + b.w && b.x < a.x + a.w && a.y < b.y + b.h && b.y < a.y + a.h;
|
||||
}
|
||||
|
||||
function tileFilter(fit: SlotFit, r: PxRect): string {
|
||||
switch (fit) {
|
||||
case "contain": return `scale=${r.w}:${r.h}:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=${r.w}:${r.h}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1`;
|
||||
case "cover": return `scale=${r.w}:${r.h}:force_original_aspect_ratio=increase:force_divisible_by=2,crop=${r.w}:${r.h},setsar=1`;
|
||||
case "crop": return `crop=${r.w}:${r.h},setsar=1`;
|
||||
}
|
||||
}
|
||||
|
||||
function db(value: number): string { return `${value}dB`; }
|
||||
|
||||
const AUDIO_FORMAT = `aformat=sample_fmts=s16:sample_rates=${GATE_SAMPLE_RATE}:channel_layouts=stereo`;
|
||||
|
||||
/**
|
||||
* The `-filter_complex` string. Input labels: video of source i is `[v{i}]`
|
||||
* after the builder's `-i` ordering, audio `[a{i}]`, asset k `[o{k}]`.
|
||||
*/
|
||||
export function buildFilterGraph(plan: CompositorPlan, inputIndex: { video: number[]; audio: Array<number | null>; assets: number[] }): string {
|
||||
const { graph } = plan;
|
||||
const { canvas } = graph;
|
||||
const chains: string[] = [];
|
||||
const black = (w: number, h: number): string => `color=c=black:s=${w}x${h}:r=${canvas.fps}`;
|
||||
|
||||
// Video sources used by several tiles are split once.
|
||||
const uses = new Map<number, number>();
|
||||
for (const tile of graph.tiles) if (tile.source !== null) uses.set(tile.source, (uses.get(tile.source) ?? 0) + 1);
|
||||
const taken = new Map<number, number>();
|
||||
const videoPad = (source: number): string => {
|
||||
const n = uses.get(source) ?? 0;
|
||||
const input = `${inputIndex.video[source]}:v`;
|
||||
if (n <= 1) return `[${input}]`;
|
||||
if (!taken.has(source)) {
|
||||
chains.push(`[${input}]split=${n}${Array.from({ length: n }, (_, i) => `[s${source}_${i}]`).join("")}`);
|
||||
taken.set(source, 0);
|
||||
}
|
||||
const i = taken.get(source) ?? 0;
|
||||
taken.set(source, i + 1);
|
||||
return `[s${source}_${i}]`;
|
||||
};
|
||||
|
||||
const tiles = graph.tiles.map((tile, i) => ({ tile, label: `t${i}` }));
|
||||
const live = tiles.filter(({ tile }) => tile.source !== null);
|
||||
for (const { tile, label } of live) chains.push(`${videoPad(tile.source ?? 0)}${tileFilter(tile.fit, tile.rect)}[${label}]`);
|
||||
|
||||
const W = canvas.width;
|
||||
const H = canvas.height;
|
||||
const stackable = graph.layout !== "pip" && live.every(({ tile }, i) => live.every((other, j) => i === j || !overlaps(tile.rect, other.tile.rect)));
|
||||
if (live.length === 0) {
|
||||
chains.push(`${black(W, H)}[base]`);
|
||||
} else if (stackable && live.length >= 2) {
|
||||
const layout = live.map(({ tile }) => `${tile.rect.x}_${tile.rect.y}`).join("|");
|
||||
// xstack sizes its output to the tiles' bounding box; pad only when that is not the canvas.
|
||||
const right = Math.max(...live.map(({ tile }) => tile.rect.x + tile.rect.w));
|
||||
const bottom = Math.max(...live.map(({ tile }) => tile.rect.y + tile.rect.h));
|
||||
const fit = right === W && bottom === H ? "" : `,pad=${W}:${H}:0:0:color=black`;
|
||||
chains.push(`${live.map(({ label }) => `[${label}]`).join("")}xstack=inputs=${live.length}:layout=${layout}:fill=black${fit}[base]`);
|
||||
} else if (stackable) {
|
||||
const only = live[0]!;
|
||||
const full = only.tile.rect.w === W && only.tile.rect.h === H;
|
||||
chains.push(full ? `[${only.label}]null[base]` : `[${only.label}]pad=${W}:${H}:${only.tile.rect.x}:${only.tile.rect.y}:color=black[base]`);
|
||||
} else {
|
||||
// PIP and overlapping custom rects: overlay chain in slot order (later slots on top). The base
|
||||
// is the first tile padded to the canvas, never a lavfi `color` source: an unpaced main input
|
||||
// runs ahead of the live tiles, so framesync queues them (~10 frames of extra video latency,
|
||||
// more than a swap's alignment margin) and keeps emitting frames after the inputs end.
|
||||
const first = live[0]!;
|
||||
const fullFirst = first.tile.rect.x === 0 && first.tile.rect.y === 0 && first.tile.rect.w === W && first.tile.rect.h === H;
|
||||
let cur = first.label;
|
||||
if (!fullFirst) {
|
||||
chains.push(`[${first.label}]pad=${W}:${H}:${first.tile.rect.x}:${first.tile.rect.y}:color=black[bg]`);
|
||||
cur = "bg";
|
||||
}
|
||||
live.slice(1).forEach(({ tile, label }, i) => {
|
||||
const next = `c${i}`;
|
||||
chains.push(`[${cur}][${label}]overlay=x=${tile.rect.x}:y=${tile.rect.y}:eof_action=repeat[${next}]`);
|
||||
cur = next;
|
||||
});
|
||||
chains.push(`[${cur}]null[base]`);
|
||||
}
|
||||
|
||||
let cur = "base";
|
||||
graph.overlays.forEach((overlay, i) => {
|
||||
const next = `ov${i}`;
|
||||
const asset = `[${inputIndex.assets[overlay.asset]}:v]`;
|
||||
if (overlay.kind === "image") {
|
||||
// Default format (yuv420, alpha from the PNG): `format=auto` breaks the 1:1 main-frame
|
||||
// mapping in FFmpeg 5.1 (~1 in 6 main frames duplicated and its neighbour dropped).
|
||||
chains.push(`[${cur}]${asset}overlay=x=${overlay.x}:y=${overlay.y}:eof_action=repeat[${next}]`);
|
||||
} else {
|
||||
// §4.3 chyron: one wide pre-rendered strip scrolled right-to-left by an x-expression of t.
|
||||
chains.push(`[${cur}]${asset}overlay=x='W-mod(t*${overlay.speedPxPerSec}\\,w+W)':y=${overlay.y}:eof_action=repeat[${next}]`);
|
||||
}
|
||||
cur = next;
|
||||
});
|
||||
chains.push(`[${cur}]format=yuv420p[vout]`);
|
||||
|
||||
// §2.3 program mix: volume per input → amix (dead sources fade out) → master gain.
|
||||
const mix = graph.audio.inputs;
|
||||
mix.forEach((m, i) => chains.push(`[${inputIndex.audio[m.source]}:a]${AUDIO_FORMAT},volume=${db(m.gainDb)}[m${i}]`));
|
||||
const master = `volume=${db(graph.audio.masterGainDb)},${AUDIO_FORMAT}`;
|
||||
if (mix.length === 0) chains.push(`anullsrc=r=${GATE_SAMPLE_RATE}:cl=stereo,${master}[aout]`);
|
||||
else if (mix.length === 1) chains.push(`[m0]${master}[aout]`);
|
||||
else chains.push(`${mix.map((_, i) => `[m${i}]`).join("")}amix=inputs=${mix.length}:duration=longest:dropout_transition=2:normalize=1,${master}[aout]`);
|
||||
return chains.join(";");
|
||||
}
|
||||
|
||||
function validatePlan(plan: CompositorPlan): void {
|
||||
const { graph } = plan;
|
||||
assertGeometry(graph.canvas);
|
||||
if (plan.sources.length > 16) fail("at most 16 compositor sources");
|
||||
plan.sources.forEach((s, i) => {
|
||||
if (s.video.kind === "rawPipe") fd(s.video.fd, `sources[${i}].video.fd`);
|
||||
else {
|
||||
if (!SYNTHETIC_PATTERNS.includes(s.video.pattern)) fail(`sources[${i}].video.pattern`);
|
||||
int(s.video.seed, `sources[${i}].video.seed`, 0, 1_000_000);
|
||||
}
|
||||
if (s.audio?.kind === "rawPipe") fd(s.audio.fd, `sources[${i}].audio.fd`);
|
||||
else if (s.audio?.kind === "synthetic") int(s.audio.toneHz, `sources[${i}].audio.toneHz`, 20, 20_000);
|
||||
});
|
||||
plan.assets.forEach((a, i) => assetPath(a, `assets[${i}]`));
|
||||
if (graph.tiles.length > 16) fail("at most 16 tiles");
|
||||
graph.tiles.forEach((t, i) => {
|
||||
if (t.source !== null) int(t.source, `tiles[${i}].source`, 0, plan.sources.length - 1);
|
||||
checkRect(t.rect, graph.canvas, `tiles[${i}].rect`);
|
||||
if (t.fit !== "contain" && t.fit !== "cover" && t.fit !== "crop") fail(`tiles[${i}].fit`);
|
||||
});
|
||||
if (graph.overlays.length > 64) fail("at most 64 overlays");
|
||||
graph.overlays.forEach((o, i) => {
|
||||
int(o.asset, `overlays[${i}].asset`, 0, plan.assets.length - 1);
|
||||
int(o.y, `overlays[${i}].y`, 0, graph.canvas.height);
|
||||
if (o.kind === "image") int(o.x, `overlays[${i}].x`, 0, graph.canvas.width);
|
||||
else if (o.kind === "chyron") int(o.speedPxPerSec, `overlays[${i}].speedPxPerSec`, CHYRON_SPEED_MIN, CHYRON_SPEED_MAX);
|
||||
else fail(`overlays[${i}].kind`);
|
||||
});
|
||||
const seen = new Set<number>();
|
||||
graph.audio.inputs.forEach((m, i) => {
|
||||
int(m.source, `audio.inputs[${i}].source`, 0, plan.sources.length - 1);
|
||||
if (plan.sources[m.source]?.audio == null) fail(`audio.inputs[${i}] names a source without audio`);
|
||||
if (seen.has(m.source)) fail(`audio.inputs[${i}] repeats a source`);
|
||||
seen.add(m.source);
|
||||
gainDb(m.gainDb, `audio.inputs[${i}].gainDb`);
|
||||
});
|
||||
if (graph.audio.masterGainDb < GAIN_DB_MIN || graph.audio.masterGainDb > GAIN_DB_MAX) fail("audio.masterGainDb out of range");
|
||||
gainDb(graph.audio.masterGainDb, "audio.masterGainDb");
|
||||
if (plan.output.kind === "framemd5") int(plan.output.frames, "output.frames", 1, 600);
|
||||
}
|
||||
|
||||
const RAW_IN = ["-thread_queue_size", "64", "-probesize", "32", "-analyzeduration", "0"];
|
||||
|
||||
function rawVideoInput(canvas: Canvas, fdNum: number): string[] {
|
||||
return [...RAW_IN, "-f", "rawvideo", "-pix_fmt", "yuv420p", "-s", `${canvas.width}x${canvas.height}`, "-framerate", String(canvas.fps), "-i", `pipe:${fdNum}`];
|
||||
}
|
||||
|
||||
function rawAudioInput(fdNum: number): string[] {
|
||||
return [...RAW_IN, "-f", "s16le", "-ar", String(GATE_SAMPLE_RATE), "-ac", String(GATE_CHANNELS), "-i", `pipe:${fdNum}`];
|
||||
}
|
||||
|
||||
function syntheticVideo(canvas: Canvas, pattern: SyntheticPattern, seed: number): string {
|
||||
// Seeded temporal noise makes every source distinct and every frame different, deterministically.
|
||||
return `${pattern}=s=${canvas.width}x${canvas.height}:r=${canvas.fps},noise=alls=12:allf=t+u:all_seed=${seed},format=yuv420p`;
|
||||
}
|
||||
|
||||
/** Full compositor argv. The process reads N sources (+ assets) and writes the gate FIFOs on fds 1/3. */
|
||||
export function buildCompositorArgs(plan: CompositorPlan): string[] {
|
||||
validatePlan(plan);
|
||||
const { canvas } = plan.graph;
|
||||
// Single-threaded filtering keeps golden frames deterministic and per-compositor CPU predictable.
|
||||
// (Slice threads were measured in the S2a CPU smoke and did not change its bus drops.)
|
||||
// eslint-disable-next-line no-restricted-syntax -- FFmpeg's -threads options, not the Threads provider
|
||||
const args = ["-hide_banner", "-nostdin", "-loglevel", "error", "-threads", "1", "-filter_threads", "1", "-filter_complex_threads", "1"];
|
||||
let next = 0;
|
||||
const video: number[] = [];
|
||||
const audio: Array<number | null> = [];
|
||||
const synthetic = plan.output.kind === "framemd5";
|
||||
for (const s of plan.sources) {
|
||||
if (s.video.kind === "rawPipe") args.push(...rawVideoInput(canvas, s.video.fd));
|
||||
else args.push("-f", "lavfi", "-i", syntheticVideo(canvas, s.video.pattern, s.video.seed));
|
||||
video.push(next++);
|
||||
if (s.audio === null) { audio.push(null); continue; }
|
||||
if (s.audio.kind === "rawPipe") args.push(...rawAudioInput(s.audio.fd));
|
||||
else args.push("-f", "lavfi", "-i", `sine=frequency=${s.audio.toneHz}:sample_rate=${GATE_SAMPLE_RATE}`);
|
||||
audio.push(next++);
|
||||
}
|
||||
const assets: number[] = [];
|
||||
for (const path of plan.assets) {
|
||||
// One decoded frame, held by overlay's eof_action=repeat. A `-loop 1` image stream would be
|
||||
// decoded every frame and framesync'd against the live main input, which jitters the main.
|
||||
args.push("-framerate", String(canvas.fps), "-i", `file:${path}`);
|
||||
assets.push(next++);
|
||||
}
|
||||
args.push("-filter_complex", buildFilterGraph(plan, { video, audio, assets }));
|
||||
if (plan.output.kind === "gate") {
|
||||
args.push("-map", "[vout]", "-f", "rawvideo", "-pix_fmt", "yuv420p", "-fps_mode", "passthrough", "-flush_packets", "1", "pipe:1",
|
||||
"-map", "[aout]", "-f", "s16le", "-ar", String(GATE_SAMPLE_RATE), "-ac", String(GATE_CHANNELS), "-flush_packets", "1", "pipe:3");
|
||||
} else {
|
||||
void synthetic;
|
||||
const seconds = (plan.output.frames / canvas.fps).toFixed(6);
|
||||
args.push("-map", "[vout]", "-map", "[aout]", "-t", seconds, "-c:v", "rawvideo", "-c:a", "pcm_s16le", "-fflags", "+bitexact", "-flags:v", "+bitexact", "-flags:a", "+bitexact", "-f", "framemd5", "pipe:1");
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Ingest ×N normalize stage
|
||||
|
||||
export type IngestInput =
|
||||
/** Live MPEG-TS from the pinned egress relay on stdin (WP7). */
|
||||
| { kind: "mpegtsPipe" }
|
||||
/**
|
||||
* Playlist source (§5.1): the asset feeder's concatenated MPEG-TS on stdin, read at native rate
|
||||
* (`-re`; assets arrive far faster than real time). The feeder shifts each asset to where the
|
||||
* previous one ended, so timestamps are monotonic and the normalize stage and the gate hide
|
||||
* asset boundaries. Audio is optional (a silent asset gets the bus's silence fill).
|
||||
*/
|
||||
| { kind: "playlistPipe" }
|
||||
/** Deterministic test source, paced at native rate. */
|
||||
| { kind: "synthetic"; pattern: SyntheticPattern; seed: number; toneHz: number };
|
||||
|
||||
export interface IngestPlan {
|
||||
input: IngestInput;
|
||||
canvas: Canvas;
|
||||
/** "none" when the source is known to have no audio track: silence is generated instead. */
|
||||
audio: "source" | "none";
|
||||
}
|
||||
|
||||
/**
|
||||
* One process per source: decode → scale/pad to the canvas, CFR at the canvas
|
||||
* fps, yuv420p; audio → 48 kHz s16le stereo. Video on fd 1, PCM on fd 3.
|
||||
*/
|
||||
export function buildIngestArgs(plan: IngestPlan): string[] {
|
||||
const { canvas, input } = plan;
|
||||
assertGeometry(canvas);
|
||||
// Auto frame-threading scales with host CPU count and adds almost a second of decode
|
||||
// latency on a 28-core host. Independently warming ingests then hold different amounts
|
||||
// of history in the compositor inputs. Bound decode/filter latency and emit raw frames
|
||||
// synchronously instead of frame-threading the rawvideo encoder as well.
|
||||
// eslint-disable-next-line no-restricted-syntax -- FFmpeg options, not the Threads provider
|
||||
const args = ["-hide_banner", "-nostdin", "-loglevel", "error", "-threads", "2", "-filter_threads", "1"];
|
||||
if (input.kind === "mpegtsPipe") {
|
||||
args.push("-fflags", "+genpts+discardcorrupt", "-f", "mpegts", "-i", "pipe:0");
|
||||
} else if (input.kind === "playlistPipe") {
|
||||
args.push("-re", "-fflags", "+genpts+discardcorrupt", "-f", "mpegts", "-i", "pipe:0");
|
||||
} else {
|
||||
if (!SYNTHETIC_PATTERNS.includes(input.pattern)) fail("ingest pattern");
|
||||
int(input.seed, "ingest seed", 0, 1_000_000);
|
||||
int(input.toneHz, "ingest toneHz", 20, 20_000);
|
||||
args.push("-re", "-f", "lavfi", "-i", syntheticVideo(canvas, input.pattern, input.seed),
|
||||
"-re", "-f", "lavfi", "-i", `sine=frequency=${input.toneHz}:sample_rate=${GATE_SAMPLE_RATE}`);
|
||||
}
|
||||
const audioIndex = input.kind === "synthetic" ? 1 : 0;
|
||||
if (plan.audio === "none") args.push("-f", "lavfi", "-i", `anullsrc=r=${GATE_SAMPLE_RATE}:cl=stereo`);
|
||||
const silenceIndex = input.kind === "synthetic" ? 2 : 1;
|
||||
const sourceAudio = input.kind === "playlistPipe" ? `${audioIndex}:a:0?` : `${audioIndex}:a:0`;
|
||||
const W = canvas.width;
|
||||
const H = canvas.height;
|
||||
args.push(
|
||||
"-map", "0:v:0",
|
||||
"-vf", `scale=${W}:${H}:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=${W}:${H}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=${canvas.fps},format=yuv420p`,
|
||||
// eslint-disable-next-line no-restricted-syntax -- FFmpeg rawvideo thread option, not the Threads provider
|
||||
"-c:v", "rawvideo", "-threads:v", "1", "-f", "rawvideo", "-pix_fmt", "yuv420p", "-flush_packets", "1", "pipe:1",
|
||||
"-map", plan.audio === "none" ? `${silenceIndex}:a:0` : sourceAudio,
|
||||
"-af", `aresample=${GATE_SAMPLE_RATE}:async=1000,${AUDIO_FORMAT}`,
|
||||
"-f", "s16le", "-ar", String(GATE_SAMPLE_RATE), "-ac", String(GATE_CHANNELS), "-flush_packets", "1", "pipe:3",
|
||||
);
|
||||
return args;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Persistent encoder
|
||||
|
||||
export type EncoderOutput = { kind: "flvPipe"; fd: number } | { kind: "null" };
|
||||
|
||||
/**
|
||||
* Program preview tap (§2.1: "the program tap is the same mechanism applied to the gate output"):
|
||||
* the encoder also emits what it encodes, downscaled to `size`, as raw yuv420p + s16le on two
|
||||
* extra fds that a source-bus helper reads. Each tap output is a `tee` with `onfail=ignore` plus a
|
||||
* `null` anchor slave: a dead tap reader closes only that slave ("continuing with 1/2 slaves"),
|
||||
* never the encoder (with a single slave ffmpeg 5.1 fails the whole output, verified in S2b).
|
||||
*/
|
||||
export interface EncoderProgramTap { videoFd: number; audioFd: number; size: { width: number; height: number } }
|
||||
|
||||
export interface EncoderPlan { canvas: Canvas; videoBitrateKbps: number; audioBitrateKbps: number; output: EncoderOutput; programTap?: EncoderProgramTap }
|
||||
|
||||
/** Reads the gate on fds 0 (video) / 3 (audio). Started once per channel; scene changes never restart it. */
|
||||
export function buildEncoderArgs(plan: EncoderPlan): string[] {
|
||||
const { canvas, programTap: tap } = plan;
|
||||
assertGeometry(canvas);
|
||||
const vb = int(plan.videoBitrateKbps, "videoBitrateKbps", 200, 50_000);
|
||||
const ab = int(plan.audioBitrateKbps, "audioBitrateKbps", 32, 320);
|
||||
const args = ["-hide_banner", "-nostdin", "-loglevel", "error", ...rawVideoInput(canvas, 0), ...rawAudioInput(3)];
|
||||
if (tap) {
|
||||
const w = int(tap.size.width, "programTap.width", 2, canvas.width);
|
||||
const h = int(tap.size.height, "programTap.height", 2, canvas.height);
|
||||
if (w % 2 || h % 2) fail("programTap size must be even");
|
||||
args.push("-filter_complex", `[0:v]split=2[enc][tap];[tap]scale=${w}:${h}:flags=bilinear,format=yuv420p[tapv];[1:a]asplit=2[enca][tapa]`,
|
||||
"-map", "[enc]", "-map", "[enca]");
|
||||
} else {
|
||||
args.push("-map", "0:v", "-map", "1:a");
|
||||
}
|
||||
args.push(
|
||||
"-c:v", "libx264", "-preset", "veryfast", "-tune", "zerolatency", "-pix_fmt", "yuv420p",
|
||||
"-b:v", `${vb}k`, "-maxrate", `${vb}k`, "-bufsize", `${vb * 2}k`, "-g", String(canvas.fps * 2), "-keyint_min", String(canvas.fps * 2), "-sc_threshold", "0",
|
||||
"-c:a", "aac", "-b:a", `${ab}k`, "-ar", String(GATE_SAMPLE_RATE));
|
||||
if (plan.output.kind === "flvPipe") args.push("-f", "flv", `pipe:${fd(plan.output.fd, "output.fd")}`);
|
||||
else args.push("-f", "null", "-");
|
||||
if (tap) {
|
||||
const v = fd(tap.videoFd, "programTap.videoFd");
|
||||
const a = fd(tap.audioFd, "programTap.audioFd");
|
||||
args.push("-map", "[tapv]", "-c:v", "rawvideo", "-f", "tee", `[f=rawvideo:onfail=ignore]pipe:${v}|[f=null]-`,
|
||||
"-map", "[tapa]", "-c:a", "pcm_s16le", "-f", "tee", `[f=s16le:onfail=ignore]pipe:${a}|[f=null]-`);
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Playlist feeder (§5.1)
|
||||
|
||||
/** Output timestamp offset: microseconds, rendered as seconds (bounded: ~1 year of continuous playout). */
|
||||
function tsOffset(offsetUs: number): string {
|
||||
return (int(offsetUs, "offsetUs", 0, 365 * 86_400 * 1_000_000) / 1e6).toFixed(6);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remux of one playlist asset (pre-normalized at upload, so never transcoded) from stdin to MPEG-TS
|
||||
* on stdout, with its timestamps shifted to `offsetUs` (where the previous asset ended), so the
|
||||
* concatenated stream is monotonic. FFmpeg 5.1's own MPEG-TS discontinuity correction does not
|
||||
* work with `-re` (measured: 1.3–2 s stalls and bursts at every boundary). `-progress pipe:3`
|
||||
* reports the asset's end (`out_time_us`, excluding the offset) for the next asset's offset.
|
||||
* `initial_discontinuity` marks the continuity-counter restart as legitimate.
|
||||
*/
|
||||
export function buildPlaylistRemuxArgs(offsetUs: number): string[] {
|
||||
return ["-hide_banner", "-nostdin", "-loglevel", "error", "-progress", "pipe:3", "-protocol_whitelist", "pipe", "-i", "pipe:0",
|
||||
"-map", "0:v:0", "-map", "0:a:0?", "-c", "copy", "-output_ts_offset", tsOffset(offsetUs),
|
||||
"-f", "mpegts", "-mpegts_flags", "+initial_discontinuity", "-muxdelay", "0", "pipe:1"];
|
||||
}
|
||||
|
||||
/**
|
||||
* Slate floor (§5.1): `seconds` of a still (the rendered slate PNG when there is one, else black)
|
||||
* plus silence as MPEG-TS at `offsetUs`, fed when no asset can be played or a non-looping playlist
|
||||
* has ended. Encoded faster than real time and paced by the playlist ingest like any asset.
|
||||
*/
|
||||
export function buildSlateArgs(plan: { canvas: Canvas; seconds: number; image: string | null; offsetUs: number }): string[] {
|
||||
const { canvas } = plan;
|
||||
assertGeometry(canvas);
|
||||
const seconds = int(plan.seconds, "slate seconds", 1, 60);
|
||||
const args = ["-hide_banner", "-nostdin", "-loglevel", "error", "-progress", "pipe:3"];
|
||||
if (plan.image) args.push("-loop", "1", "-framerate", String(canvas.fps), "-i", `file:${assetPath(plan.image, "slate image")}`);
|
||||
else args.push("-f", "lavfi", "-i", `color=c=black:s=${canvas.width}x${canvas.height}:r=${canvas.fps}`);
|
||||
args.push("-f", "lavfi", "-i", `anullsrc=r=${GATE_SAMPLE_RATE}:cl=stereo`, "-t", String(seconds),
|
||||
"-vf", `scale=${canvas.width}:${canvas.height}:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=${canvas.width}:${canvas.height}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,format=yuv420p`,
|
||||
"-c:v", "libx264", "-preset", "ultrafast", "-tune", "stillimage", "-g", String(canvas.fps), "-b:v", "500k",
|
||||
"-c:a", "aac", "-b:a", "64k", "-output_ts_offset", tsOffset(plan.offsetUs),
|
||||
"-f", "mpegts", "-mpegts_flags", "+initial_discontinuity", "-muxdelay", "0", "pipe:1");
|
||||
return args;
|
||||
}
|
||||
|
||||
export const MediaCommandBuilder = {
|
||||
layoutRects, rectFromBasisPoints, buildFilterGraph,
|
||||
compositor: buildCompositorArgs,
|
||||
ingest: buildIngestArgs,
|
||||
encoder: buildEncoderArgs,
|
||||
} as const;
|
||||
95
apps/media-worker/src/compositor/compositor.ts
Normal file
95
apps/media-worker/src/compositor/compositor.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import type { StdioOptions } from "node:child_process";
|
||||
import type { FrameGate } from "../gate/frame-gate";
|
||||
import { spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg";
|
||||
import { buildCompositorArgs, type CompositorGraph, type CompositorSource } from "./command-builder";
|
||||
import type { BusSubscription, SourceBus } from "./source-bus";
|
||||
|
||||
/** A scene ready to compose: buses in input order plus the typed graph (source indexes refer to `inputs`). */
|
||||
export interface ComposeRequest {
|
||||
inputs: Array<{ bus: SourceBus; audio: boolean }>;
|
||||
assets: string[];
|
||||
graph: CompositorGraph;
|
||||
}
|
||||
|
||||
const FIRST_INPUT_FD = 4;
|
||||
|
||||
/**
|
||||
* One compositor process (A or B of a swap). Reads its sources from bus
|
||||
* subscription FIFOs inherited on fds 4.., writes the gate's slot FIFOs on fds 1 and 3.
|
||||
* Its lifetime is bounded by the gate: when the gate retires the slot the FIFOs
|
||||
* close and the process gets EPIPE; `stop()` also escalates to SIGKILL.
|
||||
*/
|
||||
export class CompositorInstance {
|
||||
private proc: ManagedProcess | null = null;
|
||||
/** One per input, in request order (valid after `start()`). */
|
||||
subscriptions: BusSubscription[] = [];
|
||||
/** Gate origin sent at attach (see protocol.attachLine). */
|
||||
origin: number | undefined;
|
||||
slotId = 0;
|
||||
|
||||
constructor(
|
||||
private readonly gate: FrameGate,
|
||||
private readonly request: ComposeRequest,
|
||||
private readonly redactor: StderrRedactor,
|
||||
private readonly onStderr: (message: string) => void,
|
||||
) {}
|
||||
|
||||
get pid(): number | undefined { return this.proc?.child.pid; }
|
||||
|
||||
/**
|
||||
* Subscribes, spawns and attaches to the gate as the pending slot. `originFor`
|
||||
* maps the fresh subscriptions to a gate origin. Resolve ≠ on air: use `gate.waitActive(slotId)`.
|
||||
*/
|
||||
async start(originFor?: (subscriptions: BusSubscription[]) => number | undefined): Promise<void> {
|
||||
const slot = this.gate.allocateSlot();
|
||||
this.slotId = slot.id;
|
||||
try {
|
||||
for (const input of this.request.inputs) this.subscriptions.push(await input.bus.subscribe({ video: true, audio: input.audio }));
|
||||
} catch (error) {
|
||||
slot.fifos.releaseWriters();
|
||||
slot.fifos.releaseHolders();
|
||||
slot.fifos.unlink();
|
||||
await this.stop();
|
||||
throw error;
|
||||
}
|
||||
let next = FIRST_INPUT_FD;
|
||||
const inputFds: number[] = [];
|
||||
const sources: CompositorSource[] = this.subscriptions.map((sub) => {
|
||||
if (sub.videoFd === null) throw new Error("compositor input without video");
|
||||
inputFds.push(sub.videoFd);
|
||||
const video = { kind: "rawPipe" as const, fd: next++ };
|
||||
if (sub.audioFd === null) return { video, audio: null };
|
||||
inputFds.push(sub.audioFd);
|
||||
return { video, audio: { kind: "rawPipe" as const, fd: next++ } };
|
||||
});
|
||||
const args = buildCompositorArgs({ sources, assets: this.request.assets, graph: this.request.graph, output: { kind: "gate", videoFd: 1, audioFd: 3 } });
|
||||
const stdio: StdioOptions = ["ignore", slot.fifos.videoFd, "pipe", slot.fifos.audioFd, ...inputFds];
|
||||
let proc: ManagedProcess;
|
||||
try {
|
||||
proc = spawnManaged("ffmpeg", args, stdio, this.redactor, this.onStderr);
|
||||
} finally {
|
||||
slot.fifos.releaseWriters();
|
||||
// The compositor holds the read ends now: when it exits, the bus sees EPIPE and drops it.
|
||||
for (const sub of this.subscriptions) sub.releaseReaders();
|
||||
}
|
||||
this.proc = proc;
|
||||
try {
|
||||
await proc.started;
|
||||
this.origin = originFor?.(this.subscriptions);
|
||||
await this.gate.attach(slot, this.origin);
|
||||
} catch (error) {
|
||||
slot.fifos.releaseHolders();
|
||||
slot.fifos.unlink();
|
||||
await this.stop();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
for (const sub of this.subscriptions) sub.unsubscribe();
|
||||
await this.proc?.terminate();
|
||||
this.proc = null;
|
||||
}
|
||||
|
||||
kill(signal: NodeJS.Signals): void { this.proc?.kill(signal); }
|
||||
}
|
||||
206
apps/media-worker/src/compositor/cpu-smoke.ts
Normal file
206
apps/media-worker/src/compositor/cpu-smoke.ts
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
/**
|
||||
* S2 CPU budget and sustained transport gate:
|
||||
* 4 × 1080p30 H.264 MPEG-TS sources → ingest×4 (decode + normalize) → source-bus helpers →
|
||||
* 2x2 compositor with a lower third and a chyron → Rust frame gate → x264
|
||||
* veryfast 6 Mb/s persistent encoder (-f null). CPU per process from
|
||||
* /proc/<pid>/stat over the measurement window, plus the worker (Node) itself.
|
||||
*
|
||||
* node --import tsx src/compositor/cpu-smoke.ts [--seconds 30] [--swaps-every 0]
|
||||
* [--warmup-seconds 30] [--settle-max 45]
|
||||
*/
|
||||
import { closeSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { ManagedProcess } from "../relay/ffmpeg";
|
||||
import { layoutRects, type Canvas, type CompositorGraph } from "./command-builder";
|
||||
import type { ComposeRequest } from "./compositor";
|
||||
import { IngestProcess } from "./ingest";
|
||||
import { ProgramChain } from "./program";
|
||||
import { encodeSourceClip, ensureGateBinary, makeAssets, passthroughRedactor, startPacer } from "./testing";
|
||||
|
||||
const canvas: Canvas = { width: 1920, height: 1080, fps: 30 };
|
||||
const TICKS = 100; // USER_HZ
|
||||
|
||||
function cpuTicks(pid: number | undefined): number {
|
||||
if (pid === undefined) return 0;
|
||||
try {
|
||||
const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
|
||||
const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" ");
|
||||
return Number(fields[11]) + Number(fields[12]); // utime + stime
|
||||
} catch { return 0; }
|
||||
}
|
||||
|
||||
/** Time spent runnable but not running (host contention), ns, from /proc/<pid>/schedstat (all threads). */
|
||||
function runDelayNs(pid: number | undefined): number {
|
||||
if (pid === undefined) return 0;
|
||||
try {
|
||||
let total = 0;
|
||||
for (const tid of readdirSync(`/proc/${pid}/task`)) total += Number(readFileSync(`/proc/${pid}/task/${tid}/schedstat`, "utf8").split(" ")[1] ?? 0);
|
||||
return total;
|
||||
} catch { return 0; }
|
||||
}
|
||||
|
||||
function arg(name: string, fallback: number): number {
|
||||
const i = process.argv.indexOf(`--${name}`);
|
||||
return i >= 0 ? Number(process.argv[i + 1]) : fallback;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const seconds = arg("seconds", 30);
|
||||
const swapEvery = arg("swaps-every", 0);
|
||||
const minimumWarmup = arg("warmup-seconds", 30);
|
||||
const settleCap = arg("settle-max", 45);
|
||||
if (!Number.isFinite(seconds) || seconds <= 0 || !Number.isFinite(swapEvery) || swapEvery < 0
|
||||
|| !Number.isFinite(minimumWarmup) || minimumWarmup < 0 || !Number.isFinite(settleCap) || settleCap < Math.max(3, minimumWarmup)) {
|
||||
throw new Error("invalid smoke duration, swap interval, or warmup/settle bounds");
|
||||
}
|
||||
if (!ensureGateBinary()) throw new Error("native helpers unavailable (cargo build failed)");
|
||||
const dir = mkdtempSync(join(tmpdir(), "cpu-smoke-"));
|
||||
const procs: ManagedProcess[] = [];
|
||||
const ingests: IngestProcess[] = [];
|
||||
let chain: ProgramChain | null = null;
|
||||
try {
|
||||
// A 1080p30 H.264 + AAC source clip (encoded once, before measuring).
|
||||
const clip = join(dir, "src.ts");
|
||||
encodeSourceClip(clip, canvas);
|
||||
const assets = makeAssets(dir);
|
||||
|
||||
for (let i = 0; i < 4; i += 1) {
|
||||
// Stand-in for the pinned egress relay: a paced remux of the clip into a FIFO the ingest reads.
|
||||
const { readFd, proc: pacer } = await startPacer(clip, dir, `src${i}`);
|
||||
procs.push(pacer);
|
||||
const ingest = new IngestProcess(`s${i}`, { input: { kind: "mpegtsPipe" }, canvas, audio: "source" }, readFd, join(dir, "gate"), passthroughRedactor,
|
||||
(m) => process.stderr.write(m), (id, r) => process.stderr.write(`${id} ${r}\n`));
|
||||
await ingest.start();
|
||||
closeSync(readFd);
|
||||
ingests.push(ingest);
|
||||
}
|
||||
|
||||
const graph = (i: number): CompositorGraph => ({
|
||||
canvas, layout: "2x2",
|
||||
tiles: layoutRects("2x2", canvas).map((rect, t) => ({ source: (t + i) % 4, rect, fit: "contain" as const })),
|
||||
overlays: [{ kind: "image", asset: 0, x: 96, y: 900 }, { kind: "chyron", asset: 1, y: 1040, speedPxPerSec: 120 }],
|
||||
audio: { inputs: [{ source: 0, gainDb: 0 }, { source: 1, gainDb: -6 }], masterGainDb: 0 },
|
||||
});
|
||||
const request = (i: number): ComposeRequest => ({ inputs: ingests.map((ing, k) => ({ bus: ing.bus, audio: k < 2 })), assets, graph: graph(i) });
|
||||
const fatal: string[] = [];
|
||||
chain = new ProgramChain({ canvas, scratchDir: join(dir, "gate"), videoBitrateKbps: 6000, output: { kind: "null" },
|
||||
redactor: passthroughRedactor, onStderr: (m) => process.stderr.write(m), onFatal: (r) => fatal.push(r) });
|
||||
const tStart = Date.now();
|
||||
await chain.start(request(0));
|
||||
const warmupEnd = Date.now() + minimumWarmup * 1000;
|
||||
// Warm-up: the first seconds are a startup transient on a loaded host (4 ingests bursting to
|
||||
// catch up with their sources, compositor and x264 warming up), during which the compositor's
|
||||
// bus queues stand near the backlog limit and the gate repeats. Measure the steady state: wait
|
||||
// until every subscriber queue is within its configured backlog + replay history
|
||||
// and neither bus drops nor gate fills increase for 3 consecutive seconds. Allow at least
|
||||
// 30 s for demux/decoder queues and x264 to warm, at most 45 s by default. Startup loss
|
||||
// stays visible separately; the sustained window begins only after this declared warmup.
|
||||
// A queue can remain nonempty at native rate: it reflects fixed pipeline/replay delay,
|
||||
// not overload. Absence of drops/fills at bounded depth is the steady-state criterion.
|
||||
const backlogLimit = Math.min(...ingests.map((ingest) => ingest.bus.maxBacklogFrames + ingest.bus.historyFrames));
|
||||
let calm = 0;
|
||||
const initialGate = await chain.stats();
|
||||
let lastFills = initialGate.vrep + initialGate.asil;
|
||||
let lastDrops = (await Promise.all(ingests.map((ingest) => ingest.bus.stats()))).reduce((total, bus) => total + bus.subscriberDrops, 0);
|
||||
const settleTrace: Array<[number, number, number]> = [];
|
||||
for (let s = 0; s < settleCap && (calm < 3 || Date.now() < warmupEnd); s += 1) {
|
||||
await new Promise((r) => setTimeout(r, 1_000));
|
||||
const buses = await Promise.all(ingests.map((i) => i.bus.stats()));
|
||||
const q = Math.max(0, ...buses.flatMap((bus) => bus.subscribers.map((sub) => sub.queued)));
|
||||
const drops = buses.reduce((total, bus) => total + bus.subscriberDrops, 0);
|
||||
const gate = await chain.stats();
|
||||
const fills = gate.vrep + gate.asil;
|
||||
settleTrace.push([q, fills - lastFills, drops - lastDrops]);
|
||||
calm = q <= backlogLimit && fills === lastFills && drops === lastDrops ? calm + 1 : 0;
|
||||
lastFills = fills;
|
||||
lastDrops = drops;
|
||||
}
|
||||
const settleS = (Date.now() - tStart) / 1000;
|
||||
const startupDrops = (await Promise.all(ingests.map((i) => i.bus.stats()))).reduce((a, b) => a + b.subscriberDrops, 0);
|
||||
|
||||
const pids = {
|
||||
ingest: ingests.map((i) => i.pid),
|
||||
compositor: () => chain?.activeCompositor?.pid,
|
||||
gate: chain.gate.pid,
|
||||
encoder: chain.encoderPid,
|
||||
};
|
||||
const busPids = ingests.map((i) => i.bus.pid);
|
||||
const snap = async () => {
|
||||
const buses = await Promise.all(ingests.map((i) => i.bus.stats()));
|
||||
return {
|
||||
ingest: pids.ingest.map(cpuTicks).reduce((a, b) => a + b, 0),
|
||||
compositor: cpuTicks(pids.compositor()),
|
||||
gate: cpuTicks(pids.gate),
|
||||
encoder: cpuTicks(pids.encoder),
|
||||
buses: busPids.map(cpuTicks).reduce((a, b) => a + b, 0),
|
||||
worker: process.cpuUsage(),
|
||||
busDrops: buses.reduce((a, b) => a + b.subscriberDrops, 0),
|
||||
busPairs: buses.map((b) => b.pairs),
|
||||
busStats: buses,
|
||||
delay: { compositor: runDelayNs(pids.compositor()), encoder: runDelayNs(pids.encoder), gate: runDelayNs(pids.gate) },
|
||||
};
|
||||
};
|
||||
await chain.gate.resetPeaks();
|
||||
const g0 = await chain.stats();
|
||||
const s0 = await snap();
|
||||
const t0 = Date.now();
|
||||
let swaps = 0;
|
||||
// Deepest subscriber queue seen per second (frames waiting in a source-bus helper for a consumer).
|
||||
const queueMax: number[] = [];
|
||||
while (Date.now() - t0 < seconds * 1000) {
|
||||
await new Promise((r) => setTimeout(r, swapEvery > 0 ? swapEvery * 1000 : 1000));
|
||||
if (swapEvery > 0) { await chain.swap(request(++swaps)); }
|
||||
const depths = (await Promise.all(ingests.map((i) => i.bus.stats()))).flatMap((b) => b.subscribers.map((s) => s.queued));
|
||||
queueMax.push(Math.max(0, ...depths));
|
||||
}
|
||||
const s1 = await snap();
|
||||
const g1 = await chain.stats();
|
||||
const wall = (Date.now() - t0) / 1000;
|
||||
const pct = (ticks: number): number => Number(((ticks / TICKS / wall) * 100).toFixed(1));
|
||||
const workerUs = (s1.worker.user + s1.worker.system) - (s0.worker.user + s0.worker.system);
|
||||
const result = {
|
||||
host: { cpus: (await import("node:os")).cpus().length, model: (await import("node:os")).cpus()[0]?.model, loadavg: (await import("node:os")).loadavg() },
|
||||
wallS: wall, swaps,
|
||||
// Seconds from chain start to steady state, the per-second [max queue, gate repeats] trace, and drops before the window.
|
||||
settleS, minimumWarmupS: minimumWarmup, settled: calm >= 3 && Date.now() >= warmupEnd, settleTrace, startupDrops,
|
||||
startupGate: { ticks: g0.ticks, vrep: g0.vrep, asil: g0.asil, outDrops: g0.outDrops },
|
||||
pctOfOneCore: {
|
||||
ingestX4: pct(s1.ingest - s0.ingest),
|
||||
compositor: swapEvery > 0 ? null : pct(s1.compositor - s0.compositor),
|
||||
gate: pct(s1.gate - s0.gate),
|
||||
encoderX264: pct(s1.encoder - s0.encoder),
|
||||
sourceBusX4: pct(s1.buses - s0.buses),
|
||||
workerNode: Number(((workerUs / 1e6 / wall) * 100).toFixed(1)),
|
||||
},
|
||||
// Pairs the buses dropped for a subscriber that fell behind, inside the window (a compositor not keeping up).
|
||||
busDropsInWindow: s1.busDrops - s0.busDrops,
|
||||
busQueueMaxPerSecond: queueMax,
|
||||
// Host contention: ms per second of wall time each process (all threads) sat runnable but not scheduled.
|
||||
runDelayMsPerS: {
|
||||
compositor: Number(((s1.delay.compositor - s0.delay.compositor) / 1e6 / wall).toFixed(1)),
|
||||
encoder: Number(((s1.delay.encoder - s0.delay.encoder) / 1e6 / wall).toFixed(1)),
|
||||
gate: Number(((s1.delay.gate - s0.delay.gate) / 1e6 / wall).toFixed(1)),
|
||||
},
|
||||
// Source pairs each bus produced inside the window (≈ 30 × wallS for a paced 30 fps source).
|
||||
busPairsInWindow: s1.busPairs.map((n, k) => n - (s0.busPairs[k] ?? 0)),
|
||||
gate: { ticks: g1.ticks - g0.ticks, vrep: g1.vrep - g0.vrep, asil: g1.asil - g0.asil, lateOverFrame: g1.lateOverFrame, lateMaxUs: g1.lateMaxUs, outDrops: g1.outDrops - g0.outDrops },
|
||||
buses: s1.busStats,
|
||||
fatal,
|
||||
};
|
||||
const p = result.pctOfOneCore;
|
||||
process.stdout.write(`${JSON.stringify({ ...result, totalCores: Number((((p.ingestX4 + (p.compositor ?? 0) + p.gate + p.encoderX264 + p.sourceBusX4 + p.workerNode) / 100)).toFixed(2)) })}\n`);
|
||||
if (!result.settled || result.busDropsInWindow !== 0 || result.gate.vrep !== 0 || result.gate.asil !== 0 || result.gate.outDrops !== 0 || fatal.length > 0
|
||||
|| result.busPairsInWindow.some((pairs) => pairs < canvas.fps * wall * 0.98)) {
|
||||
throw new Error("cpu-smoke failed: require settled 4×1080p30, zero bus drops, zero gate fills/output drops, and no fatal exit");
|
||||
}
|
||||
} finally {
|
||||
await chain?.stop();
|
||||
for (const i of ingests) await i.stop();
|
||||
for (const p of procs) await p.terminate(500);
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (require.main === module) void main().catch((error: unknown) => { process.stderr.write(`${String(error)}\n`); process.exit(1); });
|
||||
115
apps/media-worker/src/compositor/golden-frames.json
Normal file
115
apps/media-worker/src/compositor/golden-frames.json
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
{
|
||||
"ffmpeg": "ffmpeg version 5.1.9-0+deb12u1 Copyright (c) 2000-2026 the FFmpeg developers",
|
||||
"canvas": {
|
||||
"width": 320,
|
||||
"height": 180,
|
||||
"fps": 30
|
||||
},
|
||||
"frames": 4,
|
||||
"layouts": {
|
||||
"single": [
|
||||
"0:0:86400:473ec55d56d6cd3f85b31ef5f0b615b2",
|
||||
"1:0:4096:f858b65fe890efbc0ac6cf1e085a37d2",
|
||||
"1:1024:4096:e3ef895bb0e533a4a8a89d5a6ed843f6",
|
||||
"0:1:86400:80e9688e53079ff4623b78115e2b6b1c",
|
||||
"1:2048:4096:8da1b5054e4f15590b674af0c5239537",
|
||||
"1:3072:4096:18a3a0028363827191cd4d8885bc2b11",
|
||||
"0:2:86400:f2b6d37903c429f8ad6c4416056a8538",
|
||||
"1:4096:4096:aecafa4e5f0f3626cc00a9849cbcc3fa",
|
||||
"0:3:86400:5113494af070e29d7c5c6cbbbaa7a721",
|
||||
"1:5120:4096:749e60b205100820f0f493a79c17ee46",
|
||||
"1:6144:1024:f465c566bd65b8743119b970d5b9734e"
|
||||
],
|
||||
"pip": [
|
||||
"0:0:86400:ff2365f77b17a7437246001c3282af88",
|
||||
"1:0:4096:fd773b1ed90b3793d331bebf439d85a7",
|
||||
"1:1024:4096:e1a1133f44e0c84ae28eca0ac57333c9",
|
||||
"0:1:86400:9a238c82abca419758f64d31f3f5080e",
|
||||
"1:2048:4096:c7f8ec938afbe515acf2cc21ee07b398",
|
||||
"1:3072:4096:c99fc35fb2f9fb03c18e8ff04b5300a5",
|
||||
"0:2:86400:ef78085de2d4d6f6e20e266e40e7a07d",
|
||||
"1:4096:4096:62bc5a8e144fb37ec4e5a48fa0a3179a",
|
||||
"0:3:86400:956a390b4024b79289063d54abde9c07",
|
||||
"1:5120:4096:20dd40a44d6167e3d20631d72cca7021",
|
||||
"1:6144:1024:4aebc7614678f7837d5c2c42c7ae5049"
|
||||
],
|
||||
"2x1": [
|
||||
"0:0:86400:124ceb28ce6fb2adda306ebab85c020e",
|
||||
"1:0:4096:779110ff86bde4abc0c312c254c6a07b",
|
||||
"1:1024:4096:58bf6a6a073a28c148162edbfb1cb2f0",
|
||||
"0:1:86400:2a815118db500f1fe35af785761546d3",
|
||||
"1:2048:4096:b47c4007c2bd03d67892ed6c4d2e0ad2",
|
||||
"1:3072:4096:4c1fa084e7e962bdfccc6445944fe6bd",
|
||||
"0:2:86400:fb62f8882a43ace4e0a50f611ba5b011",
|
||||
"1:4096:4096:96331541b14dd07c95116914832e2646",
|
||||
"0:3:86400:3d22dd326e76b9ace9b605e67568c5c9",
|
||||
"1:5120:4096:8d09c0dd1dd3d94d2296b8b96a887d0f",
|
||||
"1:6144:1024:456a74bc6828cb74630049d3c3ede52b"
|
||||
],
|
||||
"2x2": [
|
||||
"0:0:86400:c1a86085e98caf0c6ddca6eecd76f996",
|
||||
"1:0:4096:f2e48ac5003bebe6980227d7984f03ee",
|
||||
"1:1024:4096:c42c6c8bd7e38a04da330614958e8ec7",
|
||||
"0:1:86400:eacfcee60a815e7098d9f2678ef3a2b8",
|
||||
"1:2048:4096:07d15edac8583afe4f0c37062f8e2e95",
|
||||
"1:3072:4096:2eeb48f1b0671b51871cca079144b9a0",
|
||||
"0:2:86400:a69202dd6be84938c0669cfb1b1aea91",
|
||||
"1:4096:4096:86b18ee36947048c70bacd00707025d0",
|
||||
"0:3:86400:339a15c6489b3af66d4cbe4311179d0f",
|
||||
"1:5120:4096:e10e40ebb313c090b2679d523299c4f4",
|
||||
"1:6144:1024:2f3a2ef3cc8c5f0206828e53fe002d1b"
|
||||
],
|
||||
"3x3": [
|
||||
"0:0:86400:dfffffffbc45583776628ba9e07d2ac7",
|
||||
"1:0:4096:95afee33956e36002cb7bc4174d41fb8",
|
||||
"1:1024:4096:4cc1e8ca719bd34775c4bc5f0f1d1149",
|
||||
"0:1:86400:495bf533902abce01a4c6213275d4a8a",
|
||||
"1:2048:4096:39a06315fe075c8d02500f7b69f9d0b1",
|
||||
"1:3072:4096:28a2f2f96cbb5947d1ad1bf6f0f656d3",
|
||||
"0:2:86400:2b721e57614c93bc1a50f486317d3a69",
|
||||
"1:4096:4096:f80381b7c6e59ac4ed8cf350cdf675ae",
|
||||
"0:3:86400:8e738a80e6b6a24d2c97f0a4a819e8b4",
|
||||
"1:5120:4096:47905d32edded2c3b98653ac97c9b92e",
|
||||
"1:6144:1024:7cdba1f0765decdaa01239f8d415ce83"
|
||||
],
|
||||
"4x4": [
|
||||
"0:0:86400:9b5bc219325405095ad5c3ed5acc1ea1",
|
||||
"1:0:4096:620f0b67a91f7f74151bc5be745b7110",
|
||||
"1:1024:4096:620f0b67a91f7f74151bc5be745b7110",
|
||||
"0:1:86400:f96bf1c80e7dfac31e8e612139c0ad40",
|
||||
"1:2048:4096:620f0b67a91f7f74151bc5be745b7110",
|
||||
"1:3072:4096:620f0b67a91f7f74151bc5be745b7110",
|
||||
"0:2:86400:0b6280a6f2b063114eb892b8fa2ef406",
|
||||
"1:4096:4096:620f0b67a91f7f74151bc5be745b7110",
|
||||
"0:3:86400:73f848d7c873b09ef45467fe2a8b5785",
|
||||
"1:5120:4096:620f0b67a91f7f74151bc5be745b7110",
|
||||
"1:6144:1024:0f343b0931126a20f133d67c2b018a3b"
|
||||
],
|
||||
"custom": [
|
||||
"0:0:86400:7151990b9ab56564d8f6f1ab63edd9ee",
|
||||
"1:0:4096:8904df5033d62a7340b98e03dea71359",
|
||||
"1:1024:4096:9fa5c863b67d9831fa7bc93cbe868e4e",
|
||||
"0:1:86400:ccfdc3ae531e8d495aa67e2989dd8e72",
|
||||
"1:2048:4096:5602eff96d58dd2c18365237b01aeac5",
|
||||
"1:3072:4096:d6702b18d7e15f6ead23e77cdb3b7be0",
|
||||
"0:2:86400:9725c1c13fccbc094cad0c2ef683fbab",
|
||||
"1:4096:4096:3090fcea0f7513e35c03b80a199925be",
|
||||
"0:3:86400:019eb04e7297c7bae18d86f47ebbfee7",
|
||||
"1:5120:4096:3bbc5f09a7275706e87668d3ccd96793",
|
||||
"1:6144:1024:2642090512e9f3b93a215ba0497dcbd8"
|
||||
],
|
||||
"custom-overlap": [
|
||||
"0:0:86400:5c78229f51b9307615a1f9e442efe391",
|
||||
"1:0:4096:2cc5270f68b8a25c1909d435272422b4",
|
||||
"1:1024:4096:3c83cd930cd96b9da3e3f29ce369adf9",
|
||||
"0:1:86400:0cd6d909f4b84e42f6ac6360f1bda3f6",
|
||||
"1:2048:4096:f6e469bc77eaa6d1042c70e91450b4ab",
|
||||
"1:3072:4096:f5e8ee7c276f02215053e8143e65861d",
|
||||
"0:2:86400:fdff300a63db4e8bb5d9d7fea498ea59",
|
||||
"1:4096:4096:d9af646ccccb557d6a065868cf2fa667",
|
||||
"0:3:86400:aeaecd38cc4cbd0f900e8a1cec209962",
|
||||
"1:5120:4096:f22c40c3beb84a93a61e834d0daad01a",
|
||||
"1:6144:1024:e93ffc6d18f786a744a9b1ac241ec306"
|
||||
]
|
||||
}
|
||||
}
|
||||
59
apps/media-worker/src/compositor/ingest.ts
Normal file
59
apps/media-worker/src/compositor/ingest.ts
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
import { spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg";
|
||||
import { buildIngestArgs, type IngestPlan } from "./command-builder";
|
||||
import { SourceBus } from "./source-bus";
|
||||
|
||||
/**
|
||||
* Ingest ×N normalize stage: one ffmpeg per source decodes and normalizes to
|
||||
* the canvas (raw yuv420p on fd 1, s16le 48 kHz stereo on fd 3) straight into
|
||||
* its source bus's FIFOs. Compositors and preview taps subscribe to buses; they
|
||||
* never touch the remote, and the raw frames never pass through the worker.
|
||||
*/
|
||||
export class IngestProcess {
|
||||
readonly bus: SourceBus;
|
||||
private proc: ManagedProcess | null = null;
|
||||
private stopping = false;
|
||||
|
||||
constructor(
|
||||
readonly sourceId: string,
|
||||
private readonly plan: IngestPlan,
|
||||
/** MPEG-TS fd: the WP7 pinned relay (`mpegtsPipe`) or the playlist feeder (`playlistPipe`); null for synthetic. */
|
||||
private readonly stdinFd: number | null,
|
||||
scratchDir: string,
|
||||
private readonly redactor: StderrRedactor,
|
||||
private readonly onStderr: (message: string) => void,
|
||||
private readonly onExit: (sourceId: string, reason: string) => void,
|
||||
) {
|
||||
// Native fan-out can retain a bounded half-second of live backlog in addition to
|
||||
// replay history. With FIFO resizing denied on shared hosts, the replay/input delay
|
||||
// otherwise leaves too little room for short decoder/filter scheduling bursts.
|
||||
this.bus = new SourceBus({ geometry: plan.canvas, scratchDir, redactor, onStderr,
|
||||
maxBacklogFrames: Math.max(4, Math.ceil(plan.canvas.fps / 2)),
|
||||
onExit: (reason) => { if (!this.stopping) onExit(sourceId, reason); } });
|
||||
}
|
||||
|
||||
get pid(): number | undefined { return this.proc?.child.pid; }
|
||||
|
||||
async start(): Promise<void> {
|
||||
if ((this.plan.input.kind !== "synthetic") !== (this.stdinFd !== null)) throw new Error("ingest input and fd disagree");
|
||||
this.stopping = false;
|
||||
const out = await this.bus.start();
|
||||
try {
|
||||
const proc = spawnManaged("ffmpeg", buildIngestArgs(this.plan), [this.stdinFd ?? "ignore", out.videoFd, "pipe", out.audioFd], this.redactor, this.onStderr);
|
||||
this.proc = proc;
|
||||
void proc.exited.then(({ code, signal }) => { if (!this.stopping) this.onExit(this.sourceId, `ingest exited code=${code} signal=${signal}`); });
|
||||
await proc.started;
|
||||
} catch (error) {
|
||||
await this.stop();
|
||||
throw error;
|
||||
} finally {
|
||||
this.bus.releaseIngestWriters();
|
||||
}
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.stopping = true;
|
||||
await this.proc?.terminate();
|
||||
this.proc = null;
|
||||
await this.bus.stop();
|
||||
}
|
||||
}
|
||||
160
apps/media-worker/src/compositor/playlist-feeder.test.ts
Normal file
160
apps/media-worker/src/compositor/playlist-feeder.test.ts
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
/**
|
||||
* Playlist feeder (streaming-architecture.md §5.1): argv goldens, and real
|
||||
* ffmpeg runs of feeder → `playlistPipe` ingest → source bus, checking that
|
||||
* asset boundaries keep the bus at frame rate with A/V paired, that a broken
|
||||
* asset is skipped, that the slate floor covers an unplayable or ended list,
|
||||
* and that `update()` adopts a new list at the next asset boundary.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createReadStream, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
import test from "node:test";
|
||||
import { buildIngestArgs, buildPlaylistRemuxArgs, buildSlateArgs } from "./command-builder";
|
||||
import { IngestProcess } from "./ingest";
|
||||
import { PlaylistFeeder, type FeederEvent } from "./playlist-feeder";
|
||||
import { ensureGateBinary, passthroughRedactor } from "./testing";
|
||||
|
||||
const canvas = { width: 320, height: 180, fps: 30 };
|
||||
|
||||
test("playlist argv: copy-only remux from stdin at a monotonic offset, paced ingest, slate floor", () => {
|
||||
assert.deepEqual(buildPlaylistRemuxArgs(2_020_133), ["-hide_banner", "-nostdin", "-loglevel", "error", "-progress", "pipe:3", "-protocol_whitelist", "pipe", "-i", "pipe:0",
|
||||
"-map", "0:v:0", "-map", "0:a:0?", "-c", "copy", "-output_ts_offset", "2.020133",
|
||||
"-f", "mpegts", "-mpegts_flags", "+initial_discontinuity", "-muxdelay", "0", "pipe:1"]);
|
||||
assert.throws(() => buildPlaylistRemuxArgs(-1));
|
||||
assert.throws(() => buildPlaylistRemuxArgs(1.5));
|
||||
const ingest = buildIngestArgs({ input: { kind: "playlistPipe" }, canvas, audio: "source" });
|
||||
assert.deepEqual(ingest.slice(ingest.indexOf("-re"), ingest.indexOf("-re") + 8), ["-re", "-fflags", "+genpts+discardcorrupt", "-f", "mpegts", "-i", "pipe:0", "-map"]);
|
||||
assert.ok(ingest.includes("0:a:0?"), "a silent asset cannot fail the ingest");
|
||||
const slate = buildSlateArgs({ canvas, seconds: 5, image: null, offsetUs: 0 });
|
||||
assert.ok(slate.includes("color=c=black:s=320x180:r=30") && slate.includes("anullsrc=r=48000:cl=stereo") && slate.at(-1) === "pipe:1");
|
||||
const png = `/scratch/${"a".repeat(64)}.png`;
|
||||
assert.ok(buildSlateArgs({ canvas, seconds: 5, image: png, offsetUs: 7_000_000 }).includes(`file:${png}`));
|
||||
assert.ok(buildSlateArgs({ canvas, seconds: 5, image: null, offsetUs: 7_000_000 }).includes("7.000000"));
|
||||
assert.throws(() => buildSlateArgs({ canvas, seconds: 5, image: "/etc/passwd", offsetUs: 0 }));
|
||||
assert.throws(() => buildSlateArgs({ canvas, seconds: 0, image: null, offsetUs: 0 }));
|
||||
const all = [...buildPlaylistRemuxArgs(0), ...ingest, ...slate].join(" ");
|
||||
assert.ok(!/https?:|concat|movie=|tcp:|udp:/.test(all), "pipes and lavfi only");
|
||||
});
|
||||
|
||||
function clip(dir: string, name: string, seconds: number, hz: number): string {
|
||||
const path = join(dir, name);
|
||||
const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", `testsrc2=s=320x180:r=30:d=${seconds}`, "-f", "lavfi", "-i", `sine=frequency=${hz}:d=${seconds}`,
|
||||
"-c:v", "libx264", "-preset", "ultrafast", "-g", "30", "-c:a", "aac", "-movflags", "+faststart", path]);
|
||||
assert.equal(r.status, 0);
|
||||
return path;
|
||||
}
|
||||
|
||||
async function run(t: { skip(msg: string): void }, assetIds: string[], loop: boolean, files: Map<string, string>, seconds: number,
|
||||
during?: (feeder: PlaylistFeeder) => void): Promise<{ events: FeederEvent[]; samples: number[]; stats: { pairs: number; silenceFilled: number }; feeder: PlaylistFeeder; exits: string[] } | null> {
|
||||
if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return null; }
|
||||
const dir = mkdtempSync(join(tmpdir(), "feeder-"));
|
||||
const events: FeederEvent[] = [];
|
||||
const exits: string[] = [];
|
||||
const feeder = new PlaylistFeeder("src", { assetIds, loop }, {
|
||||
assets: { async open(id) { const path = files.get(id); if (!path) throw new Error("no such asset"); return id === "broken" ? Readable.from([Buffer.from("not media")]) : createReadStream(path); } },
|
||||
canvas, scratchDir: dir, redactor: passthroughRedactor, onStderr: () => undefined, onEvent: (e) => events.push(e), slateSeconds: 2,
|
||||
});
|
||||
const ingest = new IngestProcess("src", { input: { kind: "playlistPipe" }, canvas, audio: "source" }, feeder.start(), dir, passthroughRedactor, () => undefined, (_id, r) => exits.push(r));
|
||||
const samples: number[] = [];
|
||||
try {
|
||||
await ingest.start();
|
||||
feeder.releaseReader();
|
||||
// Measure playback once FFmpeg has probed the TS input and delivered a pair.
|
||||
// Process startup is bounded separately, rather than consuming the playback window.
|
||||
const startupDeadline = Date.now() + 10_000;
|
||||
while ((await ingest.bus.stats()).pairs === 0) {
|
||||
assert.ok(Date.now() < startupDeadline, "playlist ingest produced its first A/V pair within 10 s");
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
}
|
||||
const t0 = Date.now();
|
||||
while (Date.now() - t0 < seconds * 1000) {
|
||||
await new Promise((r) => setTimeout(r, 250));
|
||||
samples.push((await ingest.bus.stats()).pairs);
|
||||
if (during && samples.length === 4) during(feeder);
|
||||
}
|
||||
const stats = await ingest.bus.stats();
|
||||
return { events, samples, stats, feeder, exits };
|
||||
} finally {
|
||||
await feeder.stop();
|
||||
await ingest.stop();
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Longest stretch (in 250 ms samples) without a new pair once the stream is up. */
|
||||
function longestStall(samples: number[]): number {
|
||||
let worst = 0;
|
||||
let run = 0;
|
||||
const first = samples.findIndex((n) => n > 0);
|
||||
for (let i = Math.max(1, first + 1); i < samples.length; i += 1) {
|
||||
run = samples[i] === samples[i - 1] ? run + 1 : 0;
|
||||
worst = Math.max(worst, run);
|
||||
}
|
||||
return worst;
|
||||
}
|
||||
|
||||
test("feeder: asset boundaries stay at frame rate with A/V paired; a broken asset is skipped; the list loops", { timeout: 120_000 }, async (t) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "feeder-clips-"));
|
||||
try {
|
||||
const files = new Map([["a", clip(dir, "a.mp4", 2, 300)], ["b", clip(dir, "b.mp4", 1, 600)], ["broken", join(dir, "broken")]]);
|
||||
writeFileSync(join(dir, "broken"), "x");
|
||||
const r = await run(t, ["a", "broken", "b"], true, files, 7);
|
||||
if (!r) return;
|
||||
const played = r.events.filter((e): e is Extract<FeederEvent, { type: "asset" }> => e.type === "asset").map((e) => e.assetId);
|
||||
assert.deepEqual(played.slice(0, 4), ["a", "broken", "b", "a"], "in order, looping");
|
||||
assert.ok(r.events.some((e) => e.type === "asset-failed" && e.assetId === "broken"));
|
||||
assert.ok(!r.events.some((e) => e.type === "slate"), "one bad asset does not drop to the slate");
|
||||
assert.equal(r.stats.silenceFilled, 0, "audio continuous across boundaries");
|
||||
assert.ok(longestStall(r.samples) <= 1, `no stall at boundaries (samples ${r.samples.join(",")})`);
|
||||
const up = r.samples.findIndex((n) => n > 0);
|
||||
const rate = ((r.samples.at(-1) ?? 0) - (r.samples[up] ?? 0)) / ((r.samples.length - 1 - up) * 0.25);
|
||||
assert.ok(rate > 26 && rate < 36, `paced at the frame rate (${rate.toFixed(1)} pairs/s)`);
|
||||
assert.deepEqual(r.exits, []);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("feeder: nothing playable → slate floor (the bus keeps running); a finished non-looping list → slate", { timeout: 120_000 }, async (t) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "feeder-clips-"));
|
||||
try {
|
||||
writeFileSync(join(dir, "broken"), "x");
|
||||
const dead = await run(t, ["broken"], true, new Map([["broken", join(dir, "broken")]]), 4);
|
||||
if (!dead) return;
|
||||
assert.ok(dead.events.some((e) => e.type === "slate" && e.reason === "unavailable"));
|
||||
assert.ok(dead.stats.pairs >= 60, `slate keeps the source alive (${dead.stats.pairs} pairs)`);
|
||||
const once = await run(t, ["a"], false, new Map([["a", clip(dir, "a.mp4", 1, 300)]]), 4);
|
||||
assert.ok(once?.events.some((e) => e.type === "slate" && e.reason === "ended"));
|
||||
assert.equal(once?.feeder.ended, true);
|
||||
assert.ok(longestStall(once?.samples ?? []) <= 1, "the slate follows the last asset without a stall");
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("feeder: update() adopts a new list after the asset being fed (desired-state v2, no restart)", { timeout: 120_000 }, async (t) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "feeder-clips-"));
|
||||
try {
|
||||
const files = new Map([["a", clip(dir, "a.mp4", 2, 300)], ["b", clip(dir, "b.mp4", 1, 600)], ["c", clip(dir, "c.mp4", 1, 900)]]);
|
||||
// 1 s in, [a, b] becomes [c, a]. The feeder hands assets to the ingest ahead of playback (FIFO and
|
||||
// demuxer buffers: tens of ms at production bitrates, several of these tiny clips), so "the asset
|
||||
// being fed" is the boundary: after it, playback follows the new list from the position after it.
|
||||
let adoptedAfter: string | null = null;
|
||||
const r = await run(t, ["a", "b"], true, files, 12, (feeder) => {
|
||||
adoptedAfter = feeder.nowPlaying;
|
||||
feeder.update({ assetIds: ["c", "a"], loop: true });
|
||||
});
|
||||
if (!r) return;
|
||||
const order = r.events.filter((e) => e.type === "asset" || e.type === "adopted").map((e) => (e.type === "asset" ? e.assetId : "|"));
|
||||
const cut = order.indexOf("|");
|
||||
assert.ok(cut > 0, `adopted (events ${order.join(",")})`);
|
||||
const before = order.slice(0, cut);
|
||||
const after = order.slice(cut + 1);
|
||||
assert.deepEqual(before, ["a", "b", "a", "b", "a", "b"].slice(0, before.length), "the old list until the update");
|
||||
assert.ok(after.length >= 2, `the new list plays (${order.join(",")})`);
|
||||
// Position after the asset being fed: after "a" (index 1 in [c, a]) comes "c"; after "b" (gone) the list restarts at "c".
|
||||
assert.equal(after[0], "c", `after ${String(adoptedAfter)} the new list continues at c (${order.join(",")})`);
|
||||
assert.deepEqual(after.slice(0, 4), ["c", "a", "c", "a"].slice(0, Math.min(4, after.length)));
|
||||
assert.ok(longestStall(r.samples) <= 1, `no stall when a prepared asset is replaced (samples ${r.samples.join(",")})`);
|
||||
assert.deepEqual(r.exits, []);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
250
apps/media-worker/src/compositor/playlist-feeder.ts
Normal file
250
apps/media-worker/src/compositor/playlist-feeder.ts
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
/**
|
||||
* Playlist feeder (streaming-architecture.md §5.1): a `kind: "playlist"` source
|
||||
* becomes a continuous input like any other. The feeder walks the ordered asset
|
||||
* list, opens each asset (a presigned GET through the ingest egress relay),
|
||||
* remuxes it to MPEG-TS (pre-normalized at upload: copy only, never a
|
||||
* transcode) and pipes the remuxes one after another into a FIFO that one
|
||||
* `playlistPipe` ingest reads at native rate. FFmpeg reads only pipes (WP9);
|
||||
* no concat protocol.
|
||||
*
|
||||
* - Timestamps: each remux is shifted to where the previous asset ended (its
|
||||
* `-progress` end time), so the concatenated stream is monotonic. FFmpeg
|
||||
* 5.1's own discontinuity correction stalls `-re` for 1–2 s per boundary.
|
||||
* - Boundaries: two overlapped stages. Asset i+1's remux starts as soon as
|
||||
* asset i's remux has exited (its end time is then known), while asset i is
|
||||
* still being fed from its capture buffer (~4 MiB ahead, several seconds), so
|
||||
* the next asset's bytes are waiting when the current one ends: no gap.
|
||||
* - Failure: an asset that cannot be opened or remuxed is skipped. When a whole
|
||||
* pass of the list fails (or the list is empty), the slate floor plays for a
|
||||
* few seconds and the list is retried.
|
||||
* - End: `loop: false` plays the list once, then the slate for good.
|
||||
* - Desired-state v2: `update()` adopts a new asset list without a restart. The
|
||||
* asset on air finishes; playback continues after it in the new list (or at
|
||||
* the new list's start when it is no longer there). An already prepared next
|
||||
* asset that is no longer next is discarded.
|
||||
*/
|
||||
import { closeSync, constants, openSync, rmSync } from "node:fs";
|
||||
import { Socket } from "node:net";
|
||||
import { PassThrough, type Readable } from "node:stream";
|
||||
import { makeFifo, openBlockingReader } from "../gate/fifo";
|
||||
import { spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg";
|
||||
import { buildPlaylistRemuxArgs, buildSlateArgs, type Canvas } from "./command-builder";
|
||||
|
||||
/** Opens one playlist asset's bytes (WP7: presigned GET through the pinned ingest egress relay). */
|
||||
export interface PlaylistAssetSource { open(assetId: string): Promise<Readable> }
|
||||
|
||||
export interface PlaylistSpec { assetIds: string[]; loop: boolean }
|
||||
|
||||
export type FeederEvent =
|
||||
| { type: "asset"; assetId: string; index: number }
|
||||
| { type: "asset-failed"; assetId: string; reason: string }
|
||||
| { type: "slate"; reason: "unavailable" | "ended" }
|
||||
| { type: "adopted"; assetIds: number };
|
||||
|
||||
export interface PlaylistFeederDeps {
|
||||
assets: PlaylistAssetSource;
|
||||
canvas: Canvas;
|
||||
scratchDir: string;
|
||||
redactor: StderrRedactor;
|
||||
onStderr(message: string): void;
|
||||
onEvent?(event: FeederEvent): void;
|
||||
/** Rendered slate PNG (S4, overlay renderer); null = black. */
|
||||
slateImage?: string | null;
|
||||
slateSeconds?: number;
|
||||
}
|
||||
|
||||
/** Bytes of produced MPEG-TS held ahead of the FIFO per producer (~5 s at 6 Mb/s). */
|
||||
const CAPTURE_BYTES = 4 << 20;
|
||||
|
||||
/** One remux (or slate) run: its captured output and, once it exits, its end time. */
|
||||
interface Job {
|
||||
proc: ManagedProcess;
|
||||
output: Readable;
|
||||
/** Resolves when the producer exits: success, and the end time it reported (µs, excluding the offset). */
|
||||
done: Promise<{ ok: boolean; endUs: number; reason: string }>;
|
||||
input: Readable | null;
|
||||
cancelled: boolean;
|
||||
}
|
||||
|
||||
let nextFeeder = 1;
|
||||
|
||||
export class PlaylistFeeder {
|
||||
private spec: PlaylistSpec;
|
||||
private out: Socket | null = null;
|
||||
private reader: { fd: number; releaseHolder(): void } | null = null;
|
||||
private stopping = false;
|
||||
private running: Promise<void> = Promise.resolve();
|
||||
private current: { assetId: string; index: number } | null = null;
|
||||
private nextIndex = 0;
|
||||
/** Bumped by `update()`: a job prepared under an older list is discarded. */
|
||||
private generation = 0;
|
||||
private readonly jobs = new Set<Job>();
|
||||
private abortFeed: (() => void) | null = null;
|
||||
/** Assets started, assets failed, slate chunks fed. */
|
||||
readonly stats = { assets: 0, failures: 0, slates: 0 };
|
||||
ended = false;
|
||||
|
||||
constructor(readonly sourceId: string, spec: PlaylistSpec, private readonly deps: PlaylistFeederDeps) {
|
||||
this.spec = { assetIds: [...spec.assetIds], loop: spec.loop };
|
||||
}
|
||||
|
||||
get nowPlaying(): string | null { return this.current?.assetId ?? null; }
|
||||
|
||||
/**
|
||||
* Creates the FIFO and starts feeding. Returns the blocking read end for the ingest's stdin;
|
||||
* call `releaseReader()` once the ingest has it.
|
||||
*/
|
||||
start(): number {
|
||||
const path = makeFifo(this.deps.scratchDir, `playlist-${nextFeeder++}.fifo`);
|
||||
const reader = openBlockingReader(path);
|
||||
this.reader = reader;
|
||||
const out = new Socket({ fd: openSync(path, constants.O_WRONLY | constants.O_NONBLOCK), readable: false, writable: true });
|
||||
out.on("error", () => { if (!this.stopping) void this.stop(); }); // the ingest is gone; IngestProcess reports its exit
|
||||
this.out = out;
|
||||
reader.releaseHolder();
|
||||
rmSync(path, { force: true });
|
||||
this.running = this.run().catch((error: unknown) => this.deps.onStderr(`playlist feeder ${this.sourceId}: ${String(error)}`));
|
||||
return reader.fd;
|
||||
}
|
||||
|
||||
releaseReader(): void {
|
||||
if (this.reader) { closeSync(this.reader.fd); this.reader = null; }
|
||||
}
|
||||
|
||||
/** Desired-state v2: adopt a new list at the next asset boundary (no restart, no mid-asset cut). */
|
||||
update(spec: PlaylistSpec): void {
|
||||
this.spec = { assetIds: [...spec.assetIds], loop: spec.loop };
|
||||
const at = this.current ? this.spec.assetIds.indexOf(this.current.assetId) : -1;
|
||||
this.nextIndex = at >= 0 ? at + 1 : 0;
|
||||
this.ended = false;
|
||||
this.generation += 1;
|
||||
this.deps.onEvent?.({ type: "adopted", assetIds: this.spec.assetIds.length });
|
||||
}
|
||||
|
||||
private peekNext(): { assetId: string; index: number } | null {
|
||||
const ids = this.spec.assetIds;
|
||||
if (ids.length === 0) return null;
|
||||
let index = this.nextIndex;
|
||||
if (index >= ids.length) {
|
||||
if (!this.spec.loop) return null;
|
||||
index = 0;
|
||||
}
|
||||
return { assetId: ids[index]!, index };
|
||||
}
|
||||
|
||||
/** Spawns a remux of `asset` (or the slate) at `offsetUs`, capturing its output from the start. */
|
||||
private produce(asset: { assetId: string; index: number } | null, offsetUs: number): Job {
|
||||
const args = asset ? buildPlaylistRemuxArgs(offsetUs)
|
||||
: buildSlateArgs({ canvas: this.deps.canvas, seconds: this.deps.slateSeconds ?? 5, image: this.deps.slateImage ?? null, offsetUs });
|
||||
const proc = spawnManaged("ffmpeg", args, [asset ? "pipe" : "ignore", "pipe", "pipe", "pipe"], this.deps.redactor, this.deps.onStderr);
|
||||
// A producer may exit before its turn (a broken asset at once, a small one that fits in the
|
||||
// buffers), and Node then resumes its unconsumed stdout itself, dropping data and 'end'. The
|
||||
// capture keeps both until fed; backpressure holds a large asset's remux CAPTURE_BYTES ahead.
|
||||
const output = new PassThrough({ highWaterMark: CAPTURE_BYTES });
|
||||
const stdout = proc.child.stdout as Readable | null;
|
||||
if (stdout) stdout.pipe(output);
|
||||
else output.end();
|
||||
let progress = "";
|
||||
(proc.child.stdio[3] as Readable | null)?.on("data", (chunk: Buffer) => { progress = (progress + chunk.toString("latin1")).slice(-4096); });
|
||||
let failed: string | null = null;
|
||||
const done = proc.exited.then(({ code }) => {
|
||||
const endUs = Number([...progress.matchAll(/out_time_us=(\d+)/g)].at(-1)?.[1] ?? 0);
|
||||
return { ok: failed === null && code === 0 && endUs > 0, endUs, reason: failed ?? `producer exited code=${code} end=${endUs}us` };
|
||||
});
|
||||
const job: Job = { proc, output, input: null, done, cancelled: false };
|
||||
if (asset) {
|
||||
proc.child.stdin?.on("error", () => undefined); // EPIPE when the remux rejects its input
|
||||
void this.deps.assets.open(asset.assetId).then((input) => {
|
||||
if (job.cancelled || this.stopping || proc.child.exitCode !== null || proc.child.signalCode !== null) { input.destroy(); return; }
|
||||
job.input = input;
|
||||
input.on("error", (error: Error) => { failed = `asset read failed: ${error.name}`; proc.kill("SIGKILL"); });
|
||||
if (proc.child.stdin) input.pipe(proc.child.stdin);
|
||||
}, (error: unknown) => {
|
||||
failed = `asset open failed: ${error instanceof Error ? error.name : "error"}`;
|
||||
proc.kill("SIGKILL");
|
||||
});
|
||||
}
|
||||
this.jobs.add(job);
|
||||
void done.then(() => this.jobs.delete(job));
|
||||
return job;
|
||||
}
|
||||
|
||||
private discard(job: Job): void {
|
||||
job.cancelled = true;
|
||||
job.input?.destroy();
|
||||
job.output.destroy();
|
||||
job.proc.kill("SIGKILL");
|
||||
}
|
||||
|
||||
/** Pipes a job's captured output into the FIFO; resolves once all of it has been handed over. */
|
||||
private async feed(job: Job): Promise<void> {
|
||||
const out = this.out;
|
||||
if (!out) return;
|
||||
job.output.pipe(out, { end: false });
|
||||
// Until the producer's last byte ('end'). On stop the FIFO socket is destroyed, which unpipes
|
||||
// and pauses the capture, so 'end' would never come: stop() aborts the wait instead.
|
||||
await new Promise<void>((resolve) => {
|
||||
this.abortFeed = resolve;
|
||||
job.output.once("end", resolve);
|
||||
job.output.once("error", () => resolve());
|
||||
});
|
||||
this.abortFeed = null;
|
||||
job.output.unpipe(out);
|
||||
}
|
||||
|
||||
private async run(): Promise<void> {
|
||||
let offsetUs = 0;
|
||||
let failedInARow = 0;
|
||||
let slateNext = false;
|
||||
let feeding: Promise<void> = Promise.resolve();
|
||||
while (!this.stopping) {
|
||||
const generation = this.generation;
|
||||
const next = slateNext ? null : this.peekNext();
|
||||
slateNext = false;
|
||||
// Prepared while the previous job is still being fed (its producer has exited: offsetUs is final).
|
||||
const job = this.produce(next, offsetUs);
|
||||
await feeding;
|
||||
if (this.stopping) { this.discard(job); break; }
|
||||
if (generation !== this.generation) { this.discard(job); continue; } // re-plan under the new list
|
||||
if (next) {
|
||||
this.nextIndex = next.index + 1;
|
||||
this.current = next;
|
||||
this.stats.assets += 1;
|
||||
this.deps.onEvent?.({ type: "asset", assetId: next.assetId, index: next.index });
|
||||
} else {
|
||||
this.current = null;
|
||||
this.ended = this.spec.assetIds.length > 0 && !this.spec.loop && this.nextIndex >= this.spec.assetIds.length;
|
||||
this.stats.slates += 1;
|
||||
this.deps.onEvent?.({ type: "slate", reason: this.ended ? "ended" : "unavailable" });
|
||||
}
|
||||
feeding = this.feed(job);
|
||||
const result = await job.done;
|
||||
if (result.ok) {
|
||||
offsetUs += result.endUs;
|
||||
failedInARow = 0;
|
||||
continue;
|
||||
}
|
||||
if (!next) continue; // a failed slate is retried (offset unchanged)
|
||||
this.stats.failures += 1;
|
||||
failedInARow += 1;
|
||||
this.deps.onEvent?.({ type: "asset-failed", assetId: next.assetId, reason: result.reason });
|
||||
if (failedInARow >= Math.max(1, this.spec.assetIds.length)) {
|
||||
failedInARow = 0;
|
||||
slateNext = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
if (this.stopping) { await this.running; return; }
|
||||
this.stopping = true;
|
||||
const jobs = [...this.jobs];
|
||||
for (const job of jobs) this.discard(job);
|
||||
this.abortFeed?.();
|
||||
this.out?.destroy();
|
||||
this.out = null;
|
||||
this.releaseReader();
|
||||
await this.running;
|
||||
await Promise.all(jobs.map((job) => job.done));
|
||||
}
|
||||
}
|
||||
181
apps/media-worker/src/compositor/production-job.ts
Normal file
181
apps/media-worker/src/compositor/production-job.ts
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import type { GateImpl } from "../gate/frame-gate";
|
||||
import { snapshotHlsUrls, OUTPUT_PROFILES, type DesiredChannel } from "../types";
|
||||
import { validateEndpointShape, type EndpointValidator } from "../relay/endpoints";
|
||||
import type { StderrRedactor, VaultHandleBroker } from "../relay/ffmpeg";
|
||||
import type { Canvas, EncoderOutput, IngestInput } from "./command-builder";
|
||||
import type { ComposeRequest } from "./compositor";
|
||||
import { ManifestPreviewPublisher, type PreviewManifestStore, type PreviewObjectStore } from "../preview/contract";
|
||||
import type { HlsRendition } from "../preview/argv";
|
||||
import { PreviewTaps } from "../preview/taps";
|
||||
import { IngestProcess } from "./ingest";
|
||||
import { PlaylistFeeder, type PlaylistAssetSource } from "./playlist-feeder";
|
||||
import { ProgramChain } from "./program";
|
||||
import type { SourceBus } from "./source-bus";
|
||||
import { planScene, programScene } from "./scene-plan";
|
||||
|
||||
/** WP7 egress for a production snapshot: one pinned MPEG-TS fd per HLS source, plus the push fd. */
|
||||
export interface ProductionEgress {
|
||||
prepare(config: DesiredChannel, validator: EndpointValidator, vault: VaultHandleBroker): Promise<{
|
||||
sources: Map<string, { input: IngestInput; fd: number | null }>;
|
||||
output: EncoderOutput;
|
||||
close(): Promise<void>;
|
||||
}>;
|
||||
}
|
||||
|
||||
/** Overlay PNGs by content id, fetched through assignment-scoped presigned GETs (§4.3) into scratch. */
|
||||
export interface AssetStore { localPath(assetId: string): Promise<string> }
|
||||
|
||||
export interface ProductionJobDeps {
|
||||
validator: EndpointValidator;
|
||||
egress: ProductionEgress;
|
||||
vault: VaultHandleBroker;
|
||||
assets: AssetStore;
|
||||
redactor: StderrRedactor;
|
||||
onStderr(message: string): void;
|
||||
onExit(reason: string): void;
|
||||
scratchDir: string;
|
||||
gateImpl?: GateImpl;
|
||||
/** Playlist sources (§5.1). Without it, playlist sources render black and are reported in `lastSkipped`. */
|
||||
playlists?: PlaylistAssetSource;
|
||||
/** Preview taps (§2.1) publish through the API's manifest contract. Without it, no taps run. */
|
||||
preview?: { objects: PreviewObjectStore; manifests: PreviewManifestStore };
|
||||
}
|
||||
|
||||
/**
|
||||
* What may change by swap. Anything else (the set of sources, an HLS URL, output, profile, mode)
|
||||
* restarts the channel. A playlist's asset list and loop flag are not part of it: the feeder
|
||||
* adopts them in place (desired-state v2: an edited playlist never interrupts the program).
|
||||
*/
|
||||
function pathKey(c: DesiredChannel): string {
|
||||
const sources = c.production?.sources.map((src) => (src.kind === "hls" ? { id: src.id, kind: src.kind, url: src.url } : { id: src.id, kind: src.kind })) ?? null;
|
||||
return createHash("sha256").update(JSON.stringify({ profile: c.profile, mode: c.mode, output: c.output, sources, source: c.source })).digest("hex");
|
||||
}
|
||||
|
||||
/**
|
||||
* Supervisor RelayJob for a desired-state v2 snapshot. `start()` brings up
|
||||
* ingest×N (HLS through the egress relay, playlists through the asset feeder)
|
||||
* → gate → encoder, puts the program scene on air, and starts the preview taps.
|
||||
* `reconfigure()` is the meaning of an accepted RUNNING snapshot while already
|
||||
* RUNNING: a TAKE, layout, overlay or gain change is an A/B compositor swap,
|
||||
* playlist edits are adopted by their feeders, and it resolves only once the
|
||||
* new compositor is on air at a frame boundary.
|
||||
*/
|
||||
export class ProductionJob {
|
||||
private ingests = new Map<string, IngestProcess>();
|
||||
private feeders = new Map<string, PlaylistFeeder>();
|
||||
private chain: ProgramChain | null = null;
|
||||
private taps: PreviewTaps | null = null;
|
||||
private closeEgress: (() => Promise<void>) | null = null;
|
||||
private key = "";
|
||||
private canvas: Canvas | null = null;
|
||||
lastSkipped: string[] = [];
|
||||
|
||||
constructor(private readonly deps: ProductionJobDeps) {}
|
||||
|
||||
get program(): ProgramChain | null { return this.chain; }
|
||||
get previews(): PreviewTaps | null { return this.taps; }
|
||||
feeder(sourceId: string): PlaylistFeeder | undefined { return this.feeders.get(sourceId); }
|
||||
/** The source's bus (its ingest's fan-out), for stats. */
|
||||
sourceBus(sourceId: string): SourceBus | undefined { return this.ingests.get(sourceId)?.bus; }
|
||||
|
||||
async start(config: DesiredChannel): Promise<void> {
|
||||
const spec = config.production;
|
||||
if (!spec || !config.output) throw new Error("production job needs a production snapshot with an output");
|
||||
for (const url of snapshotHlsUrls(config)) validateEndpointShape(url, "hls");
|
||||
validateEndpointShape(config.output.endpointUrl, "output");
|
||||
const profile = OUTPUT_PROFILES[config.profile];
|
||||
const canvas: Canvas = { width: profile.width, height: profile.height, fps: profile.fps };
|
||||
this.canvas = canvas;
|
||||
const egress = await this.deps.egress.prepare(config, this.deps.validator, this.deps.vault);
|
||||
this.closeEgress = egress.close;
|
||||
const lost = (id: string, reason: string): void => this.deps.onExit(`source ${id}: ${reason}`);
|
||||
try {
|
||||
for (const [sourceId, { input, fd }] of egress.sources) {
|
||||
const ingest = new IngestProcess(sourceId, { input, canvas, audio: "source" }, fd, this.deps.scratchDir, this.deps.redactor, this.deps.onStderr, lost);
|
||||
this.ingests.set(sourceId, ingest);
|
||||
await ingest.start();
|
||||
}
|
||||
const playlists = this.deps.playlists;
|
||||
for (const src of spec.sources) {
|
||||
if (src.kind !== "playlist" || !playlists) continue;
|
||||
const feeder = new PlaylistFeeder(src.id, { assetIds: src.assetIds, loop: src.loop }, {
|
||||
assets: playlists, canvas, scratchDir: this.deps.scratchDir, redactor: this.deps.redactor, onStderr: this.deps.onStderr,
|
||||
onEvent: (e) => { if (e.type === "asset-failed" || e.type === "slate") this.deps.onStderr(`playlist ${src.id}: ${e.type}${"reason" in e ? ` ${e.reason}` : ""}`); },
|
||||
});
|
||||
this.feeders.set(src.id, feeder);
|
||||
const fd = feeder.start();
|
||||
const ingest = new IngestProcess(src.id, { input: { kind: "playlistPipe" }, canvas, audio: "source" }, fd, this.deps.scratchDir, this.deps.redactor, this.deps.onStderr, lost);
|
||||
this.ingests.set(src.id, ingest);
|
||||
try { await ingest.start(); } finally { feeder.releaseReader(); }
|
||||
}
|
||||
this.chain = new ProgramChain({
|
||||
canvas, scratchDir: this.deps.scratchDir, videoBitrateKbps: Number.parseInt(profile.videoBitrate, 10),
|
||||
output: egress.output, redactor: this.deps.redactor, onStderr: this.deps.onStderr,
|
||||
onFatal: (reason) => this.deps.onExit(reason), gateImpl: this.deps.gateImpl, programTap: this.deps.preview !== undefined,
|
||||
});
|
||||
await this.chain.start(await this.compose(config));
|
||||
this.key = pathKey(config);
|
||||
} catch (error) {
|
||||
await this.stop();
|
||||
throw error;
|
||||
}
|
||||
await this.startPreviews(config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Preview taps never gate the program: a tap that cannot start is reported and the channel runs on.
|
||||
* The program tap's renditions are always on (§2.2: the PGM tile is always live); source renditions
|
||||
* start on demand (`previews.demand()`).
|
||||
*/
|
||||
private async startPreviews(config: DesiredChannel): Promise<void> {
|
||||
const preview = this.deps.preview;
|
||||
if (!preview || !this.chain?.programBus) return;
|
||||
const taps = new PreviewTaps({
|
||||
publisher: new ManifestPreviewPublisher({ workspaceId: config.workspaceId, channelId: config.channelId }, preview.objects, preview.manifests),
|
||||
scratchDir: this.deps.scratchDir, redactor: this.deps.redactor, onStderr: this.deps.onStderr,
|
||||
onError: (tap, reason) => this.deps.onStderr(`preview ${tap}: ${reason}`),
|
||||
});
|
||||
this.taps = taps;
|
||||
const add = async (tap: string, bus: SourceBus, alwaysOn: HlsRendition[]): Promise<void> => {
|
||||
try { await taps.add(tap, bus, { alwaysOn }); } catch (error) { this.deps.onStderr(`preview ${tap}: ${String(error)}`); }
|
||||
};
|
||||
await add("program", this.chain.programBus, ["video", "audio"]);
|
||||
for (const [sourceId, ingest] of this.ingests) await add(sourceId, ingest.bus, []);
|
||||
}
|
||||
|
||||
/** false = this change cannot be a swap (sources/output/profile changed): the supervisor restarts. */
|
||||
async reconfigure(config: DesiredChannel): Promise<boolean> {
|
||||
if (!this.chain || !config.production || pathKey(config) !== this.key) return false;
|
||||
for (const src of config.production.sources) {
|
||||
if (src.kind === "playlist") this.feeders.get(src.id)?.update({ assetIds: src.assetIds, loop: src.loop });
|
||||
}
|
||||
await this.chain.swap(await this.compose(config));
|
||||
return true;
|
||||
}
|
||||
private async compose(config: DesiredChannel): Promise<ComposeRequest> {
|
||||
if (!config.production || !this.canvas) throw new Error("no production spec");
|
||||
const plan = planScene(config.production, programScene(config.production), this.canvas, (id) => this.ingests.has(id));
|
||||
this.lastSkipped = plan.skipped;
|
||||
const inputs = plan.sourceIds.map((id, i) => {
|
||||
const ingest = this.ingests.get(id);
|
||||
if (!ingest) throw new Error("planned source has no ingest");
|
||||
return { bus: ingest.bus, audio: plan.audio[i] ?? false };
|
||||
});
|
||||
const assets = await Promise.all(plan.assetIds.map((id) => this.deps.assets.localPath(id)));
|
||||
return { inputs, assets, graph: plan.graph };
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
await this.taps?.stop();
|
||||
this.taps = null;
|
||||
await this.chain?.stop();
|
||||
this.chain = null;
|
||||
for (const feeder of this.feeders.values()) await feeder.stop();
|
||||
this.feeders.clear();
|
||||
for (const ingest of this.ingests.values()) await ingest.stop();
|
||||
this.ingests.clear();
|
||||
await this.closeEgress?.();
|
||||
this.closeEgress = null;
|
||||
}
|
||||
}
|
||||
194
apps/media-worker/src/compositor/program.ts
Normal file
194
apps/media-worker/src/compositor/program.ts
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
import { CompositorLagError, FrameGate, type GateImpl } from "../gate/frame-gate";
|
||||
import type { GateEvent, GateStats } from "../gate/protocol";
|
||||
import { spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg";
|
||||
import { programTapGeometry } from "../preview/argv";
|
||||
import { buildEncoderArgs, type Canvas, type EncoderOutput, type EncoderProgramTap } from "./command-builder";
|
||||
import { CompositorInstance, type ComposeRequest } from "./compositor";
|
||||
import { SourceBus, type BusSubscription } from "./source-bus";
|
||||
|
||||
export interface ProgramChainOptions {
|
||||
canvas: Canvas;
|
||||
scratchDir: string;
|
||||
videoBitrateKbps: number;
|
||||
/**
|
||||
* flvPipe: the push relay's inherited fd (WP9/WP13); null: x264 into `-f null` (CPU smoke);
|
||||
* external: no encoder is spawned, the caller has opened the gate's encoder FIFOs (verifying sinks).
|
||||
*/
|
||||
output: EncoderOutput | { kind: "external" };
|
||||
redactor: StderrRedactor;
|
||||
onStderr(message: string): void;
|
||||
/** Encoder or gate died: the RTMP session is gone and the supervisor must restart the channel. */
|
||||
onFatal(reason: string): void;
|
||||
/** The on-air compositor died (gate is repeating its last frame); the chain respawns it. */
|
||||
onCompositorLost?(reason: string): void;
|
||||
gateImpl?: GateImpl;
|
||||
/** Swap must go on air within this long, or B is abandoned and A stays (default 10 s). */
|
||||
swapTimeoutMs?: number;
|
||||
/**
|
||||
* Program preview tap (§2.1): the encoder also emits a 360p copy of what it encodes into
|
||||
* `programBus`, which preview taps subscribe to. Needs an encoder (not `external`).
|
||||
*/
|
||||
programTap?: boolean;
|
||||
}
|
||||
|
||||
/** Child fd of the push relay's pipe in the encoder (flvPipe), and of the program tap outputs after it. */
|
||||
const ENCODER_OUTPUT_FD = 4;
|
||||
|
||||
const LAG_RETRIES = 2;
|
||||
|
||||
/**
|
||||
* The persistent part of a channel's program path (§1.4): frame gate + encoder,
|
||||
* started once, plus the A/B compositor swap. `swap()` starts B for the new
|
||||
* scene, waits until the gate splices it in at a frame boundary, then stops A.
|
||||
* The encoder and its push session are never restarted by a swap.
|
||||
*/
|
||||
export class ProgramChain {
|
||||
readonly gate: FrameGate;
|
||||
/** The program tap's bus (360p of the encoder input), when `programTap` is set. */
|
||||
readonly programBus: SourceBus | null;
|
||||
private encoder: ManagedProcess | null = null;
|
||||
private active: CompositorInstance | null = null;
|
||||
private activeRequest: ComposeRequest | null = null;
|
||||
private queue: Promise<unknown> = Promise.resolve();
|
||||
private stopping = false;
|
||||
swaps = 0;
|
||||
/** Swap-in compositors replaced after a gate `LAG` report. */
|
||||
lagRetries = 0;
|
||||
|
||||
constructor(private readonly options: ProgramChainOptions) {
|
||||
// The Node reference gate has neither A/V pair lockstep nor origin alignment (s2a-report §5.3):
|
||||
// swaps through it can step backwards and skew A/V. Spike/contract comparison only (gate/spike.ts, gate.test.ts).
|
||||
if (options.gateImpl === "node") throw new Error("the Node reference gate is spike-only; a program chain needs the Rust gate");
|
||||
this.gate = new FrameGate({
|
||||
geometry: options.canvas, scratchDir: options.scratchDir, impl: options.gateImpl ?? "rust",
|
||||
redactor: options.redactor, onStderr: options.onStderr,
|
||||
onEvent: (event) => this.onGateEvent(event),
|
||||
onExit: (reason) => { if (!this.stopping) options.onFatal(reason); },
|
||||
});
|
||||
if (options.programTap && options.output.kind === "external") throw new Error("the program tap needs an encoder");
|
||||
this.programBus = options.programTap ? new SourceBus({
|
||||
geometry: programTapGeometry(options.canvas), scratchDir: options.scratchDir, redactor: options.redactor, onStderr: options.onStderr,
|
||||
// Losing the tap bus loses previews, never the program: the encoder's tee drops that slave.
|
||||
onExit: (reason) => options.onStderr(`program tap bus: ${reason}`),
|
||||
}) : null;
|
||||
}
|
||||
|
||||
get activeCompositor(): CompositorInstance | null { return this.active; }
|
||||
get encoderPid(): number | undefined { return this.encoder?.child.pid; }
|
||||
|
||||
async start(first: ComposeRequest): Promise<void> {
|
||||
const { canvas, output } = this.options;
|
||||
if (output.kind === "external") {
|
||||
await this.gate.start();
|
||||
await this.swap(first);
|
||||
return;
|
||||
}
|
||||
const inputs = this.gate.encoderInputs();
|
||||
const stdio: Array<number | "pipe" | "ignore"> = [inputs.videoFd, "ignore", "pipe", inputs.audioFd];
|
||||
// The push relay's fd is inherited as child fd 4, whatever its number in the worker.
|
||||
let encoderOutput: EncoderOutput = output;
|
||||
if (output.kind === "flvPipe") { stdio.push(output.fd); encoderOutput = { kind: "flvPipe", fd: ENCODER_OUTPUT_FD }; }
|
||||
let programTap: EncoderProgramTap | undefined;
|
||||
if (this.programBus) {
|
||||
const tap = await this.programBus.start();
|
||||
const first = stdio.length;
|
||||
stdio.push(tap.videoFd, tap.audioFd);
|
||||
programTap = { videoFd: first, audioFd: first + 1, size: programTapGeometry(canvas) };
|
||||
}
|
||||
let encoder: ManagedProcess;
|
||||
try {
|
||||
encoder = spawnManaged("ffmpeg", buildEncoderArgs({ canvas, videoBitrateKbps: this.options.videoBitrateKbps, audioBitrateKbps: 128, output: encoderOutput, programTap }),
|
||||
stdio, this.options.redactor, this.options.onStderr);
|
||||
} finally {
|
||||
this.programBus?.releaseIngestWriters();
|
||||
}
|
||||
this.encoder = encoder;
|
||||
void encoder.exited.then(({ code, signal }) => { if (!this.stopping) this.options.onFatal(`encoder exited code=${code} signal=${signal}`); });
|
||||
await encoder.started;
|
||||
await this.gate.start();
|
||||
await this.swap(first);
|
||||
}
|
||||
|
||||
/** Serialized A/B swap. Resolves with the gate tick at which the new compositor went on air. */
|
||||
swap(request: ComposeRequest): Promise<number> {
|
||||
const run = this.queue.catch(() => undefined).then(() => this.swapNow(request));
|
||||
this.queue = run;
|
||||
return run;
|
||||
}
|
||||
|
||||
/**
|
||||
* A B the gate reports as lagging (still behind A's on-air source position, typically a spawn
|
||||
* starved of CPU while draining its startup backlog) is replaced by a fresh one; A stays on air.
|
||||
*/
|
||||
private async swapNow(request: ComposeRequest): Promise<number> {
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
try {
|
||||
return await this.swapOnce(request);
|
||||
} catch (error) {
|
||||
if (!(error instanceof CompositorLagError) || attempt >= LAG_RETRIES) throw error;
|
||||
this.lagRetries += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async swapOnce(request: ComposeRequest): Promise<number> {
|
||||
const next = new CompositorInstance(this.gate, request, this.options.redactor, this.options.onStderr);
|
||||
const current = this.active;
|
||||
await next.start((subs) => (current ? this.originAfter(current, subs) : 0));
|
||||
let tick: number;
|
||||
try {
|
||||
tick = await this.gate.waitActive(next.slotId, this.options.swapTimeoutMs ?? 10_000);
|
||||
} catch (error) {
|
||||
try { this.gate.detach(next.slotId); } catch { /* gate gone */ }
|
||||
await next.stop();
|
||||
throw error;
|
||||
}
|
||||
const old = this.active;
|
||||
this.active = next;
|
||||
this.activeRequest = request;
|
||||
this.swaps += 1;
|
||||
if (old) void old.stop();
|
||||
return tick;
|
||||
}
|
||||
|
||||
/**
|
||||
* Origin of B relative to on-air A, in A's numbering: through a bus both use
|
||||
* (exact: pair counters, minus pairs A had lost to backlog when B subscribed), else by wall clock.
|
||||
* Pairs B loses after attach only make B *ahead*: a forward skip, never backwards.
|
||||
*/
|
||||
private originAfter(a: CompositorInstance, subs: BusSubscription[]): number | undefined {
|
||||
if (a.origin === undefined) return undefined;
|
||||
for (const sb of subs) {
|
||||
const sa = a.subscriptions.find((x) => x.bus === sb.bus);
|
||||
if (sa) return a.origin + sb.startPair - sa.startPair - (sb.droppedBefore(sa) ?? sa.dropped);
|
||||
}
|
||||
const sa = a.subscriptions[0];
|
||||
const sb = subs[0];
|
||||
if (!sa || !sb) return undefined;
|
||||
return a.origin + Math.round((Number(sb.startedAt - sa.startedAt) / 1e9) * this.options.canvas.fps);
|
||||
}
|
||||
|
||||
private onGateEvent(event: GateEvent): void {
|
||||
if (event.type !== "EOF" || this.stopping || !this.active || event.id !== this.active.slotId || !this.activeRequest) return;
|
||||
// The on-air compositor died (crash, kill -9). The gate is already holding
|
||||
// the last frame + silence, so the encoder never notices; respawn the scene.
|
||||
const lost = this.active;
|
||||
this.active = null;
|
||||
void lost.stop();
|
||||
this.options.onCompositorLost?.(`compositor ${event.id} ended on air`);
|
||||
void this.swap(this.activeRequest).catch((error: unknown) => this.options.onCompositorLost?.(`respawn failed: ${String(error)}`));
|
||||
}
|
||||
|
||||
stats(): Promise<GateStats> { return this.gate.stats(); }
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.stopping = true;
|
||||
await this.queue.catch(() => undefined);
|
||||
await this.active?.stop();
|
||||
this.active = null;
|
||||
await this.gate.stop();
|
||||
await this.encoder?.terminate();
|
||||
this.encoder = null;
|
||||
await this.programBus?.stop();
|
||||
}
|
||||
}
|
||||
68
apps/media-worker/src/compositor/scene-plan.test.ts
Normal file
68
apps/media-worker/src/compositor/scene-plan.test.ts
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { parseProductionSpec } from "../production";
|
||||
import { buildCompositorArgs } from "./command-builder";
|
||||
import { planScene, programScene } from "./scene-plan";
|
||||
|
||||
const id = (n: number) => `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`;
|
||||
const sha = (c: string) => c.repeat(64);
|
||||
|
||||
const spec = parseProductionSpec({
|
||||
sources: [
|
||||
{ id: id(1), label: "Cam A", kind: "hls", url: "https://a.example/live.m3u8" },
|
||||
{ id: id(2), label: "Cam B", kind: "hls", url: "https://b.example/live.m3u8" },
|
||||
{ id: id(3), label: "Loop", kind: "playlist", assetIds: [id(9)], loop: true },
|
||||
],
|
||||
scenes: [
|
||||
{ id: id(10), label: "Grid", layout: "2x2", slots: [
|
||||
{ index: 0, sourceId: id(1), rect: null, fit: "cover", audioMode: "primary", gainDb: 0 },
|
||||
{ index: 1, sourceId: id(2), rect: null, fit: "contain", audioMode: "on", gainDb: -6 },
|
||||
{ index: 2, sourceId: id(3), rect: null, fit: "contain", audioMode: "on", gainDb: 0 },
|
||||
{ index: 3, sourceId: id(1), rect: null, fit: "crop", audioMode: "on", gainDb: -12 },
|
||||
] },
|
||||
{ id: id(11), label: "PIP", layout: "pip", slots: [
|
||||
{ index: 0, sourceId: id(2), rect: null, fit: "cover", audioMode: "primary", gainDb: -3 },
|
||||
{ index: 1, sourceId: id(1), rect: { x: 6500, y: 6500, w: 3000, h: 3000 }, fit: "contain", audioMode: "off", gainDb: 0 },
|
||||
] },
|
||||
],
|
||||
buses: { preview: id(11), program: id(10) },
|
||||
overlays: [
|
||||
{ id: id(20), sceneId: id(10), kind: "chyron", enabled: true, layer: 2, position: { x: 0, y: 9000 }, revisionId: id(30), assetId: sha("c"), chyron: { speedPxPerSec: 120, stripWidthPx: 4000 }, clock: null },
|
||||
{ id: id(21), sceneId: id(10), kind: "lower_third", enabled: true, layer: 1, position: { x: 500, y: 7500 }, revisionId: id(31), assetId: sha("a"), chyron: null, clock: null },
|
||||
{ id: id(22), sceneId: id(10), kind: "logo", enabled: false, layer: 3, position: { x: 9000, y: 500 }, revisionId: id(32), assetId: sha("b"), chyron: null, clock: null },
|
||||
{ id: id(23), sceneId: id(10), kind: "clock", enabled: true, layer: 4, position: { x: 9000, y: 500 }, revisionId: null, assetId: null, chyron: null, clock: { format: "HH:MM" } },
|
||||
{ id: id(24), sceneId: id(11), kind: "lower_third", enabled: true, layer: 0, position: { x: 0, y: 0 }, revisionId: id(33), assetId: sha("d"), chyron: null, clock: null },
|
||||
],
|
||||
transition: { type: "cut" },
|
||||
audio: { masterGainDb: -3 },
|
||||
});
|
||||
const canvas = { width: 1920, height: 1080, fps: 30 };
|
||||
const hls = (sourceId: string) => spec.sources.find((s) => s.id === sourceId)?.kind === "hls";
|
||||
|
||||
test("program scene → typed graph: grid cells, shared source, playlist black, one mix input per source", () => {
|
||||
const plan = planScene(spec, programScene(spec), canvas, hls);
|
||||
assert.deepEqual(plan.sourceIds, [id(1), id(2)]);
|
||||
assert.deepEqual(plan.audio, [true, true]);
|
||||
assert.deepEqual(plan.graph.tiles.map((t) => t.source), [0, 1, null, 0]);
|
||||
assert.deepEqual(plan.graph.tiles[3]?.rect, { x: 960, y: 540, w: 960, h: 540 });
|
||||
// Cam A is primary at 0 dB and "on" at -12 dB in slot 3: one input at the louder gain.
|
||||
assert.deepEqual(plan.graph.audio, { inputs: [{ source: 0, gainDb: 0 }, { source: 1, gainDb: -6 }], masterGainDb: -3 });
|
||||
// Layer order (lower third under chyron); disabled logo and S2a-unsupported clock skipped.
|
||||
assert.deepEqual(plan.assetIds, [sha("a"), sha("c")]);
|
||||
assert.deepEqual(plan.graph.overlays, [{ kind: "image", asset: 0, x: 96, y: 810 }, { kind: "chyron", asset: 1, y: 972, speedPxPerSec: 120 }]);
|
||||
assert.equal(plan.skipped.length, 2);
|
||||
const args = buildCompositorArgs({
|
||||
sources: plan.sourceIds.map((_, i) => ({ video: { kind: "rawPipe" as const, fd: 4 + 2 * i }, audio: { kind: "rawPipe" as const, fd: 5 + 2 * i } })),
|
||||
assets: plan.assetIds.map((a) => `/scratch/assets/${a}.png`), graph: plan.graph, output: { kind: "gate", videoFd: 1, audioFd: 3 },
|
||||
});
|
||||
assert.ok(!args.join(" ").includes("Cam A"), "labels never reach argv");
|
||||
assert.ok(!args.join(" ").includes("example"), "source URLs never reach the compositor");
|
||||
});
|
||||
|
||||
test("PIP scene: inset rect from basis points, audio off slot not mixed", () => {
|
||||
const plan = planScene(spec, spec.scenes[1]!, canvas, hls);
|
||||
assert.deepEqual(plan.sourceIds, [id(2), id(1)]);
|
||||
assert.deepEqual(plan.audio, [true, false]);
|
||||
assert.deepEqual(plan.graph.tiles[1]?.rect, { x: 1248, y: 702, w: 576, h: 324 });
|
||||
assert.deepEqual(plan.graph.audio.inputs, [{ source: 0, gainDb: -3 }]);
|
||||
});
|
||||
81
apps/media-worker/src/compositor/scene-plan.ts
Normal file
81
apps/media-worker/src/compositor/scene-plan.ts
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
import type { ProductionOverlay, ProductionScene, ProductionSpec } from "../production";
|
||||
import { layoutRects, rectFromBasisPoints, type Canvas, type CompositorGraph, type MixInput, type OverlaySpec, type TileSpec } from "./command-builder";
|
||||
|
||||
/** Desired-state v2 scene → typed compositor graph over a list of source ids (input order). */
|
||||
export interface ScenePlan {
|
||||
sourceIds: string[];
|
||||
/** Same order as sourceIds: whether the compositor needs that source's audio. */
|
||||
audio: boolean[];
|
||||
assetIds: string[];
|
||||
graph: CompositorGraph;
|
||||
/** Things in the snapshot this slice renders as black/skips, for the observed reason. */
|
||||
skipped: string[];
|
||||
}
|
||||
|
||||
export function programScene(spec: ProductionSpec): ProductionScene {
|
||||
const scene = spec.scenes.find((s) => s.id === spec.buses.program);
|
||||
if (!scene) throw new Error("program bus names no scene");
|
||||
return scene;
|
||||
}
|
||||
|
||||
/**
|
||||
* `available(sourceId)` says whether the worker has an ingest for that source:
|
||||
* S2a ingests HLS sources only; playlist sources (S2b feeder) render black.
|
||||
* Overlay order: `layer` ascending, snapshot order within a layer.
|
||||
*/
|
||||
export function planScene(spec: ProductionSpec, scene: ProductionScene, canvas: Canvas, available: (sourceId: string) => boolean): ScenePlan {
|
||||
const skipped: string[] = [];
|
||||
const sourceIds: string[] = [];
|
||||
const indexOf = (id: string): number => {
|
||||
let i = sourceIds.indexOf(id);
|
||||
if (i < 0) { sourceIds.push(id); i = sourceIds.length - 1; }
|
||||
return i;
|
||||
};
|
||||
const grid = layoutRects(scene.layout, canvas);
|
||||
const slots = [...scene.slots].sort((a, b) => a.index - b.index);
|
||||
const tiles: TileSpec[] = [];
|
||||
const gains = new Map<number, number>();
|
||||
for (const slot of slots) {
|
||||
const rect = slot.rect ? rectFromBasisPoints(slot.rect, canvas) : grid[slot.index];
|
||||
if (!rect) continue;
|
||||
let source: number | null = null;
|
||||
if (slot.sourceId !== null) {
|
||||
if (available(slot.sourceId)) source = indexOf(slot.sourceId);
|
||||
else skipped.push(`slot ${slot.index}: source ${slot.sourceId} has no ingest in this worker build`);
|
||||
}
|
||||
tiles.push({ source, rect, fit: slot.fit });
|
||||
if (source !== null && slot.audioMode !== "off") gains.set(source, Math.max(gains.get(source) ?? -Infinity, slot.gainDb));
|
||||
}
|
||||
const audio = sourceIds.map((_, i) => gains.has(i));
|
||||
const mix: MixInput[] = [...gains.entries()].sort((a, b) => a[0] - b[0]).map(([source, gainDb]) => ({ source, gainDb }));
|
||||
|
||||
const assetIds: string[] = [];
|
||||
const overlays: OverlaySpec[] = [];
|
||||
const ordered = spec.overlays
|
||||
.map((o, i) => ({ o, i }))
|
||||
.filter(({ o }) => o.sceneId === scene.id && o.enabled)
|
||||
.sort((a, b) => a.o.layer - b.o.layer || a.i - b.i)
|
||||
.map(({ o }) => o);
|
||||
for (const o of ordered) {
|
||||
const spec = overlaySpec(o, canvas, (assetId) => {
|
||||
let i = assetIds.indexOf(assetId);
|
||||
if (i < 0) { assetIds.push(assetId); i = assetIds.length - 1; }
|
||||
return i;
|
||||
});
|
||||
if (spec) overlays.push(spec);
|
||||
else skipped.push(`overlay ${o.id}: ${o.kind} is not composited in S2a`);
|
||||
}
|
||||
return {
|
||||
sourceIds, audio, assetIds, skipped,
|
||||
graph: { canvas, layout: scene.layout, tiles, overlays, audio: { inputs: mix, masterGainDb: spec.audio.masterGainDb } },
|
||||
};
|
||||
}
|
||||
|
||||
function overlaySpec(o: ProductionOverlay, canvas: Canvas, asset: (id: string) => number): OverlaySpec | null {
|
||||
if (o.assetId === null) return null; // clock: server-generated drawtext is a later slice
|
||||
const x = Math.min(canvas.width, Math.round((o.position.x * canvas.width) / 10_000));
|
||||
const y = Math.min(canvas.height, Math.round((o.position.y * canvas.height) / 10_000));
|
||||
if (o.kind === "chyron" && o.chyron) return { kind: "chyron", asset: asset(o.assetId), y, speedPxPerSec: o.chyron.speedPxPerSec };
|
||||
if (o.kind === "lower_third" || o.kind === "logo") return { kind: "image", asset: asset(o.assetId), x, y };
|
||||
return null;
|
||||
}
|
||||
275
apps/media-worker/src/compositor/soak.ts
Normal file
275
apps/media-worker/src/compositor/soak.ts
Normal file
|
|
@ -0,0 +1,275 @@
|
|||
/**
|
||||
* S2 soak (streaming-architecture.md §6 S2: "the ≥8 h soak is wired into the
|
||||
* nightly run, not blocking merge"). Drives the production path for hours and
|
||||
* fails on drift or leaks:
|
||||
*
|
||||
* ProductionJob (4 paced 1080p30 MPEG-TS sources standing in for the WP7
|
||||
* relay + one looping playlist source through the asset feeder) → source-bus
|
||||
* helpers → A/B compositors → Rust gate → x264 (-f null) + program tap, with
|
||||
* preview taps (program LL-HLS always on, one source rendition kept demanded)
|
||||
* publishing into an in-memory manifest store, and a scripted scene / overlay
|
||||
* / gain swap every `--swap-every` s through `reconfigure()` (the RUNNING
|
||||
* snapshot path the supervisor uses).
|
||||
*
|
||||
* Every `--sample-every` s one JSON line goes to `--out`: gate continuity
|
||||
* counters, bus drops, swap latency, feeder and preview counters, and RSS + open
|
||||
* fds per process. The last line is the summary; the exit code is 0 only if:
|
||||
* - no process of the chain exited on its own and every swap went on air;
|
||||
* - after the settle window: bus drops ≤ --max-bus-drops (default 0) and gate
|
||||
* repeat/silence fills ≤ 1 per 1 000 ticks;
|
||||
* - from the end of the first hour (or the first quarter of a shorter run) to
|
||||
* the end: no process grows RSS by more than 15 % (+16 MiB) and fd counts
|
||||
* move by at most 8.
|
||||
*
|
||||
* node --import tsx src/compositor/soak.ts [--hours 8] [--profile 1080p30|720p30]
|
||||
* [--swap-every 30] [--sample-every 10] [--max-bus-drops 0] [--out soak.jsonl]
|
||||
*
|
||||
* Nightly: `pnpm --filter @tactical-media-broadcast-hub/media-worker soak -- --hours 8 --out
|
||||
* runs/soak/$(date +%F).jsonl` on the reference host (see runs/wp4/s2b-report.md).
|
||||
*/
|
||||
import { appendFileSync, closeSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { basename, join } from "node:path";
|
||||
import { createReadStream } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { parseProductionSpec, type ProductionSpec } from "../production";
|
||||
import type { PreviewContentType } from "../preview/contract";
|
||||
import type { ManagedProcess } from "../relay/ffmpeg";
|
||||
import { OUTPUT_PROFILES, type DesiredChannel, type OutputPreset } from "../types";
|
||||
import { ProductionJob } from "./production-job";
|
||||
import { encodeSourceClip, ensureGateBinary, makeAssets, passthroughRedactor, startPacer } from "./testing";
|
||||
|
||||
function opt(name: string, fallback: string): string {
|
||||
const i = process.argv.indexOf(`--${name}`);
|
||||
return i >= 0 ? (process.argv[i + 1] ?? fallback) : fallback;
|
||||
}
|
||||
|
||||
const id = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`;
|
||||
const WS = "8f0c5a8e-3b1d-4c3e-9a5b-0c1d2e3f4a5b";
|
||||
const CH = "1a2b3c4d-5e6f-4a1b-8c2d-3e4f5a6b7c8d";
|
||||
const PLAYLIST = id(5);
|
||||
|
||||
/** Swap i of the script: layouts over the 4 live sources and the playlist, overlays on/off, gains in 3 dB steps. */
|
||||
function scene(i: number, assetIds: [string, string]): ProductionSpec {
|
||||
const layouts = ["2x2", "pip", "single", "3x3", "2x1"] as const;
|
||||
const layout = layouts[i % layouts.length]!;
|
||||
const order = [id(1), id(2), id(3), id(4), PLAYLIST];
|
||||
const rot = (k: number): string => order[(k + i) % order.length]!;
|
||||
const count = { "2x2": 4, pip: 2, single: 1, "3x3": 5, "2x1": 2 }[layout];
|
||||
const slots = Array.from({ length: count }, (_, k) => ({
|
||||
index: k, sourceId: rot(k),
|
||||
rect: layout === "pip" && k === 1 ? { x: 6500, y: 6500, w: 3000, h: 3000 } : null,
|
||||
fit: (["cover", "contain", "crop"] as const)[(i + k) % 3]!,
|
||||
audioMode: k === 0 ? "primary" as const : (k === 1 ? "on" as const : "off" as const),
|
||||
gainDb: -3 * ((i + k) % 9),
|
||||
}));
|
||||
return parseProductionSpec({
|
||||
sources: [
|
||||
...[1, 2, 3, 4].map((n) => ({ id: id(n), label: `Cam ${n}`, kind: "hls", url: `https://cam${n}.example/live.m3u8` })),
|
||||
{ id: PLAYLIST, label: "Loop", kind: "playlist", assetIds: [id(40), id(41), id(42)], loop: true },
|
||||
],
|
||||
scenes: [{ id: id(10), label: "Program", layout, slots }, { id: id(11), label: "Preview", layout: "single", slots: [] }],
|
||||
buses: { preview: id(11), program: id(10) },
|
||||
overlays: [
|
||||
{ id: id(20), sceneId: id(10), kind: "lower_third", enabled: i % 3 !== 0, layer: 1, position: { x: 500, y: 8000 }, revisionId: id(30), assetId: assetIds[0], chyron: null, clock: null },
|
||||
{ id: id(21), sceneId: id(10), kind: "chyron", enabled: i % 2 === 0, layer: 2, position: { x: 0, y: 9400 }, revisionId: id(31), assetId: assetIds[1], chyron: { speedPxPerSec: 40 + ((i * 20) % 360), stripWidthPx: 600 }, clock: null },
|
||||
],
|
||||
transition: { type: "cut" },
|
||||
audio: { masterGainDb: -3 * (i % 3) },
|
||||
});
|
||||
}
|
||||
|
||||
/** VmRSS (KiB) and open fds of a process. */
|
||||
function procStat(pid: number | undefined): { rssKiB: number; fds: number } | null {
|
||||
if (pid === undefined) return null;
|
||||
try {
|
||||
const rss = /VmRSS:\s+(\d+)/.exec(readFileSync(`/proc/${pid}/status`, "utf8"))?.[1];
|
||||
return { rssKiB: Number(rss ?? 0), fds: readdirSync(`/proc/${pid}/fd`).length };
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
function pids(name: string): number[] {
|
||||
const r = spawnSync("pgrep", ["-f", name], { encoding: "utf8" });
|
||||
return r.stdout.split("\n").filter(Boolean).map(Number);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const hours = Number(opt("hours", "8"));
|
||||
const profile = opt("profile", "1080p30") as OutputPreset;
|
||||
const swapEvery = Number(opt("swap-every", "30"));
|
||||
const sampleEvery = Number(opt("sample-every", "10"));
|
||||
const maxBusDrops = Number(opt("max-bus-drops", "0"));
|
||||
const out = opt("out", join(process.cwd(), `soak-${new Date().toISOString().replace(/[:.]/g, "-")}.jsonl`));
|
||||
if (!(hours > 0) || !Number.isFinite(hours) || !OUTPUT_PROFILES[profile]) throw new Error("--hours must be > 0 and --profile one of 720p30|1080p30|1080p60");
|
||||
if (!(swapEvery > 0) || !Number.isFinite(swapEvery) || !(sampleEvery > 0) || !Number.isFinite(sampleEvery) || !Number.isSafeInteger(maxBusDrops) || maxBusDrops < 0) throw new Error("swap/sample intervals must be positive and max-bus-drops a non-negative integer");
|
||||
if (!ensureGateBinary()) throw new Error("native helpers unavailable (cargo build failed)");
|
||||
const { width, height, fps } = OUTPUT_PROFILES[profile];
|
||||
const canvas = { width, height, fps };
|
||||
const dir = mkdtempSync(join(tmpdir(), "soak-"));
|
||||
let job: ProductionJob | null = null;
|
||||
let exitCode = 2;
|
||||
try {
|
||||
writeFileSync(out, "");
|
||||
const log = (record: object): void => appendFileSync(out, `${JSON.stringify(record)}\n`);
|
||||
|
||||
// Fixtures, prepared before the clock starts.
|
||||
const clip = join(dir, "src.ts");
|
||||
encodeSourceClip(clip, canvas);
|
||||
const clips = new Map<string, string>();
|
||||
for (const [n, hz] of [[40, 300], [41, 600], [42, 900]] as const) {
|
||||
const path = join(dir, `asset-${n}.mp4`);
|
||||
const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", `testsrc2=s=${width}x${height}:r=${fps}:d=6`, "-f", "lavfi", "-i", `sine=frequency=${hz}:d=6`,
|
||||
"-c:v", "libx264", "-preset", "veryfast", "-b:v", "4000k", "-g", String(fps), "-c:a", "aac", "-movflags", "+faststart", path]);
|
||||
if (r.status !== 0) throw new Error("playlist asset encode failed");
|
||||
clips.set(id(n), path);
|
||||
}
|
||||
const assets = makeAssets(dir);
|
||||
const assetIds = assets.map((p) => basename(p, ".png")) as [string, string];
|
||||
const pacers: ManagedProcess[] = [];
|
||||
const pacerFds: number[] = [];
|
||||
const exits: string[] = [];
|
||||
const published = { count: 0, bytes: 0 };
|
||||
const manifests = new Map<string, number>();
|
||||
|
||||
job = new ProductionJob({
|
||||
validator: { async validate() { throw new Error("not used: synthetic egress"); } },
|
||||
vault: { async deliverPushTarget() { throw new Error("not used: null output"); } },
|
||||
egress: {
|
||||
async prepare(config) {
|
||||
const sources = new Map<string, { input: { kind: "mpegtsPipe" }; fd: number }>();
|
||||
for (const src of config.production?.sources ?? []) {
|
||||
if (src.kind !== "hls") continue;
|
||||
const pacer = await startPacer(clip, dir, `pacer-${src.id.slice(-2)}`);
|
||||
pacers.push(pacer.proc);
|
||||
pacerFds.push(pacer.readFd);
|
||||
sources.set(src.id, { input: { kind: "mpegtsPipe" }, fd: pacer.readFd });
|
||||
}
|
||||
return { sources, output: { kind: "null" as const }, async close() { for (const p of pacers) await p.terminate(500); for (const fd of pacerFds.splice(0)) closeSync(fd); } };
|
||||
},
|
||||
},
|
||||
assets: { async localPath(assetId) { const p = assets.find((a) => basename(a, ".png") === assetId); if (!p) throw new Error("unknown overlay asset"); return p; } },
|
||||
playlists: { async open(assetId) { const p = clips.get(assetId); if (!p) throw new Error("unknown asset"); return createReadStream(p); } },
|
||||
preview: {
|
||||
objects: { async putContent(body: Buffer, contentType: PreviewContentType) { published.count += 1; published.bytes += body.length; return { key: `sha256/${"0".repeat(64)}.${contentType === "image/jpeg" ? "jpg" : "bin"}` }; } },
|
||||
manifests: { async set(key: string) { manifests.set(key, (manifests.get(key) ?? 0) + 1); } },
|
||||
},
|
||||
redactor: passthroughRedactor, onStderr: () => undefined,
|
||||
onExit: (reason) => exits.push(reason),
|
||||
scratchDir: join(dir, "scratch"),
|
||||
});
|
||||
const config = (version: number, i: number): DesiredChannel => ({
|
||||
workspaceId: WS, channelId: CH, workerId: "soak", assignmentEpoch: 1, desiredVersion: version, desiredStatus: "RUNNING",
|
||||
source: null, output: { endpointUrl: "rtmp://push.example/live", credentialHandleId: "keyref-soak" }, profile, mode: "normalize",
|
||||
production: scene(i, assetIds),
|
||||
});
|
||||
|
||||
const t0 = Date.now();
|
||||
await job.start(config(1, 0));
|
||||
const runningJob = job;
|
||||
const chain = runningJob.program!;
|
||||
await runningJob.previews?.demand(id(1), "video").catch(() => undefined);
|
||||
|
||||
const settleMs = Math.min(60_000, (hours * 3_600_000) / 10);
|
||||
const endAt = t0 + hours * 3_600_000;
|
||||
const swapLatency: number[] = [];
|
||||
let swaps = 0;
|
||||
let swapFailures = 0;
|
||||
let nextSwap = Date.now() + swapEvery * 1000;
|
||||
let nextSample = Date.now() + sampleEvery * 1000;
|
||||
let settled: { ticks: number; fills: number; busDrops: number } | null = null;
|
||||
const samples: Array<{ t: number; procs: Record<string, { rssKiB: number; fds: number } | null> }> = [];
|
||||
const busDrops = async (): Promise<number> => {
|
||||
let total = 0;
|
||||
for (const sid of [id(1), id(2), id(3), id(4), PLAYLIST]) {
|
||||
const bus = runningJob.sourceBus(sid);
|
||||
if (bus) total += (await bus.stats()).subscriberDrops;
|
||||
}
|
||||
return total;
|
||||
};
|
||||
|
||||
while (Date.now() < endAt && exits.length === 0) {
|
||||
const now = Date.now();
|
||||
if (now >= nextSwap) {
|
||||
nextSwap += swapEvery * 1000;
|
||||
const started = Date.now();
|
||||
try {
|
||||
await runningJob.reconfigure(config(swaps + 2, swaps + 1));
|
||||
swaps += 1;
|
||||
swapLatency.push(Date.now() - started);
|
||||
} catch (error) {
|
||||
swapFailures += 1;
|
||||
log({ type: "swap-failed", t: (Date.now() - t0) / 1000, error: String(error) });
|
||||
}
|
||||
}
|
||||
if (now >= nextSample) {
|
||||
nextSample += sampleEvery * 1000;
|
||||
const gate = await chain.stats();
|
||||
const drops = await busDrops();
|
||||
if (!settled && now - t0 >= settleMs) settled = { ticks: gate.ticks, fills: gate.vrep + gate.asil, busDrops: drops };
|
||||
const procs: Record<string, { rssKiB: number; fds: number } | null> = {
|
||||
worker: { rssKiB: Math.round(process.memoryUsage().rss / 1024), fds: readdirSync("/proc/self/fd").length },
|
||||
gate: procStat(chain.gate.pid),
|
||||
encoder: procStat(chain.encoderPid),
|
||||
compositor: procStat(chain.activeCompositor?.pid),
|
||||
programBus: procStat(chain.programBus?.pid ?? undefined),
|
||||
};
|
||||
pids(join(dir, "scratch", "bus-")).forEach((pid, k) => { procs[`bus${k}`] = procStat(pid); });
|
||||
samples.push({ t: now - t0, procs });
|
||||
const sorted = [...swapLatency].sort((a, b) => a - b);
|
||||
log({
|
||||
type: "sample", t: (now - t0) / 1000, ticks: gate.ticks, vrep: gate.vrep, asil: gate.asil, outDrops: gate.outDrops, lagged: gate.lagged,
|
||||
lateOverFrame: gate.lateOverFrame, busDrops: drops, swaps, swapFailures, lagRetries: chain.lagRetries,
|
||||
swapMs: { p50: sorted[Math.floor(sorted.length / 2)] ?? null, max: sorted.at(-1) ?? null },
|
||||
feeder: runningJob.feeder(PLAYLIST)?.stats, previews: { ...published, manifests: manifests.size }, procs,
|
||||
});
|
||||
// Keep the demanded source rendition alive (a viewer watching that tile).
|
||||
await runningJob.previews?.demand(id(1), "video").catch(() => undefined);
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 250));
|
||||
}
|
||||
|
||||
// Verdict.
|
||||
const gate = await chain.stats();
|
||||
const drops = await busDrops();
|
||||
const failures: string[] = [];
|
||||
if (!settled) failures.push("no settled sample collected");
|
||||
if (gate.outDrops !== 0) failures.push(`encoder output drops: ${gate.outDrops}`);
|
||||
if (exits.length) failures.push(`chain exited: ${exits.join("; ")}`);
|
||||
if (swapFailures) failures.push(`${swapFailures} swaps never went on air`);
|
||||
if (settled) {
|
||||
const ticks = gate.ticks - settled.ticks;
|
||||
const fills = gate.vrep + gate.asil - settled.fills;
|
||||
if (drops - settled.busDrops > maxBusDrops) failures.push(`bus drops after settle: ${drops - settled.busDrops}`);
|
||||
if (fills > Math.ceil(ticks / 1000)) failures.push(`gate fills after settle: ${fills} over ${ticks} ticks`);
|
||||
}
|
||||
const baseAt = Math.min(3_600_000, (hours * 3_600_000) / 4);
|
||||
const base = samples.find((s) => s.t >= baseAt);
|
||||
const last = samples.at(-1);
|
||||
const growth: Record<string, { rssPct: number; fds: number }> = {};
|
||||
if (base && last && last !== base) {
|
||||
for (const [name, b] of Object.entries(base.procs)) {
|
||||
const l = last.procs[name];
|
||||
if (!b || !l) continue;
|
||||
// Compositors are replaced by swaps; their size is not a leak signal.
|
||||
if (name === "compositor") continue;
|
||||
growth[name] = { rssPct: Number((((l.rssKiB - b.rssKiB) / b.rssKiB) * 100).toFixed(1)), fds: l.fds - b.fds };
|
||||
if (l.rssKiB > b.rssKiB * 1.15 + 16 * 1024) failures.push(`${name} RSS ${b.rssKiB} → ${l.rssKiB} KiB`);
|
||||
if (Math.abs(l.fds - b.fds) > 8) failures.push(`${name} fds ${b.fds} → ${l.fds}`);
|
||||
}
|
||||
}
|
||||
const summary = {
|
||||
type: "summary", hours: (Date.now() - t0) / 3_600_000, profile, swaps, swapFailures, lagRetries: chain.lagRetries,
|
||||
gate: { ticks: gate.ticks, vrep: gate.vrep, asil: gate.asil, outDrops: gate.outDrops, lagged: gate.lagged }, busDrops: drops, settled,
|
||||
growth, previews: published, feeder: runningJob.feeder(PLAYLIST)?.stats, pass: failures.length === 0, failures,
|
||||
};
|
||||
log(summary);
|
||||
process.stdout.write(`${JSON.stringify(summary)}\n`);
|
||||
exitCode = failures.length === 0 ? 0 : 1;
|
||||
} finally {
|
||||
await job?.stop();
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
process.exitCode = exitCode;
|
||||
}
|
||||
|
||||
if (require.main === module) void main().catch((error: unknown) => { process.stderr.write(`${String(error)}\n`); process.exit(2); });
|
||||
153
apps/media-worker/src/compositor/source-bus.test.ts
Normal file
153
apps/media-worker/src/compositor/source-bus.test.ts
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
/**
|
||||
* Source bus helper (src/gate/rust/src/bin/source-bus.rs) through its Node
|
||||
* manager: the S2a bus semantics on the new out-of-process path. Frames are
|
||||
* tagged (first byte = pair index) so delivery can be checked exactly.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { Socket } from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { audioFrameBytes, videoFrameBytes } from "../gate/protocol";
|
||||
import { ProgramChain } from "./program";
|
||||
import { SourceBus, type BusSubscription } from "./source-bus";
|
||||
import { ensureGateBinary, passthroughRedactor } from "./testing";
|
||||
|
||||
const g = { width: 64, height: 36, fps: 30 };
|
||||
const V = videoFrameBytes(g);
|
||||
const A = audioFrameBytes(g);
|
||||
|
||||
const frame = (bytes: number, tag: number): Buffer => Buffer.alloc(bytes, tag & 0xff);
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
async function until(cond: () => boolean | Promise<boolean>, ms: number, what: string): Promise<void> {
|
||||
const t0 = Date.now();
|
||||
while (!(await cond())) {
|
||||
if (Date.now() - t0 > ms) throw new Error(`timeout: ${what}`);
|
||||
await sleep(20);
|
||||
}
|
||||
}
|
||||
|
||||
type Harness = { bus: SourceBus; video: Socket; audio: Socket; dir: string; close(): Promise<void> };
|
||||
|
||||
/** Writes pair i as an ingest does (interleaved): a burst of one stream would be trimmed as skew. */
|
||||
async function pair(h: Harness, i: number, audio = true): Promise<void> {
|
||||
await Promise.all([
|
||||
new Promise<void>((r) => h.video.write(frame(V, i), () => r())),
|
||||
audio ? new Promise<void>((r) => h.audio.write(frame(A, i), () => r())) : Promise.resolve(),
|
||||
]);
|
||||
}
|
||||
|
||||
async function harness(): Promise<Harness> {
|
||||
const dir = mkdtempSync(join(tmpdir(), "bus-"));
|
||||
const bus = new SourceBus({ geometry: g, scratchDir: dir, redactor: passthroughRedactor, onStderr: () => undefined });
|
||||
const fds = await bus.start();
|
||||
bus.adoptIngestWriters();
|
||||
const video = new Socket({ fd: fds.videoFd, readable: false, writable: true });
|
||||
const audio = new Socket({ fd: fds.audioFd, readable: false, writable: true });
|
||||
return { bus, video, audio, dir, async close() { video.destroy(); audio.destroy(); await bus.stop(); rmSync(dir, { recursive: true, force: true }); } };
|
||||
}
|
||||
|
||||
/** Reads a subscriber's stream into whole frames (first byte of each). */
|
||||
function collect(fd: number | null, bytes: number): { tags: number[]; socket: Socket } {
|
||||
assert.ok(fd !== null);
|
||||
const tags: number[] = [];
|
||||
let pending = Buffer.alloc(0);
|
||||
const socket = new Socket({ fd, readable: true, writable: false });
|
||||
socket.on("data", (chunk: Buffer) => {
|
||||
pending = Buffer.concat([pending, chunk]);
|
||||
while (pending.length >= bytes) { tags.push(pending[0]!); pending = pending.subarray(bytes); }
|
||||
});
|
||||
return { tags, socket };
|
||||
}
|
||||
|
||||
test("history replay, start pair and A/V pairs delivered frame-exact", { timeout: 60_000 }, async (t) => {
|
||||
if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; }
|
||||
const h = await harness();
|
||||
try {
|
||||
for (let i = 0; i < 20; i += 1) await pair(h, i);
|
||||
await until(async () => (await h.bus.stats()).pairs === 20, 5_000, "20 pairs");
|
||||
const sub = await h.bus.subscribe({ video: true, audio: true });
|
||||
assert.equal(sub.startPair, 20 - h.bus.historyFrames, "starts at the oldest replayed pair");
|
||||
const v = collect(sub.videoFd, V);
|
||||
const a = collect(sub.audioFd, A);
|
||||
sub.adoptReaders();
|
||||
for (let i = 20; i < 25; i += 1) await pair(h, i);
|
||||
await until(() => v.tags.length >= 17 && a.tags.length >= 17, 5_000, "replay + live");
|
||||
const expected = Array.from({ length: 17 }, (_, k) => 8 + k);
|
||||
assert.deepEqual(v.tags.slice(0, 17), expected, "video: history then live, no gap");
|
||||
assert.deepEqual(a.tags.slice(0, 17), expected, "audio pairs with video");
|
||||
const stats = await h.bus.stats();
|
||||
assert.equal(stats.subscriberDrops, 0);
|
||||
assert.equal(stats.subscribers.length, 1);
|
||||
v.socket.destroy(); a.socket.destroy();
|
||||
// The reader is gone: the helper drops the subscriber on EPIPE.
|
||||
await pair(h, 25).catch(() => undefined);
|
||||
await until(async () => (await h.bus.stats()).subscribers.length === 0, 5_000, "closed subscriber removed");
|
||||
} finally { await h.close(); }
|
||||
});
|
||||
|
||||
test("stride subscribers get the newest pair then every n-th; audio-only and silence fill", { timeout: 60_000 }, async (t) => {
|
||||
if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; }
|
||||
const h = await harness();
|
||||
try {
|
||||
for (let i = 0; i < 10; i += 1) await pair(h, i);
|
||||
await until(async () => (await h.bus.stats()).pairs === 10, 5_000, "10 pairs");
|
||||
const thumb = await h.bus.subscribe({ video: true, audio: false, stride: 5 });
|
||||
assert.equal(thumb.audioFd, null);
|
||||
assert.equal(thumb.startPair, 9, "the newest pair first");
|
||||
const ears = await h.bus.subscribe({ video: false, audio: true });
|
||||
const tv = collect(thumb.videoFd, V);
|
||||
const ea = collect(ears.audioFd, A);
|
||||
thumb.adoptReaders(); ears.adoptReaders();
|
||||
for (let i = 10; i < 21; i += 1) await pair(h, i);
|
||||
// The audio-only subscriber replays the 10 pairs of history there are, then 11 live ones.
|
||||
await until(() => tv.tags.length >= 3 && ea.tags.length >= 10 + 11, 5_000, "strided + audio");
|
||||
assert.deepEqual(tv.tags.slice(0, 3), [9, 14, 19]);
|
||||
assert.deepEqual(ea.tags.slice(-3), [18, 19, 20]);
|
||||
// Video without audio: once video runs maxSkew ahead, pairs carry silence.
|
||||
for (let i = 21; i < 21 + h.bus.maxSkewFrames + 5; i += 1) await pair(h, i, false);
|
||||
await until(async () => (await h.bus.stats()).silenceFilled >= 5, 5_000, "silence fill");
|
||||
await until(() => ea.tags.at(-1) === 0, 5_000, "silence reaches the audio subscriber");
|
||||
tv.socket.destroy(); ea.socket.destroy();
|
||||
} finally { await h.close(); }
|
||||
});
|
||||
|
||||
test("a stalled subscriber loses whole pairs past its backlog; peers' drop counts are snapshotted at subscribe", { timeout: 60_000 }, async (t) => {
|
||||
if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; }
|
||||
const h = await harness();
|
||||
let stalled: BusSubscription | null = null;
|
||||
try {
|
||||
stalled = await h.bus.subscribe({ video: true, audio: true });
|
||||
// Never read: each 1 MiB pipe fills (audio first: 163 frames here), then the helper queue does.
|
||||
const limit = h.bus.maxBacklogFrames + h.bus.historyFrames;
|
||||
const youngLimit = limit + g.fps;
|
||||
const produced = Math.ceil((1 << 20) / A) + youngLimit + 20;
|
||||
for (let i = 0; i < produced; i += 1) await pair(h, i);
|
||||
await until(async () => (await h.bus.stats()).pairs === produced, 10_000, "entire burst consumed");
|
||||
await until(async () => (await h.bus.stats()).subscriberDrops > 0, 10_000, "drops while young");
|
||||
const young = await h.bus.stats();
|
||||
const queued = young.subscribers[0]?.queued ?? 0;
|
||||
assert.ok(queued > limit && queued <= youngLimit + 1, `a young subscriber queues up to ~1 s more than the limit (${queued} queued, limit ${limit})`);
|
||||
const youngDrops = young.subscriberDrops;
|
||||
await sleep(2_100); // young period over: the normal limit applies, and the queue is still above it
|
||||
for (let i = 0; i < 10; i += 1) await pair(h, i);
|
||||
await until(async () => (await h.bus.stats()).subscriberDrops >= youngDrops + 10, 5_000, "every new pair dropped");
|
||||
await until(() => stalled?.dropped === (youngDrops + 10), 5_000, "DROP events reach the subscription");
|
||||
const peer = await h.bus.subscribe({ video: true, audio: false });
|
||||
assert.equal(peer.droppedBefore(stalled), youngDrops + 10, "exact at the peer's start pair");
|
||||
await pair(h, 1);
|
||||
await until(() => (stalled?.dropped ?? 0) > youngDrops + 10, 5_000, "live count moves on");
|
||||
assert.equal(peer.droppedBefore(stalled), youngDrops + 10, "the snapshot does not");
|
||||
peer.unsubscribe();
|
||||
} finally {
|
||||
stalled?.unsubscribe();
|
||||
await h.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("program chains refuse the Node reference gate (no pair lockstep, no origin alignment)", () => {
|
||||
assert.throws(() => new ProgramChain({ canvas: g, scratchDir: join(tmpdir(), "never"), videoBitrateKbps: 500, output: { kind: "external" }, gateImpl: "node",
|
||||
redactor: passthroughRedactor, onStderr: () => undefined, onFatal: () => undefined }), /spike-only/);
|
||||
});
|
||||
274
apps/media-worker/src/compositor/source-bus.ts
Normal file
274
apps/media-worker/src/compositor/source-bus.ts
Normal file
|
|
@ -0,0 +1,274 @@
|
|||
import { closeSync, mkdirSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { resolveHelperBinary } from "../gate/binary";
|
||||
import { createFifoPair, makeFifoPair, openBlockingReader, type FifoPair } from "../gate/fifo";
|
||||
import { assertGeometry, type FrameGeometry } from "../gate/protocol";
|
||||
import { spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg";
|
||||
|
||||
/**
|
||||
* Per-source fan-out between one ingest and every consumer of the source's raw
|
||||
* frames: the compositors that use it (two during an A/B swap) and its preview
|
||||
* taps.
|
||||
*
|
||||
* The bytes never pass through Node. A `source-bus` helper process
|
||||
* (src/gate/rust/src/bin/source-bus.rs) reads the ingest's FIFOs and writes
|
||||
* each subscriber's FIFOs; this class is its process manager and speaks its
|
||||
* control lines. (S2a moved the frames on the worker's event loop: at
|
||||
* 4 × 1080p30 that left no headroom and the buses dropped ~7.5 % of source
|
||||
* pairs; runs/wp4/s2b-report.md item 1.)
|
||||
*
|
||||
* Semantics are unchanged from S2a: frames are delivered as A/V *pairs*, a
|
||||
* subscriber more than `maxBacklogFrames + historyFrames` behind loses whole
|
||||
* pairs, and the last `historyFrames` pairs are replayed to a new subscriber so
|
||||
* the gate can splice a swap-in compositor at the on-air one's next source frame.
|
||||
*/
|
||||
export interface SourceBusOptions {
|
||||
geometry: FrameGeometry;
|
||||
scratchDir: string;
|
||||
redactor: StderrRedactor;
|
||||
onStderr(message: string): void;
|
||||
/** The helper died (not on `stop()`): the source is gone. */
|
||||
onExit?(reason: string): void;
|
||||
maxSkewFrames?: number;
|
||||
maxBacklogFrames?: number;
|
||||
historyFrames?: number;
|
||||
}
|
||||
|
||||
export interface SubscribeOptions {
|
||||
video: boolean;
|
||||
audio: boolean;
|
||||
/** Take every `stride`-th pair only (thumbnails). Default 1. */
|
||||
stride?: number;
|
||||
}
|
||||
|
||||
/** A subscription: the pair index it starts at, and the pairs it has lost to backlog since. */
|
||||
export interface BusSubscription {
|
||||
readonly bus: SourceBus;
|
||||
readonly id: number;
|
||||
readonly startPair: number;
|
||||
readonly startedAt: bigint;
|
||||
readonly dropped: number;
|
||||
/** Blocking read ends for the consumer's stdio (null when not subscribed to that stream). */
|
||||
readonly videoFd: number | null;
|
||||
readonly audioFd: number | null;
|
||||
/** Pairs `other` had lost when this subscription started (exact, unlike its live `dropped`). */
|
||||
droppedBefore(other: BusSubscription): number | undefined;
|
||||
/** Close the worker's copies of the read ends (after spawning the consumer). */
|
||||
releaseReaders(): void;
|
||||
/** The caller takes ownership of the read ends (an in-process consumer that closes them itself). */
|
||||
adoptReaders(): void;
|
||||
unsubscribe(): void;
|
||||
}
|
||||
|
||||
export interface SourceBusStats {
|
||||
pairs: number; silenceFilled: number; audioTrimmed: number; subscriberDrops: number; allocs: number;
|
||||
input: { videoWaiting: number; audioWaiting: number; videoEof: boolean; audioEof: boolean };
|
||||
/** Per subscriber: pairs dropped to backlog, and frames queued in the helper right now. */
|
||||
subscribers: Array<{ id: number; dropped: number; queued: number }>;
|
||||
}
|
||||
|
||||
interface Waiter { match(line: string[]): boolean; resolve(line: string[]): void; reject(error: Error): void; timer: ReturnType<typeof setTimeout> }
|
||||
|
||||
let nextBus = 1;
|
||||
|
||||
export class SourceBus {
|
||||
private proc: ManagedProcess | null = null;
|
||||
private readonly busId = nextBus++;
|
||||
private nextSub = 1;
|
||||
private lineBuf = "";
|
||||
private waiters: Waiter[] = [];
|
||||
private readonly drops = new Map<number, number>();
|
||||
private stopping = false;
|
||||
private exited = false;
|
||||
private ingest: FifoPair | null = null;
|
||||
readonly maxSkewFrames: number;
|
||||
readonly maxBacklogFrames: number;
|
||||
readonly historyFrames: number;
|
||||
/** Set when the helper reports the ingest closed both streams. */
|
||||
ended = false;
|
||||
|
||||
constructor(private readonly options: SourceBusOptions) {
|
||||
assertGeometry(options.geometry);
|
||||
this.maxSkewFrames = options.maxSkewFrames ?? 15;
|
||||
this.maxBacklogFrames = options.maxBacklogFrames ?? 4;
|
||||
this.historyFrames = options.historyFrames ?? 12;
|
||||
mkdirSync(options.scratchDir, { recursive: true, mode: 0o700 });
|
||||
}
|
||||
|
||||
get geometry(): FrameGeometry { return this.options.geometry; }
|
||||
get pid(): number | undefined { return this.proc?.child.pid; }
|
||||
|
||||
/**
|
||||
* Starts the helper on fresh ingest FIFOs. The returned fds are blocking write ends for the
|
||||
* producer's stdio (ingest: video on fd 1, PCM on fd 3); call `releaseIngestWriters()` once
|
||||
* the producer has them. Until a producer writes, the bus simply has no pairs.
|
||||
*/
|
||||
async start(): Promise<{ videoFd: number; audioFd: number }> {
|
||||
const g = this.options.geometry;
|
||||
const fifos = createFifoPair(this.options.scratchDir, this.busId, "bus");
|
||||
this.ingest = fifos;
|
||||
const argv = [String(g.width), String(g.height), String(g.fps), String(this.maxSkewFrames), String(this.maxBacklogFrames), String(this.historyFrames),
|
||||
fifos.videoPath, fifos.audioPath];
|
||||
const proc = spawnManaged(resolveHelperBinary("source-bus"), argv, ["pipe", "pipe", "pipe"], this.options.redactor, this.options.onStderr);
|
||||
this.proc = proc;
|
||||
proc.child.stdout?.setEncoding("utf8");
|
||||
proc.child.stdout?.on("data", (chunk: string) => this.onData(chunk));
|
||||
// EPIPE on a control line to a helper that just died: its exit is reported below, never as a crash.
|
||||
proc.child.stdin?.on("error", () => undefined);
|
||||
void proc.exited.then(({ code, signal }) => {
|
||||
this.exited = true;
|
||||
const reason = `source bus exited code=${code} signal=${signal}`;
|
||||
for (const w of this.waiters.splice(0)) { clearTimeout(w.timer); w.reject(new Error(reason)); }
|
||||
if (!this.stopping) this.options.onExit?.(reason);
|
||||
});
|
||||
const ready = this.waitFor((l) => l[0] === "READY", 10_000);
|
||||
try {
|
||||
await proc.started;
|
||||
await ready;
|
||||
} catch (error) {
|
||||
ready.catch(() => undefined);
|
||||
fifos.releaseHolders();
|
||||
fifos.releaseWriters();
|
||||
fifos.unlink();
|
||||
throw error;
|
||||
} finally {
|
||||
// The helper holds the read ends now; from here only the producer keeps the FIFOs open.
|
||||
fifos.releaseHolders();
|
||||
fifos.unlink();
|
||||
}
|
||||
return { videoFd: fifos.videoFd, audioFd: fifos.audioFd };
|
||||
}
|
||||
|
||||
/** Close the worker's copies of the ingest write ends (after spawning the producer). */
|
||||
releaseIngestWriters(): void { this.ingest?.releaseWriters(); }
|
||||
|
||||
/** The caller takes ownership of the ingest write ends (an in-process producer that closes them itself). */
|
||||
adoptIngestWriters(): void { this.ingest = null; }
|
||||
|
||||
private onData(chunk: string): void {
|
||||
this.lineBuf += chunk;
|
||||
let nl = this.lineBuf.indexOf("\n");
|
||||
while (nl >= 0) {
|
||||
const line = this.lineBuf.slice(0, nl).trim().split(/\s+/);
|
||||
this.lineBuf = this.lineBuf.slice(nl + 1);
|
||||
nl = this.lineBuf.indexOf("\n");
|
||||
if (line[0] === "DROP") this.drops.set(Number(line[1]), Number(line[2]));
|
||||
if (line[0] === "EOF") this.ended = true;
|
||||
const i = this.waiters.findIndex((w) => w.match(line));
|
||||
if (i >= 0) {
|
||||
const [w] = this.waiters.splice(i, 1);
|
||||
if (w) { clearTimeout(w.timer); w.resolve(line); }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private waitFor(match: (line: string[]) => boolean, timeoutMs: number): Promise<string[]> {
|
||||
return new Promise<string[]>((resolve, reject) => {
|
||||
const waiter: Waiter = {
|
||||
match, resolve, reject,
|
||||
timer: setTimeout(() => { this.waiters = this.waiters.filter((w) => w !== waiter); reject(new Error("source bus reply timeout")); }, timeoutMs),
|
||||
};
|
||||
this.waiters.push(waiter);
|
||||
});
|
||||
}
|
||||
|
||||
private send(line: string): void {
|
||||
const stdin = this.proc?.child.stdin;
|
||||
if (this.exited || !stdin || stdin.destroyed) throw new Error("source bus is not running");
|
||||
stdin.write(line);
|
||||
}
|
||||
|
||||
/** Sends a control line and waits for its reply; a failed send never leaves a waiter behind. */
|
||||
private request(line: string, match: (l: string[]) => boolean, timeoutMs: number): Promise<string[]> {
|
||||
const reply = this.waitFor(match, timeoutMs);
|
||||
try {
|
||||
this.send(line);
|
||||
} catch (error) {
|
||||
reply.catch(() => undefined);
|
||||
const waiter = this.waiters.at(-1);
|
||||
if (waiter) { clearTimeout(waiter.timer); this.waiters.pop(); }
|
||||
throw error;
|
||||
}
|
||||
return reply;
|
||||
}
|
||||
|
||||
/**
|
||||
* New consumer. Delivery starts with the replayed history (a strided subscriber gets only the
|
||||
* newest pair), then live pairs; `startPair` is the first pair's index. Spawn the consumer with
|
||||
* `videoFd`/`audioFd`, then `releaseReaders()`.
|
||||
*/
|
||||
async subscribe(opts: SubscribeOptions): Promise<BusSubscription> {
|
||||
if (!opts.video && !opts.audio) throw new Error("subscription needs video or audio");
|
||||
const stride = opts.stride ?? 1;
|
||||
if (!Number.isSafeInteger(stride) || stride < 1 || stride > 3600) throw new Error("stride out of range");
|
||||
const id = this.nextSub++;
|
||||
const paths = makeFifoPair(this.options.scratchDir, id, `sub-${this.busId}`);
|
||||
// The consumer (ffmpeg) needs *blocking* read ends; a temporary writer keeps each open() from
|
||||
// blocking and is released once the helper has opened its non-blocking write end.
|
||||
const video = opts.video ? openBlockingReader(paths.videoPath) : null;
|
||||
const audio = opts.audio ? openBlockingReader(paths.audioPath) : null;
|
||||
const unlink = (): void => { rmSync(paths.videoPath, { force: true }); rmSync(paths.audioPath, { force: true }); };
|
||||
let line: string[];
|
||||
try {
|
||||
line = await this.request(`SUB ${id} ${video ? paths.videoPath : "-"} ${audio ? paths.audioPath : "-"} ${stride}\n`,
|
||||
(l) => (l[0] === "SUBSCRIBED" || l[0] === "ERROR") && Number(l[1]) === id, 5_000);
|
||||
if (line[0] !== "SUBSCRIBED") throw new Error("source bus could not open subscriber fifos");
|
||||
} catch (error) {
|
||||
for (const r of [video, audio]) if (r) { r.releaseHolder(); closeSync(r.fd); }
|
||||
throw error;
|
||||
} finally {
|
||||
video?.releaseHolder();
|
||||
audio?.releaseHolder();
|
||||
unlink();
|
||||
}
|
||||
const peers = new Map<number, number>();
|
||||
if (line[3] && line[3] !== "-") {
|
||||
for (const entry of line[3].split(",")) {
|
||||
const [peer, dropped] = entry.split(":").map(Number);
|
||||
if (peer !== undefined && dropped !== undefined) peers.set(peer, dropped);
|
||||
}
|
||||
}
|
||||
const drops = this.drops;
|
||||
let readersOpen = true;
|
||||
const subscription: BusSubscription = {
|
||||
bus: this, id, startPair: Number(line[2]), startedAt: process.hrtime.bigint(),
|
||||
get dropped() { return drops.get(id) ?? 0; },
|
||||
videoFd: video?.fd ?? null, audioFd: audio?.fd ?? null,
|
||||
droppedBefore: (other) => (other.bus === this ? peers.get(other.id) : undefined),
|
||||
releaseReaders() { if (readersOpen) { readersOpen = false; for (const r of [video, audio]) if (r) closeSync(r.fd); } },
|
||||
adoptReaders() { readersOpen = false; },
|
||||
unsubscribe: () => {
|
||||
subscription.releaseReaders();
|
||||
drops.delete(id);
|
||||
try { this.send(`UNSUB ${id}\n`); } catch { /* helper gone */ }
|
||||
},
|
||||
};
|
||||
return subscription;
|
||||
}
|
||||
|
||||
async stats(): Promise<SourceBusStats> {
|
||||
const line = await this.request("STATS\n", (l) => l[0] === "STATS", 5_000);
|
||||
const kv = new Map(line.slice(1).map((p) => p.split("=") as [string, string]));
|
||||
const n = (k: string): number => Number(kv.get(k) ?? 0);
|
||||
return { pairs: n("pairs"), silenceFilled: n("silence_filled"), audioTrimmed: n("audio_trimmed"), subscriberDrops: n("subscriber_drops"), allocs: n("allocs"),
|
||||
input: { videoWaiting: n("video_waiting"), audioWaiting: n("audio_waiting"), videoEof: n("video_eof") !== 0, audioEof: n("audio_eof") !== 0 },
|
||||
subscribers: (kv.get("subs") ?? "-").split(",").filter((e) => e !== "-").map((e) => {
|
||||
const [id, dropped, queued] = e.split(":").map(Number);
|
||||
return { id: id ?? 0, dropped: dropped ?? 0, queued: queued ?? 0 };
|
||||
}) };
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.stopping = true;
|
||||
const proc = this.proc;
|
||||
this.ingest?.releaseWriters();
|
||||
if (!proc) return;
|
||||
try { this.send("QUIT\n"); } catch { /* already gone */ }
|
||||
const timer = setTimeout(() => proc.kill("SIGKILL"), 2_000);
|
||||
await proc.exited;
|
||||
clearTimeout(timer);
|
||||
this.proc = null;
|
||||
rmSync(join(this.options.scratchDir, `bus-${this.busId}.v.fifo`), { force: true });
|
||||
rmSync(join(this.options.scratchDir, `bus-${this.busId}.a.fifo`), { force: true });
|
||||
}
|
||||
}
|
||||
211
apps/media-worker/src/compositor/testing.ts
Normal file
211
apps/media-worker/src/compositor/testing.ts
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
/**
|
||||
* Test/spike fixtures for the compositor chain (not used in production paths).
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { closeSync, copyFileSync, openSync, readFileSync } from "node:fs";
|
||||
import { Socket } from "node:net";
|
||||
import { join } from "node:path";
|
||||
import { SCENE_LAYOUTS, type SceneLayout } from "../production";
|
||||
import { ensureHelperBinaries } from "../gate/binary";
|
||||
import { layoutRects, type Canvas, type CompositorGraph, type OverlaySpec, type PxRect } from "./command-builder";
|
||||
import type { FrameGate } from "../gate/frame-gate";
|
||||
import type { ComposeRequest } from "./compositor";
|
||||
import { audioFrameBytes, videoFrameBytes, type FrameGeometry } from "../gate/protocol";
|
||||
import type { SinkReport } from "../gate/sink";
|
||||
import { spawnManaged, type ManagedProcess } from "../relay/ffmpeg";
|
||||
import { SourceBus } from "./source-bus";
|
||||
|
||||
export const passthroughRedactor = { redact: (s: string): string => s };
|
||||
|
||||
/**
|
||||
* Luma of source frame n: a frame counter that survives scaling. Solid frames
|
||||
* scale to (almost) the same value; swscale can round ±1 at some ratios
|
||||
* (4x4 tiles), so the counter moves in steps of 4 and the sink rounds.
|
||||
*/
|
||||
export function counterLuma(n: number): number { return 20 + 4 * (n % 50); }
|
||||
|
||||
/**
|
||||
* A live source paced by the wall clock that writes into a bus's ingest FIFOs
|
||||
* (the bytes an ingest process would write): solid frames whose luma is the
|
||||
* frame counter, and a 1500 Hz sine that holds a whole number of periods per
|
||||
* frame at 24/25/30/60 fps — any splice that is not on a frame boundary, and
|
||||
* any inserted silence, shows up at the sink.
|
||||
*/
|
||||
export class CounterSource {
|
||||
readonly bus: SourceBus;
|
||||
private timer: ReturnType<typeof setTimeout> | null = null;
|
||||
private n = 0;
|
||||
private readonly frames: Buffer[];
|
||||
private readonly pcm: Buffer;
|
||||
private out: { video: Socket; audio: Socket } | null = null;
|
||||
|
||||
constructor(private readonly g: FrameGeometry, scratchDir: string, amplitude = 4000) {
|
||||
this.bus = new SourceBus({ geometry: g, scratchDir, redactor: passthroughRedactor, onStderr: (m) => process.stderr.write(m) });
|
||||
const luma = g.width * g.height;
|
||||
this.frames = Array.from({ length: 50 }, (_, i) => {
|
||||
const f = Buffer.alloc(videoFrameBytes(g), 128);
|
||||
f.fill(counterLuma(i), 0, luma);
|
||||
return f;
|
||||
});
|
||||
const samples = audioFrameBytes(g) / 4;
|
||||
this.pcm = Buffer.alloc(audioFrameBytes(g));
|
||||
for (let s = 0; s < samples; s += 1) {
|
||||
const v = Math.round(amplitude * Math.sin((2 * Math.PI * 1500 * s) / 48_000));
|
||||
this.pcm.writeInt16LE(v, s * 4);
|
||||
this.pcm.writeInt16LE(v, s * 4 + 2);
|
||||
}
|
||||
}
|
||||
|
||||
get framesSent(): number { return this.n; }
|
||||
|
||||
async start(): Promise<void> {
|
||||
const fds = await this.bus.start();
|
||||
this.bus.adoptIngestWriters();
|
||||
const open = (fd: number): Socket => new Socket({ fd, readable: false, writable: true }).on("error", () => undefined);
|
||||
const out = { video: open(fds.videoFd), audio: open(fds.audioFd) };
|
||||
this.out = out;
|
||||
const t0 = process.hrtime.bigint();
|
||||
const period = 1_000_000_000n / BigInt(this.g.fps);
|
||||
const step = (): void => {
|
||||
const now = process.hrtime.bigint();
|
||||
while (t0 + BigInt(this.n) * period <= now) {
|
||||
out.video.write(this.frames[this.n % 50]!);
|
||||
out.audio.write(this.pcm);
|
||||
this.n += 1;
|
||||
}
|
||||
const wait = Number((t0 + BigInt(this.n) * period - process.hrtime.bigint()) / 1_000_000n);
|
||||
this.timer = setTimeout(step, Math.max(0, wait));
|
||||
};
|
||||
step();
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
if (this.timer) clearTimeout(this.timer);
|
||||
this.timer = null;
|
||||
this.out?.video.destroy();
|
||||
this.out?.audio.destroy();
|
||||
this.out = null;
|
||||
await this.bus.stop();
|
||||
}
|
||||
}
|
||||
|
||||
/** The verifying null encoder (src/gate/sink-cli.ts) as its own process, reading the gate's encoder FIFOs. */
|
||||
export class SinkProcess {
|
||||
private proc: ManagedProcess | null = null;
|
||||
private out = "";
|
||||
|
||||
async start(gate: FrameGate, g: FrameGeometry): Promise<void> {
|
||||
const cwd = join(__dirname, "../..");
|
||||
const proc = spawnManaged(process.execPath, ["--import", "tsx", join(__dirname, "../gate/sink-cli.ts"),
|
||||
gate.encoderFifos.videoPath, gate.encoderFifos.audioPath, String(g.width), String(g.height), String(g.fps)],
|
||||
["pipe", "pipe", "pipe"], passthroughRedactor, (m) => process.stderr.write(m), cwd);
|
||||
this.proc = proc;
|
||||
proc.child.stdout?.setEncoding("utf8");
|
||||
proc.child.stdout?.on("data", (c: string) => { this.out += c; });
|
||||
const t0 = Date.now();
|
||||
while (!this.out.includes("OPEN\n")) {
|
||||
if (Date.now() - t0 > 15_000) throw new Error("sink did not open");
|
||||
await new Promise((r) => setTimeout(r, 10));
|
||||
}
|
||||
}
|
||||
|
||||
async report(firstActiveTick: number): Promise<SinkReport & { silentAfterFirstActive: number }> {
|
||||
const proc = this.proc;
|
||||
if (!proc) throw new Error("sink not started");
|
||||
proc.child.stdin?.write(`REPORT ${firstActiveTick}\n`);
|
||||
await proc.exited;
|
||||
const line = this.out.split("\n").find((l) => l.startsWith("{"));
|
||||
if (!line) throw new Error("sink produced no report");
|
||||
return JSON.parse(line) as SinkReport & { silentAfterFirstActive: number };
|
||||
}
|
||||
|
||||
/** Also close the sink when a gate assertion fails before REPORT is requested. */
|
||||
async stop(): Promise<void> {
|
||||
await this.proc?.terminate(500);
|
||||
this.proc = null;
|
||||
}
|
||||
}
|
||||
|
||||
/** H.264 + AAC MPEG-TS clip at the canvas size (encoded once, before measuring): the stand-in for a live HLS source. */
|
||||
export function encodeSourceClip(path: string, canvas: Canvas, seconds = 12): void {
|
||||
const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", `testsrc2=s=${canvas.width}x${canvas.height}:r=${canvas.fps}:d=${seconds}`,
|
||||
"-f", "lavfi", "-i", `sine=frequency=500:sample_rate=48000:d=${seconds}`,
|
||||
"-c:v", "libx264", "-preset", "veryfast", "-b:v", "6000k", "-g", String(2 * canvas.fps), "-c:a", "aac", "-b:a", "128k", "-f", "mpegts", path]);
|
||||
if (r.status !== 0) throw new Error("source clip encode failed");
|
||||
}
|
||||
|
||||
/**
|
||||
* Stand-in for the pinned egress relay (WP7): a paced, looping remux of `clip` into a FIFO.
|
||||
* Returns the read end for an `mpegtsPipe` ingest's stdin (close it after spawning the ingest).
|
||||
*/
|
||||
export async function startPacer(clip: string, dir: string, name: string): Promise<{ readFd: number; proc: ManagedProcess }> {
|
||||
const fifo = join(dir, `${name}.fifo`);
|
||||
spawnSync("mkfifo", [fifo]);
|
||||
const readFd = openSync(fifo, "r+");
|
||||
const writeFd = openSync(fifo, "w");
|
||||
const proc = spawnManaged("ffmpeg", ["-hide_banner", "-loglevel", "error", "-re", "-stream_loop", "-1", "-i", clip, "-c", "copy", "-f", "mpegts", "pipe:1"],
|
||||
["ignore", writeFd, "pipe"], passthroughRedactor, () => undefined);
|
||||
await proc.started;
|
||||
closeSync(writeFd);
|
||||
return { readFd, proc };
|
||||
}
|
||||
|
||||
export const ZERO_GAP_CANVAS: Canvas = { width: 320, height: 180, fps: 30 };
|
||||
const canvas = ZERO_GAP_CANVAS;
|
||||
|
||||
/** Both native helpers (frame gate + source bus), built offline if missing or stale (src/gate/binary.ts). */
|
||||
export function ensureGateBinary(): boolean { return ensureHelperBinaries(); }
|
||||
|
||||
export function makeAssets(dir: string): string[] {
|
||||
const out: string[] = [];
|
||||
for (const src of ["color=c=red@0.6:s=160x24", "testsrc2=s=600x16"]) {
|
||||
const tmp = join(dir, `tmp-${out.length}.png`);
|
||||
const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", `${src},format=rgba`, "-frames:v", "1", "-fflags", "+bitexact", "-flags", "+bitexact", tmp]);
|
||||
assert.equal(r.status, 0, "asset render");
|
||||
const path = join(dir, `${createHash("sha256").update(readFileSync(tmp)).digest("hex")}.png`);
|
||||
copyFileSync(tmp, path);
|
||||
out.push(path);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const CUSTOM: PxRect[] = [{ x: 0, y: 0, w: 160, h: 90 }, { x: 160, y: 90, w: 160, h: 90 }, { x: 100, y: 40, w: 120, h: 68 }];
|
||||
|
||||
/** Swap i: cycles layouts × overlay sets × gains. Slot 0 always shows source 0 at the top-left (the counter pixel). */
|
||||
export function scriptedGraph(i: number): CompositorGraph {
|
||||
const layouts = SCENE_LAYOUTS;
|
||||
const layout: SceneLayout = layouts[i % layouts.length]!;
|
||||
const rects = layout === "pip" ? [{ x: 0, y: 0, w: 320, h: 180 }, { x: 200, y: 100, w: 100, h: 56 }]
|
||||
: layout === "custom" ? CUSTOM : layoutRects(layout, canvas);
|
||||
const tiles = rects.map((rect, t) => ({ source: t === 0 ? 0 : (t % 5 === 4 ? null : t % 4), rect, fit: (["contain", "cover", "crop"] as const)[(i + t) % 3]! }));
|
||||
if (tiles[0]) tiles[0].fit = "cover"; // cover/crop keep the counter pixel (0,0) on picture, contain letterboxes it
|
||||
const overlays: OverlaySpec[] = [];
|
||||
if (i % 3 !== 0) overlays.push({ kind: "image", asset: 0, x: 10, y: 130 });
|
||||
if (i % 2 === 0) overlays.push({ kind: "chyron", asset: 1, y: 160, speedPxPerSec: 40 + ((i * 20) % 360) });
|
||||
const gain = -3 * (i % 9);
|
||||
const audible = [...new Set(tiles.map((t) => t.source).filter((s): s is number => s !== null))].slice(0, 1 + (i % 3));
|
||||
return { canvas, layout, tiles, overlays, audio: { inputs: audible.map((source, k) => ({ source, gainDb: k === 0 ? gain : -6 })), masterGainDb: -3 * (i % 3) } };
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Compose request over only the sources the graph uses (renumbered), as
|
||||
* `planScene` does in production: ffmpeg never reads an input that is not wired
|
||||
* into the filtergraph, so an unused bus subscription would just stall.
|
||||
*/
|
||||
export function composeOver(buses: SourceBus[], assets: string[], graph: CompositorGraph): ComposeRequest {
|
||||
const used = [...new Set(graph.tiles.map((t) => t.source).concat(graph.audio.inputs.map((m) => m.source)).filter((s): s is number => s !== null))].sort((a, b) => a - b);
|
||||
const index = (s: number): number => used.indexOf(s);
|
||||
const audible = new Set(graph.audio.inputs.map((m) => m.source));
|
||||
return {
|
||||
inputs: used.map((s) => ({ bus: buses[s]!, audio: audible.has(s) })),
|
||||
assets,
|
||||
graph: {
|
||||
...graph,
|
||||
tiles: graph.tiles.map((t) => ({ ...t, source: t.source === null ? null : index(t.source) })),
|
||||
audio: { ...graph.audio, inputs: graph.audio.inputs.map((m) => ({ ...m, source: index(m.source) })) },
|
||||
},
|
||||
};
|
||||
}
|
||||
492
apps/media-worker/src/compositor/wp12-hardening.test.ts
Normal file
492
apps/media-worker/src/compositor/wp12-hardening.test.ts
Normal file
|
|
@ -0,0 +1,492 @@
|
|||
/**
|
||||
* WP12 hardening tests (flash slice) — apps/media-worker/src/compositor only.
|
||||
*
|
||||
* Two layers of the §4.3 "Injection" defense are exercised here:
|
||||
*
|
||||
* 1. Graph-builder injection at the type/argv layer. Hostile payloads
|
||||
* (`movie=`, `amovie=`, `filename=`, protocol prefixes, shell
|
||||
* metacharacters, newline/graph-chaining) are injected into every
|
||||
* string-typed input surface of the compositor plan. Either the typed
|
||||
* validators reject the plan (CommandBuildError) or the produced argv is
|
||||
* proven inert: the payload never appears, not even escaped.
|
||||
*
|
||||
* 2. Crop-mode full matrix: contain/cover/crop × every scene layout —
|
||||
* per-tile scale/pad/crop chains asserted from the generated filtergraph
|
||||
* and argv only (no ffmpeg execution).
|
||||
*
|
||||
* Vitest suite: `node_modules/.bin/vitest` of a sibling workspace app resolves
|
||||
* this file (the media-worker devDependencies install only tsx/node:test);
|
||||
* see the run note at the bottom of this file.
|
||||
*/
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
buildCompositorArgs,
|
||||
buildFilterGraph,
|
||||
CommandBuildError,
|
||||
layoutRects,
|
||||
rectFromBasisPoints,
|
||||
SYNTHETIC_PATTERNS,
|
||||
type Canvas,
|
||||
type CompositorPlan,
|
||||
type OverlaySpec,
|
||||
} from "./command-builder";
|
||||
|
||||
const canvas: Canvas = { width: 320, height: 180, fps: 30 };
|
||||
const W = canvas.width;
|
||||
const H = canvas.height;
|
||||
|
||||
function synthetic(i: number) {
|
||||
return {
|
||||
video: { kind: "synthetic" as const, pattern: SYNTHETIC_PATTERNS[i % SYNTHETIC_PATTERNS.length]!, seed: 1000 + i },
|
||||
audio: { kind: "synthetic" as const, toneHz: 220 + 110 * i },
|
||||
};
|
||||
}
|
||||
|
||||
const sha = (c: string) => c.repeat(64);
|
||||
/** Valid absolute scratch asset paths (what `assetPath` accepts). */
|
||||
const ASSET_A = `/scratch/assets/${sha("a")}.png`;
|
||||
const ASSET_B = `/scratch/assets/${sha("b")}.png`;
|
||||
|
||||
/** Minimal valid plan: one live full-frame tile, one image overlay, silent mix. */
|
||||
function basePlan(): CompositorPlan {
|
||||
return {
|
||||
sources: [synthetic(0)],
|
||||
assets: [ASSET_A],
|
||||
graph: {
|
||||
canvas,
|
||||
layout: "single",
|
||||
tiles: [{ source: 0, rect: { x: 0, y: 0, w: W, h: H }, fit: "contain" }],
|
||||
overlays: [{ kind: "image", asset: 0, x: 8, y: 8 }],
|
||||
audio: { inputs: [], masterGainDb: 0 },
|
||||
},
|
||||
output: { kind: "gate", videoFd: 1, audioFd: 3 },
|
||||
};
|
||||
}
|
||||
|
||||
/** Input-index map that buildFilterGraph expects for a one-source/one-asset plan. */
|
||||
const IDX1 = { video: [0], audio: [1], assets: [2] };
|
||||
|
||||
function graphOf(plan: CompositorPlan): string {
|
||||
return buildFilterGraph(plan, IDX1);
|
||||
}
|
||||
|
||||
function argvOf(plan: CompositorPlan): string[] {
|
||||
return buildCompositorArgs(plan);
|
||||
}
|
||||
|
||||
/**
|
||||
* A payload is inert when it is absent from every argv element — including the
|
||||
* escaped forms a filtergraph string could need (`\;`, `\'`, `\\\,` …): the
|
||||
* builder either drops or rejects it, it never transcodes it into a graph.
|
||||
*/
|
||||
function expectInert(plan: CompositorPlan, payload: string): void {
|
||||
const argv = argvOf(plan);
|
||||
const escaped = payload.replace(/([[\];,'\\])/g, "\\$1");
|
||||
for (const arg of argv) {
|
||||
assert.ok(!(arg).includes(payload));
|
||||
assert.ok(!(arg).includes(escaped));
|
||||
}
|
||||
assert.ok(!(argv.join(" ")).includes(escaped));
|
||||
}
|
||||
|
||||
/** Every mutation must either throw CommandBuildError or leave the payload out of argv. */
|
||||
function expectRejectedOrInert(mutate: (plan: CompositorPlan) => CompositorPlan, payload: string): void {
|
||||
const plan = mutate(basePlan());
|
||||
try {
|
||||
argvOf(plan);
|
||||
} catch (error) {
|
||||
assert.ok((error) instanceof CommandBuildError);
|
||||
return;
|
||||
}
|
||||
expectInert(plan, payload);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Graph-builder injection: every string-typed input surface
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** [wb] WP12 payload classes, each must be rejected or proven inert. */
|
||||
const PAYLOADS: Array<[string, string]> = [
|
||||
["lavfi movie/amovie", "movie=evil.mp4"],
|
||||
["lavfi amovie", "amovie=evil.mp3"],
|
||||
["lavfi filename", "filename=/etc/passwd"],
|
||||
["lavfi chained filter", "color=c=black:s=320x180,drawtext=text=pwned"],
|
||||
["file protocol", "file:/etc/passwd"],
|
||||
["http url", "http://evil.example/x.mp4"],
|
||||
["https url", "https://evil.example/x.m3u8"],
|
||||
["pipe protocol", "pipe:0"],
|
||||
["subfile protocol", "subfile,,start=0,end=0,,:/etc/passwd"],
|
||||
["concat protocol", "concat:/etc/passwd|/etc/shadow"],
|
||||
["data protocol", "data:text/plain,pwned"],
|
||||
["hls protocol", "hls+http://evil.example/x.m3u8"],
|
||||
["rtsp url", "rtsp://evil.example:8554/live"],
|
||||
["shell separator", ";rm -rf /"],
|
||||
["shell pipe", "|tee /tmp/pwned"],
|
||||
["shell command substitution", "$(id -u)"],
|
||||
["shell backtick substitution", "`id -u`"],
|
||||
["shell redirect", ">/tmp/pwned"],
|
||||
["shell and-chain", "&&rm -rf /"],
|
||||
["shell or-chain", "||rm -rf /"],
|
||||
["graph pad link", "[0:v]null[sink]"],
|
||||
["graph output pad", "[vout]"],
|
||||
["graph chain separator", "null;nullsink"],
|
||||
["filter option comma", "scale=1:1,scale=2:2"],
|
||||
["filter option equals", "rate=999"],
|
||||
["quoted value", "'$(id -u)'"],
|
||||
["newline chaining", "a\nb"],
|
||||
["crlf chaining", "a\r\nb"],
|
||||
["newline + filter", "\nmovie=evil.mp4"],
|
||||
["tab separator", "\tmovie=evil.mp4"],
|
||||
["unicode line separator", "a\u2028b"],
|
||||
["control char", "a\x01b"],
|
||||
["backslash escape", "\\;movie=evil.mp4"],
|
||||
];
|
||||
|
||||
/** One maximal payload spanning every class at once. */
|
||||
const KITCHEN_SINK = "movie=x|amovie=y|filename=z;[0:v]null[v];$(id)|`id`|\nhttp://e/x|file:/e";
|
||||
|
||||
describe("WP12 injection: overlay asset paths (plan.assets[i])", () => {
|
||||
const withAsset = (path: string): CompositorPlan => ({ ...basePlan(), assets: [path] });
|
||||
|
||||
for (const [_name, payload] of PAYLOADS) {
|
||||
it(`rejects or neutralizes: ${_name}`, () => {
|
||||
// Raw path form…
|
||||
expectRejectedOrInert(withAsset(payload), payload);
|
||||
// …and smuggled inside an otherwise-valid-looking scratch path shape.
|
||||
const smuggle = `/scratch/assets/${payload.replace(/[/\s]/g, "-")}-${sha("p")}.png`;
|
||||
expectRejectedOrInert(withAsset(smuggle), payload);
|
||||
});
|
||||
|
||||
it("rejects traversal and traversal-lookalikes and non-scratch shapes", () => {
|
||||
for (const path of [
|
||||
"/scratch/../etc/passwd.png",
|
||||
`/scratch/assets/../${sha("a")}.png`,
|
||||
`/scratch/assets/${sha("a")}.png/../../${sha("b")}.png`,
|
||||
`relative/${sha("a")}.png`,
|
||||
`${sha("a")}.png`,
|
||||
`/scratch/assets/${sha("a")}.PNG`,
|
||||
`/scratch/assets/${sha("a").slice(0, 63)}.png`,
|
||||
`/scratch/assets/${sha("a")} extra.png`,
|
||||
"",
|
||||
]) {
|
||||
assert.throws(() => argvOf(withAsset(path)), path, CommandBuildError);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects every protocol prefix as an asset path", () => {
|
||||
for (const prefix of ["file:", "http:", "https:", "pipe:", "fd:", "concat:", "subfile,", "data:", "hls+", "rtsp:", "movie=", "amovie=", "filename="]) {
|
||||
assert.throws(() => argvOf(withAsset(`${prefix}${sha("a")}.png`)), prefix, CommandBuildError);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects shell and graph metacharacters anywhere in an asset path", () => {
|
||||
for (const ch of [";", "&", "|", "<", ">", "`", "$", "(", ")", "'", '"', "\\", "[", "]", ",", "\n", "\r", "\t", " ", "=", ":"]) {
|
||||
const path = `/scratch/assets/x${ch}${sha("a").slice(0, 60)}.png`;
|
||||
assert.throws(() => argvOf(withAsset(path)), CommandBuildError);
|
||||
}
|
||||
});
|
||||
|
||||
it("argv carries valid assets only as closed file:<sha>.png inputs", () => {
|
||||
const argv = argvOf(basePlan());
|
||||
assert.ok((argv).includes(`file:${ASSET_A}`));
|
||||
const fileArgs = argv.filter((a) => a.startsWith("file:"));
|
||||
assert.deepStrictEqual(fileArgs, [`file:${ASSET_A}`]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("WP12 injection: overlay geometry and asset indexes (typed, no string surface)", () => {
|
||||
const withOverlay = (o: OverlaySpec): CompositorPlan => ({ ...basePlan(), graph: { ...basePlan().graph, overlays: [o] } });
|
||||
|
||||
it("rejects an overlay naming a nonexistent asset index", () => {
|
||||
assert.throws(() => argvOf(withOverlay({ kind: "image", asset: 9, x: 0, y: 0 })), /asset/);
|
||||
assert.throws(() => argvOf(withOverlay({ kind: "image", asset: -1, x: 0, y: 0 })), /asset/);
|
||||
assert.throws(() => argvOf(withOverlay({ kind: "chyron", asset: 2, y: 0, speedPxPerSec: 120 })), /asset/);
|
||||
});
|
||||
|
||||
it("chyron x-expression is built only from a bounded integer speed — payloads cannot ride the expression", () => {
|
||||
const plan = basePlan();
|
||||
plan.graph.overlays = [{ kind: "chyron", asset: 0, y: 160, speedPxPerSec: 120 }];
|
||||
const graph = graphOf(plan);
|
||||
assert.ok(!/overlay=x='W-mod/.test(graph));
|
||||
assert.ok(!(graph).includes("movie="));
|
||||
// Out-of-band speeds are rejected before any expression string is built.
|
||||
for (const speed of [0, 39, 401, 1e9, -120, 12.5, Number.NaN]) {
|
||||
const bad: CompositorPlan = { ...plan, graph: { ...plan.graph, overlays: [{ kind: "chyron", asset: 0, y: 160, speedPxPerSec: speed }] } };
|
||||
assert.throws(() => argvOf(bad), String(speed), CommandBuildError);
|
||||
}
|
||||
});
|
||||
|
||||
it("overlay geometry is integer-bounded; hostile numbers are type errors, not payload carriers", () => {
|
||||
for (const x of [-1, W + 2, 0.5, Number.NaN, Number.POSITIVE_INFINITY]) {
|
||||
assert.throws(() => argvOf(withOverlay({ kind: "image", asset: 0, x, y: 8 })), String(x), CommandBuildError);
|
||||
}
|
||||
for (const y of [-2, H + 4, 0.5, Number.NaN]) {
|
||||
assert.throws(() => argvOf(withOverlay({ kind: "chyron", asset: 0, y, speedPxPerSec: 120 })), String(y), CommandBuildError);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("WP12 injection: labels and source ids never reach the graph or argv", () => {
|
||||
it("hostile labels on a desired-state snapshot leave no trace in compositor argv", async () => {
|
||||
const { parseProductionSpec } = await import("../production");
|
||||
const { planScene, programScene } = await import("./scene-plan");
|
||||
const id = (n: number) => `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`;
|
||||
const payloads = [
|
||||
"movie=evil.mp4",
|
||||
"amovie=evil.mp3",
|
||||
"filename=/etc/passwd",
|
||||
"; rm -rf /",
|
||||
"$(id -u)",
|
||||
"`id`",
|
||||
"[0:v]null[sink]",
|
||||
"http://evil.example/x.m3u8",
|
||||
"file:/etc/passwd",
|
||||
"a\nmovie=evil.mp4",
|
||||
"pipe:0",
|
||||
KITCHEN_SINK,
|
||||
];
|
||||
// parseProductionSpec must reject all of these as labels (strict allowlist).
|
||||
for (const label of payloads) {
|
||||
assert.throws(() => parseProductionSpec(labelSpec(id, label)));
|
||||
}
|
||||
// A spec with clean labels plans/compiles; ids, labels and URLs stay out of argv.
|
||||
const spec = parseProductionSpec(labelSpec(id, "Cam A"));
|
||||
const plan = planScene(spec, programScene(spec), canvas, (sourceId) => sourceId === id(1));
|
||||
const argv = argvOf({
|
||||
sources: plan.sourceIds.map((_, i) => ({ video: { kind: "rawPipe" as const, fd: 4 + 2 * i }, audio: null })),
|
||||
assets: plan.assetIds.map((a) => `/scratch/assets/${a}.png`),
|
||||
graph: plan.graph,
|
||||
output: { kind: "gate", videoFd: 1, audioFd: 3 },
|
||||
});
|
||||
const joined = argv.join(" ");
|
||||
for (const payload of payloads) {
|
||||
assert.ok(!(joined).includes(payload));
|
||||
assert.ok(!(joined).includes(payload.replace(/([[\];,'\\])/g, "\\$1")));
|
||||
}
|
||||
assert.ok(!(joined).includes("Cam"));
|
||||
assert.ok(!(joined).includes("example"));
|
||||
});
|
||||
|
||||
it("source ids are never argv material: synthetic plans reference patterns and integers only", () => {
|
||||
const argv = argvOf(basePlan());
|
||||
const patternArg = argv.find((a) => a.includes(`testsrc2=s=${W}x${H}`));
|
||||
assert.notStrictEqual(patternArg, undefined);
|
||||
assert.ok(!/[;|`$]|\bmovie=|\bamovie=|\bfilename=/.test(patternArg));
|
||||
// Input references are structured (`<index>:v`, `<index>:a`) — never id strings.
|
||||
assert.ok(new RegExp(/\[0:v\]/).test(graphOf(basePlan())));
|
||||
});
|
||||
});
|
||||
|
||||
describe("WP12 injection: filtergraph token provenance (labels, splits, pads)", () => {
|
||||
it("graph labels are builder-owned; a payload cannot introduce new pad labels", () => {
|
||||
const plan = basePlan();
|
||||
plan.graph.tiles.push({ source: 0, rect: rectFromBasisPoints({ x: 6000, y: 6000, w: 3500, h: 3500 }, canvas), fit: "contain" });
|
||||
const graph = graphOf(plan);
|
||||
// split outputs and tile labels follow the fixed grammar [s<src>_<n>] / [t<i>] / [ov<i>] / [base] / [vout] / [aout]
|
||||
assert.ok(new RegExp(/\[0:v\]split=2\[s0_0\]\[s0_1\]/).test(graph));
|
||||
assert.ok(new RegExp(/\[s0_1\]scale=96:96/).test(graph));
|
||||
assert.ok(new RegExp(/\[t1\]overlay=x=192:y=108/).test(graph));
|
||||
const labels = graph.match(/\[[a-z0-9_:]+\]/g) ?? [];
|
||||
assert.ok((labels.length) > (0));
|
||||
assert.strictEqual(labels.every((m) => /^(?:\d+:v|\d+:a|s\d+_\d+|t\d+|ov\d+|c\d+|m\d+|bg|base|vout|aout)$/.test(m.slice(1, -1))), true);
|
||||
assert.ok(!(graph).includes("[v]"));
|
||||
assert.ok(!(graph).includes("[sink]"));
|
||||
});
|
||||
|
||||
it("the full filtergraph never contains a payload in raw or escaped form", () => {
|
||||
const plan = basePlan();
|
||||
plan.graph.tiles.push({ source: 0, rect: rectFromBasisPoints({ x: 6000, y: 6000, w: 3500, h: 3500 }, canvas), fit: "cover" });
|
||||
plan.graph.overlays = [
|
||||
{ kind: "image", asset: 0, x: 8, y: 8 },
|
||||
{ kind: "chyron", asset: 0, y: 160, speedPxPerSec: 200 },
|
||||
];
|
||||
plan.graph.audio = { inputs: [{ source: 0, gainDb: -6 }], masterGainDb: -3 };
|
||||
const graph = graphOf(plan);
|
||||
for (const [, payload] of PAYLOADS) {
|
||||
assert.ok(!(graph).includes(payload));
|
||||
assert.ok(!(graph).includes(payload.replace(/([\[\];,'\\])/g, "\\$1")));
|
||||
}
|
||||
assert.ok(!(graph).includes(KITCHEN_SINK));
|
||||
});
|
||||
|
||||
it("kitchen-sink payload stays inert end to end (argv layer)", () => {
|
||||
const argv = argvOf(basePlan());
|
||||
const joined = argv.join(" ");
|
||||
assert.ok(!(joined).includes(KITCHEN_SINK));
|
||||
assert.ok(!(joined).includes("movie="));
|
||||
assert.ok(!(joined).includes("amovie="));
|
||||
assert.ok(!(joined).includes("filename="));
|
||||
assert.ok(!(joined).includes("$(id"));
|
||||
assert.ok(!(joined).includes("`id`"));
|
||||
});
|
||||
|
||||
it("single quotes appear only as the chyron x-expression quoting around the builder-owned expression", () => {
|
||||
const single = (plan: CompositorPlan): CompositorPlan => {
|
||||
plan.graph.overlays = [{ kind: "chyron", asset: 0, y: 160, speedPxPerSec: 80 }];
|
||||
return plan;
|
||||
};
|
||||
const graph = graphOf(single(basePlan()));
|
||||
const quoted = graph.match(/'[^']*'/g) ?? [];
|
||||
assert.deepStrictEqual(quoted, ["W-mod(t*80\\,w+W)"]);
|
||||
// No quote-escape sequences exist anywhere in the graph.
|
||||
assert.ok(!(graph).includes("\\'"));
|
||||
assert.ok(!(graph).includes("''"));
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Crop-mode full matrix: contain/cover/crop × layouts
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* `tileFilter` chains per fit (from command-builder.ts):
|
||||
* contain → scale(w,h:force_original_aspect_ratio=decrease:force_divisible_by=2) → pad(w,h,centered,black) → setsar=1
|
||||
* cover → scale(w,h:force_original_aspect_ratio=increase:force_divisible_by=2) → crop(w,h) → setsar=1
|
||||
* crop → crop(w,h) → setsar=1
|
||||
*/
|
||||
const FIT_CHAIN: Record<"contain" | "cover" | "crop", (w: number, h: number) => RegExp> = {
|
||||
contain: (w, h) => new RegExp(`^scale=${w}:${h}:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=${w}:${h}:\\(ow-iw\)/2:\\(oh-ih\)/2:color=black,setsar=1$`),
|
||||
cover: (w, h) => new RegExp(`^scale=${w}:${h}:force_original_aspect_ratio=increase:force_divisible_by=2,crop=${w}:${h},setsar=1$`),
|
||||
crop: (w, h) => new RegExp(`^crop=${w}:${h},setsar=1$`),
|
||||
};
|
||||
|
||||
interface MatrixCase {
|
||||
name: string;
|
||||
layout: "single" | "pip" | "2x1" | "2x2" | "3x3" | "4x4" | "custom";
|
||||
rects: PxRect[];
|
||||
/** Slot indexes rendered black (source=null) — excluded from the live-tile assertions. */
|
||||
empty?: number[];
|
||||
}
|
||||
|
||||
const MATRIX: MatrixCase[] = [
|
||||
{ name: "single", layout: "single", rects: layoutRects("single", canvas) },
|
||||
{ name: "pip", layout: "pip", rects: [{ x: 0, y: 0, w: 320, h: 180 }, rectFromBasisPoints({ x: 6000, y: 6000, w: 3500, h: 3500 }, canvas)] },
|
||||
{ name: "2x1", layout: "2x1", rects: layoutRects("2x1", canvas) },
|
||||
{ name: "2x2", layout: "2x2", rects: layoutRects("2x2", canvas), empty: [3] },
|
||||
{ name: "3x3", layout: "3x3", rects: layoutRects("3x3", canvas), empty: [4, 8] },
|
||||
{ name: "4x4", layout: "4x4", rects: layoutRects("4x4", canvas), empty: [5, 10, 15] },
|
||||
{ name: "custom (non-overlapping, xstack)", layout: "custom", rects: [{ x: 0, y: 0, w: 160, h: 90 }, { x: 160, y: 90, w: 160, h: 90 }] },
|
||||
{ name: "custom (overlapping, overlay chain)", layout: "custom", rects: [{ x: 0, y: 0, w: 200, h: 112 }, { x: 120, y: 68, w: 200, h: 112 }] },
|
||||
];
|
||||
|
||||
/** Every layout × every fit, via buildCompositorArgs (the production entry point). */
|
||||
function matrixPlan(c: MatrixCase, fit: "contain" | "cover" | "crop"): CompositorPlan {
|
||||
const plan = basePlan();
|
||||
plan.sources = [0, 1, 2, 3].map(synthetic);
|
||||
plan.graph.layout = c.layout;
|
||||
plan.graph.tiles = c.rects.map((rect, i) => ({ source: c.empty?.includes(i) ? null : i % 4, rect, fit }));
|
||||
plan.graph.overlays = [];
|
||||
plan.graph.audio = { inputs: [], masterGainDb: 0 };
|
||||
plan.assets = [];
|
||||
return plan;
|
||||
}
|
||||
|
||||
describe("WP12 crop-mode matrix: contain/cover/crop × single/pip/two_up/split grids/custom", () => {
|
||||
it("layout cell rects are even-aligned and tile the canvas exactly", () => {
|
||||
for (const layout of ["2x1", "2x2", "3x3", "4x4"] as const) {
|
||||
const rects = layoutRects(layout, canvas);
|
||||
assert.strictEqual(rects.reduce((a, r) => a + r.w * r.h, 0), W * H);
|
||||
for (const r of rects) for (const v of [r.x, r.y, r.w, r.h]) assert.strictEqual(v % 2, 0);
|
||||
}
|
||||
});
|
||||
|
||||
for (const { c, fit } of MATRIX.flatMap((c) => (["contain", "cover", "crop"] as const).map((fit) => ({ c, fit })))) {
|
||||
const plan = matrixPlan(c, fit);
|
||||
const argv = argvOf(plan);
|
||||
const graph = argv[argv.indexOf("-filter_complex") + 1]!;
|
||||
|
||||
const live = c.rects.filter((_, i) => !c.empty?.includes(i));
|
||||
live.forEach((rect, i) => {
|
||||
const chain = new RegExp(`\\[t${i}\\]`.replace("[", "\\[") + `\\]`); // placeholder, replaced below
|
||||
void chain;
|
||||
const tileChain = new RegExp(`\\[s?\\d*(?::v)?\\][^;]*\\[t${i}\\]`);
|
||||
const match = graph.match(tileChain);
|
||||
assert.notStrictEqual(match, null);
|
||||
assert.strictEqual(FIT_CHAIN[fit](rect.w, rect.h).test(match![0].slice(match![0].indexOf("]") + 1)), true);
|
||||
// Geometry is exactly the layout cell: no source- or fit-dependent drift.
|
||||
assert.ok((graph).includes(`[t${i}]`));
|
||||
});
|
||||
|
||||
// The assembly stage per layout family:
|
||||
if (c.layout === "pip" || c.name.includes("overlapping")) {
|
||||
// PIP and overlapping customs are overlay chains; xstack must not appear.
|
||||
assert.ok(!(graph).includes("xstack"));
|
||||
assert.ok(new RegExp(/overlay=x=\d+:y=\d+:eof_action=repeat/).test(graph));
|
||||
} else if (live.length >= 2) {
|
||||
const layout = live.map((r) => `${r.x}_${r.y}`).join("|");
|
||||
assert.ok(graph.includes(`xstack=inputs=${live.length}:layout=${layout.replace(/([|_])/g, "\\$1")}:fill=black`));
|
||||
} else {
|
||||
assert.ok(!(graph).includes("xstack"));
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("argv structure per matrix row: -i ordering, -filter_complex once, gate outputs on pipe:1/pipe:3", () => {
|
||||
for (const c of MATRIX) {
|
||||
for (const fit of ["contain", "cover", "crop"] as const) {
|
||||
const argv = argvOf(matrixPlan(c, fit));
|
||||
assert.strictEqual((argv.filter((a) => a === "-filter_complex")).length, 1);
|
||||
assert.strictEqual((argv.filter((a) => a === "-i")).length, c.rects.filter((_, i) => !c.empty?.includes(i)).length || 1);
|
||||
assert.ok(!/[&<>]/.test(argv[argv.indexOf("-filter_complex") + 1]));
|
||||
assert.ok(!/[;&|`$<>]/.test(argv[argv.indexOf("-filter_complex") + 1]));
|
||||
assert.strictEqual(argv.at(-1), "pipe:3");
|
||||
assert.ok((argv).includes("pipe:1"));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("split_horizontal/split_vertical equivalents (2x1 / custom row-split) keep per-half fit chains", () => {
|
||||
// Horizontal split = 2x1 grid: left/right halves, each independently cropped.
|
||||
const row = matrixPlan({ name: "2x1", layout: "2x1", rects: layoutRects("2x1", canvas) }, "crop");
|
||||
const rowGraph = argvOf(row)[argvOf(row).indexOf("-filter_complex") + 1]!;
|
||||
assert.ok(new RegExp(/\[t0\]$/m).test(rowGraph));
|
||||
assert.ok((rowGraph).includes("crop=160:180,setsar=1[t0]"));
|
||||
assert.ok((rowGraph).includes("crop=160:180,setsar=1[t1]"));
|
||||
// Vertical split = a custom top/bottom split: same halves stacked, xstack fills black.
|
||||
const col = matrixPlan({ name: "custom vsplit", layout: "custom", rects: [{ x: 0, y: 0, w: 320, h: 90 }, { x: 0, y: 90, w: 320, h: 90 }] }, "cover");
|
||||
const colGraph = argvOf(col)[argvOf(col).indexOf("-filter_complex") + 1]!;
|
||||
assert.ok((colGraph).includes("scale=320:90:force_original_aspect_ratio=increase:force_divisible_by=2,crop=320:90,setsar=1[t0]"));
|
||||
assert.ok((colGraph).includes("scale=320:90:force_original_aspect_ratio=increase:force_divisible_by=2,crop=320:90,setsar=1[t1]"));
|
||||
assert.ok(new RegExp(/xstack=inputs=2:layout=0_0\|0_90:fill=black/).test(colGraph));
|
||||
});
|
||||
|
||||
it("mixed fits inside one layout keep each tile's own chain (cover base + contain PIP inset)", () => {
|
||||
const plan = matrixPlan({ name: "pip", layout: "pip", rects: [{ x: 0, y: 0, w: 320, h: 180 }, rectFromBasisPoints({ x: 6000, y: 6000, w: 3500, h: 3500 }, canvas)] }, "cover");
|
||||
plan.graph.tiles[1]!.fit = "contain";
|
||||
const graph = argvOf(plan)[argvOf(plan).indexOf("-filter_complex") + 1]!;
|
||||
assert.ok((graph).includes("scale=320:180:force_original_aspect_ratio=increase:force_divisible_by=2,crop=320:180,setsar=1[t0]"));
|
||||
assert.ok((graph).includes("scale=96:96:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=96:96:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1[t1]"));
|
||||
assert.ok(new RegExp(/overlay=x=192:y=108:eof_action=repeat/).test(graph));
|
||||
});
|
||||
|
||||
it("contain letterboxes black and never crops; crop never pads; cover never letterboxes", () => {
|
||||
const rect = { x: 0, y: 0, w: 160, h: 180 };
|
||||
const graph = (fit: "contain" | "cover" | "crop") => graphOf({ ...basePlan(), graph: { ...basePlan().graph, tiles: [{ source: 0, rect, fit }] } });
|
||||
const contain = graph("contain");
|
||||
assert.ok((contain).includes("pad=160:180:(ow-iw)/2:(oh-ih)/2:color=black"));
|
||||
assert.ok(!(contain).includes(`crop=${rect.w}:${rect.h}`));
|
||||
const cover = graph("cover");
|
||||
assert.ok((cover).includes(`crop=${rect.w}:${rect.h},setsar=1`));
|
||||
assert.ok(!(cover).includes("pad="));
|
||||
const cropOnly = graph("crop");
|
||||
assert.ok((cropOnly).includes(`crop=${rect.w}:${rect.h},setsar=1`));
|
||||
assert.ok(!(cropOnly).includes("scale="));
|
||||
assert.ok(!(cropOnly).includes("pad="));
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Run note
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// This suite runs under the repo's vitest 2.1.9 (installed with the api app;
|
||||
// the media-worker package itself has no vitest dependency):
|
||||
//
|
||||
// cd apps/api && ./node_modules/.bin/vitest run \
|
||||
// --root ../media-worker src/compositor/wp12-hardening.test.ts
|
||||
//
|
||||
// The media-worker `npm test` glob (node --test) also picks this file up as a
|
||||
// bare `describe/it` import failure — it is authored for vitest only, per the
|
||||
// WP12 deliverable.
|
||||
116
apps/media-worker/src/compositor/zero-gap.test.ts
Normal file
116
apps/media-worker/src/compositor/zero-gap.test.ts
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
/**
|
||||
* S2 gate: zero gap at the encoder input across 1 000 scripted scene, overlay
|
||||
* and gain swaps (streaming-architecture.md §6 S2), plus kill -9 of the
|
||||
* compositor mid-swap leaving the encoder input up.
|
||||
*
|
||||
* Real compositor ffmpeg processes built by MediaCommandBuilder, the Rust frame
|
||||
* gate, and a verifying null encoder in its own process. Sources are
|
||||
* CounterSources (luma = frame counter, phase-locked 1500 Hz sine).
|
||||
* The regular suite runs 50 swaps. The standalone acceptance gate runs
|
||||
* ZERO_GAP_SWAPS=1000, keeping the extended media workload separate from other live-process tests.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import type { ComposeRequest } from "./compositor";
|
||||
import { ProgramChain } from "./program";
|
||||
import { composeOver, CounterSource, ensureGateBinary, makeAssets, passthroughRedactor, scriptedGraph, SinkProcess, ZERO_GAP_CANVAS as canvas } from "./testing";
|
||||
|
||||
const SWAPS = Number(process.env.ZERO_GAP_SWAPS ?? 50);
|
||||
|
||||
test("zero gap at the encoder input across scripted scene/overlay/gain swaps", { timeout: 1_800_000 }, async (t) => {
|
||||
if (!ensureGateBinary()) { t.skip("frame-gate binary unavailable (cargo build failed)"); return; }
|
||||
const scratch = mkdtempSync(join(tmpdir(), "zero-gap-"));
|
||||
const assets = makeAssets(scratch);
|
||||
const sources = Array.from({ length: 4 }, () => new CounterSource(canvas, scratch));
|
||||
for (const s of sources) await s.start();
|
||||
const lost: string[] = [];
|
||||
const chain = new ProgramChain({ canvas, scratchDir: scratch, videoBitrateKbps: 1000, output: { kind: "external" },
|
||||
redactor: passthroughRedactor, onStderr: () => undefined, onFatal: (r) => lost.push(`fatal ${r}`), onCompositorLost: (r) => lost.push(r) });
|
||||
const sink = new SinkProcess();
|
||||
await sink.start(chain.gate, canvas);
|
||||
const request = (i: number): ComposeRequest => composeOver(sources.map((s) => s.bus), assets, scriptedGraph(i));
|
||||
try {
|
||||
await chain.start(request(0));
|
||||
const first = await chain.stats();
|
||||
const firstTick = first.ticks;
|
||||
const swapTicks: number[] = [];
|
||||
for (let i = 1; i <= SWAPS; i += 1) swapTicks.push(await chain.swap(request(i)));
|
||||
// Let the last compositor run, then read the counters.
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
const stats = await chain.stats();
|
||||
// Report before teardown: stopping the chain makes the gate hold its last frame.
|
||||
const report = await sink.report(firstTick);
|
||||
await chain.stop();
|
||||
|
||||
assert.equal(chain.swaps, SWAPS + 1);
|
||||
assert.equal(stats.swaps, SWAPS + 1);
|
||||
assert.deepEqual(lost.filter((l) => !l.startsWith("fatal frame gate exited")), [], "no compositor lost on air");
|
||||
assert.ok(swapTicks.every((tick, k) => k === 0 || tick > (swapTicks[k - 1] ?? 0)), "swap ticks strictly increase");
|
||||
// Encoder-side continuity: every tick delivered exactly one video and one audio frame.
|
||||
assert.ok(Math.abs(report.videoFrames - report.audioFrames) <= 1, "video/audio lockstep at the encoder input");
|
||||
assert.ok(report.videoFrames >= stats.ticks - 2, "encoder received every tick");
|
||||
assert.equal(stats.outDrops, 0, "gate never dropped toward the encoder");
|
||||
// Every delivered frame advances by exactly one, including across every swap.
|
||||
assert.deepEqual(Object.keys(report.tagDeltas), ["1"], `frame counter all +1 (deltas ${JSON.stringify(report.tagDeltas)}, gate lagged=${stats.lagged} behindMax=${stats.behindMax} drops=${stats.drops} vrep=${stats.vrep})`);
|
||||
// PCM continuity: no inserted silence after the first compositor and no step larger than the sine allows.
|
||||
const bound = Math.ceil(report.maxPcmAbs * Math.sin((2 * Math.PI * 1500) / 48_000)) + 4;
|
||||
assert.ok(report.maxPcmStep <= bound, `PCM step ${report.maxPcmStep} <= ${bound}`);
|
||||
const fills = stats.vrep + stats.asil;
|
||||
// On an overloaded host a fill can come from the gate itself not being scheduled; it must stay rare.
|
||||
assert.ok(fills <= Math.ceil(stats.ticks / 1000), `repeat/silence fills ${fills} over ${stats.ticks} ticks`);
|
||||
t.diagnostic(JSON.stringify({ swaps: SWAPS, lagRetries: chain.lagRetries, lagged: stats.lagged, behindMax: stats.behindMax, ticks: stats.ticks, vrep: stats.vrep, asil: stats.asil, lateMaxUs: stats.lateMaxUs, lateOverFrame: stats.lateOverFrame,
|
||||
tagDeltas: report.tagDeltas, maxPcmStep: report.maxPcmStep, bound, maxGapMs: report.maxGapMs }));
|
||||
} finally {
|
||||
await sink.stop();
|
||||
for (const s of sources) await s.stop();
|
||||
await chain.stop();
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("kill -9 of the compositor mid-swap leaves the encoder input up", { timeout: 120_000 }, async (t) => {
|
||||
if (!ensureGateBinary()) { t.skip("frame-gate binary unavailable"); return; }
|
||||
const scratch = mkdtempSync(join(tmpdir(), "kill9-"));
|
||||
const assets = makeAssets(scratch);
|
||||
const sources = Array.from({ length: 4 }, () => new CounterSource(canvas, scratch));
|
||||
for (const s of sources) await s.start();
|
||||
const lost: string[] = [];
|
||||
const fatal: string[] = [];
|
||||
const chain = new ProgramChain({ canvas, scratchDir: scratch, videoBitrateKbps: 1000, output: { kind: "external" },
|
||||
redactor: passthroughRedactor, onStderr: () => undefined, onFatal: (r) => fatal.push(r), onCompositorLost: (r) => lost.push(r) });
|
||||
const sink = new SinkProcess();
|
||||
await sink.start(chain.gate, canvas);
|
||||
const request = (i: number): ComposeRequest => composeOver(sources.map((s) => s.bus), assets, { ...scriptedGraph(i), audio: { inputs: [{ source: 0, gainDb: 0 }], masterGainDb: 0 } });
|
||||
try {
|
||||
await chain.start(request(0));
|
||||
const firstTick = (await chain.stats()).ticks;
|
||||
// Mid-swap: B is starting while A is SIGKILLed.
|
||||
const a = chain.activeCompositor;
|
||||
const swapping = chain.swap(request(1));
|
||||
a?.kill("SIGKILL");
|
||||
await swapping;
|
||||
// On air: SIGKILL the active one; the chain respawns it behind the gate.
|
||||
chain.activeCompositor?.kill("SIGKILL");
|
||||
await new Promise((r) => setTimeout(r, 2_000));
|
||||
await chain.swap(request(2));
|
||||
const before = await chain.stats();
|
||||
await new Promise((r) => setTimeout(r, 1_000));
|
||||
const after = await chain.stats();
|
||||
const report = await sink.report(firstTick);
|
||||
await chain.stop();
|
||||
assert.deepEqual(fatal.filter((f) => !f.startsWith("frame gate exited")), [], "gate and encoder input never went down");
|
||||
assert.ok(lost.length >= 1, "the on-air kill was detected");
|
||||
assert.ok(after.ticks - before.ticks >= canvas.fps * 0.8, "gate kept ticking");
|
||||
assert.ok(Math.abs(report.videoFrames - report.audioFrames) <= 1);
|
||||
assert.ok(report.videoFrames >= after.ticks - 2, "encoder input never starved");
|
||||
t.diagnostic(JSON.stringify({ lost, vrep: after.vrep, asil: after.asil, ticks: after.ticks }));
|
||||
} finally {
|
||||
await sink.stop();
|
||||
for (const s of sources) await s.stop();
|
||||
await chain.stop();
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue