Tactical Media Broadcast Hub — initial release
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
This commit is contained in:
commit
aa5e262f3a
880 changed files with 111042 additions and 0 deletions
14
apps/media-worker/src/bus/control-signer.ts
Normal file
14
apps/media-worker/src/bus/control-signer.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
import type { KeyObject } from "node:crypto";
|
||||
import { signCommand, type CommandBody, type SignedCommand } from "./envelope";
|
||||
|
||||
/** API-side stub: caller must commit desired state and re-authorize before signing. */
|
||||
export interface ControlCommandSigner {
|
||||
signCommittedSnapshot(body: CommandBody): Promise<SignedCommand>;
|
||||
}
|
||||
|
||||
export class Ed25519ControlSigner implements ControlCommandSigner {
|
||||
constructor(private readonly kid: string, private readonly privateKey: KeyObject) {}
|
||||
async signCommittedSnapshot(body: CommandBody): Promise<SignedCommand> {
|
||||
return signCommand(body, this.kid, this.privateKey);
|
||||
}
|
||||
}
|
||||
61
apps/media-worker/src/bus/envelope.ts
Normal file
61
apps/media-worker/src/bus/envelope.ts
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
import { sign, verify, type KeyObject } from "node:crypto";
|
||||
import { isDesiredChannel, snapshotHlsUrls, type DesiredChannel } from "../types";
|
||||
import { validateEndpointShape } from "../relay/endpoints";
|
||||
|
||||
export interface CommandBody {
|
||||
workspaceId: string;
|
||||
channelId: string;
|
||||
workerId: string;
|
||||
assignmentEpoch: number;
|
||||
commandId: string;
|
||||
desiredVersion: number;
|
||||
issuedAt: string;
|
||||
expiresAt: string;
|
||||
snapshot: DesiredChannel;
|
||||
}
|
||||
export interface SignedCommand { kid: string; body: CommandBody; signature: string }
|
||||
|
||||
/** Fixed field order is the wire-format contract for both signing and verifying. */
|
||||
export function commandBytes(body: CommandBody): Buffer {
|
||||
return Buffer.from(JSON.stringify({
|
||||
workspaceId: body.workspaceId, channelId: body.channelId, workerId: body.workerId,
|
||||
assignmentEpoch: body.assignmentEpoch, commandId: body.commandId,
|
||||
desiredVersion: body.desiredVersion, issuedAt: body.issuedAt,
|
||||
expiresAt: body.expiresAt, snapshot: body.snapshot,
|
||||
}));
|
||||
}
|
||||
|
||||
export function signCommand(body: CommandBody, kid: string, key: KeyObject): SignedCommand {
|
||||
if (!isDesiredChannel(body.snapshot)) throw new Error("invalid desired snapshot");
|
||||
validateSnapshotEndpoints(body.snapshot);
|
||||
return { kid, body, signature: sign(null, commandBytes(body), key).toString("base64") };
|
||||
}
|
||||
|
||||
function validateSnapshotEndpoints(snapshot: DesiredChannel): void {
|
||||
for (const url of snapshotHlsUrls(snapshot)) validateEndpointShape(url, "hls");
|
||||
if (snapshot.output) validateEndpointShape(snapshot.output.endpointUrl, "output");
|
||||
}
|
||||
|
||||
export function verifyCommand(value: unknown, keys: ReadonlyMap<string, KeyObject>, workerId: string, now = Date.now()): SignedCommand {
|
||||
if (typeof value !== "object" || value === null) throw new Error("invalid command");
|
||||
const command = value as Partial<SignedCommand>;
|
||||
const body = command.body;
|
||||
if (typeof command.kid !== "string" || typeof command.signature !== "string" || !body ||
|
||||
typeof body.workspaceId !== "string" || typeof body.channelId !== "string" ||
|
||||
body.workerId !== workerId || typeof body.commandId !== "string" || body.commandId.length === 0 ||
|
||||
!Number.isSafeInteger(body.assignmentEpoch) || (body.assignmentEpoch ?? -1) < 0 ||
|
||||
!Number.isSafeInteger(body.desiredVersion) || (body.desiredVersion ?? -1) < 0 ||
|
||||
typeof body.issuedAt !== "string" || typeof body.expiresAt !== "string" ||
|
||||
!isDesiredChannel(body.snapshot)) throw new Error("invalid command fields");
|
||||
const issued = Date.parse(body.issuedAt);
|
||||
const expires = Date.parse(body.expiresAt);
|
||||
if (!Number.isFinite(issued) || !Number.isFinite(expires) || issued > now + 30_000 || expires <= now || expires <= issued) throw new Error("expired command");
|
||||
const snapshot = body.snapshot;
|
||||
if (snapshot.workspaceId !== body.workspaceId || snapshot.channelId !== body.channelId ||
|
||||
snapshot.workerId !== body.workerId || snapshot.assignmentEpoch !== body.assignmentEpoch ||
|
||||
snapshot.desiredVersion !== body.desiredVersion) throw new Error("snapshot mismatch");
|
||||
validateSnapshotEndpoints(snapshot);
|
||||
const key = keys.get(command.kid);
|
||||
if (!key || !verify(null, commandBytes(body as CommandBody), key, Buffer.from(command.signature, "base64"))) throw new Error("invalid signature");
|
||||
return command as SignedCommand;
|
||||
}
|
||||
68
apps/media-worker/src/bus/fence.ts
Normal file
68
apps/media-worker/src/bus/fence.ts
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { CommandBody } from "./envelope";
|
||||
|
||||
interface FenceEntry { assignmentEpoch: number; acceptedVersion: number; seen: Record<string, number> }
|
||||
type FenceData = Record<string, FenceEntry>;
|
||||
|
||||
/** Local durable fence. Atomic rename plus fsync of file and directory precedes execute/ack. */
|
||||
export class DurableFence {
|
||||
private data: FenceData;
|
||||
constructor(private readonly path: string) {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
this.data = existsSync(path) ? this.read() : {};
|
||||
}
|
||||
|
||||
private read(): FenceData {
|
||||
const parsed: unknown = JSON.parse(readFileSync(this.path, "utf8"));
|
||||
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error("invalid fence file");
|
||||
for (const entry of Object.values(parsed)) {
|
||||
if (typeof entry !== "object" || entry === null) throw new Error("invalid fence entry");
|
||||
const record = entry as Partial<FenceEntry>;
|
||||
if (!Number.isSafeInteger(record.assignmentEpoch) || !Number.isSafeInteger(record.acceptedVersion) ||
|
||||
(record.assignmentEpoch ?? -1) < 0 || (record.acceptedVersion ?? -1) < 0 ||
|
||||
typeof record.seen !== "object" || record.seen === null || Array.isArray(record.seen) ||
|
||||
Object.values(record.seen).some((expiry) => !Number.isFinite(expiry))) throw new Error("invalid fence entry");
|
||||
}
|
||||
return parsed as FenceData;
|
||||
}
|
||||
|
||||
accepted(workspaceId: string, channelId: string): { assignmentEpoch: number; acceptedVersion: number } | null {
|
||||
const entry = this.data[`${workspaceId}:${channelId}`];
|
||||
return entry ? { assignmentEpoch: entry.assignmentEpoch, acceptedVersion: entry.acceptedVersion } : null;
|
||||
}
|
||||
|
||||
accept(body: CommandBody, now = Date.now()): "accepted" | "replay" | "stale" {
|
||||
const key = `${body.workspaceId}:${body.channelId}`;
|
||||
const previous = this.data[key];
|
||||
if (previous && body.assignmentEpoch < previous.assignmentEpoch) return "stale";
|
||||
if (previous && body.assignmentEpoch === previous.assignmentEpoch && body.desiredVersion < previous.acceptedVersion) return "stale";
|
||||
if (previous?.seen[body.commandId] !== undefined) return "replay";
|
||||
const seen = previous ? { ...previous.seen } : {};
|
||||
for (const [id, expiry] of Object.entries(seen)) if (expiry <= now) delete seen[id];
|
||||
seen[body.commandId] = Date.parse(body.expiresAt);
|
||||
const next: FenceData = { ...this.data, [key]: {
|
||||
assignmentEpoch: body.assignmentEpoch,
|
||||
acceptedVersion: previous?.assignmentEpoch === body.assignmentEpoch
|
||||
? Math.max(previous.acceptedVersion, body.desiredVersion) : body.desiredVersion,
|
||||
seen,
|
||||
} };
|
||||
this.persist(next);
|
||||
this.data = next;
|
||||
return "accepted";
|
||||
}
|
||||
|
||||
private persist(data: FenceData): void {
|
||||
const temp = `${this.path}.${randomUUID()}.tmp`;
|
||||
const fd = openSync(temp, "wx", 0o600);
|
||||
try {
|
||||
writeFileSync(fd, JSON.stringify(data));
|
||||
fsyncSync(fd);
|
||||
} catch (error) { unlinkSync(temp); throw error; }
|
||||
finally { closeSync(fd); }
|
||||
renameSync(temp, this.path);
|
||||
const dir = openSync(dirname(this.path), "r");
|
||||
try { fsyncSync(dir); } finally { closeSync(dir); }
|
||||
}
|
||||
}
|
||||
137
apps/media-worker/src/bus/redis.ts
Normal file
137
apps/media-worker/src/bus/redis.ts
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
import net, { type Socket } from "node:net";
|
||||
import tls from "node:tls";
|
||||
import type { ObservedChannel } from "../types";
|
||||
|
||||
type Reply = string | number | null | Reply[];
|
||||
function encode(parts: string[]): Buffer {
|
||||
return Buffer.from(`*${parts.length}\r\n${parts.map((part) => `\$${Buffer.byteLength(part)}\r\n${part}\r\n`).join("")}`);
|
||||
}
|
||||
function parse(buffer: Buffer, offset = 0): { reply: Reply; end: number } | null {
|
||||
if (offset >= buffer.length) return null;
|
||||
const endLine = buffer.indexOf("\r\n", offset);
|
||||
if (endLine < 0) return null;
|
||||
const length = Number(buffer.subarray(offset + 1, endLine).toString());
|
||||
const type = String.fromCharCode(buffer[offset] ?? 0);
|
||||
let pos = endLine + 2;
|
||||
if (type === "+" || type === ":" || type === "-") return { reply: type === ":" ? length : buffer.subarray(offset + 1, endLine).toString(), end: pos };
|
||||
if (type === "$") {
|
||||
if (length === -1) return { reply: null, end: pos };
|
||||
if (!Number.isInteger(length) || length < 0 || length > 2_000_000) throw new Error("invalid Redis bulk length");
|
||||
if (buffer.length < pos + length + 2) return null;
|
||||
const reply = buffer.subarray(pos, pos + length).toString();
|
||||
return { reply, end: pos + length + 2 };
|
||||
}
|
||||
if (type === "*") {
|
||||
if (length === -1) return { reply: null, end: pos };
|
||||
if (!Number.isInteger(length) || length < 0 || length > 100) throw new Error("invalid Redis array length");
|
||||
const replies: Reply[] = [];
|
||||
for (let i = 0; i < length; i += 1) {
|
||||
const item = parse(buffer, pos);
|
||||
if (!item) return null;
|
||||
replies.push(item.reply);
|
||||
pos = item.end;
|
||||
}
|
||||
return { reply: replies, end: pos };
|
||||
}
|
||||
throw new Error("invalid Redis reply");
|
||||
}
|
||||
|
||||
class RedisConnection {
|
||||
private socket: Socket;
|
||||
private buffer = Buffer.alloc(0);
|
||||
private pending: Array<{ resolve: (reply: Reply) => void; reject: (error: Error) => void }> = [];
|
||||
onPush: ((reply: Reply) => void) | null = null;
|
||||
private subscribing = false;
|
||||
|
||||
private constructor(socket: Socket) {
|
||||
this.socket = socket;
|
||||
socket.on("data", (chunk: Buffer) => {
|
||||
this.buffer = Buffer.concat([this.buffer, chunk]);
|
||||
if (this.buffer.length > 2_000_000) { socket.destroy(new Error("Redis reply too large")); return; }
|
||||
while (true) {
|
||||
const prefix = this.buffer[0];
|
||||
const parsed = parse(this.buffer);
|
||||
if (!parsed) break;
|
||||
this.buffer = this.buffer.subarray(parsed.end);
|
||||
if (prefix === 45) {
|
||||
if (this.subscribing) this.onPush?.(parsed.reply);
|
||||
else this.pending.shift()?.reject(new Error(`Redis: ${String(parsed.reply)}`));
|
||||
continue;
|
||||
}
|
||||
if (this.subscribing) this.onPush?.(parsed.reply);
|
||||
else this.pending.shift()?.resolve(parsed.reply);
|
||||
}
|
||||
});
|
||||
socket.on("error", (error) => { for (const item of this.pending.splice(0)) item.reject(error); });
|
||||
socket.on("close", () => { for (const item of this.pending.splice(0)) item.reject(new Error("Redis disconnected")); });
|
||||
}
|
||||
|
||||
static async connect(url: URL): Promise<RedisConnection> {
|
||||
const port = Number(url.port || (url.protocol === "rediss:" ? 6380 : 6379));
|
||||
const socket = url.protocol === "rediss:"
|
||||
? tls.connect({ host: url.hostname, port, servername: url.hostname, rejectUnauthorized: true })
|
||||
: net.connect({ host: url.hostname, port });
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
socket.once(url.protocol === "rediss:" ? "secureConnect" : "connect", () => resolve());
|
||||
socket.once("error", reject);
|
||||
});
|
||||
const connection = new RedisConnection(socket);
|
||||
if (url.password) await connection.send(["AUTH", decodeURIComponent(url.username || "default"), decodeURIComponent(url.password)]);
|
||||
return connection;
|
||||
}
|
||||
|
||||
send(parts: string[]): Promise<Reply> {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.pending.push({ resolve, reject });
|
||||
this.socket.write(encode(parts));
|
||||
});
|
||||
}
|
||||
subscribe(channel: string, onMessage: (message: string) => void): Promise<void> {
|
||||
this.subscribing = true;
|
||||
return new Promise((resolve, reject) => {
|
||||
this.onPush = (reply) => {
|
||||
if (typeof reply === "string") reject(new Error("Redis subscription rejected"));
|
||||
else if (Array.isArray(reply) && reply[0] === "subscribe" && reply[1] === channel) resolve();
|
||||
else if (Array.isArray(reply) && reply[0] === "message" && reply[1] === channel && typeof reply[2] === "string") onMessage(reply[2]);
|
||||
};
|
||||
this.socket.write(encode(["SUBSCRIBE", channel]));
|
||||
});
|
||||
}
|
||||
close(): void { this.socket.destroy(); }
|
||||
}
|
||||
|
||||
export interface ObservedStore {
|
||||
get(workspaceId: string, channelId: string): Promise<ObservedChannel | null>;
|
||||
put(observed: ObservedChannel): Promise<void>;
|
||||
heartbeat(workerId: string, channels: ObservedChannel[]): Promise<void>;
|
||||
}
|
||||
|
||||
export class RedisBus implements ObservedStore {
|
||||
private subscribers: RedisConnection[] = [];
|
||||
private constructor(private readonly url: URL, private readonly client: RedisConnection) {}
|
||||
static async connect(rawUrl: string): Promise<RedisBus> {
|
||||
const url = new URL(rawUrl);
|
||||
if (url.protocol !== "redis:" && url.protocol !== "rediss:") throw new Error("invalid Redis URL");
|
||||
return new RedisBus(url, await RedisConnection.connect(url));
|
||||
}
|
||||
async get(workspaceId: string, channelId: string): Promise<ObservedChannel | null> {
|
||||
const result = await this.client.send(["GET", `obs:${workspaceId}:${channelId}`]);
|
||||
return typeof result === "string" ? JSON.parse(result) as ObservedChannel : null;
|
||||
}
|
||||
async put(observed: ObservedChannel): Promise<void> {
|
||||
await this.client.send(["SET", `obs:${observed.workspaceId}:${observed.channelId}`, JSON.stringify(observed), "EX", "60"]);
|
||||
}
|
||||
async heartbeat(workerId: string, channels: ObservedChannel[]): Promise<void> {
|
||||
await this.client.send(["SET", `hb:${workerId}`, JSON.stringify({ workerId, sentAt: new Date().toISOString(), channels: channels.map((c) => ({ workspaceId: c.workspaceId, channelId: c.channelId, assignmentEpoch: c.assignmentEpoch, realizedVersion: c.realizedVersion })) }), "EX", "15"]);
|
||||
for (const channel of channels) await this.put(channel);
|
||||
}
|
||||
async subscribe(workerId: string, workspaces: string[], onCommand: (message: string) => void): Promise<void> {
|
||||
// A worker uses one subscription per assigned workspace; all channels are signature checked.
|
||||
for (const workspaceId of workspaces) {
|
||||
const subscriber = await RedisConnection.connect(this.url);
|
||||
await subscriber.subscribe(`cmd:${workspaceId}:${workerId}`, onCommand);
|
||||
this.subscribers.push(subscriber);
|
||||
}
|
||||
}
|
||||
close(): void { for (const subscriber of this.subscribers) subscriber.close(); this.client.close(); }
|
||||
}
|
||||
203
apps/media-worker/src/compositor/command-builder.test.ts
Normal file
203
apps/media-worker/src/compositor/command-builder.test.ts
Normal file
|
|
@ -0,0 +1,203 @@
|
|||
/**
|
||||
* MediaCommandBuilder: argv shape, typed-param rejection, and golden-frame
|
||||
* hashes per layout (deterministic synthetic inputs: seeded noise over test
|
||||
* patterns; framemd5 of the composited output, video + program audio).
|
||||
*
|
||||
* Hashes are pinned to the system FFmpeg (5.1.x here). UPDATE_GOLDEN=1 rewrites
|
||||
* src/compositor/golden-frames.json after an intentional graph change or an
|
||||
* FFmpeg upgrade.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import {
|
||||
buildCompositorArgs, buildEncoderArgs, buildFilterGraph, buildIngestArgs, CommandBuildError, layoutRects, rectFromBasisPoints,
|
||||
SYNTHETIC_PATTERNS, type Canvas, type CompositorPlan, type PxRect,
|
||||
} from "./command-builder";
|
||||
import type { SceneLayout } from "../production";
|
||||
import { makeAssets } from "./testing";
|
||||
|
||||
const canvas: Canvas = { width: 320, height: 180, fps: 30 };
|
||||
const GOLDEN = join(__dirname, "golden-frames.json");
|
||||
|
||||
function synthetic(i: number) {
|
||||
return { video: { kind: "synthetic" as const, pattern: SYNTHETIC_PATTERNS[i % SYNTHETIC_PATTERNS.length]!, seed: 1000 + i }, audio: { kind: "synthetic" as const, toneHz: 220 + 110 * i } };
|
||||
}
|
||||
|
||||
const CUSTOM: PxRect[] = [{ x: 0, y: 0, w: 160, h: 90 }, { x: 160, y: 0, w: 160, h: 90 }, { x: 80, y: 90, w: 160, h: 90 }];
|
||||
const CUSTOM_OVERLAP: PxRect[] = [{ x: 0, y: 0, w: 200, h: 112 }, { x: 120, y: 68, w: 200, h: 112 }];
|
||||
|
||||
interface Case { name: string; layout: SceneLayout; rects: PxRect[]; empty?: number[]; overlays: "none" | "lower" | "chyron" | "both"; mix: Array<[number, number]>; master: number }
|
||||
|
||||
const CASES: Case[] = [
|
||||
{ name: "single", layout: "single", rects: layoutRects("single", canvas), overlays: "none", mix: [[0, 0]], master: 0 },
|
||||
{ name: "pip", layout: "pip", rects: [{ x: 0, y: 0, w: 320, h: 180 }, rectFromBasisPoints({ x: 6000, y: 6000, w: 3500, h: 3500 }, canvas)], overlays: "lower", mix: [[0, 0], [1, -12]], master: -3 },
|
||||
{ name: "2x1", layout: "2x1", rects: layoutRects("2x1", canvas), overlays: "none", mix: [[0, -3], [1, -3]], master: 0 },
|
||||
{ name: "2x2", layout: "2x2", rects: layoutRects("2x2", canvas), empty: [3], overlays: "chyron", mix: [[0, 0]], master: -6 },
|
||||
{ name: "3x3", layout: "3x3", rects: layoutRects("3x3", canvas), empty: [4, 8], overlays: "both", mix: [[0, 0], [1, -6], [2, -24]], master: 0 },
|
||||
{ name: "4x4", layout: "4x4", rects: layoutRects("4x4", canvas), empty: [5, 10, 15], overlays: "none", mix: [], master: 0 },
|
||||
{ name: "custom", layout: "custom", rects: CUSTOM, overlays: "lower", mix: [[2, 0]], master: 0 },
|
||||
{ name: "custom-overlap", layout: "custom", rects: CUSTOM_OVERLAP, overlays: "both", mix: [[0, 0], [1, 0]], master: -9 },
|
||||
];
|
||||
|
||||
function plan(c: Case, assets: string[]): CompositorPlan {
|
||||
const tiles = c.rects.map((rect, i) => ({ source: c.empty?.includes(i) ? null : i % 4, rect, fit: (["contain", "cover", "crop"] as const)[i % 3]! }));
|
||||
const overlays = [
|
||||
...(c.overlays === "lower" || c.overlays === "both" ? [{ kind: "image" as const, asset: 0, x: 12, y: 140 }] : []),
|
||||
...(c.overlays === "chyron" || c.overlays === "both" ? [{ kind: "chyron" as const, asset: 1, y: 164, speedPxPerSec: 120 }] : []),
|
||||
];
|
||||
return {
|
||||
sources: [0, 1, 2, 3].map(synthetic), assets,
|
||||
graph: { canvas, layout: c.layout, tiles, overlays, audio: { inputs: c.mix.map(([source, gainDb]) => ({ source, gainDb })), masterGainDb: c.master } },
|
||||
output: { kind: "framemd5", frames: 4 },
|
||||
};
|
||||
}
|
||||
|
||||
function hashes(p: CompositorPlan): string[] {
|
||||
const r = spawnSync("ffmpeg", buildCompositorArgs(p), { encoding: "utf8", timeout: 60_000 });
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
// framemd5 lines: stream, dts, pts, duration, size, hash — keep stream/pts/size/hash.
|
||||
return r.stdout.split("\n").filter((l) => l && !l.startsWith("#")).map((l) => l.split(",").map((x) => x.trim())).map((f) => `${f[0]}:${f[2]}:${f[4]}:${f[5]}`);
|
||||
}
|
||||
|
||||
test("golden-frame hashes per layout (deterministic synthetic inputs)", { timeout: 300_000 }, () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "golden-"));
|
||||
try {
|
||||
const assets = makeAssets(dir);
|
||||
const got: Record<string, string[]> = {};
|
||||
for (const c of CASES) {
|
||||
const p = plan(c, assets);
|
||||
const h = hashes(p);
|
||||
assert.ok(h.some((l) => l.startsWith("0:")) && h.some((l) => l.startsWith("1:")), `${c.name}: video and audio frames`);
|
||||
assert.deepEqual(hashes(p), h, `${c.name}: two runs hash identically`);
|
||||
got[c.name] = h;
|
||||
}
|
||||
if (process.env.UPDATE_GOLDEN || !existsSync(GOLDEN)) {
|
||||
writeFileSync(GOLDEN, `${JSON.stringify({ ffmpeg: ffmpegVersion(), canvas, frames: 4, layouts: got }, null, 2)}\n`);
|
||||
}
|
||||
const golden = JSON.parse(readFileSync(GOLDEN, "utf8")) as { ffmpeg: string; layouts: Record<string, string[]> };
|
||||
for (const c of CASES) assert.deepEqual(got[c.name], golden.layouts[c.name], `${c.name} matches golden (pinned to ${golden.ffmpeg}; UPDATE_GOLDEN=1 after an intentional change)`);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function ffmpegVersion(): string {
|
||||
return spawnSync("ffmpeg", ["-version"], { encoding: "utf8" }).stdout.split("\n")[0] ?? "unknown";
|
||||
}
|
||||
|
||||
test("layout cells are even, cover the canvas, and xstack/overlay is chosen by geometry", () => {
|
||||
for (const layout of ["2x1", "2x2", "3x3", "4x4"] as const) {
|
||||
for (const c of [canvas, { width: 1920, height: 1080, fps: 30 }, { width: 1280, height: 720, fps: 30 }]) {
|
||||
const rects = layoutRects(layout, c);
|
||||
assert.equal(rects.reduce((a, r) => a + r.w * r.h, 0), c.width * c.height, `${layout} ${c.width}x${c.height} tiles cover the canvas`);
|
||||
for (const r of rects) for (const v of [r.x, r.y, r.w, r.h]) assert.equal(v % 2, 0);
|
||||
}
|
||||
}
|
||||
const idx = { video: [0, 1, 2, 3], audio: [null, null, null, null], assets: [] };
|
||||
const base = { sources: [0, 1, 2, 3].map(synthetic), assets: [], output: { kind: "framemd5" as const, frames: 1 } };
|
||||
const audio = { inputs: [], masterGainDb: 0 };
|
||||
const grid = buildFilterGraph({ ...base, graph: { canvas, layout: "2x2", tiles: layoutRects("2x2", canvas).map((rect, i) => ({ source: i, rect, fit: "contain" as const })), overlays: [], audio } }, idx);
|
||||
assert.match(grid, /xstack=inputs=4:layout=0_0\|160_0\|0_90\|160_90:fill=black/);
|
||||
assert.match(grid, /anullsrc=r=48000:cl=stereo/);
|
||||
const pip = buildFilterGraph({ ...base, graph: { canvas, layout: "pip", tiles: [{ source: 0, rect: { x: 0, y: 0, w: 320, h: 180 }, fit: "cover" }, { source: 1, rect: { x: 200, y: 100, w: 100, h: 56 }, fit: "contain" }], overlays: [], audio } }, idx);
|
||||
assert.match(pip, /overlay=x=200:y=100/);
|
||||
assert.doesNotMatch(pip, /xstack/);
|
||||
// Overlapping custom rects: the base is the first tile padded (paced by a live input), not a lavfi color source.
|
||||
const overlap = buildFilterGraph({ ...base, graph: { canvas, layout: "custom", tiles: CUSTOM_OVERLAP.map((rect, i) => ({ source: i, rect, fit: "cover" as const })), overlays: [], audio } }, idx);
|
||||
assert.match(overlap, /\[t0\]pad=320:180:0:0:color=black\[bg\];\[bg\]\[t1\]overlay=x=120:y=68/);
|
||||
assert.doesNotMatch(overlap, /color=c=black/);
|
||||
const shared = buildFilterGraph({ ...base, graph: { canvas, layout: "2x1", tiles: layoutRects("2x1", canvas).map((rect) => ({ source: 0, rect, fit: "crop" as const })), overlays: [], audio } }, idx);
|
||||
assert.match(shared, /\[0:v\]split=2\[s0_0\]\[s0_1\]/);
|
||||
});
|
||||
|
||||
test("program mix follows §2.3: volume per input → amix → master gain, 3 dB steps only", () => {
|
||||
const idx = { video: [0, 2, 4], audio: [1, 3, 5], assets: [] };
|
||||
const p = (inputs: Array<{ source: number; gainDb: number }>, masterGainDb = 0): CompositorPlan => ({
|
||||
sources: [0, 1, 2].map(synthetic), assets: [], output: { kind: "framemd5", frames: 1 },
|
||||
graph: { canvas, layout: "single", tiles: [{ source: 0, rect: { x: 0, y: 0, w: 320, h: 180 }, fit: "contain" }], overlays: [], audio: { inputs, masterGainDb } },
|
||||
});
|
||||
const g = buildFilterGraph(p([{ source: 0, gainDb: -6 }, { source: 2, gainDb: 0 }], -3), idx);
|
||||
assert.match(g, /\[1:a\]aformat=[^;]*,volume=-6dB\[m0\]/);
|
||||
assert.match(g, /\[5:a\]aformat=[^;]*,volume=0dB\[m1\]/);
|
||||
assert.match(g, /\[m0\]\[m1\]amix=inputs=2:duration=longest:dropout_transition=2:normalize=1,volume=-3dB/);
|
||||
assert.doesNotMatch(buildFilterGraph(p([{ source: 0, gainDb: 0 }]), idx), /amix/);
|
||||
for (const bad of [-1, -25, 3, 1.5]) assert.throws(() => buildCompositorArgs(p([{ source: 0, gainDb: bad }])), /gainDb/);
|
||||
assert.throws(() => buildCompositorArgs(p([{ source: 0, gainDb: 0 }, { source: 0, gainDb: -3 }])), /repeats a source/);
|
||||
assert.throws(() => buildCompositorArgs(p([{ source: 0, gainDb: 0 }], -2)), /masterGainDb/);
|
||||
});
|
||||
|
||||
test("typed params only: hostile values never reach argv", () => {
|
||||
const good: CompositorPlan = {
|
||||
sources: [{ video: { kind: "rawPipe", fd: 4 }, audio: { kind: "rawPipe", fd: 5 } }],
|
||||
assets: [`/scratch/assets/${"a".repeat(64)}.png`],
|
||||
graph: { canvas, layout: "single", tiles: [{ source: 0, rect: { x: 0, y: 0, w: 320, h: 180 }, fit: "contain" }], overlays: [{ kind: "chyron", asset: 0, y: 160, speedPxPerSec: 100 }], audio: { inputs: [{ source: 0, gainDb: 0 }], masterGainDb: 0 } },
|
||||
output: { kind: "gate", videoFd: 1, audioFd: 3 },
|
||||
};
|
||||
const args = buildCompositorArgs(good);
|
||||
assert.ok(args.includes("pipe:4") && args.includes("pipe:5") && args.includes(`file:/scratch/assets/${"a".repeat(64)}.png`));
|
||||
assert.deepEqual(args.slice(-1), ["pipe:3"]);
|
||||
const hostileAssets = ["/scratch/../etc/passwd.png", "/scratch/x;rm.png", "relative/" + "a".repeat(64) + ".png", `/scratch/${"a".repeat(64)}.png'`, `/s/movie=${"a".repeat(58)}.png`, "/scratch/assets/notahash.png"];
|
||||
for (const asset of hostileAssets) assert.throws(() => buildCompositorArgs({ ...good, assets: [asset] }), CommandBuildError, asset);
|
||||
const g = good.graph;
|
||||
assert.throws(() => buildCompositorArgs({ ...good, graph: { ...g, tiles: [{ ...g.tiles[0]!, rect: { x: 1, y: 0, w: 320, h: 180 } }] } }), /even/);
|
||||
assert.throws(() => buildCompositorArgs({ ...good, graph: { ...g, tiles: [{ ...g.tiles[0]!, rect: { x: 0, y: 0, w: 322, h: 180 } }] } }), CommandBuildError);
|
||||
assert.throws(() => buildCompositorArgs({ ...good, graph: { ...g, overlays: [{ kind: "chyron", asset: 0, y: 160, speedPxPerSec: 39 }] } }), /speedPxPerSec/);
|
||||
assert.throws(() => buildCompositorArgs({ ...good, graph: { ...g, overlays: [{ kind: "image", asset: 1, x: 0, y: 0 }] } }), /asset/);
|
||||
assert.throws(() => buildCompositorArgs({ ...good, sources: [{ video: { kind: "rawPipe", fd: 1.5 }, audio: null }] }), /fd/);
|
||||
assert.throws(() => buildCompositorArgs({ ...good, graph: { ...g, canvas: { width: 321, height: 180, fps: 30 } } }), /even/);
|
||||
assert.throws(() => buildCompositorArgs({ ...good, graph: { ...g, canvas: { width: 320, height: 180, fps: 7 } } }), /fps/);
|
||||
// Nothing in argv is a shell: no metacharacters outside the filtergraph and the expression quotes.
|
||||
for (const a of args) if (!a.includes("[")) assert.doesNotMatch(a, /[;&|`$<>]/, a);
|
||||
});
|
||||
|
||||
test("ingest normalize and persistent encoder argv", () => {
|
||||
const ingest = buildIngestArgs({ input: { kind: "mpegtsPipe" }, canvas: { width: 1920, height: 1080, fps: 30 }, audio: "source" });
|
||||
assert.deepEqual(ingest.slice(ingest.indexOf("-f"), ingest.indexOf("-f") + 4), ["-f", "mpegts", "-i", "pipe:0"]);
|
||||
assert.ok(ingest.includes("pipe:1") && ingest.includes("pipe:3"));
|
||||
assert.ok(ingest.some((a) => a.startsWith("scale=1920:1080:force_original_aspect_ratio=decrease") && a.includes("fps=30")));
|
||||
const silent = buildIngestArgs({ input: { kind: "mpegtsPipe" }, canvas, audio: "none" });
|
||||
assert.ok(silent.includes("anullsrc=r=48000:cl=stereo") && silent.includes("1:a:0"));
|
||||
const enc = buildEncoderArgs({ canvas: { width: 1920, height: 1080, fps: 30 }, videoBitrateKbps: 6000, audioBitrateKbps: 128, output: { kind: "flvPipe", fd: 4 } });
|
||||
assert.deepEqual(enc.slice(-3), ["-f", "flv", "pipe:4"]);
|
||||
assert.ok(enc.includes("pipe:0") && enc.includes("pipe:3") && enc.includes("6000k") && enc.includes("60"));
|
||||
assert.throws(() => buildEncoderArgs({ canvas, videoBitrateKbps: 1e9, audioBitrateKbps: 128, output: { kind: "null" } }));
|
||||
});
|
||||
|
||||
test("ingest ×N normalize stage end to end: MPEG-TS pipe (4:3, no audio) and synthetic sources fill their buses", { timeout: 60_000 }, async () => {
|
||||
const { IngestProcess } = await import("./ingest");
|
||||
const { ensureGateBinary, passthroughRedactor } = await import("./testing");
|
||||
assert.ok(ensureGateBinary(), "native helpers (source bus) build");
|
||||
// 4:3 MPEG-TS without audio on the ingest's stdin, as the egress relay would deliver it.
|
||||
const ts = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=s=640x480:r=25:d=2", "-c:v", "mpeg2video", "-f", "mpegts", "-"], { maxBuffer: 64 << 20 });
|
||||
assert.equal(ts.status, 0);
|
||||
const dir = mkdtempSync(join(tmpdir(), "ingest-"));
|
||||
const tsPath = join(dir, "in.ts");
|
||||
writeFileSync(tsPath, ts.stdout);
|
||||
const { openSync, closeSync } = await import("node:fs");
|
||||
const fd = openSync(tsPath, "r");
|
||||
const exits: string[] = [];
|
||||
const pipeIngest = new IngestProcess("a", { input: { kind: "mpegtsPipe" }, canvas, audio: "none" }, fd, join(dir, "scratch"), passthroughRedactor, () => undefined, (_id, r) => exits.push(r));
|
||||
const synth = new IngestProcess("b", { input: { kind: "synthetic", pattern: "smptehdbars", seed: 7, toneHz: 440 }, canvas, audio: "source" }, null, join(dir, "scratch"), passthroughRedactor, () => undefined, (_id, r) => exits.push(r));
|
||||
try {
|
||||
await pipeIngest.start();
|
||||
closeSync(fd);
|
||||
await synth.start();
|
||||
await new Promise((r) => setTimeout(r, 1_500));
|
||||
// 2 s of 25 fps input → 30 fps CFR on the canvas (decoded faster than real time).
|
||||
const piped = await pipeIngest.bus.stats();
|
||||
assert.ok(piped.pairs >= 50, `mpegts pairs ${piped.pairs}`);
|
||||
assert.equal(piped.silenceFilled, 0, "anullsrc supplies audio for a silent source");
|
||||
// -re synthetic: real-time pace.
|
||||
const synthetic = await synth.bus.stats();
|
||||
assert.ok(synthetic.pairs >= 30 && synthetic.pairs <= 60, `synthetic pairs ${synthetic.pairs}`);
|
||||
assert.deepEqual(exits, []);
|
||||
} finally {
|
||||
await pipeIngest.stop();
|
||||
await synth.stop();
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
486
apps/media-worker/src/compositor/command-builder.ts
Normal file
486
apps/media-worker/src/compositor/command-builder.ts
Normal file
|
|
@ -0,0 +1,486 @@
|
|||
/**
|
||||
* MediaCommandBuilder (docs/design/streaming-architecture.md §1.4, §2.3, §4, §6 S2).
|
||||
*
|
||||
* Typed params in, argv out. Nothing here takes a free-form string: sources are
|
||||
* fds or closed synthetic patterns, assets are validated scratch paths to PNGs
|
||||
* named by content id, geometry is integers, gains are 3 dB steps. Every
|
||||
* filtergraph token is produced from those values, so no operator text can
|
||||
* reach a graph or argv (§4.3 "Injection").
|
||||
*/
|
||||
import { CHYRON_SPEED_MAX, CHYRON_SPEED_MIN, GAIN_DB_MAX, GAIN_DB_MIN, gainDb, type SceneLayout, type SlotFit } from "../production";
|
||||
import { assertGeometry, GATE_CHANNELS, GATE_SAMPLE_RATE, type FrameGeometry } from "../gate/protocol";
|
||||
|
||||
export type Canvas = FrameGeometry;
|
||||
|
||||
export interface PxRect { x: number; y: number; w: number; h: number }
|
||||
|
||||
/** Closed set of deterministic test patterns (golden frames, spike, CPU smoke). */
|
||||
export type SyntheticPattern = "testsrc2" | "smptehdbars" | "mandelbrot" | "rgbtestsrc";
|
||||
export const SYNTHETIC_PATTERNS: readonly SyntheticPattern[] = ["testsrc2", "smptehdbars", "mandelbrot", "rgbtestsrc"];
|
||||
|
||||
export type VideoInput = { kind: "rawPipe"; fd: number } | { kind: "synthetic"; pattern: SyntheticPattern; seed: number };
|
||||
export type AudioInput = { kind: "rawPipe"; fd: number } | { kind: "synthetic"; toneHz: number };
|
||||
|
||||
export interface CompositorSource { video: VideoInput; audio: AudioInput | null }
|
||||
|
||||
export interface TileSpec {
|
||||
/** Index into `sources`; null renders black. */
|
||||
source: number | null;
|
||||
/** Pixel rect inside the canvas; computed by `layoutRects` for grid layouts. */
|
||||
rect: PxRect;
|
||||
fit: SlotFit;
|
||||
}
|
||||
|
||||
export type OverlaySpec =
|
||||
| { kind: "image"; asset: number; x: number; y: number }
|
||||
| { kind: "chyron"; asset: number; y: number; speedPxPerSec: number };
|
||||
|
||||
export interface MixInput { source: number; gainDb: number }
|
||||
|
||||
export interface CompositorGraph {
|
||||
canvas: Canvas;
|
||||
layout: SceneLayout;
|
||||
tiles: TileSpec[];
|
||||
/** Drawn in order (lowest layer first). */
|
||||
overlays: OverlaySpec[];
|
||||
audio: { inputs: MixInput[]; masterGainDb: number };
|
||||
}
|
||||
|
||||
export type CompositorOutput =
|
||||
| { kind: "gate"; videoFd: 1; audioFd: 3 }
|
||||
/** Golden-frame hashing: framemd5 of the first `frames` video frames (and the audio in that span) on stdout. */
|
||||
| { kind: "framemd5"; frames: number };
|
||||
|
||||
export interface CompositorPlan {
|
||||
sources: CompositorSource[];
|
||||
/** Absolute scratch paths of rendered overlay PNGs, by content id. */
|
||||
assets: string[];
|
||||
graph: CompositorGraph;
|
||||
output: CompositorOutput;
|
||||
}
|
||||
|
||||
export class CommandBuildError extends Error {
|
||||
constructor(message: string) { super(message); this.name = "CommandBuildError"; }
|
||||
}
|
||||
function fail(message: string): never { throw new CommandBuildError(message); }
|
||||
|
||||
function int(value: number, field: string, min: number, max: number): number {
|
||||
if (!Number.isSafeInteger(value) || value < min || value > max) fail(`${field} must be an integer in ${min}..${max}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function fd(value: number, field: string): number { return int(value, field, 0, 1023); }
|
||||
|
||||
/** Rendered assets live in the worker's scratch dir and are named by sha256 (§4.3). */
|
||||
const ASSET_PATH = /^\/(?:[A-Za-z0-9._-]+\/)*[0-9a-f]{64}\.png$/;
|
||||
function assetPath(path: string, field: string): string {
|
||||
if (!ASSET_PATH.test(path) || path.split("/").includes("..")) fail(`${field} must be an absolute scratch path to <sha256>.png`);
|
||||
return path;
|
||||
}
|
||||
|
||||
function even(n: number): number { return Math.round(n / 2) * 2; }
|
||||
|
||||
/** Even cell boundaries so every tile has even size and offset (yuv420p chroma alignment). */
|
||||
function boundaries(total: number, parts: number): number[] {
|
||||
return Array.from({ length: parts + 1 }, (_, i) => (i === parts ? total : even((i * total) / parts)));
|
||||
}
|
||||
|
||||
const GRID: Partial<Record<SceneLayout, [cols: number, rows: number]>> = { "2x1": [2, 1], "2x2": [2, 2], "3x3": [3, 3], "4x4": [4, 4] };
|
||||
|
||||
/** Cell rects for grid layouts, in slot-index order (row-major). `single`/`pip` slot 0 is the full canvas. */
|
||||
export function layoutRects(layout: SceneLayout, canvas: Canvas): PxRect[] {
|
||||
if (layout === "single" || layout === "pip") return [{ x: 0, y: 0, w: canvas.width, h: canvas.height }];
|
||||
const grid = GRID[layout];
|
||||
if (!grid) return [];
|
||||
const xs = boundaries(canvas.width, grid[0]);
|
||||
const ys = boundaries(canvas.height, grid[1]);
|
||||
const rects: PxRect[] = [];
|
||||
for (let r = 0; r < grid[1]; r += 1) {
|
||||
for (let c = 0; c < grid[0]; c += 1) rects.push({ x: xs[c] ?? 0, y: ys[r] ?? 0, w: (xs[c + 1] ?? 0) - (xs[c] ?? 0), h: (ys[r + 1] ?? 0) - (ys[r] ?? 0) });
|
||||
}
|
||||
return rects;
|
||||
}
|
||||
|
||||
/** Basis points (0..10000) of the canvas to an even pixel rect. */
|
||||
export function rectFromBasisPoints(r: { x: number; y: number; w: number; h: number }, canvas: Canvas): PxRect {
|
||||
const x = even((r.x * canvas.width) / 10_000);
|
||||
const y = even((r.y * canvas.height) / 10_000);
|
||||
const w = Math.max(2, Math.min(canvas.width - x, even((r.w * canvas.width) / 10_000)));
|
||||
const h = Math.max(2, Math.min(canvas.height - y, even((r.h * canvas.height) / 10_000)));
|
||||
return { x, y, w, h };
|
||||
}
|
||||
|
||||
function checkRect(r: PxRect, canvas: Canvas, field: string): void {
|
||||
int(r.x, `${field}.x`, 0, canvas.width - 2);
|
||||
int(r.y, `${field}.y`, 0, canvas.height - 2);
|
||||
int(r.w, `${field}.w`, 2, canvas.width);
|
||||
int(r.h, `${field}.h`, 2, canvas.height);
|
||||
if (r.x % 2 || r.y % 2 || r.w % 2 || r.h % 2) fail(`${field} must be even-aligned`);
|
||||
if (r.x + r.w > canvas.width || r.y + r.h > canvas.height) fail(`${field} must lie inside the canvas`);
|
||||
}
|
||||
|
||||
function overlaps(a: PxRect, b: PxRect): boolean {
|
||||
return a.x < b.x + b.w && b.x < a.x + a.w && a.y < b.y + b.h && b.y < a.y + a.h;
|
||||
}
|
||||
|
||||
function tileFilter(fit: SlotFit, r: PxRect): string {
|
||||
switch (fit) {
|
||||
case "contain": return `scale=${r.w}:${r.h}:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=${r.w}:${r.h}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1`;
|
||||
case "cover": return `scale=${r.w}:${r.h}:force_original_aspect_ratio=increase:force_divisible_by=2,crop=${r.w}:${r.h},setsar=1`;
|
||||
case "crop": return `crop=${r.w}:${r.h},setsar=1`;
|
||||
}
|
||||
}
|
||||
|
||||
function db(value: number): string { return `${value}dB`; }
|
||||
|
||||
const AUDIO_FORMAT = `aformat=sample_fmts=s16:sample_rates=${GATE_SAMPLE_RATE}:channel_layouts=stereo`;
|
||||
|
||||
/**
|
||||
* The `-filter_complex` string. Input labels: video of source i is `[v{i}]`
|
||||
* after the builder's `-i` ordering, audio `[a{i}]`, asset k `[o{k}]`.
|
||||
*/
|
||||
export function buildFilterGraph(plan: CompositorPlan, inputIndex: { video: number[]; audio: Array<number | null>; assets: number[] }): string {
|
||||
const { graph } = plan;
|
||||
const { canvas } = graph;
|
||||
const chains: string[] = [];
|
||||
const black = (w: number, h: number): string => `color=c=black:s=${w}x${h}:r=${canvas.fps}`;
|
||||
|
||||
// Video sources used by several tiles are split once.
|
||||
const uses = new Map<number, number>();
|
||||
for (const tile of graph.tiles) if (tile.source !== null) uses.set(tile.source, (uses.get(tile.source) ?? 0) + 1);
|
||||
const taken = new Map<number, number>();
|
||||
const videoPad = (source: number): string => {
|
||||
const n = uses.get(source) ?? 0;
|
||||
const input = `${inputIndex.video[source]}:v`;
|
||||
if (n <= 1) return `[${input}]`;
|
||||
if (!taken.has(source)) {
|
||||
chains.push(`[${input}]split=${n}${Array.from({ length: n }, (_, i) => `[s${source}_${i}]`).join("")}`);
|
||||
taken.set(source, 0);
|
||||
}
|
||||
const i = taken.get(source) ?? 0;
|
||||
taken.set(source, i + 1);
|
||||
return `[s${source}_${i}]`;
|
||||
};
|
||||
|
||||
const tiles = graph.tiles.map((tile, i) => ({ tile, label: `t${i}` }));
|
||||
const live = tiles.filter(({ tile }) => tile.source !== null);
|
||||
for (const { tile, label } of live) chains.push(`${videoPad(tile.source ?? 0)}${tileFilter(tile.fit, tile.rect)}[${label}]`);
|
||||
|
||||
const W = canvas.width;
|
||||
const H = canvas.height;
|
||||
const stackable = graph.layout !== "pip" && live.every(({ tile }, i) => live.every((other, j) => i === j || !overlaps(tile.rect, other.tile.rect)));
|
||||
if (live.length === 0) {
|
||||
chains.push(`${black(W, H)}[base]`);
|
||||
} else if (stackable && live.length >= 2) {
|
||||
const layout = live.map(({ tile }) => `${tile.rect.x}_${tile.rect.y}`).join("|");
|
||||
// xstack sizes its output to the tiles' bounding box; pad only when that is not the canvas.
|
||||
const right = Math.max(...live.map(({ tile }) => tile.rect.x + tile.rect.w));
|
||||
const bottom = Math.max(...live.map(({ tile }) => tile.rect.y + tile.rect.h));
|
||||
const fit = right === W && bottom === H ? "" : `,pad=${W}:${H}:0:0:color=black`;
|
||||
chains.push(`${live.map(({ label }) => `[${label}]`).join("")}xstack=inputs=${live.length}:layout=${layout}:fill=black${fit}[base]`);
|
||||
} else if (stackable) {
|
||||
const only = live[0]!;
|
||||
const full = only.tile.rect.w === W && only.tile.rect.h === H;
|
||||
chains.push(full ? `[${only.label}]null[base]` : `[${only.label}]pad=${W}:${H}:${only.tile.rect.x}:${only.tile.rect.y}:color=black[base]`);
|
||||
} else {
|
||||
// PIP and overlapping custom rects: overlay chain in slot order (later slots on top). The base
|
||||
// is the first tile padded to the canvas, never a lavfi `color` source: an unpaced main input
|
||||
// runs ahead of the live tiles, so framesync queues them (~10 frames of extra video latency,
|
||||
// more than a swap's alignment margin) and keeps emitting frames after the inputs end.
|
||||
const first = live[0]!;
|
||||
const fullFirst = first.tile.rect.x === 0 && first.tile.rect.y === 0 && first.tile.rect.w === W && first.tile.rect.h === H;
|
||||
let cur = first.label;
|
||||
if (!fullFirst) {
|
||||
chains.push(`[${first.label}]pad=${W}:${H}:${first.tile.rect.x}:${first.tile.rect.y}:color=black[bg]`);
|
||||
cur = "bg";
|
||||
}
|
||||
live.slice(1).forEach(({ tile, label }, i) => {
|
||||
const next = `c${i}`;
|
||||
chains.push(`[${cur}][${label}]overlay=x=${tile.rect.x}:y=${tile.rect.y}:eof_action=repeat[${next}]`);
|
||||
cur = next;
|
||||
});
|
||||
chains.push(`[${cur}]null[base]`);
|
||||
}
|
||||
|
||||
let cur = "base";
|
||||
graph.overlays.forEach((overlay, i) => {
|
||||
const next = `ov${i}`;
|
||||
const asset = `[${inputIndex.assets[overlay.asset]}:v]`;
|
||||
if (overlay.kind === "image") {
|
||||
// Default format (yuv420, alpha from the PNG): `format=auto` breaks the 1:1 main-frame
|
||||
// mapping in FFmpeg 5.1 (~1 in 6 main frames duplicated and its neighbour dropped).
|
||||
chains.push(`[${cur}]${asset}overlay=x=${overlay.x}:y=${overlay.y}:eof_action=repeat[${next}]`);
|
||||
} else {
|
||||
// §4.3 chyron: one wide pre-rendered strip scrolled right-to-left by an x-expression of t.
|
||||
chains.push(`[${cur}]${asset}overlay=x='W-mod(t*${overlay.speedPxPerSec}\\,w+W)':y=${overlay.y}:eof_action=repeat[${next}]`);
|
||||
}
|
||||
cur = next;
|
||||
});
|
||||
chains.push(`[${cur}]format=yuv420p[vout]`);
|
||||
|
||||
// §2.3 program mix: volume per input → amix (dead sources fade out) → master gain.
|
||||
const mix = graph.audio.inputs;
|
||||
mix.forEach((m, i) => chains.push(`[${inputIndex.audio[m.source]}:a]${AUDIO_FORMAT},volume=${db(m.gainDb)}[m${i}]`));
|
||||
const master = `volume=${db(graph.audio.masterGainDb)},${AUDIO_FORMAT}`;
|
||||
if (mix.length === 0) chains.push(`anullsrc=r=${GATE_SAMPLE_RATE}:cl=stereo,${master}[aout]`);
|
||||
else if (mix.length === 1) chains.push(`[m0]${master}[aout]`);
|
||||
else chains.push(`${mix.map((_, i) => `[m${i}]`).join("")}amix=inputs=${mix.length}:duration=longest:dropout_transition=2:normalize=1,${master}[aout]`);
|
||||
return chains.join(";");
|
||||
}
|
||||
|
||||
function validatePlan(plan: CompositorPlan): void {
|
||||
const { graph } = plan;
|
||||
assertGeometry(graph.canvas);
|
||||
if (plan.sources.length > 16) fail("at most 16 compositor sources");
|
||||
plan.sources.forEach((s, i) => {
|
||||
if (s.video.kind === "rawPipe") fd(s.video.fd, `sources[${i}].video.fd`);
|
||||
else {
|
||||
if (!SYNTHETIC_PATTERNS.includes(s.video.pattern)) fail(`sources[${i}].video.pattern`);
|
||||
int(s.video.seed, `sources[${i}].video.seed`, 0, 1_000_000);
|
||||
}
|
||||
if (s.audio?.kind === "rawPipe") fd(s.audio.fd, `sources[${i}].audio.fd`);
|
||||
else if (s.audio?.kind === "synthetic") int(s.audio.toneHz, `sources[${i}].audio.toneHz`, 20, 20_000);
|
||||
});
|
||||
plan.assets.forEach((a, i) => assetPath(a, `assets[${i}]`));
|
||||
if (graph.tiles.length > 16) fail("at most 16 tiles");
|
||||
graph.tiles.forEach((t, i) => {
|
||||
if (t.source !== null) int(t.source, `tiles[${i}].source`, 0, plan.sources.length - 1);
|
||||
checkRect(t.rect, graph.canvas, `tiles[${i}].rect`);
|
||||
if (t.fit !== "contain" && t.fit !== "cover" && t.fit !== "crop") fail(`tiles[${i}].fit`);
|
||||
});
|
||||
if (graph.overlays.length > 64) fail("at most 64 overlays");
|
||||
graph.overlays.forEach((o, i) => {
|
||||
int(o.asset, `overlays[${i}].asset`, 0, plan.assets.length - 1);
|
||||
int(o.y, `overlays[${i}].y`, 0, graph.canvas.height);
|
||||
if (o.kind === "image") int(o.x, `overlays[${i}].x`, 0, graph.canvas.width);
|
||||
else if (o.kind === "chyron") int(o.speedPxPerSec, `overlays[${i}].speedPxPerSec`, CHYRON_SPEED_MIN, CHYRON_SPEED_MAX);
|
||||
else fail(`overlays[${i}].kind`);
|
||||
});
|
||||
const seen = new Set<number>();
|
||||
graph.audio.inputs.forEach((m, i) => {
|
||||
int(m.source, `audio.inputs[${i}].source`, 0, plan.sources.length - 1);
|
||||
if (plan.sources[m.source]?.audio == null) fail(`audio.inputs[${i}] names a source without audio`);
|
||||
if (seen.has(m.source)) fail(`audio.inputs[${i}] repeats a source`);
|
||||
seen.add(m.source);
|
||||
gainDb(m.gainDb, `audio.inputs[${i}].gainDb`);
|
||||
});
|
||||
if (graph.audio.masterGainDb < GAIN_DB_MIN || graph.audio.masterGainDb > GAIN_DB_MAX) fail("audio.masterGainDb out of range");
|
||||
gainDb(graph.audio.masterGainDb, "audio.masterGainDb");
|
||||
if (plan.output.kind === "framemd5") int(plan.output.frames, "output.frames", 1, 600);
|
||||
}
|
||||
|
||||
const RAW_IN = ["-thread_queue_size", "64", "-probesize", "32", "-analyzeduration", "0"];
|
||||
|
||||
function rawVideoInput(canvas: Canvas, fdNum: number): string[] {
|
||||
return [...RAW_IN, "-f", "rawvideo", "-pix_fmt", "yuv420p", "-s", `${canvas.width}x${canvas.height}`, "-framerate", String(canvas.fps), "-i", `pipe:${fdNum}`];
|
||||
}
|
||||
|
||||
function rawAudioInput(fdNum: number): string[] {
|
||||
return [...RAW_IN, "-f", "s16le", "-ar", String(GATE_SAMPLE_RATE), "-ac", String(GATE_CHANNELS), "-i", `pipe:${fdNum}`];
|
||||
}
|
||||
|
||||
function syntheticVideo(canvas: Canvas, pattern: SyntheticPattern, seed: number): string {
|
||||
// Seeded temporal noise makes every source distinct and every frame different, deterministically.
|
||||
return `${pattern}=s=${canvas.width}x${canvas.height}:r=${canvas.fps},noise=alls=12:allf=t+u:all_seed=${seed},format=yuv420p`;
|
||||
}
|
||||
|
||||
/** Full compositor argv. The process reads N sources (+ assets) and writes the gate FIFOs on fds 1/3. */
|
||||
export function buildCompositorArgs(plan: CompositorPlan): string[] {
|
||||
validatePlan(plan);
|
||||
const { canvas } = plan.graph;
|
||||
// Single-threaded filtering keeps golden frames deterministic and per-compositor CPU predictable.
|
||||
// (Slice threads were measured in the S2a CPU smoke and did not change its bus drops.)
|
||||
// eslint-disable-next-line no-restricted-syntax -- FFmpeg's -threads options, not the Threads provider
|
||||
const args = ["-hide_banner", "-nostdin", "-loglevel", "error", "-threads", "1", "-filter_threads", "1", "-filter_complex_threads", "1"];
|
||||
let next = 0;
|
||||
const video: number[] = [];
|
||||
const audio: Array<number | null> = [];
|
||||
const synthetic = plan.output.kind === "framemd5";
|
||||
for (const s of plan.sources) {
|
||||
if (s.video.kind === "rawPipe") args.push(...rawVideoInput(canvas, s.video.fd));
|
||||
else args.push("-f", "lavfi", "-i", syntheticVideo(canvas, s.video.pattern, s.video.seed));
|
||||
video.push(next++);
|
||||
if (s.audio === null) { audio.push(null); continue; }
|
||||
if (s.audio.kind === "rawPipe") args.push(...rawAudioInput(s.audio.fd));
|
||||
else args.push("-f", "lavfi", "-i", `sine=frequency=${s.audio.toneHz}:sample_rate=${GATE_SAMPLE_RATE}`);
|
||||
audio.push(next++);
|
||||
}
|
||||
const assets: number[] = [];
|
||||
for (const path of plan.assets) {
|
||||
// One decoded frame, held by overlay's eof_action=repeat. A `-loop 1` image stream would be
|
||||
// decoded every frame and framesync'd against the live main input, which jitters the main.
|
||||
args.push("-framerate", String(canvas.fps), "-i", `file:${path}`);
|
||||
assets.push(next++);
|
||||
}
|
||||
args.push("-filter_complex", buildFilterGraph(plan, { video, audio, assets }));
|
||||
if (plan.output.kind === "gate") {
|
||||
args.push("-map", "[vout]", "-f", "rawvideo", "-pix_fmt", "yuv420p", "-fps_mode", "passthrough", "-flush_packets", "1", "pipe:1",
|
||||
"-map", "[aout]", "-f", "s16le", "-ar", String(GATE_SAMPLE_RATE), "-ac", String(GATE_CHANNELS), "-flush_packets", "1", "pipe:3");
|
||||
} else {
|
||||
void synthetic;
|
||||
const seconds = (plan.output.frames / canvas.fps).toFixed(6);
|
||||
args.push("-map", "[vout]", "-map", "[aout]", "-t", seconds, "-c:v", "rawvideo", "-c:a", "pcm_s16le", "-fflags", "+bitexact", "-flags:v", "+bitexact", "-flags:a", "+bitexact", "-f", "framemd5", "pipe:1");
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Ingest ×N normalize stage
|
||||
|
||||
export type IngestInput =
|
||||
/** Live MPEG-TS from the pinned egress relay on stdin (WP7). */
|
||||
| { kind: "mpegtsPipe" }
|
||||
/**
|
||||
* Playlist source (§5.1): the asset feeder's concatenated MPEG-TS on stdin, read at native rate
|
||||
* (`-re`; assets arrive far faster than real time). The feeder shifts each asset to where the
|
||||
* previous one ended, so timestamps are monotonic and the normalize stage and the gate hide
|
||||
* asset boundaries. Audio is optional (a silent asset gets the bus's silence fill).
|
||||
*/
|
||||
| { kind: "playlistPipe" }
|
||||
/** Deterministic test source, paced at native rate. */
|
||||
| { kind: "synthetic"; pattern: SyntheticPattern; seed: number; toneHz: number };
|
||||
|
||||
export interface IngestPlan {
|
||||
input: IngestInput;
|
||||
canvas: Canvas;
|
||||
/** "none" when the source is known to have no audio track: silence is generated instead. */
|
||||
audio: "source" | "none";
|
||||
}
|
||||
|
||||
/**
|
||||
* One process per source: decode → scale/pad to the canvas, CFR at the canvas
|
||||
* fps, yuv420p; audio → 48 kHz s16le stereo. Video on fd 1, PCM on fd 3.
|
||||
*/
|
||||
export function buildIngestArgs(plan: IngestPlan): string[] {
|
||||
const { canvas, input } = plan;
|
||||
assertGeometry(canvas);
|
||||
// Auto frame-threading scales with host CPU count and adds almost a second of decode
|
||||
// latency on a 28-core host. Independently warming ingests then hold different amounts
|
||||
// of history in the compositor inputs. Bound decode/filter latency and emit raw frames
|
||||
// synchronously instead of frame-threading the rawvideo encoder as well.
|
||||
// eslint-disable-next-line no-restricted-syntax -- FFmpeg options, not the Threads provider
|
||||
const args = ["-hide_banner", "-nostdin", "-loglevel", "error", "-threads", "2", "-filter_threads", "1"];
|
||||
if (input.kind === "mpegtsPipe") {
|
||||
args.push("-fflags", "+genpts+discardcorrupt", "-f", "mpegts", "-i", "pipe:0");
|
||||
} else if (input.kind === "playlistPipe") {
|
||||
args.push("-re", "-fflags", "+genpts+discardcorrupt", "-f", "mpegts", "-i", "pipe:0");
|
||||
} else {
|
||||
if (!SYNTHETIC_PATTERNS.includes(input.pattern)) fail("ingest pattern");
|
||||
int(input.seed, "ingest seed", 0, 1_000_000);
|
||||
int(input.toneHz, "ingest toneHz", 20, 20_000);
|
||||
args.push("-re", "-f", "lavfi", "-i", syntheticVideo(canvas, input.pattern, input.seed),
|
||||
"-re", "-f", "lavfi", "-i", `sine=frequency=${input.toneHz}:sample_rate=${GATE_SAMPLE_RATE}`);
|
||||
}
|
||||
const audioIndex = input.kind === "synthetic" ? 1 : 0;
|
||||
if (plan.audio === "none") args.push("-f", "lavfi", "-i", `anullsrc=r=${GATE_SAMPLE_RATE}:cl=stereo`);
|
||||
const silenceIndex = input.kind === "synthetic" ? 2 : 1;
|
||||
const sourceAudio = input.kind === "playlistPipe" ? `${audioIndex}:a:0?` : `${audioIndex}:a:0`;
|
||||
const W = canvas.width;
|
||||
const H = canvas.height;
|
||||
args.push(
|
||||
"-map", "0:v:0",
|
||||
"-vf", `scale=${W}:${H}:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=${W}:${H}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=${canvas.fps},format=yuv420p`,
|
||||
// eslint-disable-next-line no-restricted-syntax -- FFmpeg rawvideo thread option, not the Threads provider
|
||||
"-c:v", "rawvideo", "-threads:v", "1", "-f", "rawvideo", "-pix_fmt", "yuv420p", "-flush_packets", "1", "pipe:1",
|
||||
"-map", plan.audio === "none" ? `${silenceIndex}:a:0` : sourceAudio,
|
||||
"-af", `aresample=${GATE_SAMPLE_RATE}:async=1000,${AUDIO_FORMAT}`,
|
||||
"-f", "s16le", "-ar", String(GATE_SAMPLE_RATE), "-ac", String(GATE_CHANNELS), "-flush_packets", "1", "pipe:3",
|
||||
);
|
||||
return args;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Persistent encoder
|
||||
|
||||
export type EncoderOutput = { kind: "flvPipe"; fd: number } | { kind: "null" };
|
||||
|
||||
/**
|
||||
* Program preview tap (§2.1: "the program tap is the same mechanism applied to the gate output"):
|
||||
* the encoder also emits what it encodes, downscaled to `size`, as raw yuv420p + s16le on two
|
||||
* extra fds that a source-bus helper reads. Each tap output is a `tee` with `onfail=ignore` plus a
|
||||
* `null` anchor slave: a dead tap reader closes only that slave ("continuing with 1/2 slaves"),
|
||||
* never the encoder (with a single slave ffmpeg 5.1 fails the whole output, verified in S2b).
|
||||
*/
|
||||
export interface EncoderProgramTap { videoFd: number; audioFd: number; size: { width: number; height: number } }
|
||||
|
||||
export interface EncoderPlan { canvas: Canvas; videoBitrateKbps: number; audioBitrateKbps: number; output: EncoderOutput; programTap?: EncoderProgramTap }
|
||||
|
||||
/** Reads the gate on fds 0 (video) / 3 (audio). Started once per channel; scene changes never restart it. */
|
||||
export function buildEncoderArgs(plan: EncoderPlan): string[] {
|
||||
const { canvas, programTap: tap } = plan;
|
||||
assertGeometry(canvas);
|
||||
const vb = int(plan.videoBitrateKbps, "videoBitrateKbps", 200, 50_000);
|
||||
const ab = int(plan.audioBitrateKbps, "audioBitrateKbps", 32, 320);
|
||||
const args = ["-hide_banner", "-nostdin", "-loglevel", "error", ...rawVideoInput(canvas, 0), ...rawAudioInput(3)];
|
||||
if (tap) {
|
||||
const w = int(tap.size.width, "programTap.width", 2, canvas.width);
|
||||
const h = int(tap.size.height, "programTap.height", 2, canvas.height);
|
||||
if (w % 2 || h % 2) fail("programTap size must be even");
|
||||
args.push("-filter_complex", `[0:v]split=2[enc][tap];[tap]scale=${w}:${h}:flags=bilinear,format=yuv420p[tapv];[1:a]asplit=2[enca][tapa]`,
|
||||
"-map", "[enc]", "-map", "[enca]");
|
||||
} else {
|
||||
args.push("-map", "0:v", "-map", "1:a");
|
||||
}
|
||||
args.push(
|
||||
"-c:v", "libx264", "-preset", "veryfast", "-tune", "zerolatency", "-pix_fmt", "yuv420p",
|
||||
"-b:v", `${vb}k`, "-maxrate", `${vb}k`, "-bufsize", `${vb * 2}k`, "-g", String(canvas.fps * 2), "-keyint_min", String(canvas.fps * 2), "-sc_threshold", "0",
|
||||
"-c:a", "aac", "-b:a", `${ab}k`, "-ar", String(GATE_SAMPLE_RATE));
|
||||
if (plan.output.kind === "flvPipe") args.push("-f", "flv", `pipe:${fd(plan.output.fd, "output.fd")}`);
|
||||
else args.push("-f", "null", "-");
|
||||
if (tap) {
|
||||
const v = fd(tap.videoFd, "programTap.videoFd");
|
||||
const a = fd(tap.audioFd, "programTap.audioFd");
|
||||
args.push("-map", "[tapv]", "-c:v", "rawvideo", "-f", "tee", `[f=rawvideo:onfail=ignore]pipe:${v}|[f=null]-`,
|
||||
"-map", "[tapa]", "-c:a", "pcm_s16le", "-f", "tee", `[f=s16le:onfail=ignore]pipe:${a}|[f=null]-`);
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Playlist feeder (§5.1)
|
||||
|
||||
/** Output timestamp offset: microseconds, rendered as seconds (bounded: ~1 year of continuous playout). */
|
||||
function tsOffset(offsetUs: number): string {
|
||||
return (int(offsetUs, "offsetUs", 0, 365 * 86_400 * 1_000_000) / 1e6).toFixed(6);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remux of one playlist asset (pre-normalized at upload, so never transcoded) from stdin to MPEG-TS
|
||||
* on stdout, with its timestamps shifted to `offsetUs` (where the previous asset ended), so the
|
||||
* concatenated stream is monotonic. FFmpeg 5.1's own MPEG-TS discontinuity correction does not
|
||||
* work with `-re` (measured: 1.3–2 s stalls and bursts at every boundary). `-progress pipe:3`
|
||||
* reports the asset's end (`out_time_us`, excluding the offset) for the next asset's offset.
|
||||
* `initial_discontinuity` marks the continuity-counter restart as legitimate.
|
||||
*/
|
||||
export function buildPlaylistRemuxArgs(offsetUs: number): string[] {
|
||||
return ["-hide_banner", "-nostdin", "-loglevel", "error", "-progress", "pipe:3", "-protocol_whitelist", "pipe", "-i", "pipe:0",
|
||||
"-map", "0:v:0", "-map", "0:a:0?", "-c", "copy", "-output_ts_offset", tsOffset(offsetUs),
|
||||
"-f", "mpegts", "-mpegts_flags", "+initial_discontinuity", "-muxdelay", "0", "pipe:1"];
|
||||
}
|
||||
|
||||
/**
|
||||
* Slate floor (§5.1): `seconds` of a still (the rendered slate PNG when there is one, else black)
|
||||
* plus silence as MPEG-TS at `offsetUs`, fed when no asset can be played or a non-looping playlist
|
||||
* has ended. Encoded faster than real time and paced by the playlist ingest like any asset.
|
||||
*/
|
||||
export function buildSlateArgs(plan: { canvas: Canvas; seconds: number; image: string | null; offsetUs: number }): string[] {
|
||||
const { canvas } = plan;
|
||||
assertGeometry(canvas);
|
||||
const seconds = int(plan.seconds, "slate seconds", 1, 60);
|
||||
const args = ["-hide_banner", "-nostdin", "-loglevel", "error", "-progress", "pipe:3"];
|
||||
if (plan.image) args.push("-loop", "1", "-framerate", String(canvas.fps), "-i", `file:${assetPath(plan.image, "slate image")}`);
|
||||
else args.push("-f", "lavfi", "-i", `color=c=black:s=${canvas.width}x${canvas.height}:r=${canvas.fps}`);
|
||||
args.push("-f", "lavfi", "-i", `anullsrc=r=${GATE_SAMPLE_RATE}:cl=stereo`, "-t", String(seconds),
|
||||
"-vf", `scale=${canvas.width}:${canvas.height}:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=${canvas.width}:${canvas.height}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,format=yuv420p`,
|
||||
"-c:v", "libx264", "-preset", "ultrafast", "-tune", "stillimage", "-g", String(canvas.fps), "-b:v", "500k",
|
||||
"-c:a", "aac", "-b:a", "64k", "-output_ts_offset", tsOffset(plan.offsetUs),
|
||||
"-f", "mpegts", "-mpegts_flags", "+initial_discontinuity", "-muxdelay", "0", "pipe:1");
|
||||
return args;
|
||||
}
|
||||
|
||||
export const MediaCommandBuilder = {
|
||||
layoutRects, rectFromBasisPoints, buildFilterGraph,
|
||||
compositor: buildCompositorArgs,
|
||||
ingest: buildIngestArgs,
|
||||
encoder: buildEncoderArgs,
|
||||
} as const;
|
||||
95
apps/media-worker/src/compositor/compositor.ts
Normal file
95
apps/media-worker/src/compositor/compositor.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import type { StdioOptions } from "node:child_process";
|
||||
import type { FrameGate } from "../gate/frame-gate";
|
||||
import { spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg";
|
||||
import { buildCompositorArgs, type CompositorGraph, type CompositorSource } from "./command-builder";
|
||||
import type { BusSubscription, SourceBus } from "./source-bus";
|
||||
|
||||
/** A scene ready to compose: buses in input order plus the typed graph (source indexes refer to `inputs`). */
|
||||
export interface ComposeRequest {
|
||||
inputs: Array<{ bus: SourceBus; audio: boolean }>;
|
||||
assets: string[];
|
||||
graph: CompositorGraph;
|
||||
}
|
||||
|
||||
const FIRST_INPUT_FD = 4;
|
||||
|
||||
/**
|
||||
* One compositor process (A or B of a swap). Reads its sources from bus
|
||||
* subscription FIFOs inherited on fds 4.., writes the gate's slot FIFOs on fds 1 and 3.
|
||||
* Its lifetime is bounded by the gate: when the gate retires the slot the FIFOs
|
||||
* close and the process gets EPIPE; `stop()` also escalates to SIGKILL.
|
||||
*/
|
||||
export class CompositorInstance {
|
||||
private proc: ManagedProcess | null = null;
|
||||
/** One per input, in request order (valid after `start()`). */
|
||||
subscriptions: BusSubscription[] = [];
|
||||
/** Gate origin sent at attach (see protocol.attachLine). */
|
||||
origin: number | undefined;
|
||||
slotId = 0;
|
||||
|
||||
constructor(
|
||||
private readonly gate: FrameGate,
|
||||
private readonly request: ComposeRequest,
|
||||
private readonly redactor: StderrRedactor,
|
||||
private readonly onStderr: (message: string) => void,
|
||||
) {}
|
||||
|
||||
get pid(): number | undefined { return this.proc?.child.pid; }
|
||||
|
||||
/**
|
||||
* Subscribes, spawns and attaches to the gate as the pending slot. `originFor`
|
||||
* maps the fresh subscriptions to a gate origin. Resolve ≠ on air: use `gate.waitActive(slotId)`.
|
||||
*/
|
||||
async start(originFor?: (subscriptions: BusSubscription[]) => number | undefined): Promise<void> {
|
||||
const slot = this.gate.allocateSlot();
|
||||
this.slotId = slot.id;
|
||||
try {
|
||||
for (const input of this.request.inputs) this.subscriptions.push(await input.bus.subscribe({ video: true, audio: input.audio }));
|
||||
} catch (error) {
|
||||
slot.fifos.releaseWriters();
|
||||
slot.fifos.releaseHolders();
|
||||
slot.fifos.unlink();
|
||||
await this.stop();
|
||||
throw error;
|
||||
}
|
||||
let next = FIRST_INPUT_FD;
|
||||
const inputFds: number[] = [];
|
||||
const sources: CompositorSource[] = this.subscriptions.map((sub) => {
|
||||
if (sub.videoFd === null) throw new Error("compositor input without video");
|
||||
inputFds.push(sub.videoFd);
|
||||
const video = { kind: "rawPipe" as const, fd: next++ };
|
||||
if (sub.audioFd === null) return { video, audio: null };
|
||||
inputFds.push(sub.audioFd);
|
||||
return { video, audio: { kind: "rawPipe" as const, fd: next++ } };
|
||||
});
|
||||
const args = buildCompositorArgs({ sources, assets: this.request.assets, graph: this.request.graph, output: { kind: "gate", videoFd: 1, audioFd: 3 } });
|
||||
const stdio: StdioOptions = ["ignore", slot.fifos.videoFd, "pipe", slot.fifos.audioFd, ...inputFds];
|
||||
let proc: ManagedProcess;
|
||||
try {
|
||||
proc = spawnManaged("ffmpeg", args, stdio, this.redactor, this.onStderr);
|
||||
} finally {
|
||||
slot.fifos.releaseWriters();
|
||||
// The compositor holds the read ends now: when it exits, the bus sees EPIPE and drops it.
|
||||
for (const sub of this.subscriptions) sub.releaseReaders();
|
||||
}
|
||||
this.proc = proc;
|
||||
try {
|
||||
await proc.started;
|
||||
this.origin = originFor?.(this.subscriptions);
|
||||
await this.gate.attach(slot, this.origin);
|
||||
} catch (error) {
|
||||
slot.fifos.releaseHolders();
|
||||
slot.fifos.unlink();
|
||||
await this.stop();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
for (const sub of this.subscriptions) sub.unsubscribe();
|
||||
await this.proc?.terminate();
|
||||
this.proc = null;
|
||||
}
|
||||
|
||||
kill(signal: NodeJS.Signals): void { this.proc?.kill(signal); }
|
||||
}
|
||||
206
apps/media-worker/src/compositor/cpu-smoke.ts
Normal file
206
apps/media-worker/src/compositor/cpu-smoke.ts
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
/**
|
||||
* S2 CPU budget and sustained transport gate:
|
||||
* 4 × 1080p30 H.264 MPEG-TS sources → ingest×4 (decode + normalize) → source-bus helpers →
|
||||
* 2x2 compositor with a lower third and a chyron → Rust frame gate → x264
|
||||
* veryfast 6 Mb/s persistent encoder (-f null). CPU per process from
|
||||
* /proc/<pid>/stat over the measurement window, plus the worker (Node) itself.
|
||||
*
|
||||
* node --import tsx src/compositor/cpu-smoke.ts [--seconds 30] [--swaps-every 0]
|
||||
* [--warmup-seconds 30] [--settle-max 45]
|
||||
*/
|
||||
import { closeSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { ManagedProcess } from "../relay/ffmpeg";
|
||||
import { layoutRects, type Canvas, type CompositorGraph } from "./command-builder";
|
||||
import type { ComposeRequest } from "./compositor";
|
||||
import { IngestProcess } from "./ingest";
|
||||
import { ProgramChain } from "./program";
|
||||
import { encodeSourceClip, ensureGateBinary, makeAssets, passthroughRedactor, startPacer } from "./testing";
|
||||
|
||||
const canvas: Canvas = { width: 1920, height: 1080, fps: 30 };
|
||||
const TICKS = 100; // USER_HZ
|
||||
|
||||
function cpuTicks(pid: number | undefined): number {
|
||||
if (pid === undefined) return 0;
|
||||
try {
|
||||
const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
|
||||
const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" ");
|
||||
return Number(fields[11]) + Number(fields[12]); // utime + stime
|
||||
} catch { return 0; }
|
||||
}
|
||||
|
||||
/** Time spent runnable but not running (host contention), ns, from /proc/<pid>/schedstat (all threads). */
|
||||
function runDelayNs(pid: number | undefined): number {
|
||||
if (pid === undefined) return 0;
|
||||
try {
|
||||
let total = 0;
|
||||
for (const tid of readdirSync(`/proc/${pid}/task`)) total += Number(readFileSync(`/proc/${pid}/task/${tid}/schedstat`, "utf8").split(" ")[1] ?? 0);
|
||||
return total;
|
||||
} catch { return 0; }
|
||||
}
|
||||
|
||||
function arg(name: string, fallback: number): number {
|
||||
const i = process.argv.indexOf(`--${name}`);
|
||||
return i >= 0 ? Number(process.argv[i + 1]) : fallback;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const seconds = arg("seconds", 30);
|
||||
const swapEvery = arg("swaps-every", 0);
|
||||
const minimumWarmup = arg("warmup-seconds", 30);
|
||||
const settleCap = arg("settle-max", 45);
|
||||
if (!Number.isFinite(seconds) || seconds <= 0 || !Number.isFinite(swapEvery) || swapEvery < 0
|
||||
|| !Number.isFinite(minimumWarmup) || minimumWarmup < 0 || !Number.isFinite(settleCap) || settleCap < Math.max(3, minimumWarmup)) {
|
||||
throw new Error("invalid smoke duration, swap interval, or warmup/settle bounds");
|
||||
}
|
||||
if (!ensureGateBinary()) throw new Error("native helpers unavailable (cargo build failed)");
|
||||
const dir = mkdtempSync(join(tmpdir(), "cpu-smoke-"));
|
||||
const procs: ManagedProcess[] = [];
|
||||
const ingests: IngestProcess[] = [];
|
||||
let chain: ProgramChain | null = null;
|
||||
try {
|
||||
// A 1080p30 H.264 + AAC source clip (encoded once, before measuring).
|
||||
const clip = join(dir, "src.ts");
|
||||
encodeSourceClip(clip, canvas);
|
||||
const assets = makeAssets(dir);
|
||||
|
||||
for (let i = 0; i < 4; i += 1) {
|
||||
// Stand-in for the pinned egress relay: a paced remux of the clip into a FIFO the ingest reads.
|
||||
const { readFd, proc: pacer } = await startPacer(clip, dir, `src${i}`);
|
||||
procs.push(pacer);
|
||||
const ingest = new IngestProcess(`s${i}`, { input: { kind: "mpegtsPipe" }, canvas, audio: "source" }, readFd, join(dir, "gate"), passthroughRedactor,
|
||||
(m) => process.stderr.write(m), (id, r) => process.stderr.write(`${id} ${r}\n`));
|
||||
await ingest.start();
|
||||
closeSync(readFd);
|
||||
ingests.push(ingest);
|
||||
}
|
||||
|
||||
const graph = (i: number): CompositorGraph => ({
|
||||
canvas, layout: "2x2",
|
||||
tiles: layoutRects("2x2", canvas).map((rect, t) => ({ source: (t + i) % 4, rect, fit: "contain" as const })),
|
||||
overlays: [{ kind: "image", asset: 0, x: 96, y: 900 }, { kind: "chyron", asset: 1, y: 1040, speedPxPerSec: 120 }],
|
||||
audio: { inputs: [{ source: 0, gainDb: 0 }, { source: 1, gainDb: -6 }], masterGainDb: 0 },
|
||||
});
|
||||
const request = (i: number): ComposeRequest => ({ inputs: ingests.map((ing, k) => ({ bus: ing.bus, audio: k < 2 })), assets, graph: graph(i) });
|
||||
const fatal: string[] = [];
|
||||
chain = new ProgramChain({ canvas, scratchDir: join(dir, "gate"), videoBitrateKbps: 6000, output: { kind: "null" },
|
||||
redactor: passthroughRedactor, onStderr: (m) => process.stderr.write(m), onFatal: (r) => fatal.push(r) });
|
||||
const tStart = Date.now();
|
||||
await chain.start(request(0));
|
||||
const warmupEnd = Date.now() + minimumWarmup * 1000;
|
||||
// Warm-up: the first seconds are a startup transient on a loaded host (4 ingests bursting to
|
||||
// catch up with their sources, compositor and x264 warming up), during which the compositor's
|
||||
// bus queues stand near the backlog limit and the gate repeats. Measure the steady state: wait
|
||||
// until every subscriber queue is within its configured backlog + replay history
|
||||
// and neither bus drops nor gate fills increase for 3 consecutive seconds. Allow at least
|
||||
// 30 s for demux/decoder queues and x264 to warm, at most 45 s by default. Startup loss
|
||||
// stays visible separately; the sustained window begins only after this declared warmup.
|
||||
// A queue can remain nonempty at native rate: it reflects fixed pipeline/replay delay,
|
||||
// not overload. Absence of drops/fills at bounded depth is the steady-state criterion.
|
||||
const backlogLimit = Math.min(...ingests.map((ingest) => ingest.bus.maxBacklogFrames + ingest.bus.historyFrames));
|
||||
let calm = 0;
|
||||
const initialGate = await chain.stats();
|
||||
let lastFills = initialGate.vrep + initialGate.asil;
|
||||
let lastDrops = (await Promise.all(ingests.map((ingest) => ingest.bus.stats()))).reduce((total, bus) => total + bus.subscriberDrops, 0);
|
||||
const settleTrace: Array<[number, number, number]> = [];
|
||||
for (let s = 0; s < settleCap && (calm < 3 || Date.now() < warmupEnd); s += 1) {
|
||||
await new Promise((r) => setTimeout(r, 1_000));
|
||||
const buses = await Promise.all(ingests.map((i) => i.bus.stats()));
|
||||
const q = Math.max(0, ...buses.flatMap((bus) => bus.subscribers.map((sub) => sub.queued)));
|
||||
const drops = buses.reduce((total, bus) => total + bus.subscriberDrops, 0);
|
||||
const gate = await chain.stats();
|
||||
const fills = gate.vrep + gate.asil;
|
||||
settleTrace.push([q, fills - lastFills, drops - lastDrops]);
|
||||
calm = q <= backlogLimit && fills === lastFills && drops === lastDrops ? calm + 1 : 0;
|
||||
lastFills = fills;
|
||||
lastDrops = drops;
|
||||
}
|
||||
const settleS = (Date.now() - tStart) / 1000;
|
||||
const startupDrops = (await Promise.all(ingests.map((i) => i.bus.stats()))).reduce((a, b) => a + b.subscriberDrops, 0);
|
||||
|
||||
const pids = {
|
||||
ingest: ingests.map((i) => i.pid),
|
||||
compositor: () => chain?.activeCompositor?.pid,
|
||||
gate: chain.gate.pid,
|
||||
encoder: chain.encoderPid,
|
||||
};
|
||||
const busPids = ingests.map((i) => i.bus.pid);
|
||||
const snap = async () => {
|
||||
const buses = await Promise.all(ingests.map((i) => i.bus.stats()));
|
||||
return {
|
||||
ingest: pids.ingest.map(cpuTicks).reduce((a, b) => a + b, 0),
|
||||
compositor: cpuTicks(pids.compositor()),
|
||||
gate: cpuTicks(pids.gate),
|
||||
encoder: cpuTicks(pids.encoder),
|
||||
buses: busPids.map(cpuTicks).reduce((a, b) => a + b, 0),
|
||||
worker: process.cpuUsage(),
|
||||
busDrops: buses.reduce((a, b) => a + b.subscriberDrops, 0),
|
||||
busPairs: buses.map((b) => b.pairs),
|
||||
busStats: buses,
|
||||
delay: { compositor: runDelayNs(pids.compositor()), encoder: runDelayNs(pids.encoder), gate: runDelayNs(pids.gate) },
|
||||
};
|
||||
};
|
||||
await chain.gate.resetPeaks();
|
||||
const g0 = await chain.stats();
|
||||
const s0 = await snap();
|
||||
const t0 = Date.now();
|
||||
let swaps = 0;
|
||||
// Deepest subscriber queue seen per second (frames waiting in a source-bus helper for a consumer).
|
||||
const queueMax: number[] = [];
|
||||
while (Date.now() - t0 < seconds * 1000) {
|
||||
await new Promise((r) => setTimeout(r, swapEvery > 0 ? swapEvery * 1000 : 1000));
|
||||
if (swapEvery > 0) { await chain.swap(request(++swaps)); }
|
||||
const depths = (await Promise.all(ingests.map((i) => i.bus.stats()))).flatMap((b) => b.subscribers.map((s) => s.queued));
|
||||
queueMax.push(Math.max(0, ...depths));
|
||||
}
|
||||
const s1 = await snap();
|
||||
const g1 = await chain.stats();
|
||||
const wall = (Date.now() - t0) / 1000;
|
||||
const pct = (ticks: number): number => Number(((ticks / TICKS / wall) * 100).toFixed(1));
|
||||
const workerUs = (s1.worker.user + s1.worker.system) - (s0.worker.user + s0.worker.system);
|
||||
const result = {
|
||||
host: { cpus: (await import("node:os")).cpus().length, model: (await import("node:os")).cpus()[0]?.model, loadavg: (await import("node:os")).loadavg() },
|
||||
wallS: wall, swaps,
|
||||
// Seconds from chain start to steady state, the per-second [max queue, gate repeats] trace, and drops before the window.
|
||||
settleS, minimumWarmupS: minimumWarmup, settled: calm >= 3 && Date.now() >= warmupEnd, settleTrace, startupDrops,
|
||||
startupGate: { ticks: g0.ticks, vrep: g0.vrep, asil: g0.asil, outDrops: g0.outDrops },
|
||||
pctOfOneCore: {
|
||||
ingestX4: pct(s1.ingest - s0.ingest),
|
||||
compositor: swapEvery > 0 ? null : pct(s1.compositor - s0.compositor),
|
||||
gate: pct(s1.gate - s0.gate),
|
||||
encoderX264: pct(s1.encoder - s0.encoder),
|
||||
sourceBusX4: pct(s1.buses - s0.buses),
|
||||
workerNode: Number(((workerUs / 1e6 / wall) * 100).toFixed(1)),
|
||||
},
|
||||
// Pairs the buses dropped for a subscriber that fell behind, inside the window (a compositor not keeping up).
|
||||
busDropsInWindow: s1.busDrops - s0.busDrops,
|
||||
busQueueMaxPerSecond: queueMax,
|
||||
// Host contention: ms per second of wall time each process (all threads) sat runnable but not scheduled.
|
||||
runDelayMsPerS: {
|
||||
compositor: Number(((s1.delay.compositor - s0.delay.compositor) / 1e6 / wall).toFixed(1)),
|
||||
encoder: Number(((s1.delay.encoder - s0.delay.encoder) / 1e6 / wall).toFixed(1)),
|
||||
gate: Number(((s1.delay.gate - s0.delay.gate) / 1e6 / wall).toFixed(1)),
|
||||
},
|
||||
// Source pairs each bus produced inside the window (≈ 30 × wallS for a paced 30 fps source).
|
||||
busPairsInWindow: s1.busPairs.map((n, k) => n - (s0.busPairs[k] ?? 0)),
|
||||
gate: { ticks: g1.ticks - g0.ticks, vrep: g1.vrep - g0.vrep, asil: g1.asil - g0.asil, lateOverFrame: g1.lateOverFrame, lateMaxUs: g1.lateMaxUs, outDrops: g1.outDrops - g0.outDrops },
|
||||
buses: s1.busStats,
|
||||
fatal,
|
||||
};
|
||||
const p = result.pctOfOneCore;
|
||||
process.stdout.write(`${JSON.stringify({ ...result, totalCores: Number((((p.ingestX4 + (p.compositor ?? 0) + p.gate + p.encoderX264 + p.sourceBusX4 + p.workerNode) / 100)).toFixed(2)) })}\n`);
|
||||
if (!result.settled || result.busDropsInWindow !== 0 || result.gate.vrep !== 0 || result.gate.asil !== 0 || result.gate.outDrops !== 0 || fatal.length > 0
|
||||
|| result.busPairsInWindow.some((pairs) => pairs < canvas.fps * wall * 0.98)) {
|
||||
throw new Error("cpu-smoke failed: require settled 4×1080p30, zero bus drops, zero gate fills/output drops, and no fatal exit");
|
||||
}
|
||||
} finally {
|
||||
await chain?.stop();
|
||||
for (const i of ingests) await i.stop();
|
||||
for (const p of procs) await p.terminate(500);
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (require.main === module) void main().catch((error: unknown) => { process.stderr.write(`${String(error)}\n`); process.exit(1); });
|
||||
115
apps/media-worker/src/compositor/golden-frames.json
Normal file
115
apps/media-worker/src/compositor/golden-frames.json
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
{
|
||||
"ffmpeg": "ffmpeg version 5.1.9-0+deb12u1 Copyright (c) 2000-2026 the FFmpeg developers",
|
||||
"canvas": {
|
||||
"width": 320,
|
||||
"height": 180,
|
||||
"fps": 30
|
||||
},
|
||||
"frames": 4,
|
||||
"layouts": {
|
||||
"single": [
|
||||
"0:0:86400:473ec55d56d6cd3f85b31ef5f0b615b2",
|
||||
"1:0:4096:f858b65fe890efbc0ac6cf1e085a37d2",
|
||||
"1:1024:4096:e3ef895bb0e533a4a8a89d5a6ed843f6",
|
||||
"0:1:86400:80e9688e53079ff4623b78115e2b6b1c",
|
||||
"1:2048:4096:8da1b5054e4f15590b674af0c5239537",
|
||||
"1:3072:4096:18a3a0028363827191cd4d8885bc2b11",
|
||||
"0:2:86400:f2b6d37903c429f8ad6c4416056a8538",
|
||||
"1:4096:4096:aecafa4e5f0f3626cc00a9849cbcc3fa",
|
||||
"0:3:86400:5113494af070e29d7c5c6cbbbaa7a721",
|
||||
"1:5120:4096:749e60b205100820f0f493a79c17ee46",
|
||||
"1:6144:1024:f465c566bd65b8743119b970d5b9734e"
|
||||
],
|
||||
"pip": [
|
||||
"0:0:86400:ff2365f77b17a7437246001c3282af88",
|
||||
"1:0:4096:fd773b1ed90b3793d331bebf439d85a7",
|
||||
"1:1024:4096:e1a1133f44e0c84ae28eca0ac57333c9",
|
||||
"0:1:86400:9a238c82abca419758f64d31f3f5080e",
|
||||
"1:2048:4096:c7f8ec938afbe515acf2cc21ee07b398",
|
||||
"1:3072:4096:c99fc35fb2f9fb03c18e8ff04b5300a5",
|
||||
"0:2:86400:ef78085de2d4d6f6e20e266e40e7a07d",
|
||||
"1:4096:4096:62bc5a8e144fb37ec4e5a48fa0a3179a",
|
||||
"0:3:86400:956a390b4024b79289063d54abde9c07",
|
||||
"1:5120:4096:20dd40a44d6167e3d20631d72cca7021",
|
||||
"1:6144:1024:4aebc7614678f7837d5c2c42c7ae5049"
|
||||
],
|
||||
"2x1": [
|
||||
"0:0:86400:124ceb28ce6fb2adda306ebab85c020e",
|
||||
"1:0:4096:779110ff86bde4abc0c312c254c6a07b",
|
||||
"1:1024:4096:58bf6a6a073a28c148162edbfb1cb2f0",
|
||||
"0:1:86400:2a815118db500f1fe35af785761546d3",
|
||||
"1:2048:4096:b47c4007c2bd03d67892ed6c4d2e0ad2",
|
||||
"1:3072:4096:4c1fa084e7e962bdfccc6445944fe6bd",
|
||||
"0:2:86400:fb62f8882a43ace4e0a50f611ba5b011",
|
||||
"1:4096:4096:96331541b14dd07c95116914832e2646",
|
||||
"0:3:86400:3d22dd326e76b9ace9b605e67568c5c9",
|
||||
"1:5120:4096:8d09c0dd1dd3d94d2296b8b96a887d0f",
|
||||
"1:6144:1024:456a74bc6828cb74630049d3c3ede52b"
|
||||
],
|
||||
"2x2": [
|
||||
"0:0:86400:c1a86085e98caf0c6ddca6eecd76f996",
|
||||
"1:0:4096:f2e48ac5003bebe6980227d7984f03ee",
|
||||
"1:1024:4096:c42c6c8bd7e38a04da330614958e8ec7",
|
||||
"0:1:86400:eacfcee60a815e7098d9f2678ef3a2b8",
|
||||
"1:2048:4096:07d15edac8583afe4f0c37062f8e2e95",
|
||||
"1:3072:4096:2eeb48f1b0671b51871cca079144b9a0",
|
||||
"0:2:86400:a69202dd6be84938c0669cfb1b1aea91",
|
||||
"1:4096:4096:86b18ee36947048c70bacd00707025d0",
|
||||
"0:3:86400:339a15c6489b3af66d4cbe4311179d0f",
|
||||
"1:5120:4096:e10e40ebb313c090b2679d523299c4f4",
|
||||
"1:6144:1024:2f3a2ef3cc8c5f0206828e53fe002d1b"
|
||||
],
|
||||
"3x3": [
|
||||
"0:0:86400:dfffffffbc45583776628ba9e07d2ac7",
|
||||
"1:0:4096:95afee33956e36002cb7bc4174d41fb8",
|
||||
"1:1024:4096:4cc1e8ca719bd34775c4bc5f0f1d1149",
|
||||
"0:1:86400:495bf533902abce01a4c6213275d4a8a",
|
||||
"1:2048:4096:39a06315fe075c8d02500f7b69f9d0b1",
|
||||
"1:3072:4096:28a2f2f96cbb5947d1ad1bf6f0f656d3",
|
||||
"0:2:86400:2b721e57614c93bc1a50f486317d3a69",
|
||||
"1:4096:4096:f80381b7c6e59ac4ed8cf350cdf675ae",
|
||||
"0:3:86400:8e738a80e6b6a24d2c97f0a4a819e8b4",
|
||||
"1:5120:4096:47905d32edded2c3b98653ac97c9b92e",
|
||||
"1:6144:1024:7cdba1f0765decdaa01239f8d415ce83"
|
||||
],
|
||||
"4x4": [
|
||||
"0:0:86400:9b5bc219325405095ad5c3ed5acc1ea1",
|
||||
"1:0:4096:620f0b67a91f7f74151bc5be745b7110",
|
||||
"1:1024:4096:620f0b67a91f7f74151bc5be745b7110",
|
||||
"0:1:86400:f96bf1c80e7dfac31e8e612139c0ad40",
|
||||
"1:2048:4096:620f0b67a91f7f74151bc5be745b7110",
|
||||
"1:3072:4096:620f0b67a91f7f74151bc5be745b7110",
|
||||
"0:2:86400:0b6280a6f2b063114eb892b8fa2ef406",
|
||||
"1:4096:4096:620f0b67a91f7f74151bc5be745b7110",
|
||||
"0:3:86400:73f848d7c873b09ef45467fe2a8b5785",
|
||||
"1:5120:4096:620f0b67a91f7f74151bc5be745b7110",
|
||||
"1:6144:1024:0f343b0931126a20f133d67c2b018a3b"
|
||||
],
|
||||
"custom": [
|
||||
"0:0:86400:7151990b9ab56564d8f6f1ab63edd9ee",
|
||||
"1:0:4096:8904df5033d62a7340b98e03dea71359",
|
||||
"1:1024:4096:9fa5c863b67d9831fa7bc93cbe868e4e",
|
||||
"0:1:86400:ccfdc3ae531e8d495aa67e2989dd8e72",
|
||||
"1:2048:4096:5602eff96d58dd2c18365237b01aeac5",
|
||||
"1:3072:4096:d6702b18d7e15f6ead23e77cdb3b7be0",
|
||||
"0:2:86400:9725c1c13fccbc094cad0c2ef683fbab",
|
||||
"1:4096:4096:3090fcea0f7513e35c03b80a199925be",
|
||||
"0:3:86400:019eb04e7297c7bae18d86f47ebbfee7",
|
||||
"1:5120:4096:3bbc5f09a7275706e87668d3ccd96793",
|
||||
"1:6144:1024:2642090512e9f3b93a215ba0497dcbd8"
|
||||
],
|
||||
"custom-overlap": [
|
||||
"0:0:86400:5c78229f51b9307615a1f9e442efe391",
|
||||
"1:0:4096:2cc5270f68b8a25c1909d435272422b4",
|
||||
"1:1024:4096:3c83cd930cd96b9da3e3f29ce369adf9",
|
||||
"0:1:86400:0cd6d909f4b84e42f6ac6360f1bda3f6",
|
||||
"1:2048:4096:f6e469bc77eaa6d1042c70e91450b4ab",
|
||||
"1:3072:4096:f5e8ee7c276f02215053e8143e65861d",
|
||||
"0:2:86400:fdff300a63db4e8bb5d9d7fea498ea59",
|
||||
"1:4096:4096:d9af646ccccb557d6a065868cf2fa667",
|
||||
"0:3:86400:aeaecd38cc4cbd0f900e8a1cec209962",
|
||||
"1:5120:4096:f22c40c3beb84a93a61e834d0daad01a",
|
||||
"1:6144:1024:e93ffc6d18f786a744a9b1ac241ec306"
|
||||
]
|
||||
}
|
||||
}
|
||||
59
apps/media-worker/src/compositor/ingest.ts
Normal file
59
apps/media-worker/src/compositor/ingest.ts
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
import { spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg";
|
||||
import { buildIngestArgs, type IngestPlan } from "./command-builder";
|
||||
import { SourceBus } from "./source-bus";
|
||||
|
||||
/**
|
||||
* Ingest ×N normalize stage: one ffmpeg per source decodes and normalizes to
|
||||
* the canvas (raw yuv420p on fd 1, s16le 48 kHz stereo on fd 3) straight into
|
||||
* its source bus's FIFOs. Compositors and preview taps subscribe to buses; they
|
||||
* never touch the remote, and the raw frames never pass through the worker.
|
||||
*/
|
||||
export class IngestProcess {
|
||||
readonly bus: SourceBus;
|
||||
private proc: ManagedProcess | null = null;
|
||||
private stopping = false;
|
||||
|
||||
constructor(
|
||||
readonly sourceId: string,
|
||||
private readonly plan: IngestPlan,
|
||||
/** MPEG-TS fd: the WP7 pinned relay (`mpegtsPipe`) or the playlist feeder (`playlistPipe`); null for synthetic. */
|
||||
private readonly stdinFd: number | null,
|
||||
scratchDir: string,
|
||||
private readonly redactor: StderrRedactor,
|
||||
private readonly onStderr: (message: string) => void,
|
||||
private readonly onExit: (sourceId: string, reason: string) => void,
|
||||
) {
|
||||
// Native fan-out can retain a bounded half-second of live backlog in addition to
|
||||
// replay history. With FIFO resizing denied on shared hosts, the replay/input delay
|
||||
// otherwise leaves too little room for short decoder/filter scheduling bursts.
|
||||
this.bus = new SourceBus({ geometry: plan.canvas, scratchDir, redactor, onStderr,
|
||||
maxBacklogFrames: Math.max(4, Math.ceil(plan.canvas.fps / 2)),
|
||||
onExit: (reason) => { if (!this.stopping) onExit(sourceId, reason); } });
|
||||
}
|
||||
|
||||
get pid(): number | undefined { return this.proc?.child.pid; }
|
||||
|
||||
async start(): Promise<void> {
|
||||
if ((this.plan.input.kind !== "synthetic") !== (this.stdinFd !== null)) throw new Error("ingest input and fd disagree");
|
||||
this.stopping = false;
|
||||
const out = await this.bus.start();
|
||||
try {
|
||||
const proc = spawnManaged("ffmpeg", buildIngestArgs(this.plan), [this.stdinFd ?? "ignore", out.videoFd, "pipe", out.audioFd], this.redactor, this.onStderr);
|
||||
this.proc = proc;
|
||||
void proc.exited.then(({ code, signal }) => { if (!this.stopping) this.onExit(this.sourceId, `ingest exited code=${code} signal=${signal}`); });
|
||||
await proc.started;
|
||||
} catch (error) {
|
||||
await this.stop();
|
||||
throw error;
|
||||
} finally {
|
||||
this.bus.releaseIngestWriters();
|
||||
}
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.stopping = true;
|
||||
await this.proc?.terminate();
|
||||
this.proc = null;
|
||||
await this.bus.stop();
|
||||
}
|
||||
}
|
||||
160
apps/media-worker/src/compositor/playlist-feeder.test.ts
Normal file
160
apps/media-worker/src/compositor/playlist-feeder.test.ts
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
/**
|
||||
* Playlist feeder (streaming-architecture.md §5.1): argv goldens, and real
|
||||
* ffmpeg runs of feeder → `playlistPipe` ingest → source bus, checking that
|
||||
* asset boundaries keep the bus at frame rate with A/V paired, that a broken
|
||||
* asset is skipped, that the slate floor covers an unplayable or ended list,
|
||||
* and that `update()` adopts a new list at the next asset boundary.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createReadStream, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
import test from "node:test";
|
||||
import { buildIngestArgs, buildPlaylistRemuxArgs, buildSlateArgs } from "./command-builder";
|
||||
import { IngestProcess } from "./ingest";
|
||||
import { PlaylistFeeder, type FeederEvent } from "./playlist-feeder";
|
||||
import { ensureGateBinary, passthroughRedactor } from "./testing";
|
||||
|
||||
const canvas = { width: 320, height: 180, fps: 30 };
|
||||
|
||||
test("playlist argv: copy-only remux from stdin at a monotonic offset, paced ingest, slate floor", () => {
|
||||
assert.deepEqual(buildPlaylistRemuxArgs(2_020_133), ["-hide_banner", "-nostdin", "-loglevel", "error", "-progress", "pipe:3", "-protocol_whitelist", "pipe", "-i", "pipe:0",
|
||||
"-map", "0:v:0", "-map", "0:a:0?", "-c", "copy", "-output_ts_offset", "2.020133",
|
||||
"-f", "mpegts", "-mpegts_flags", "+initial_discontinuity", "-muxdelay", "0", "pipe:1"]);
|
||||
assert.throws(() => buildPlaylistRemuxArgs(-1));
|
||||
assert.throws(() => buildPlaylistRemuxArgs(1.5));
|
||||
const ingest = buildIngestArgs({ input: { kind: "playlistPipe" }, canvas, audio: "source" });
|
||||
assert.deepEqual(ingest.slice(ingest.indexOf("-re"), ingest.indexOf("-re") + 8), ["-re", "-fflags", "+genpts+discardcorrupt", "-f", "mpegts", "-i", "pipe:0", "-map"]);
|
||||
assert.ok(ingest.includes("0:a:0?"), "a silent asset cannot fail the ingest");
|
||||
const slate = buildSlateArgs({ canvas, seconds: 5, image: null, offsetUs: 0 });
|
||||
assert.ok(slate.includes("color=c=black:s=320x180:r=30") && slate.includes("anullsrc=r=48000:cl=stereo") && slate.at(-1) === "pipe:1");
|
||||
const png = `/scratch/${"a".repeat(64)}.png`;
|
||||
assert.ok(buildSlateArgs({ canvas, seconds: 5, image: png, offsetUs: 7_000_000 }).includes(`file:${png}`));
|
||||
assert.ok(buildSlateArgs({ canvas, seconds: 5, image: null, offsetUs: 7_000_000 }).includes("7.000000"));
|
||||
assert.throws(() => buildSlateArgs({ canvas, seconds: 5, image: "/etc/passwd", offsetUs: 0 }));
|
||||
assert.throws(() => buildSlateArgs({ canvas, seconds: 0, image: null, offsetUs: 0 }));
|
||||
const all = [...buildPlaylistRemuxArgs(0), ...ingest, ...slate].join(" ");
|
||||
assert.ok(!/https?:|concat|movie=|tcp:|udp:/.test(all), "pipes and lavfi only");
|
||||
});
|
||||
|
||||
function clip(dir: string, name: string, seconds: number, hz: number): string {
|
||||
const path = join(dir, name);
|
||||
const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", `testsrc2=s=320x180:r=30:d=${seconds}`, "-f", "lavfi", "-i", `sine=frequency=${hz}:d=${seconds}`,
|
||||
"-c:v", "libx264", "-preset", "ultrafast", "-g", "30", "-c:a", "aac", "-movflags", "+faststart", path]);
|
||||
assert.equal(r.status, 0);
|
||||
return path;
|
||||
}
|
||||
|
||||
async function run(t: { skip(msg: string): void }, assetIds: string[], loop: boolean, files: Map<string, string>, seconds: number,
|
||||
during?: (feeder: PlaylistFeeder) => void): Promise<{ events: FeederEvent[]; samples: number[]; stats: { pairs: number; silenceFilled: number }; feeder: PlaylistFeeder; exits: string[] } | null> {
|
||||
if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return null; }
|
||||
const dir = mkdtempSync(join(tmpdir(), "feeder-"));
|
||||
const events: FeederEvent[] = [];
|
||||
const exits: string[] = [];
|
||||
const feeder = new PlaylistFeeder("src", { assetIds, loop }, {
|
||||
assets: { async open(id) { const path = files.get(id); if (!path) throw new Error("no such asset"); return id === "broken" ? Readable.from([Buffer.from("not media")]) : createReadStream(path); } },
|
||||
canvas, scratchDir: dir, redactor: passthroughRedactor, onStderr: () => undefined, onEvent: (e) => events.push(e), slateSeconds: 2,
|
||||
});
|
||||
const ingest = new IngestProcess("src", { input: { kind: "playlistPipe" }, canvas, audio: "source" }, feeder.start(), dir, passthroughRedactor, () => undefined, (_id, r) => exits.push(r));
|
||||
const samples: number[] = [];
|
||||
try {
|
||||
await ingest.start();
|
||||
feeder.releaseReader();
|
||||
// Measure playback once FFmpeg has probed the TS input and delivered a pair.
|
||||
// Process startup is bounded separately, rather than consuming the playback window.
|
||||
const startupDeadline = Date.now() + 10_000;
|
||||
while ((await ingest.bus.stats()).pairs === 0) {
|
||||
assert.ok(Date.now() < startupDeadline, "playlist ingest produced its first A/V pair within 10 s");
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
}
|
||||
const t0 = Date.now();
|
||||
while (Date.now() - t0 < seconds * 1000) {
|
||||
await new Promise((r) => setTimeout(r, 250));
|
||||
samples.push((await ingest.bus.stats()).pairs);
|
||||
if (during && samples.length === 4) during(feeder);
|
||||
}
|
||||
const stats = await ingest.bus.stats();
|
||||
return { events, samples, stats, feeder, exits };
|
||||
} finally {
|
||||
await feeder.stop();
|
||||
await ingest.stop();
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Longest stretch (in 250 ms samples) without a new pair once the stream is up. */
|
||||
function longestStall(samples: number[]): number {
|
||||
let worst = 0;
|
||||
let run = 0;
|
||||
const first = samples.findIndex((n) => n > 0);
|
||||
for (let i = Math.max(1, first + 1); i < samples.length; i += 1) {
|
||||
run = samples[i] === samples[i - 1] ? run + 1 : 0;
|
||||
worst = Math.max(worst, run);
|
||||
}
|
||||
return worst;
|
||||
}
|
||||
|
||||
test("feeder: asset boundaries stay at frame rate with A/V paired; a broken asset is skipped; the list loops", { timeout: 120_000 }, async (t) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "feeder-clips-"));
|
||||
try {
|
||||
const files = new Map([["a", clip(dir, "a.mp4", 2, 300)], ["b", clip(dir, "b.mp4", 1, 600)], ["broken", join(dir, "broken")]]);
|
||||
writeFileSync(join(dir, "broken"), "x");
|
||||
const r = await run(t, ["a", "broken", "b"], true, files, 7);
|
||||
if (!r) return;
|
||||
const played = r.events.filter((e): e is Extract<FeederEvent, { type: "asset" }> => e.type === "asset").map((e) => e.assetId);
|
||||
assert.deepEqual(played.slice(0, 4), ["a", "broken", "b", "a"], "in order, looping");
|
||||
assert.ok(r.events.some((e) => e.type === "asset-failed" && e.assetId === "broken"));
|
||||
assert.ok(!r.events.some((e) => e.type === "slate"), "one bad asset does not drop to the slate");
|
||||
assert.equal(r.stats.silenceFilled, 0, "audio continuous across boundaries");
|
||||
assert.ok(longestStall(r.samples) <= 1, `no stall at boundaries (samples ${r.samples.join(",")})`);
|
||||
const up = r.samples.findIndex((n) => n > 0);
|
||||
const rate = ((r.samples.at(-1) ?? 0) - (r.samples[up] ?? 0)) / ((r.samples.length - 1 - up) * 0.25);
|
||||
assert.ok(rate > 26 && rate < 36, `paced at the frame rate (${rate.toFixed(1)} pairs/s)`);
|
||||
assert.deepEqual(r.exits, []);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("feeder: nothing playable → slate floor (the bus keeps running); a finished non-looping list → slate", { timeout: 120_000 }, async (t) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "feeder-clips-"));
|
||||
try {
|
||||
writeFileSync(join(dir, "broken"), "x");
|
||||
const dead = await run(t, ["broken"], true, new Map([["broken", join(dir, "broken")]]), 4);
|
||||
if (!dead) return;
|
||||
assert.ok(dead.events.some((e) => e.type === "slate" && e.reason === "unavailable"));
|
||||
assert.ok(dead.stats.pairs >= 60, `slate keeps the source alive (${dead.stats.pairs} pairs)`);
|
||||
const once = await run(t, ["a"], false, new Map([["a", clip(dir, "a.mp4", 1, 300)]]), 4);
|
||||
assert.ok(once?.events.some((e) => e.type === "slate" && e.reason === "ended"));
|
||||
assert.equal(once?.feeder.ended, true);
|
||||
assert.ok(longestStall(once?.samples ?? []) <= 1, "the slate follows the last asset without a stall");
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("feeder: update() adopts a new list after the asset being fed (desired-state v2, no restart)", { timeout: 120_000 }, async (t) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "feeder-clips-"));
|
||||
try {
|
||||
const files = new Map([["a", clip(dir, "a.mp4", 2, 300)], ["b", clip(dir, "b.mp4", 1, 600)], ["c", clip(dir, "c.mp4", 1, 900)]]);
|
||||
// 1 s in, [a, b] becomes [c, a]. The feeder hands assets to the ingest ahead of playback (FIFO and
|
||||
// demuxer buffers: tens of ms at production bitrates, several of these tiny clips), so "the asset
|
||||
// being fed" is the boundary: after it, playback follows the new list from the position after it.
|
||||
let adoptedAfter: string | null = null;
|
||||
const r = await run(t, ["a", "b"], true, files, 12, (feeder) => {
|
||||
adoptedAfter = feeder.nowPlaying;
|
||||
feeder.update({ assetIds: ["c", "a"], loop: true });
|
||||
});
|
||||
if (!r) return;
|
||||
const order = r.events.filter((e) => e.type === "asset" || e.type === "adopted").map((e) => (e.type === "asset" ? e.assetId : "|"));
|
||||
const cut = order.indexOf("|");
|
||||
assert.ok(cut > 0, `adopted (events ${order.join(",")})`);
|
||||
const before = order.slice(0, cut);
|
||||
const after = order.slice(cut + 1);
|
||||
assert.deepEqual(before, ["a", "b", "a", "b", "a", "b"].slice(0, before.length), "the old list until the update");
|
||||
assert.ok(after.length >= 2, `the new list plays (${order.join(",")})`);
|
||||
// Position after the asset being fed: after "a" (index 1 in [c, a]) comes "c"; after "b" (gone) the list restarts at "c".
|
||||
assert.equal(after[0], "c", `after ${String(adoptedAfter)} the new list continues at c (${order.join(",")})`);
|
||||
assert.deepEqual(after.slice(0, 4), ["c", "a", "c", "a"].slice(0, Math.min(4, after.length)));
|
||||
assert.ok(longestStall(r.samples) <= 1, `no stall when a prepared asset is replaced (samples ${r.samples.join(",")})`);
|
||||
assert.deepEqual(r.exits, []);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
250
apps/media-worker/src/compositor/playlist-feeder.ts
Normal file
250
apps/media-worker/src/compositor/playlist-feeder.ts
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
/**
|
||||
* Playlist feeder (streaming-architecture.md §5.1): a `kind: "playlist"` source
|
||||
* becomes a continuous input like any other. The feeder walks the ordered asset
|
||||
* list, opens each asset (a presigned GET through the ingest egress relay),
|
||||
* remuxes it to MPEG-TS (pre-normalized at upload: copy only, never a
|
||||
* transcode) and pipes the remuxes one after another into a FIFO that one
|
||||
* `playlistPipe` ingest reads at native rate. FFmpeg reads only pipes (WP9);
|
||||
* no concat protocol.
|
||||
*
|
||||
* - Timestamps: each remux is shifted to where the previous asset ended (its
|
||||
* `-progress` end time), so the concatenated stream is monotonic. FFmpeg
|
||||
* 5.1's own discontinuity correction stalls `-re` for 1–2 s per boundary.
|
||||
* - Boundaries: two overlapped stages. Asset i+1's remux starts as soon as
|
||||
* asset i's remux has exited (its end time is then known), while asset i is
|
||||
* still being fed from its capture buffer (~4 MiB ahead, several seconds), so
|
||||
* the next asset's bytes are waiting when the current one ends: no gap.
|
||||
* - Failure: an asset that cannot be opened or remuxed is skipped. When a whole
|
||||
* pass of the list fails (or the list is empty), the slate floor plays for a
|
||||
* few seconds and the list is retried.
|
||||
* - End: `loop: false` plays the list once, then the slate for good.
|
||||
* - Desired-state v2: `update()` adopts a new asset list without a restart. The
|
||||
* asset on air finishes; playback continues after it in the new list (or at
|
||||
* the new list's start when it is no longer there). An already prepared next
|
||||
* asset that is no longer next is discarded.
|
||||
*/
|
||||
import { closeSync, constants, openSync, rmSync } from "node:fs";
|
||||
import { Socket } from "node:net";
|
||||
import { PassThrough, type Readable } from "node:stream";
|
||||
import { makeFifo, openBlockingReader } from "../gate/fifo";
|
||||
import { spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg";
|
||||
import { buildPlaylistRemuxArgs, buildSlateArgs, type Canvas } from "./command-builder";
|
||||
|
||||
/** Opens one playlist asset's bytes (WP7: presigned GET through the pinned ingest egress relay). */
|
||||
export interface PlaylistAssetSource { open(assetId: string): Promise<Readable> }
|
||||
|
||||
export interface PlaylistSpec { assetIds: string[]; loop: boolean }
|
||||
|
||||
export type FeederEvent =
|
||||
| { type: "asset"; assetId: string; index: number }
|
||||
| { type: "asset-failed"; assetId: string; reason: string }
|
||||
| { type: "slate"; reason: "unavailable" | "ended" }
|
||||
| { type: "adopted"; assetIds: number };
|
||||
|
||||
export interface PlaylistFeederDeps {
|
||||
assets: PlaylistAssetSource;
|
||||
canvas: Canvas;
|
||||
scratchDir: string;
|
||||
redactor: StderrRedactor;
|
||||
onStderr(message: string): void;
|
||||
onEvent?(event: FeederEvent): void;
|
||||
/** Rendered slate PNG (S4, overlay renderer); null = black. */
|
||||
slateImage?: string | null;
|
||||
slateSeconds?: number;
|
||||
}
|
||||
|
||||
/** Bytes of produced MPEG-TS held ahead of the FIFO per producer (~5 s at 6 Mb/s). */
|
||||
const CAPTURE_BYTES = 4 << 20;
|
||||
|
||||
/** One remux (or slate) run: its captured output and, once it exits, its end time. */
|
||||
interface Job {
|
||||
proc: ManagedProcess;
|
||||
output: Readable;
|
||||
/** Resolves when the producer exits: success, and the end time it reported (µs, excluding the offset). */
|
||||
done: Promise<{ ok: boolean; endUs: number; reason: string }>;
|
||||
input: Readable | null;
|
||||
cancelled: boolean;
|
||||
}
|
||||
|
||||
let nextFeeder = 1;
|
||||
|
||||
export class PlaylistFeeder {
|
||||
private spec: PlaylistSpec;
|
||||
private out: Socket | null = null;
|
||||
private reader: { fd: number; releaseHolder(): void } | null = null;
|
||||
private stopping = false;
|
||||
private running: Promise<void> = Promise.resolve();
|
||||
private current: { assetId: string; index: number } | null = null;
|
||||
private nextIndex = 0;
|
||||
/** Bumped by `update()`: a job prepared under an older list is discarded. */
|
||||
private generation = 0;
|
||||
private readonly jobs = new Set<Job>();
|
||||
private abortFeed: (() => void) | null = null;
|
||||
/** Assets started, assets failed, slate chunks fed. */
|
||||
readonly stats = { assets: 0, failures: 0, slates: 0 };
|
||||
ended = false;
|
||||
|
||||
constructor(readonly sourceId: string, spec: PlaylistSpec, private readonly deps: PlaylistFeederDeps) {
|
||||
this.spec = { assetIds: [...spec.assetIds], loop: spec.loop };
|
||||
}
|
||||
|
||||
get nowPlaying(): string | null { return this.current?.assetId ?? null; }
|
||||
|
||||
/**
|
||||
* Creates the FIFO and starts feeding. Returns the blocking read end for the ingest's stdin;
|
||||
* call `releaseReader()` once the ingest has it.
|
||||
*/
|
||||
start(): number {
|
||||
const path = makeFifo(this.deps.scratchDir, `playlist-${nextFeeder++}.fifo`);
|
||||
const reader = openBlockingReader(path);
|
||||
this.reader = reader;
|
||||
const out = new Socket({ fd: openSync(path, constants.O_WRONLY | constants.O_NONBLOCK), readable: false, writable: true });
|
||||
out.on("error", () => { if (!this.stopping) void this.stop(); }); // the ingest is gone; IngestProcess reports its exit
|
||||
this.out = out;
|
||||
reader.releaseHolder();
|
||||
rmSync(path, { force: true });
|
||||
this.running = this.run().catch((error: unknown) => this.deps.onStderr(`playlist feeder ${this.sourceId}: ${String(error)}`));
|
||||
return reader.fd;
|
||||
}
|
||||
|
||||
releaseReader(): void {
|
||||
if (this.reader) { closeSync(this.reader.fd); this.reader = null; }
|
||||
}
|
||||
|
||||
/** Desired-state v2: adopt a new list at the next asset boundary (no restart, no mid-asset cut). */
|
||||
update(spec: PlaylistSpec): void {
|
||||
this.spec = { assetIds: [...spec.assetIds], loop: spec.loop };
|
||||
const at = this.current ? this.spec.assetIds.indexOf(this.current.assetId) : -1;
|
||||
this.nextIndex = at >= 0 ? at + 1 : 0;
|
||||
this.ended = false;
|
||||
this.generation += 1;
|
||||
this.deps.onEvent?.({ type: "adopted", assetIds: this.spec.assetIds.length });
|
||||
}
|
||||
|
||||
private peekNext(): { assetId: string; index: number } | null {
|
||||
const ids = this.spec.assetIds;
|
||||
if (ids.length === 0) return null;
|
||||
let index = this.nextIndex;
|
||||
if (index >= ids.length) {
|
||||
if (!this.spec.loop) return null;
|
||||
index = 0;
|
||||
}
|
||||
return { assetId: ids[index]!, index };
|
||||
}
|
||||
|
||||
/** Spawns a remux of `asset` (or the slate) at `offsetUs`, capturing its output from the start. */
|
||||
private produce(asset: { assetId: string; index: number } | null, offsetUs: number): Job {
|
||||
const args = asset ? buildPlaylistRemuxArgs(offsetUs)
|
||||
: buildSlateArgs({ canvas: this.deps.canvas, seconds: this.deps.slateSeconds ?? 5, image: this.deps.slateImage ?? null, offsetUs });
|
||||
const proc = spawnManaged("ffmpeg", args, [asset ? "pipe" : "ignore", "pipe", "pipe", "pipe"], this.deps.redactor, this.deps.onStderr);
|
||||
// A producer may exit before its turn (a broken asset at once, a small one that fits in the
|
||||
// buffers), and Node then resumes its unconsumed stdout itself, dropping data and 'end'. The
|
||||
// capture keeps both until fed; backpressure holds a large asset's remux CAPTURE_BYTES ahead.
|
||||
const output = new PassThrough({ highWaterMark: CAPTURE_BYTES });
|
||||
const stdout = proc.child.stdout as Readable | null;
|
||||
if (stdout) stdout.pipe(output);
|
||||
else output.end();
|
||||
let progress = "";
|
||||
(proc.child.stdio[3] as Readable | null)?.on("data", (chunk: Buffer) => { progress = (progress + chunk.toString("latin1")).slice(-4096); });
|
||||
let failed: string | null = null;
|
||||
const done = proc.exited.then(({ code }) => {
|
||||
const endUs = Number([...progress.matchAll(/out_time_us=(\d+)/g)].at(-1)?.[1] ?? 0);
|
||||
return { ok: failed === null && code === 0 && endUs > 0, endUs, reason: failed ?? `producer exited code=${code} end=${endUs}us` };
|
||||
});
|
||||
const job: Job = { proc, output, input: null, done, cancelled: false };
|
||||
if (asset) {
|
||||
proc.child.stdin?.on("error", () => undefined); // EPIPE when the remux rejects its input
|
||||
void this.deps.assets.open(asset.assetId).then((input) => {
|
||||
if (job.cancelled || this.stopping || proc.child.exitCode !== null || proc.child.signalCode !== null) { input.destroy(); return; }
|
||||
job.input = input;
|
||||
input.on("error", (error: Error) => { failed = `asset read failed: ${error.name}`; proc.kill("SIGKILL"); });
|
||||
if (proc.child.stdin) input.pipe(proc.child.stdin);
|
||||
}, (error: unknown) => {
|
||||
failed = `asset open failed: ${error instanceof Error ? error.name : "error"}`;
|
||||
proc.kill("SIGKILL");
|
||||
});
|
||||
}
|
||||
this.jobs.add(job);
|
||||
void done.then(() => this.jobs.delete(job));
|
||||
return job;
|
||||
}
|
||||
|
||||
private discard(job: Job): void {
|
||||
job.cancelled = true;
|
||||
job.input?.destroy();
|
||||
job.output.destroy();
|
||||
job.proc.kill("SIGKILL");
|
||||
}
|
||||
|
||||
/** Pipes a job's captured output into the FIFO; resolves once all of it has been handed over. */
|
||||
private async feed(job: Job): Promise<void> {
|
||||
const out = this.out;
|
||||
if (!out) return;
|
||||
job.output.pipe(out, { end: false });
|
||||
// Until the producer's last byte ('end'). On stop the FIFO socket is destroyed, which unpipes
|
||||
// and pauses the capture, so 'end' would never come: stop() aborts the wait instead.
|
||||
await new Promise<void>((resolve) => {
|
||||
this.abortFeed = resolve;
|
||||
job.output.once("end", resolve);
|
||||
job.output.once("error", () => resolve());
|
||||
});
|
||||
this.abortFeed = null;
|
||||
job.output.unpipe(out);
|
||||
}
|
||||
|
||||
private async run(): Promise<void> {
|
||||
let offsetUs = 0;
|
||||
let failedInARow = 0;
|
||||
let slateNext = false;
|
||||
let feeding: Promise<void> = Promise.resolve();
|
||||
while (!this.stopping) {
|
||||
const generation = this.generation;
|
||||
const next = slateNext ? null : this.peekNext();
|
||||
slateNext = false;
|
||||
// Prepared while the previous job is still being fed (its producer has exited: offsetUs is final).
|
||||
const job = this.produce(next, offsetUs);
|
||||
await feeding;
|
||||
if (this.stopping) { this.discard(job); break; }
|
||||
if (generation !== this.generation) { this.discard(job); continue; } // re-plan under the new list
|
||||
if (next) {
|
||||
this.nextIndex = next.index + 1;
|
||||
this.current = next;
|
||||
this.stats.assets += 1;
|
||||
this.deps.onEvent?.({ type: "asset", assetId: next.assetId, index: next.index });
|
||||
} else {
|
||||
this.current = null;
|
||||
this.ended = this.spec.assetIds.length > 0 && !this.spec.loop && this.nextIndex >= this.spec.assetIds.length;
|
||||
this.stats.slates += 1;
|
||||
this.deps.onEvent?.({ type: "slate", reason: this.ended ? "ended" : "unavailable" });
|
||||
}
|
||||
feeding = this.feed(job);
|
||||
const result = await job.done;
|
||||
if (result.ok) {
|
||||
offsetUs += result.endUs;
|
||||
failedInARow = 0;
|
||||
continue;
|
||||
}
|
||||
if (!next) continue; // a failed slate is retried (offset unchanged)
|
||||
this.stats.failures += 1;
|
||||
failedInARow += 1;
|
||||
this.deps.onEvent?.({ type: "asset-failed", assetId: next.assetId, reason: result.reason });
|
||||
if (failedInARow >= Math.max(1, this.spec.assetIds.length)) {
|
||||
failedInARow = 0;
|
||||
slateNext = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
if (this.stopping) { await this.running; return; }
|
||||
this.stopping = true;
|
||||
const jobs = [...this.jobs];
|
||||
for (const job of jobs) this.discard(job);
|
||||
this.abortFeed?.();
|
||||
this.out?.destroy();
|
||||
this.out = null;
|
||||
this.releaseReader();
|
||||
await this.running;
|
||||
await Promise.all(jobs.map((job) => job.done));
|
||||
}
|
||||
}
|
||||
181
apps/media-worker/src/compositor/production-job.ts
Normal file
181
apps/media-worker/src/compositor/production-job.ts
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import type { GateImpl } from "../gate/frame-gate";
|
||||
import { snapshotHlsUrls, OUTPUT_PROFILES, type DesiredChannel } from "../types";
|
||||
import { validateEndpointShape, type EndpointValidator } from "../relay/endpoints";
|
||||
import type { StderrRedactor, VaultHandleBroker } from "../relay/ffmpeg";
|
||||
import type { Canvas, EncoderOutput, IngestInput } from "./command-builder";
|
||||
import type { ComposeRequest } from "./compositor";
|
||||
import { ManifestPreviewPublisher, type PreviewManifestStore, type PreviewObjectStore } from "../preview/contract";
|
||||
import type { HlsRendition } from "../preview/argv";
|
||||
import { PreviewTaps } from "../preview/taps";
|
||||
import { IngestProcess } from "./ingest";
|
||||
import { PlaylistFeeder, type PlaylistAssetSource } from "./playlist-feeder";
|
||||
import { ProgramChain } from "./program";
|
||||
import type { SourceBus } from "./source-bus";
|
||||
import { planScene, programScene } from "./scene-plan";
|
||||
|
||||
/** WP7 egress for a production snapshot: one pinned MPEG-TS fd per HLS source, plus the push fd. */
|
||||
export interface ProductionEgress {
|
||||
prepare(config: DesiredChannel, validator: EndpointValidator, vault: VaultHandleBroker): Promise<{
|
||||
sources: Map<string, { input: IngestInput; fd: number | null }>;
|
||||
output: EncoderOutput;
|
||||
close(): Promise<void>;
|
||||
}>;
|
||||
}
|
||||
|
||||
/** Overlay PNGs by content id, fetched through assignment-scoped presigned GETs (§4.3) into scratch. */
|
||||
export interface AssetStore { localPath(assetId: string): Promise<string> }
|
||||
|
||||
export interface ProductionJobDeps {
|
||||
validator: EndpointValidator;
|
||||
egress: ProductionEgress;
|
||||
vault: VaultHandleBroker;
|
||||
assets: AssetStore;
|
||||
redactor: StderrRedactor;
|
||||
onStderr(message: string): void;
|
||||
onExit(reason: string): void;
|
||||
scratchDir: string;
|
||||
gateImpl?: GateImpl;
|
||||
/** Playlist sources (§5.1). Without it, playlist sources render black and are reported in `lastSkipped`. */
|
||||
playlists?: PlaylistAssetSource;
|
||||
/** Preview taps (§2.1) publish through the API's manifest contract. Without it, no taps run. */
|
||||
preview?: { objects: PreviewObjectStore; manifests: PreviewManifestStore };
|
||||
}
|
||||
|
||||
/**
|
||||
* What may change by swap. Anything else (the set of sources, an HLS URL, output, profile, mode)
|
||||
* restarts the channel. A playlist's asset list and loop flag are not part of it: the feeder
|
||||
* adopts them in place (desired-state v2: an edited playlist never interrupts the program).
|
||||
*/
|
||||
function pathKey(c: DesiredChannel): string {
|
||||
const sources = c.production?.sources.map((src) => (src.kind === "hls" ? { id: src.id, kind: src.kind, url: src.url } : { id: src.id, kind: src.kind })) ?? null;
|
||||
return createHash("sha256").update(JSON.stringify({ profile: c.profile, mode: c.mode, output: c.output, sources, source: c.source })).digest("hex");
|
||||
}
|
||||
|
||||
/**
|
||||
* Supervisor RelayJob for a desired-state v2 snapshot. `start()` brings up
|
||||
* ingest×N (HLS through the egress relay, playlists through the asset feeder)
|
||||
* → gate → encoder, puts the program scene on air, and starts the preview taps.
|
||||
* `reconfigure()` is the meaning of an accepted RUNNING snapshot while already
|
||||
* RUNNING: a TAKE, layout, overlay or gain change is an A/B compositor swap,
|
||||
* playlist edits are adopted by their feeders, and it resolves only once the
|
||||
* new compositor is on air at a frame boundary.
|
||||
*/
|
||||
export class ProductionJob {
|
||||
private ingests = new Map<string, IngestProcess>();
|
||||
private feeders = new Map<string, PlaylistFeeder>();
|
||||
private chain: ProgramChain | null = null;
|
||||
private taps: PreviewTaps | null = null;
|
||||
private closeEgress: (() => Promise<void>) | null = null;
|
||||
private key = "";
|
||||
private canvas: Canvas | null = null;
|
||||
lastSkipped: string[] = [];
|
||||
|
||||
constructor(private readonly deps: ProductionJobDeps) {}
|
||||
|
||||
get program(): ProgramChain | null { return this.chain; }
|
||||
get previews(): PreviewTaps | null { return this.taps; }
|
||||
feeder(sourceId: string): PlaylistFeeder | undefined { return this.feeders.get(sourceId); }
|
||||
/** The source's bus (its ingest's fan-out), for stats. */
|
||||
sourceBus(sourceId: string): SourceBus | undefined { return this.ingests.get(sourceId)?.bus; }
|
||||
|
||||
async start(config: DesiredChannel): Promise<void> {
|
||||
const spec = config.production;
|
||||
if (!spec || !config.output) throw new Error("production job needs a production snapshot with an output");
|
||||
for (const url of snapshotHlsUrls(config)) validateEndpointShape(url, "hls");
|
||||
validateEndpointShape(config.output.endpointUrl, "output");
|
||||
const profile = OUTPUT_PROFILES[config.profile];
|
||||
const canvas: Canvas = { width: profile.width, height: profile.height, fps: profile.fps };
|
||||
this.canvas = canvas;
|
||||
const egress = await this.deps.egress.prepare(config, this.deps.validator, this.deps.vault);
|
||||
this.closeEgress = egress.close;
|
||||
const lost = (id: string, reason: string): void => this.deps.onExit(`source ${id}: ${reason}`);
|
||||
try {
|
||||
for (const [sourceId, { input, fd }] of egress.sources) {
|
||||
const ingest = new IngestProcess(sourceId, { input, canvas, audio: "source" }, fd, this.deps.scratchDir, this.deps.redactor, this.deps.onStderr, lost);
|
||||
this.ingests.set(sourceId, ingest);
|
||||
await ingest.start();
|
||||
}
|
||||
const playlists = this.deps.playlists;
|
||||
for (const src of spec.sources) {
|
||||
if (src.kind !== "playlist" || !playlists) continue;
|
||||
const feeder = new PlaylistFeeder(src.id, { assetIds: src.assetIds, loop: src.loop }, {
|
||||
assets: playlists, canvas, scratchDir: this.deps.scratchDir, redactor: this.deps.redactor, onStderr: this.deps.onStderr,
|
||||
onEvent: (e) => { if (e.type === "asset-failed" || e.type === "slate") this.deps.onStderr(`playlist ${src.id}: ${e.type}${"reason" in e ? ` ${e.reason}` : ""}`); },
|
||||
});
|
||||
this.feeders.set(src.id, feeder);
|
||||
const fd = feeder.start();
|
||||
const ingest = new IngestProcess(src.id, { input: { kind: "playlistPipe" }, canvas, audio: "source" }, fd, this.deps.scratchDir, this.deps.redactor, this.deps.onStderr, lost);
|
||||
this.ingests.set(src.id, ingest);
|
||||
try { await ingest.start(); } finally { feeder.releaseReader(); }
|
||||
}
|
||||
this.chain = new ProgramChain({
|
||||
canvas, scratchDir: this.deps.scratchDir, videoBitrateKbps: Number.parseInt(profile.videoBitrate, 10),
|
||||
output: egress.output, redactor: this.deps.redactor, onStderr: this.deps.onStderr,
|
||||
onFatal: (reason) => this.deps.onExit(reason), gateImpl: this.deps.gateImpl, programTap: this.deps.preview !== undefined,
|
||||
});
|
||||
await this.chain.start(await this.compose(config));
|
||||
this.key = pathKey(config);
|
||||
} catch (error) {
|
||||
await this.stop();
|
||||
throw error;
|
||||
}
|
||||
await this.startPreviews(config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Preview taps never gate the program: a tap that cannot start is reported and the channel runs on.
|
||||
* The program tap's renditions are always on (§2.2: the PGM tile is always live); source renditions
|
||||
* start on demand (`previews.demand()`).
|
||||
*/
|
||||
private async startPreviews(config: DesiredChannel): Promise<void> {
|
||||
const preview = this.deps.preview;
|
||||
if (!preview || !this.chain?.programBus) return;
|
||||
const taps = new PreviewTaps({
|
||||
publisher: new ManifestPreviewPublisher({ workspaceId: config.workspaceId, channelId: config.channelId }, preview.objects, preview.manifests),
|
||||
scratchDir: this.deps.scratchDir, redactor: this.deps.redactor, onStderr: this.deps.onStderr,
|
||||
onError: (tap, reason) => this.deps.onStderr(`preview ${tap}: ${reason}`),
|
||||
});
|
||||
this.taps = taps;
|
||||
const add = async (tap: string, bus: SourceBus, alwaysOn: HlsRendition[]): Promise<void> => {
|
||||
try { await taps.add(tap, bus, { alwaysOn }); } catch (error) { this.deps.onStderr(`preview ${tap}: ${String(error)}`); }
|
||||
};
|
||||
await add("program", this.chain.programBus, ["video", "audio"]);
|
||||
for (const [sourceId, ingest] of this.ingests) await add(sourceId, ingest.bus, []);
|
||||
}
|
||||
|
||||
/** false = this change cannot be a swap (sources/output/profile changed): the supervisor restarts. */
|
||||
async reconfigure(config: DesiredChannel): Promise<boolean> {
|
||||
if (!this.chain || !config.production || pathKey(config) !== this.key) return false;
|
||||
for (const src of config.production.sources) {
|
||||
if (src.kind === "playlist") this.feeders.get(src.id)?.update({ assetIds: src.assetIds, loop: src.loop });
|
||||
}
|
||||
await this.chain.swap(await this.compose(config));
|
||||
return true;
|
||||
}
|
||||
private async compose(config: DesiredChannel): Promise<ComposeRequest> {
|
||||
if (!config.production || !this.canvas) throw new Error("no production spec");
|
||||
const plan = planScene(config.production, programScene(config.production), this.canvas, (id) => this.ingests.has(id));
|
||||
this.lastSkipped = plan.skipped;
|
||||
const inputs = plan.sourceIds.map((id, i) => {
|
||||
const ingest = this.ingests.get(id);
|
||||
if (!ingest) throw new Error("planned source has no ingest");
|
||||
return { bus: ingest.bus, audio: plan.audio[i] ?? false };
|
||||
});
|
||||
const assets = await Promise.all(plan.assetIds.map((id) => this.deps.assets.localPath(id)));
|
||||
return { inputs, assets, graph: plan.graph };
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
await this.taps?.stop();
|
||||
this.taps = null;
|
||||
await this.chain?.stop();
|
||||
this.chain = null;
|
||||
for (const feeder of this.feeders.values()) await feeder.stop();
|
||||
this.feeders.clear();
|
||||
for (const ingest of this.ingests.values()) await ingest.stop();
|
||||
this.ingests.clear();
|
||||
await this.closeEgress?.();
|
||||
this.closeEgress = null;
|
||||
}
|
||||
}
|
||||
194
apps/media-worker/src/compositor/program.ts
Normal file
194
apps/media-worker/src/compositor/program.ts
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
import { CompositorLagError, FrameGate, type GateImpl } from "../gate/frame-gate";
|
||||
import type { GateEvent, GateStats } from "../gate/protocol";
|
||||
import { spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg";
|
||||
import { programTapGeometry } from "../preview/argv";
|
||||
import { buildEncoderArgs, type Canvas, type EncoderOutput, type EncoderProgramTap } from "./command-builder";
|
||||
import { CompositorInstance, type ComposeRequest } from "./compositor";
|
||||
import { SourceBus, type BusSubscription } from "./source-bus";
|
||||
|
||||
export interface ProgramChainOptions {
|
||||
canvas: Canvas;
|
||||
scratchDir: string;
|
||||
videoBitrateKbps: number;
|
||||
/**
|
||||
* flvPipe: the push relay's inherited fd (WP9/WP13); null: x264 into `-f null` (CPU smoke);
|
||||
* external: no encoder is spawned, the caller has opened the gate's encoder FIFOs (verifying sinks).
|
||||
*/
|
||||
output: EncoderOutput | { kind: "external" };
|
||||
redactor: StderrRedactor;
|
||||
onStderr(message: string): void;
|
||||
/** Encoder or gate died: the RTMP session is gone and the supervisor must restart the channel. */
|
||||
onFatal(reason: string): void;
|
||||
/** The on-air compositor died (gate is repeating its last frame); the chain respawns it. */
|
||||
onCompositorLost?(reason: string): void;
|
||||
gateImpl?: GateImpl;
|
||||
/** Swap must go on air within this long, or B is abandoned and A stays (default 10 s). */
|
||||
swapTimeoutMs?: number;
|
||||
/**
|
||||
* Program preview tap (§2.1): the encoder also emits a 360p copy of what it encodes into
|
||||
* `programBus`, which preview taps subscribe to. Needs an encoder (not `external`).
|
||||
*/
|
||||
programTap?: boolean;
|
||||
}
|
||||
|
||||
/** Child fd of the push relay's pipe in the encoder (flvPipe), and of the program tap outputs after it. */
|
||||
const ENCODER_OUTPUT_FD = 4;
|
||||
|
||||
const LAG_RETRIES = 2;
|
||||
|
||||
/**
|
||||
* The persistent part of a channel's program path (§1.4): frame gate + encoder,
|
||||
* started once, plus the A/B compositor swap. `swap()` starts B for the new
|
||||
* scene, waits until the gate splices it in at a frame boundary, then stops A.
|
||||
* The encoder and its push session are never restarted by a swap.
|
||||
*/
|
||||
export class ProgramChain {
|
||||
readonly gate: FrameGate;
|
||||
/** The program tap's bus (360p of the encoder input), when `programTap` is set. */
|
||||
readonly programBus: SourceBus | null;
|
||||
private encoder: ManagedProcess | null = null;
|
||||
private active: CompositorInstance | null = null;
|
||||
private activeRequest: ComposeRequest | null = null;
|
||||
private queue: Promise<unknown> = Promise.resolve();
|
||||
private stopping = false;
|
||||
swaps = 0;
|
||||
/** Swap-in compositors replaced after a gate `LAG` report. */
|
||||
lagRetries = 0;
|
||||
|
||||
constructor(private readonly options: ProgramChainOptions) {
|
||||
// The Node reference gate has neither A/V pair lockstep nor origin alignment (s2a-report §5.3):
|
||||
// swaps through it can step backwards and skew A/V. Spike/contract comparison only (gate/spike.ts, gate.test.ts).
|
||||
if (options.gateImpl === "node") throw new Error("the Node reference gate is spike-only; a program chain needs the Rust gate");
|
||||
this.gate = new FrameGate({
|
||||
geometry: options.canvas, scratchDir: options.scratchDir, impl: options.gateImpl ?? "rust",
|
||||
redactor: options.redactor, onStderr: options.onStderr,
|
||||
onEvent: (event) => this.onGateEvent(event),
|
||||
onExit: (reason) => { if (!this.stopping) options.onFatal(reason); },
|
||||
});
|
||||
if (options.programTap && options.output.kind === "external") throw new Error("the program tap needs an encoder");
|
||||
this.programBus = options.programTap ? new SourceBus({
|
||||
geometry: programTapGeometry(options.canvas), scratchDir: options.scratchDir, redactor: options.redactor, onStderr: options.onStderr,
|
||||
// Losing the tap bus loses previews, never the program: the encoder's tee drops that slave.
|
||||
onExit: (reason) => options.onStderr(`program tap bus: ${reason}`),
|
||||
}) : null;
|
||||
}
|
||||
|
||||
get activeCompositor(): CompositorInstance | null { return this.active; }
|
||||
get encoderPid(): number | undefined { return this.encoder?.child.pid; }
|
||||
|
||||
async start(first: ComposeRequest): Promise<void> {
|
||||
const { canvas, output } = this.options;
|
||||
if (output.kind === "external") {
|
||||
await this.gate.start();
|
||||
await this.swap(first);
|
||||
return;
|
||||
}
|
||||
const inputs = this.gate.encoderInputs();
|
||||
const stdio: Array<number | "pipe" | "ignore"> = [inputs.videoFd, "ignore", "pipe", inputs.audioFd];
|
||||
// The push relay's fd is inherited as child fd 4, whatever its number in the worker.
|
||||
let encoderOutput: EncoderOutput = output;
|
||||
if (output.kind === "flvPipe") { stdio.push(output.fd); encoderOutput = { kind: "flvPipe", fd: ENCODER_OUTPUT_FD }; }
|
||||
let programTap: EncoderProgramTap | undefined;
|
||||
if (this.programBus) {
|
||||
const tap = await this.programBus.start();
|
||||
const first = stdio.length;
|
||||
stdio.push(tap.videoFd, tap.audioFd);
|
||||
programTap = { videoFd: first, audioFd: first + 1, size: programTapGeometry(canvas) };
|
||||
}
|
||||
let encoder: ManagedProcess;
|
||||
try {
|
||||
encoder = spawnManaged("ffmpeg", buildEncoderArgs({ canvas, videoBitrateKbps: this.options.videoBitrateKbps, audioBitrateKbps: 128, output: encoderOutput, programTap }),
|
||||
stdio, this.options.redactor, this.options.onStderr);
|
||||
} finally {
|
||||
this.programBus?.releaseIngestWriters();
|
||||
}
|
||||
this.encoder = encoder;
|
||||
void encoder.exited.then(({ code, signal }) => { if (!this.stopping) this.options.onFatal(`encoder exited code=${code} signal=${signal}`); });
|
||||
await encoder.started;
|
||||
await this.gate.start();
|
||||
await this.swap(first);
|
||||
}
|
||||
|
||||
/** Serialized A/B swap. Resolves with the gate tick at which the new compositor went on air. */
|
||||
swap(request: ComposeRequest): Promise<number> {
|
||||
const run = this.queue.catch(() => undefined).then(() => this.swapNow(request));
|
||||
this.queue = run;
|
||||
return run;
|
||||
}
|
||||
|
||||
/**
|
||||
* A B the gate reports as lagging (still behind A's on-air source position, typically a spawn
|
||||
* starved of CPU while draining its startup backlog) is replaced by a fresh one; A stays on air.
|
||||
*/
|
||||
private async swapNow(request: ComposeRequest): Promise<number> {
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
try {
|
||||
return await this.swapOnce(request);
|
||||
} catch (error) {
|
||||
if (!(error instanceof CompositorLagError) || attempt >= LAG_RETRIES) throw error;
|
||||
this.lagRetries += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async swapOnce(request: ComposeRequest): Promise<number> {
|
||||
const next = new CompositorInstance(this.gate, request, this.options.redactor, this.options.onStderr);
|
||||
const current = this.active;
|
||||
await next.start((subs) => (current ? this.originAfter(current, subs) : 0));
|
||||
let tick: number;
|
||||
try {
|
||||
tick = await this.gate.waitActive(next.slotId, this.options.swapTimeoutMs ?? 10_000);
|
||||
} catch (error) {
|
||||
try { this.gate.detach(next.slotId); } catch { /* gate gone */ }
|
||||
await next.stop();
|
||||
throw error;
|
||||
}
|
||||
const old = this.active;
|
||||
this.active = next;
|
||||
this.activeRequest = request;
|
||||
this.swaps += 1;
|
||||
if (old) void old.stop();
|
||||
return tick;
|
||||
}
|
||||
|
||||
/**
|
||||
* Origin of B relative to on-air A, in A's numbering: through a bus both use
|
||||
* (exact: pair counters, minus pairs A had lost to backlog when B subscribed), else by wall clock.
|
||||
* Pairs B loses after attach only make B *ahead*: a forward skip, never backwards.
|
||||
*/
|
||||
private originAfter(a: CompositorInstance, subs: BusSubscription[]): number | undefined {
|
||||
if (a.origin === undefined) return undefined;
|
||||
for (const sb of subs) {
|
||||
const sa = a.subscriptions.find((x) => x.bus === sb.bus);
|
||||
if (sa) return a.origin + sb.startPair - sa.startPair - (sb.droppedBefore(sa) ?? sa.dropped);
|
||||
}
|
||||
const sa = a.subscriptions[0];
|
||||
const sb = subs[0];
|
||||
if (!sa || !sb) return undefined;
|
||||
return a.origin + Math.round((Number(sb.startedAt - sa.startedAt) / 1e9) * this.options.canvas.fps);
|
||||
}
|
||||
|
||||
private onGateEvent(event: GateEvent): void {
|
||||
if (event.type !== "EOF" || this.stopping || !this.active || event.id !== this.active.slotId || !this.activeRequest) return;
|
||||
// The on-air compositor died (crash, kill -9). The gate is already holding
|
||||
// the last frame + silence, so the encoder never notices; respawn the scene.
|
||||
const lost = this.active;
|
||||
this.active = null;
|
||||
void lost.stop();
|
||||
this.options.onCompositorLost?.(`compositor ${event.id} ended on air`);
|
||||
void this.swap(this.activeRequest).catch((error: unknown) => this.options.onCompositorLost?.(`respawn failed: ${String(error)}`));
|
||||
}
|
||||
|
||||
stats(): Promise<GateStats> { return this.gate.stats(); }
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.stopping = true;
|
||||
await this.queue.catch(() => undefined);
|
||||
await this.active?.stop();
|
||||
this.active = null;
|
||||
await this.gate.stop();
|
||||
await this.encoder?.terminate();
|
||||
this.encoder = null;
|
||||
await this.programBus?.stop();
|
||||
}
|
||||
}
|
||||
68
apps/media-worker/src/compositor/scene-plan.test.ts
Normal file
68
apps/media-worker/src/compositor/scene-plan.test.ts
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { parseProductionSpec } from "../production";
|
||||
import { buildCompositorArgs } from "./command-builder";
|
||||
import { planScene, programScene } from "./scene-plan";
|
||||
|
||||
const id = (n: number) => `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`;
|
||||
const sha = (c: string) => c.repeat(64);
|
||||
|
||||
const spec = parseProductionSpec({
|
||||
sources: [
|
||||
{ id: id(1), label: "Cam A", kind: "hls", url: "https://a.example/live.m3u8" },
|
||||
{ id: id(2), label: "Cam B", kind: "hls", url: "https://b.example/live.m3u8" },
|
||||
{ id: id(3), label: "Loop", kind: "playlist", assetIds: [id(9)], loop: true },
|
||||
],
|
||||
scenes: [
|
||||
{ id: id(10), label: "Grid", layout: "2x2", slots: [
|
||||
{ index: 0, sourceId: id(1), rect: null, fit: "cover", audioMode: "primary", gainDb: 0 },
|
||||
{ index: 1, sourceId: id(2), rect: null, fit: "contain", audioMode: "on", gainDb: -6 },
|
||||
{ index: 2, sourceId: id(3), rect: null, fit: "contain", audioMode: "on", gainDb: 0 },
|
||||
{ index: 3, sourceId: id(1), rect: null, fit: "crop", audioMode: "on", gainDb: -12 },
|
||||
] },
|
||||
{ id: id(11), label: "PIP", layout: "pip", slots: [
|
||||
{ index: 0, sourceId: id(2), rect: null, fit: "cover", audioMode: "primary", gainDb: -3 },
|
||||
{ index: 1, sourceId: id(1), rect: { x: 6500, y: 6500, w: 3000, h: 3000 }, fit: "contain", audioMode: "off", gainDb: 0 },
|
||||
] },
|
||||
],
|
||||
buses: { preview: id(11), program: id(10) },
|
||||
overlays: [
|
||||
{ id: id(20), sceneId: id(10), kind: "chyron", enabled: true, layer: 2, position: { x: 0, y: 9000 }, revisionId: id(30), assetId: sha("c"), chyron: { speedPxPerSec: 120, stripWidthPx: 4000 }, clock: null },
|
||||
{ id: id(21), sceneId: id(10), kind: "lower_third", enabled: true, layer: 1, position: { x: 500, y: 7500 }, revisionId: id(31), assetId: sha("a"), chyron: null, clock: null },
|
||||
{ id: id(22), sceneId: id(10), kind: "logo", enabled: false, layer: 3, position: { x: 9000, y: 500 }, revisionId: id(32), assetId: sha("b"), chyron: null, clock: null },
|
||||
{ id: id(23), sceneId: id(10), kind: "clock", enabled: true, layer: 4, position: { x: 9000, y: 500 }, revisionId: null, assetId: null, chyron: null, clock: { format: "HH:MM" } },
|
||||
{ id: id(24), sceneId: id(11), kind: "lower_third", enabled: true, layer: 0, position: { x: 0, y: 0 }, revisionId: id(33), assetId: sha("d"), chyron: null, clock: null },
|
||||
],
|
||||
transition: { type: "cut" },
|
||||
audio: { masterGainDb: -3 },
|
||||
});
|
||||
const canvas = { width: 1920, height: 1080, fps: 30 };
|
||||
const hls = (sourceId: string) => spec.sources.find((s) => s.id === sourceId)?.kind === "hls";
|
||||
|
||||
test("program scene → typed graph: grid cells, shared source, playlist black, one mix input per source", () => {
|
||||
const plan = planScene(spec, programScene(spec), canvas, hls);
|
||||
assert.deepEqual(plan.sourceIds, [id(1), id(2)]);
|
||||
assert.deepEqual(plan.audio, [true, true]);
|
||||
assert.deepEqual(plan.graph.tiles.map((t) => t.source), [0, 1, null, 0]);
|
||||
assert.deepEqual(plan.graph.tiles[3]?.rect, { x: 960, y: 540, w: 960, h: 540 });
|
||||
// Cam A is primary at 0 dB and "on" at -12 dB in slot 3: one input at the louder gain.
|
||||
assert.deepEqual(plan.graph.audio, { inputs: [{ source: 0, gainDb: 0 }, { source: 1, gainDb: -6 }], masterGainDb: -3 });
|
||||
// Layer order (lower third under chyron); disabled logo and S2a-unsupported clock skipped.
|
||||
assert.deepEqual(plan.assetIds, [sha("a"), sha("c")]);
|
||||
assert.deepEqual(plan.graph.overlays, [{ kind: "image", asset: 0, x: 96, y: 810 }, { kind: "chyron", asset: 1, y: 972, speedPxPerSec: 120 }]);
|
||||
assert.equal(plan.skipped.length, 2);
|
||||
const args = buildCompositorArgs({
|
||||
sources: plan.sourceIds.map((_, i) => ({ video: { kind: "rawPipe" as const, fd: 4 + 2 * i }, audio: { kind: "rawPipe" as const, fd: 5 + 2 * i } })),
|
||||
assets: plan.assetIds.map((a) => `/scratch/assets/${a}.png`), graph: plan.graph, output: { kind: "gate", videoFd: 1, audioFd: 3 },
|
||||
});
|
||||
assert.ok(!args.join(" ").includes("Cam A"), "labels never reach argv");
|
||||
assert.ok(!args.join(" ").includes("example"), "source URLs never reach the compositor");
|
||||
});
|
||||
|
||||
test("PIP scene: inset rect from basis points, audio off slot not mixed", () => {
|
||||
const plan = planScene(spec, spec.scenes[1]!, canvas, hls);
|
||||
assert.deepEqual(plan.sourceIds, [id(2), id(1)]);
|
||||
assert.deepEqual(plan.audio, [true, false]);
|
||||
assert.deepEqual(plan.graph.tiles[1]?.rect, { x: 1248, y: 702, w: 576, h: 324 });
|
||||
assert.deepEqual(plan.graph.audio.inputs, [{ source: 0, gainDb: -3 }]);
|
||||
});
|
||||
81
apps/media-worker/src/compositor/scene-plan.ts
Normal file
81
apps/media-worker/src/compositor/scene-plan.ts
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
import type { ProductionOverlay, ProductionScene, ProductionSpec } from "../production";
|
||||
import { layoutRects, rectFromBasisPoints, type Canvas, type CompositorGraph, type MixInput, type OverlaySpec, type TileSpec } from "./command-builder";
|
||||
|
||||
/** Desired-state v2 scene → typed compositor graph over a list of source ids (input order). */
|
||||
export interface ScenePlan {
|
||||
sourceIds: string[];
|
||||
/** Same order as sourceIds: whether the compositor needs that source's audio. */
|
||||
audio: boolean[];
|
||||
assetIds: string[];
|
||||
graph: CompositorGraph;
|
||||
/** Things in the snapshot this slice renders as black/skips, for the observed reason. */
|
||||
skipped: string[];
|
||||
}
|
||||
|
||||
export function programScene(spec: ProductionSpec): ProductionScene {
|
||||
const scene = spec.scenes.find((s) => s.id === spec.buses.program);
|
||||
if (!scene) throw new Error("program bus names no scene");
|
||||
return scene;
|
||||
}
|
||||
|
||||
/**
|
||||
* `available(sourceId)` says whether the worker has an ingest for that source:
|
||||
* S2a ingests HLS sources only; playlist sources (S2b feeder) render black.
|
||||
* Overlay order: `layer` ascending, snapshot order within a layer.
|
||||
*/
|
||||
export function planScene(spec: ProductionSpec, scene: ProductionScene, canvas: Canvas, available: (sourceId: string) => boolean): ScenePlan {
|
||||
const skipped: string[] = [];
|
||||
const sourceIds: string[] = [];
|
||||
const indexOf = (id: string): number => {
|
||||
let i = sourceIds.indexOf(id);
|
||||
if (i < 0) { sourceIds.push(id); i = sourceIds.length - 1; }
|
||||
return i;
|
||||
};
|
||||
const grid = layoutRects(scene.layout, canvas);
|
||||
const slots = [...scene.slots].sort((a, b) => a.index - b.index);
|
||||
const tiles: TileSpec[] = [];
|
||||
const gains = new Map<number, number>();
|
||||
for (const slot of slots) {
|
||||
const rect = slot.rect ? rectFromBasisPoints(slot.rect, canvas) : grid[slot.index];
|
||||
if (!rect) continue;
|
||||
let source: number | null = null;
|
||||
if (slot.sourceId !== null) {
|
||||
if (available(slot.sourceId)) source = indexOf(slot.sourceId);
|
||||
else skipped.push(`slot ${slot.index}: source ${slot.sourceId} has no ingest in this worker build`);
|
||||
}
|
||||
tiles.push({ source, rect, fit: slot.fit });
|
||||
if (source !== null && slot.audioMode !== "off") gains.set(source, Math.max(gains.get(source) ?? -Infinity, slot.gainDb));
|
||||
}
|
||||
const audio = sourceIds.map((_, i) => gains.has(i));
|
||||
const mix: MixInput[] = [...gains.entries()].sort((a, b) => a[0] - b[0]).map(([source, gainDb]) => ({ source, gainDb }));
|
||||
|
||||
const assetIds: string[] = [];
|
||||
const overlays: OverlaySpec[] = [];
|
||||
const ordered = spec.overlays
|
||||
.map((o, i) => ({ o, i }))
|
||||
.filter(({ o }) => o.sceneId === scene.id && o.enabled)
|
||||
.sort((a, b) => a.o.layer - b.o.layer || a.i - b.i)
|
||||
.map(({ o }) => o);
|
||||
for (const o of ordered) {
|
||||
const spec = overlaySpec(o, canvas, (assetId) => {
|
||||
let i = assetIds.indexOf(assetId);
|
||||
if (i < 0) { assetIds.push(assetId); i = assetIds.length - 1; }
|
||||
return i;
|
||||
});
|
||||
if (spec) overlays.push(spec);
|
||||
else skipped.push(`overlay ${o.id}: ${o.kind} is not composited in S2a`);
|
||||
}
|
||||
return {
|
||||
sourceIds, audio, assetIds, skipped,
|
||||
graph: { canvas, layout: scene.layout, tiles, overlays, audio: { inputs: mix, masterGainDb: spec.audio.masterGainDb } },
|
||||
};
|
||||
}
|
||||
|
||||
function overlaySpec(o: ProductionOverlay, canvas: Canvas, asset: (id: string) => number): OverlaySpec | null {
|
||||
if (o.assetId === null) return null; // clock: server-generated drawtext is a later slice
|
||||
const x = Math.min(canvas.width, Math.round((o.position.x * canvas.width) / 10_000));
|
||||
const y = Math.min(canvas.height, Math.round((o.position.y * canvas.height) / 10_000));
|
||||
if (o.kind === "chyron" && o.chyron) return { kind: "chyron", asset: asset(o.assetId), y, speedPxPerSec: o.chyron.speedPxPerSec };
|
||||
if (o.kind === "lower_third" || o.kind === "logo") return { kind: "image", asset: asset(o.assetId), x, y };
|
||||
return null;
|
||||
}
|
||||
275
apps/media-worker/src/compositor/soak.ts
Normal file
275
apps/media-worker/src/compositor/soak.ts
Normal file
|
|
@ -0,0 +1,275 @@
|
|||
/**
|
||||
* S2 soak (streaming-architecture.md §6 S2: "the ≥8 h soak is wired into the
|
||||
* nightly run, not blocking merge"). Drives the production path for hours and
|
||||
* fails on drift or leaks:
|
||||
*
|
||||
* ProductionJob (4 paced 1080p30 MPEG-TS sources standing in for the WP7
|
||||
* relay + one looping playlist source through the asset feeder) → source-bus
|
||||
* helpers → A/B compositors → Rust gate → x264 (-f null) + program tap, with
|
||||
* preview taps (program LL-HLS always on, one source rendition kept demanded)
|
||||
* publishing into an in-memory manifest store, and a scripted scene / overlay
|
||||
* / gain swap every `--swap-every` s through `reconfigure()` (the RUNNING
|
||||
* snapshot path the supervisor uses).
|
||||
*
|
||||
* Every `--sample-every` s one JSON line goes to `--out`: gate continuity
|
||||
* counters, bus drops, swap latency, feeder and preview counters, and RSS + open
|
||||
* fds per process. The last line is the summary; the exit code is 0 only if:
|
||||
* - no process of the chain exited on its own and every swap went on air;
|
||||
* - after the settle window: bus drops ≤ --max-bus-drops (default 0) and gate
|
||||
* repeat/silence fills ≤ 1 per 1 000 ticks;
|
||||
* - from the end of the first hour (or the first quarter of a shorter run) to
|
||||
* the end: no process grows RSS by more than 15 % (+16 MiB) and fd counts
|
||||
* move by at most 8.
|
||||
*
|
||||
* node --import tsx src/compositor/soak.ts [--hours 8] [--profile 1080p30|720p30]
|
||||
* [--swap-every 30] [--sample-every 10] [--max-bus-drops 0] [--out soak.jsonl]
|
||||
*
|
||||
* Nightly: `pnpm --filter @tactical-media-broadcast-hub/media-worker soak -- --hours 8 --out
|
||||
* runs/soak/$(date +%F).jsonl` on the reference host (see runs/wp4/s2b-report.md).
|
||||
*/
|
||||
import { appendFileSync, closeSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { basename, join } from "node:path";
|
||||
import { createReadStream } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { parseProductionSpec, type ProductionSpec } from "../production";
|
||||
import type { PreviewContentType } from "../preview/contract";
|
||||
import type { ManagedProcess } from "../relay/ffmpeg";
|
||||
import { OUTPUT_PROFILES, type DesiredChannel, type OutputPreset } from "../types";
|
||||
import { ProductionJob } from "./production-job";
|
||||
import { encodeSourceClip, ensureGateBinary, makeAssets, passthroughRedactor, startPacer } from "./testing";
|
||||
|
||||
function opt(name: string, fallback: string): string {
|
||||
const i = process.argv.indexOf(`--${name}`);
|
||||
return i >= 0 ? (process.argv[i + 1] ?? fallback) : fallback;
|
||||
}
|
||||
|
||||
const id = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`;
|
||||
const WS = "8f0c5a8e-3b1d-4c3e-9a5b-0c1d2e3f4a5b";
|
||||
const CH = "1a2b3c4d-5e6f-4a1b-8c2d-3e4f5a6b7c8d";
|
||||
const PLAYLIST = id(5);
|
||||
|
||||
/** Swap i of the script: layouts over the 4 live sources and the playlist, overlays on/off, gains in 3 dB steps. */
|
||||
function scene(i: number, assetIds: [string, string]): ProductionSpec {
|
||||
const layouts = ["2x2", "pip", "single", "3x3", "2x1"] as const;
|
||||
const layout = layouts[i % layouts.length]!;
|
||||
const order = [id(1), id(2), id(3), id(4), PLAYLIST];
|
||||
const rot = (k: number): string => order[(k + i) % order.length]!;
|
||||
const count = { "2x2": 4, pip: 2, single: 1, "3x3": 5, "2x1": 2 }[layout];
|
||||
const slots = Array.from({ length: count }, (_, k) => ({
|
||||
index: k, sourceId: rot(k),
|
||||
rect: layout === "pip" && k === 1 ? { x: 6500, y: 6500, w: 3000, h: 3000 } : null,
|
||||
fit: (["cover", "contain", "crop"] as const)[(i + k) % 3]!,
|
||||
audioMode: k === 0 ? "primary" as const : (k === 1 ? "on" as const : "off" as const),
|
||||
gainDb: -3 * ((i + k) % 9),
|
||||
}));
|
||||
return parseProductionSpec({
|
||||
sources: [
|
||||
...[1, 2, 3, 4].map((n) => ({ id: id(n), label: `Cam ${n}`, kind: "hls", url: `https://cam${n}.example/live.m3u8` })),
|
||||
{ id: PLAYLIST, label: "Loop", kind: "playlist", assetIds: [id(40), id(41), id(42)], loop: true },
|
||||
],
|
||||
scenes: [{ id: id(10), label: "Program", layout, slots }, { id: id(11), label: "Preview", layout: "single", slots: [] }],
|
||||
buses: { preview: id(11), program: id(10) },
|
||||
overlays: [
|
||||
{ id: id(20), sceneId: id(10), kind: "lower_third", enabled: i % 3 !== 0, layer: 1, position: { x: 500, y: 8000 }, revisionId: id(30), assetId: assetIds[0], chyron: null, clock: null },
|
||||
{ id: id(21), sceneId: id(10), kind: "chyron", enabled: i % 2 === 0, layer: 2, position: { x: 0, y: 9400 }, revisionId: id(31), assetId: assetIds[1], chyron: { speedPxPerSec: 40 + ((i * 20) % 360), stripWidthPx: 600 }, clock: null },
|
||||
],
|
||||
transition: { type: "cut" },
|
||||
audio: { masterGainDb: -3 * (i % 3) },
|
||||
});
|
||||
}
|
||||
|
||||
/** VmRSS (KiB) and open fds of a process. */
|
||||
function procStat(pid: number | undefined): { rssKiB: number; fds: number } | null {
|
||||
if (pid === undefined) return null;
|
||||
try {
|
||||
const rss = /VmRSS:\s+(\d+)/.exec(readFileSync(`/proc/${pid}/status`, "utf8"))?.[1];
|
||||
return { rssKiB: Number(rss ?? 0), fds: readdirSync(`/proc/${pid}/fd`).length };
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
function pids(name: string): number[] {
|
||||
const r = spawnSync("pgrep", ["-f", name], { encoding: "utf8" });
|
||||
return r.stdout.split("\n").filter(Boolean).map(Number);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const hours = Number(opt("hours", "8"));
|
||||
const profile = opt("profile", "1080p30") as OutputPreset;
|
||||
const swapEvery = Number(opt("swap-every", "30"));
|
||||
const sampleEvery = Number(opt("sample-every", "10"));
|
||||
const maxBusDrops = Number(opt("max-bus-drops", "0"));
|
||||
const out = opt("out", join(process.cwd(), `soak-${new Date().toISOString().replace(/[:.]/g, "-")}.jsonl`));
|
||||
if (!(hours > 0) || !Number.isFinite(hours) || !OUTPUT_PROFILES[profile]) throw new Error("--hours must be > 0 and --profile one of 720p30|1080p30|1080p60");
|
||||
if (!(swapEvery > 0) || !Number.isFinite(swapEvery) || !(sampleEvery > 0) || !Number.isFinite(sampleEvery) || !Number.isSafeInteger(maxBusDrops) || maxBusDrops < 0) throw new Error("swap/sample intervals must be positive and max-bus-drops a non-negative integer");
|
||||
if (!ensureGateBinary()) throw new Error("native helpers unavailable (cargo build failed)");
|
||||
const { width, height, fps } = OUTPUT_PROFILES[profile];
|
||||
const canvas = { width, height, fps };
|
||||
const dir = mkdtempSync(join(tmpdir(), "soak-"));
|
||||
let job: ProductionJob | null = null;
|
||||
let exitCode = 2;
|
||||
try {
|
||||
writeFileSync(out, "");
|
||||
const log = (record: object): void => appendFileSync(out, `${JSON.stringify(record)}\n`);
|
||||
|
||||
// Fixtures, prepared before the clock starts.
|
||||
const clip = join(dir, "src.ts");
|
||||
encodeSourceClip(clip, canvas);
|
||||
const clips = new Map<string, string>();
|
||||
for (const [n, hz] of [[40, 300], [41, 600], [42, 900]] as const) {
|
||||
const path = join(dir, `asset-${n}.mp4`);
|
||||
const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", `testsrc2=s=${width}x${height}:r=${fps}:d=6`, "-f", "lavfi", "-i", `sine=frequency=${hz}:d=6`,
|
||||
"-c:v", "libx264", "-preset", "veryfast", "-b:v", "4000k", "-g", String(fps), "-c:a", "aac", "-movflags", "+faststart", path]);
|
||||
if (r.status !== 0) throw new Error("playlist asset encode failed");
|
||||
clips.set(id(n), path);
|
||||
}
|
||||
const assets = makeAssets(dir);
|
||||
const assetIds = assets.map((p) => basename(p, ".png")) as [string, string];
|
||||
const pacers: ManagedProcess[] = [];
|
||||
const pacerFds: number[] = [];
|
||||
const exits: string[] = [];
|
||||
const published = { count: 0, bytes: 0 };
|
||||
const manifests = new Map<string, number>();
|
||||
|
||||
job = new ProductionJob({
|
||||
validator: { async validate() { throw new Error("not used: synthetic egress"); } },
|
||||
vault: { async deliverPushTarget() { throw new Error("not used: null output"); } },
|
||||
egress: {
|
||||
async prepare(config) {
|
||||
const sources = new Map<string, { input: { kind: "mpegtsPipe" }; fd: number }>();
|
||||
for (const src of config.production?.sources ?? []) {
|
||||
if (src.kind !== "hls") continue;
|
||||
const pacer = await startPacer(clip, dir, `pacer-${src.id.slice(-2)}`);
|
||||
pacers.push(pacer.proc);
|
||||
pacerFds.push(pacer.readFd);
|
||||
sources.set(src.id, { input: { kind: "mpegtsPipe" }, fd: pacer.readFd });
|
||||
}
|
||||
return { sources, output: { kind: "null" as const }, async close() { for (const p of pacers) await p.terminate(500); for (const fd of pacerFds.splice(0)) closeSync(fd); } };
|
||||
},
|
||||
},
|
||||
assets: { async localPath(assetId) { const p = assets.find((a) => basename(a, ".png") === assetId); if (!p) throw new Error("unknown overlay asset"); return p; } },
|
||||
playlists: { async open(assetId) { const p = clips.get(assetId); if (!p) throw new Error("unknown asset"); return createReadStream(p); } },
|
||||
preview: {
|
||||
objects: { async putContent(body: Buffer, contentType: PreviewContentType) { published.count += 1; published.bytes += body.length; return { key: `sha256/${"0".repeat(64)}.${contentType === "image/jpeg" ? "jpg" : "bin"}` }; } },
|
||||
manifests: { async set(key: string) { manifests.set(key, (manifests.get(key) ?? 0) + 1); } },
|
||||
},
|
||||
redactor: passthroughRedactor, onStderr: () => undefined,
|
||||
onExit: (reason) => exits.push(reason),
|
||||
scratchDir: join(dir, "scratch"),
|
||||
});
|
||||
const config = (version: number, i: number): DesiredChannel => ({
|
||||
workspaceId: WS, channelId: CH, workerId: "soak", assignmentEpoch: 1, desiredVersion: version, desiredStatus: "RUNNING",
|
||||
source: null, output: { endpointUrl: "rtmp://push.example/live", credentialHandleId: "keyref-soak" }, profile, mode: "normalize",
|
||||
production: scene(i, assetIds),
|
||||
});
|
||||
|
||||
const t0 = Date.now();
|
||||
await job.start(config(1, 0));
|
||||
const runningJob = job;
|
||||
const chain = runningJob.program!;
|
||||
await runningJob.previews?.demand(id(1), "video").catch(() => undefined);
|
||||
|
||||
const settleMs = Math.min(60_000, (hours * 3_600_000) / 10);
|
||||
const endAt = t0 + hours * 3_600_000;
|
||||
const swapLatency: number[] = [];
|
||||
let swaps = 0;
|
||||
let swapFailures = 0;
|
||||
let nextSwap = Date.now() + swapEvery * 1000;
|
||||
let nextSample = Date.now() + sampleEvery * 1000;
|
||||
let settled: { ticks: number; fills: number; busDrops: number } | null = null;
|
||||
const samples: Array<{ t: number; procs: Record<string, { rssKiB: number; fds: number } | null> }> = [];
|
||||
const busDrops = async (): Promise<number> => {
|
||||
let total = 0;
|
||||
for (const sid of [id(1), id(2), id(3), id(4), PLAYLIST]) {
|
||||
const bus = runningJob.sourceBus(sid);
|
||||
if (bus) total += (await bus.stats()).subscriberDrops;
|
||||
}
|
||||
return total;
|
||||
};
|
||||
|
||||
while (Date.now() < endAt && exits.length === 0) {
|
||||
const now = Date.now();
|
||||
if (now >= nextSwap) {
|
||||
nextSwap += swapEvery * 1000;
|
||||
const started = Date.now();
|
||||
try {
|
||||
await runningJob.reconfigure(config(swaps + 2, swaps + 1));
|
||||
swaps += 1;
|
||||
swapLatency.push(Date.now() - started);
|
||||
} catch (error) {
|
||||
swapFailures += 1;
|
||||
log({ type: "swap-failed", t: (Date.now() - t0) / 1000, error: String(error) });
|
||||
}
|
||||
}
|
||||
if (now >= nextSample) {
|
||||
nextSample += sampleEvery * 1000;
|
||||
const gate = await chain.stats();
|
||||
const drops = await busDrops();
|
||||
if (!settled && now - t0 >= settleMs) settled = { ticks: gate.ticks, fills: gate.vrep + gate.asil, busDrops: drops };
|
||||
const procs: Record<string, { rssKiB: number; fds: number } | null> = {
|
||||
worker: { rssKiB: Math.round(process.memoryUsage().rss / 1024), fds: readdirSync("/proc/self/fd").length },
|
||||
gate: procStat(chain.gate.pid),
|
||||
encoder: procStat(chain.encoderPid),
|
||||
compositor: procStat(chain.activeCompositor?.pid),
|
||||
programBus: procStat(chain.programBus?.pid ?? undefined),
|
||||
};
|
||||
pids(join(dir, "scratch", "bus-")).forEach((pid, k) => { procs[`bus${k}`] = procStat(pid); });
|
||||
samples.push({ t: now - t0, procs });
|
||||
const sorted = [...swapLatency].sort((a, b) => a - b);
|
||||
log({
|
||||
type: "sample", t: (now - t0) / 1000, ticks: gate.ticks, vrep: gate.vrep, asil: gate.asil, outDrops: gate.outDrops, lagged: gate.lagged,
|
||||
lateOverFrame: gate.lateOverFrame, busDrops: drops, swaps, swapFailures, lagRetries: chain.lagRetries,
|
||||
swapMs: { p50: sorted[Math.floor(sorted.length / 2)] ?? null, max: sorted.at(-1) ?? null },
|
||||
feeder: runningJob.feeder(PLAYLIST)?.stats, previews: { ...published, manifests: manifests.size }, procs,
|
||||
});
|
||||
// Keep the demanded source rendition alive (a viewer watching that tile).
|
||||
await runningJob.previews?.demand(id(1), "video").catch(() => undefined);
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 250));
|
||||
}
|
||||
|
||||
// Verdict.
|
||||
const gate = await chain.stats();
|
||||
const drops = await busDrops();
|
||||
const failures: string[] = [];
|
||||
if (!settled) failures.push("no settled sample collected");
|
||||
if (gate.outDrops !== 0) failures.push(`encoder output drops: ${gate.outDrops}`);
|
||||
if (exits.length) failures.push(`chain exited: ${exits.join("; ")}`);
|
||||
if (swapFailures) failures.push(`${swapFailures} swaps never went on air`);
|
||||
if (settled) {
|
||||
const ticks = gate.ticks - settled.ticks;
|
||||
const fills = gate.vrep + gate.asil - settled.fills;
|
||||
if (drops - settled.busDrops > maxBusDrops) failures.push(`bus drops after settle: ${drops - settled.busDrops}`);
|
||||
if (fills > Math.ceil(ticks / 1000)) failures.push(`gate fills after settle: ${fills} over ${ticks} ticks`);
|
||||
}
|
||||
const baseAt = Math.min(3_600_000, (hours * 3_600_000) / 4);
|
||||
const base = samples.find((s) => s.t >= baseAt);
|
||||
const last = samples.at(-1);
|
||||
const growth: Record<string, { rssPct: number; fds: number }> = {};
|
||||
if (base && last && last !== base) {
|
||||
for (const [name, b] of Object.entries(base.procs)) {
|
||||
const l = last.procs[name];
|
||||
if (!b || !l) continue;
|
||||
// Compositors are replaced by swaps; their size is not a leak signal.
|
||||
if (name === "compositor") continue;
|
||||
growth[name] = { rssPct: Number((((l.rssKiB - b.rssKiB) / b.rssKiB) * 100).toFixed(1)), fds: l.fds - b.fds };
|
||||
if (l.rssKiB > b.rssKiB * 1.15 + 16 * 1024) failures.push(`${name} RSS ${b.rssKiB} → ${l.rssKiB} KiB`);
|
||||
if (Math.abs(l.fds - b.fds) > 8) failures.push(`${name} fds ${b.fds} → ${l.fds}`);
|
||||
}
|
||||
}
|
||||
const summary = {
|
||||
type: "summary", hours: (Date.now() - t0) / 3_600_000, profile, swaps, swapFailures, lagRetries: chain.lagRetries,
|
||||
gate: { ticks: gate.ticks, vrep: gate.vrep, asil: gate.asil, outDrops: gate.outDrops, lagged: gate.lagged }, busDrops: drops, settled,
|
||||
growth, previews: published, feeder: runningJob.feeder(PLAYLIST)?.stats, pass: failures.length === 0, failures,
|
||||
};
|
||||
log(summary);
|
||||
process.stdout.write(`${JSON.stringify(summary)}\n`);
|
||||
exitCode = failures.length === 0 ? 0 : 1;
|
||||
} finally {
|
||||
await job?.stop();
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
process.exitCode = exitCode;
|
||||
}
|
||||
|
||||
if (require.main === module) void main().catch((error: unknown) => { process.stderr.write(`${String(error)}\n`); process.exit(2); });
|
||||
153
apps/media-worker/src/compositor/source-bus.test.ts
Normal file
153
apps/media-worker/src/compositor/source-bus.test.ts
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
/**
|
||||
* Source bus helper (src/gate/rust/src/bin/source-bus.rs) through its Node
|
||||
* manager: the S2a bus semantics on the new out-of-process path. Frames are
|
||||
* tagged (first byte = pair index) so delivery can be checked exactly.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { Socket } from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { audioFrameBytes, videoFrameBytes } from "../gate/protocol";
|
||||
import { ProgramChain } from "./program";
|
||||
import { SourceBus, type BusSubscription } from "./source-bus";
|
||||
import { ensureGateBinary, passthroughRedactor } from "./testing";
|
||||
|
||||
const g = { width: 64, height: 36, fps: 30 };
|
||||
const V = videoFrameBytes(g);
|
||||
const A = audioFrameBytes(g);
|
||||
|
||||
const frame = (bytes: number, tag: number): Buffer => Buffer.alloc(bytes, tag & 0xff);
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
async function until(cond: () => boolean | Promise<boolean>, ms: number, what: string): Promise<void> {
|
||||
const t0 = Date.now();
|
||||
while (!(await cond())) {
|
||||
if (Date.now() - t0 > ms) throw new Error(`timeout: ${what}`);
|
||||
await sleep(20);
|
||||
}
|
||||
}
|
||||
|
||||
type Harness = { bus: SourceBus; video: Socket; audio: Socket; dir: string; close(): Promise<void> };
|
||||
|
||||
/** Writes pair i as an ingest does (interleaved): a burst of one stream would be trimmed as skew. */
|
||||
async function pair(h: Harness, i: number, audio = true): Promise<void> {
|
||||
await Promise.all([
|
||||
new Promise<void>((r) => h.video.write(frame(V, i), () => r())),
|
||||
audio ? new Promise<void>((r) => h.audio.write(frame(A, i), () => r())) : Promise.resolve(),
|
||||
]);
|
||||
}
|
||||
|
||||
async function harness(): Promise<Harness> {
|
||||
const dir = mkdtempSync(join(tmpdir(), "bus-"));
|
||||
const bus = new SourceBus({ geometry: g, scratchDir: dir, redactor: passthroughRedactor, onStderr: () => undefined });
|
||||
const fds = await bus.start();
|
||||
bus.adoptIngestWriters();
|
||||
const video = new Socket({ fd: fds.videoFd, readable: false, writable: true });
|
||||
const audio = new Socket({ fd: fds.audioFd, readable: false, writable: true });
|
||||
return { bus, video, audio, dir, async close() { video.destroy(); audio.destroy(); await bus.stop(); rmSync(dir, { recursive: true, force: true }); } };
|
||||
}
|
||||
|
||||
/** Reads a subscriber's stream into whole frames (first byte of each). */
|
||||
function collect(fd: number | null, bytes: number): { tags: number[]; socket: Socket } {
|
||||
assert.ok(fd !== null);
|
||||
const tags: number[] = [];
|
||||
let pending = Buffer.alloc(0);
|
||||
const socket = new Socket({ fd, readable: true, writable: false });
|
||||
socket.on("data", (chunk: Buffer) => {
|
||||
pending = Buffer.concat([pending, chunk]);
|
||||
while (pending.length >= bytes) { tags.push(pending[0]!); pending = pending.subarray(bytes); }
|
||||
});
|
||||
return { tags, socket };
|
||||
}
|
||||
|
||||
test("history replay, start pair and A/V pairs delivered frame-exact", { timeout: 60_000 }, async (t) => {
|
||||
if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; }
|
||||
const h = await harness();
|
||||
try {
|
||||
for (let i = 0; i < 20; i += 1) await pair(h, i);
|
||||
await until(async () => (await h.bus.stats()).pairs === 20, 5_000, "20 pairs");
|
||||
const sub = await h.bus.subscribe({ video: true, audio: true });
|
||||
assert.equal(sub.startPair, 20 - h.bus.historyFrames, "starts at the oldest replayed pair");
|
||||
const v = collect(sub.videoFd, V);
|
||||
const a = collect(sub.audioFd, A);
|
||||
sub.adoptReaders();
|
||||
for (let i = 20; i < 25; i += 1) await pair(h, i);
|
||||
await until(() => v.tags.length >= 17 && a.tags.length >= 17, 5_000, "replay + live");
|
||||
const expected = Array.from({ length: 17 }, (_, k) => 8 + k);
|
||||
assert.deepEqual(v.tags.slice(0, 17), expected, "video: history then live, no gap");
|
||||
assert.deepEqual(a.tags.slice(0, 17), expected, "audio pairs with video");
|
||||
const stats = await h.bus.stats();
|
||||
assert.equal(stats.subscriberDrops, 0);
|
||||
assert.equal(stats.subscribers.length, 1);
|
||||
v.socket.destroy(); a.socket.destroy();
|
||||
// The reader is gone: the helper drops the subscriber on EPIPE.
|
||||
await pair(h, 25).catch(() => undefined);
|
||||
await until(async () => (await h.bus.stats()).subscribers.length === 0, 5_000, "closed subscriber removed");
|
||||
} finally { await h.close(); }
|
||||
});
|
||||
|
||||
test("stride subscribers get the newest pair then every n-th; audio-only and silence fill", { timeout: 60_000 }, async (t) => {
|
||||
if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; }
|
||||
const h = await harness();
|
||||
try {
|
||||
for (let i = 0; i < 10; i += 1) await pair(h, i);
|
||||
await until(async () => (await h.bus.stats()).pairs === 10, 5_000, "10 pairs");
|
||||
const thumb = await h.bus.subscribe({ video: true, audio: false, stride: 5 });
|
||||
assert.equal(thumb.audioFd, null);
|
||||
assert.equal(thumb.startPair, 9, "the newest pair first");
|
||||
const ears = await h.bus.subscribe({ video: false, audio: true });
|
||||
const tv = collect(thumb.videoFd, V);
|
||||
const ea = collect(ears.audioFd, A);
|
||||
thumb.adoptReaders(); ears.adoptReaders();
|
||||
for (let i = 10; i < 21; i += 1) await pair(h, i);
|
||||
// The audio-only subscriber replays the 10 pairs of history there are, then 11 live ones.
|
||||
await until(() => tv.tags.length >= 3 && ea.tags.length >= 10 + 11, 5_000, "strided + audio");
|
||||
assert.deepEqual(tv.tags.slice(0, 3), [9, 14, 19]);
|
||||
assert.deepEqual(ea.tags.slice(-3), [18, 19, 20]);
|
||||
// Video without audio: once video runs maxSkew ahead, pairs carry silence.
|
||||
for (let i = 21; i < 21 + h.bus.maxSkewFrames + 5; i += 1) await pair(h, i, false);
|
||||
await until(async () => (await h.bus.stats()).silenceFilled >= 5, 5_000, "silence fill");
|
||||
await until(() => ea.tags.at(-1) === 0, 5_000, "silence reaches the audio subscriber");
|
||||
tv.socket.destroy(); ea.socket.destroy();
|
||||
} finally { await h.close(); }
|
||||
});
|
||||
|
||||
test("a stalled subscriber loses whole pairs past its backlog; peers' drop counts are snapshotted at subscribe", { timeout: 60_000 }, async (t) => {
|
||||
if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; }
|
||||
const h = await harness();
|
||||
let stalled: BusSubscription | null = null;
|
||||
try {
|
||||
stalled = await h.bus.subscribe({ video: true, audio: true });
|
||||
// Never read: each 1 MiB pipe fills (audio first: 163 frames here), then the helper queue does.
|
||||
const limit = h.bus.maxBacklogFrames + h.bus.historyFrames;
|
||||
const youngLimit = limit + g.fps;
|
||||
const produced = Math.ceil((1 << 20) / A) + youngLimit + 20;
|
||||
for (let i = 0; i < produced; i += 1) await pair(h, i);
|
||||
await until(async () => (await h.bus.stats()).pairs === produced, 10_000, "entire burst consumed");
|
||||
await until(async () => (await h.bus.stats()).subscriberDrops > 0, 10_000, "drops while young");
|
||||
const young = await h.bus.stats();
|
||||
const queued = young.subscribers[0]?.queued ?? 0;
|
||||
assert.ok(queued > limit && queued <= youngLimit + 1, `a young subscriber queues up to ~1 s more than the limit (${queued} queued, limit ${limit})`);
|
||||
const youngDrops = young.subscriberDrops;
|
||||
await sleep(2_100); // young period over: the normal limit applies, and the queue is still above it
|
||||
for (let i = 0; i < 10; i += 1) await pair(h, i);
|
||||
await until(async () => (await h.bus.stats()).subscriberDrops >= youngDrops + 10, 5_000, "every new pair dropped");
|
||||
await until(() => stalled?.dropped === (youngDrops + 10), 5_000, "DROP events reach the subscription");
|
||||
const peer = await h.bus.subscribe({ video: true, audio: false });
|
||||
assert.equal(peer.droppedBefore(stalled), youngDrops + 10, "exact at the peer's start pair");
|
||||
await pair(h, 1);
|
||||
await until(() => (stalled?.dropped ?? 0) > youngDrops + 10, 5_000, "live count moves on");
|
||||
assert.equal(peer.droppedBefore(stalled), youngDrops + 10, "the snapshot does not");
|
||||
peer.unsubscribe();
|
||||
} finally {
|
||||
stalled?.unsubscribe();
|
||||
await h.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("program chains refuse the Node reference gate (no pair lockstep, no origin alignment)", () => {
|
||||
assert.throws(() => new ProgramChain({ canvas: g, scratchDir: join(tmpdir(), "never"), videoBitrateKbps: 500, output: { kind: "external" }, gateImpl: "node",
|
||||
redactor: passthroughRedactor, onStderr: () => undefined, onFatal: () => undefined }), /spike-only/);
|
||||
});
|
||||
274
apps/media-worker/src/compositor/source-bus.ts
Normal file
274
apps/media-worker/src/compositor/source-bus.ts
Normal file
|
|
@ -0,0 +1,274 @@
|
|||
import { closeSync, mkdirSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { resolveHelperBinary } from "../gate/binary";
|
||||
import { createFifoPair, makeFifoPair, openBlockingReader, type FifoPair } from "../gate/fifo";
|
||||
import { assertGeometry, type FrameGeometry } from "../gate/protocol";
|
||||
import { spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg";
|
||||
|
||||
/**
|
||||
* Per-source fan-out between one ingest and every consumer of the source's raw
|
||||
* frames: the compositors that use it (two during an A/B swap) and its preview
|
||||
* taps.
|
||||
*
|
||||
* The bytes never pass through Node. A `source-bus` helper process
|
||||
* (src/gate/rust/src/bin/source-bus.rs) reads the ingest's FIFOs and writes
|
||||
* each subscriber's FIFOs; this class is its process manager and speaks its
|
||||
* control lines. (S2a moved the frames on the worker's event loop: at
|
||||
* 4 × 1080p30 that left no headroom and the buses dropped ~7.5 % of source
|
||||
* pairs; runs/wp4/s2b-report.md item 1.)
|
||||
*
|
||||
* Semantics are unchanged from S2a: frames are delivered as A/V *pairs*, a
|
||||
* subscriber more than `maxBacklogFrames + historyFrames` behind loses whole
|
||||
* pairs, and the last `historyFrames` pairs are replayed to a new subscriber so
|
||||
* the gate can splice a swap-in compositor at the on-air one's next source frame.
|
||||
*/
|
||||
export interface SourceBusOptions {
|
||||
geometry: FrameGeometry;
|
||||
scratchDir: string;
|
||||
redactor: StderrRedactor;
|
||||
onStderr(message: string): void;
|
||||
/** The helper died (not on `stop()`): the source is gone. */
|
||||
onExit?(reason: string): void;
|
||||
maxSkewFrames?: number;
|
||||
maxBacklogFrames?: number;
|
||||
historyFrames?: number;
|
||||
}
|
||||
|
||||
export interface SubscribeOptions {
|
||||
video: boolean;
|
||||
audio: boolean;
|
||||
/** Take every `stride`-th pair only (thumbnails). Default 1. */
|
||||
stride?: number;
|
||||
}
|
||||
|
||||
/** A subscription: the pair index it starts at, and the pairs it has lost to backlog since. */
|
||||
export interface BusSubscription {
|
||||
readonly bus: SourceBus;
|
||||
readonly id: number;
|
||||
readonly startPair: number;
|
||||
readonly startedAt: bigint;
|
||||
readonly dropped: number;
|
||||
/** Blocking read ends for the consumer's stdio (null when not subscribed to that stream). */
|
||||
readonly videoFd: number | null;
|
||||
readonly audioFd: number | null;
|
||||
/** Pairs `other` had lost when this subscription started (exact, unlike its live `dropped`). */
|
||||
droppedBefore(other: BusSubscription): number | undefined;
|
||||
/** Close the worker's copies of the read ends (after spawning the consumer). */
|
||||
releaseReaders(): void;
|
||||
/** The caller takes ownership of the read ends (an in-process consumer that closes them itself). */
|
||||
adoptReaders(): void;
|
||||
unsubscribe(): void;
|
||||
}
|
||||
|
||||
export interface SourceBusStats {
|
||||
pairs: number; silenceFilled: number; audioTrimmed: number; subscriberDrops: number; allocs: number;
|
||||
input: { videoWaiting: number; audioWaiting: number; videoEof: boolean; audioEof: boolean };
|
||||
/** Per subscriber: pairs dropped to backlog, and frames queued in the helper right now. */
|
||||
subscribers: Array<{ id: number; dropped: number; queued: number }>;
|
||||
}
|
||||
|
||||
interface Waiter { match(line: string[]): boolean; resolve(line: string[]): void; reject(error: Error): void; timer: ReturnType<typeof setTimeout> }
|
||||
|
||||
let nextBus = 1;
|
||||
|
||||
export class SourceBus {
|
||||
private proc: ManagedProcess | null = null;
|
||||
private readonly busId = nextBus++;
|
||||
private nextSub = 1;
|
||||
private lineBuf = "";
|
||||
private waiters: Waiter[] = [];
|
||||
private readonly drops = new Map<number, number>();
|
||||
private stopping = false;
|
||||
private exited = false;
|
||||
private ingest: FifoPair | null = null;
|
||||
readonly maxSkewFrames: number;
|
||||
readonly maxBacklogFrames: number;
|
||||
readonly historyFrames: number;
|
||||
/** Set when the helper reports the ingest closed both streams. */
|
||||
ended = false;
|
||||
|
||||
constructor(private readonly options: SourceBusOptions) {
|
||||
assertGeometry(options.geometry);
|
||||
this.maxSkewFrames = options.maxSkewFrames ?? 15;
|
||||
this.maxBacklogFrames = options.maxBacklogFrames ?? 4;
|
||||
this.historyFrames = options.historyFrames ?? 12;
|
||||
mkdirSync(options.scratchDir, { recursive: true, mode: 0o700 });
|
||||
}
|
||||
|
||||
get geometry(): FrameGeometry { return this.options.geometry; }
|
||||
get pid(): number | undefined { return this.proc?.child.pid; }
|
||||
|
||||
/**
|
||||
* Starts the helper on fresh ingest FIFOs. The returned fds are blocking write ends for the
|
||||
* producer's stdio (ingest: video on fd 1, PCM on fd 3); call `releaseIngestWriters()` once
|
||||
* the producer has them. Until a producer writes, the bus simply has no pairs.
|
||||
*/
|
||||
async start(): Promise<{ videoFd: number; audioFd: number }> {
|
||||
const g = this.options.geometry;
|
||||
const fifos = createFifoPair(this.options.scratchDir, this.busId, "bus");
|
||||
this.ingest = fifos;
|
||||
const argv = [String(g.width), String(g.height), String(g.fps), String(this.maxSkewFrames), String(this.maxBacklogFrames), String(this.historyFrames),
|
||||
fifos.videoPath, fifos.audioPath];
|
||||
const proc = spawnManaged(resolveHelperBinary("source-bus"), argv, ["pipe", "pipe", "pipe"], this.options.redactor, this.options.onStderr);
|
||||
this.proc = proc;
|
||||
proc.child.stdout?.setEncoding("utf8");
|
||||
proc.child.stdout?.on("data", (chunk: string) => this.onData(chunk));
|
||||
// EPIPE on a control line to a helper that just died: its exit is reported below, never as a crash.
|
||||
proc.child.stdin?.on("error", () => undefined);
|
||||
void proc.exited.then(({ code, signal }) => {
|
||||
this.exited = true;
|
||||
const reason = `source bus exited code=${code} signal=${signal}`;
|
||||
for (const w of this.waiters.splice(0)) { clearTimeout(w.timer); w.reject(new Error(reason)); }
|
||||
if (!this.stopping) this.options.onExit?.(reason);
|
||||
});
|
||||
const ready = this.waitFor((l) => l[0] === "READY", 10_000);
|
||||
try {
|
||||
await proc.started;
|
||||
await ready;
|
||||
} catch (error) {
|
||||
ready.catch(() => undefined);
|
||||
fifos.releaseHolders();
|
||||
fifos.releaseWriters();
|
||||
fifos.unlink();
|
||||
throw error;
|
||||
} finally {
|
||||
// The helper holds the read ends now; from here only the producer keeps the FIFOs open.
|
||||
fifos.releaseHolders();
|
||||
fifos.unlink();
|
||||
}
|
||||
return { videoFd: fifos.videoFd, audioFd: fifos.audioFd };
|
||||
}
|
||||
|
||||
/** Close the worker's copies of the ingest write ends (after spawning the producer). */
|
||||
releaseIngestWriters(): void { this.ingest?.releaseWriters(); }
|
||||
|
||||
/** The caller takes ownership of the ingest write ends (an in-process producer that closes them itself). */
|
||||
adoptIngestWriters(): void { this.ingest = null; }
|
||||
|
||||
private onData(chunk: string): void {
|
||||
this.lineBuf += chunk;
|
||||
let nl = this.lineBuf.indexOf("\n");
|
||||
while (nl >= 0) {
|
||||
const line = this.lineBuf.slice(0, nl).trim().split(/\s+/);
|
||||
this.lineBuf = this.lineBuf.slice(nl + 1);
|
||||
nl = this.lineBuf.indexOf("\n");
|
||||
if (line[0] === "DROP") this.drops.set(Number(line[1]), Number(line[2]));
|
||||
if (line[0] === "EOF") this.ended = true;
|
||||
const i = this.waiters.findIndex((w) => w.match(line));
|
||||
if (i >= 0) {
|
||||
const [w] = this.waiters.splice(i, 1);
|
||||
if (w) { clearTimeout(w.timer); w.resolve(line); }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private waitFor(match: (line: string[]) => boolean, timeoutMs: number): Promise<string[]> {
|
||||
return new Promise<string[]>((resolve, reject) => {
|
||||
const waiter: Waiter = {
|
||||
match, resolve, reject,
|
||||
timer: setTimeout(() => { this.waiters = this.waiters.filter((w) => w !== waiter); reject(new Error("source bus reply timeout")); }, timeoutMs),
|
||||
};
|
||||
this.waiters.push(waiter);
|
||||
});
|
||||
}
|
||||
|
||||
private send(line: string): void {
|
||||
const stdin = this.proc?.child.stdin;
|
||||
if (this.exited || !stdin || stdin.destroyed) throw new Error("source bus is not running");
|
||||
stdin.write(line);
|
||||
}
|
||||
|
||||
/** Sends a control line and waits for its reply; a failed send never leaves a waiter behind. */
|
||||
private request(line: string, match: (l: string[]) => boolean, timeoutMs: number): Promise<string[]> {
|
||||
const reply = this.waitFor(match, timeoutMs);
|
||||
try {
|
||||
this.send(line);
|
||||
} catch (error) {
|
||||
reply.catch(() => undefined);
|
||||
const waiter = this.waiters.at(-1);
|
||||
if (waiter) { clearTimeout(waiter.timer); this.waiters.pop(); }
|
||||
throw error;
|
||||
}
|
||||
return reply;
|
||||
}
|
||||
|
||||
/**
|
||||
* New consumer. Delivery starts with the replayed history (a strided subscriber gets only the
|
||||
* newest pair), then live pairs; `startPair` is the first pair's index. Spawn the consumer with
|
||||
* `videoFd`/`audioFd`, then `releaseReaders()`.
|
||||
*/
|
||||
async subscribe(opts: SubscribeOptions): Promise<BusSubscription> {
|
||||
if (!opts.video && !opts.audio) throw new Error("subscription needs video or audio");
|
||||
const stride = opts.stride ?? 1;
|
||||
if (!Number.isSafeInteger(stride) || stride < 1 || stride > 3600) throw new Error("stride out of range");
|
||||
const id = this.nextSub++;
|
||||
const paths = makeFifoPair(this.options.scratchDir, id, `sub-${this.busId}`);
|
||||
// The consumer (ffmpeg) needs *blocking* read ends; a temporary writer keeps each open() from
|
||||
// blocking and is released once the helper has opened its non-blocking write end.
|
||||
const video = opts.video ? openBlockingReader(paths.videoPath) : null;
|
||||
const audio = opts.audio ? openBlockingReader(paths.audioPath) : null;
|
||||
const unlink = (): void => { rmSync(paths.videoPath, { force: true }); rmSync(paths.audioPath, { force: true }); };
|
||||
let line: string[];
|
||||
try {
|
||||
line = await this.request(`SUB ${id} ${video ? paths.videoPath : "-"} ${audio ? paths.audioPath : "-"} ${stride}\n`,
|
||||
(l) => (l[0] === "SUBSCRIBED" || l[0] === "ERROR") && Number(l[1]) === id, 5_000);
|
||||
if (line[0] !== "SUBSCRIBED") throw new Error("source bus could not open subscriber fifos");
|
||||
} catch (error) {
|
||||
for (const r of [video, audio]) if (r) { r.releaseHolder(); closeSync(r.fd); }
|
||||
throw error;
|
||||
} finally {
|
||||
video?.releaseHolder();
|
||||
audio?.releaseHolder();
|
||||
unlink();
|
||||
}
|
||||
const peers = new Map<number, number>();
|
||||
if (line[3] && line[3] !== "-") {
|
||||
for (const entry of line[3].split(",")) {
|
||||
const [peer, dropped] = entry.split(":").map(Number);
|
||||
if (peer !== undefined && dropped !== undefined) peers.set(peer, dropped);
|
||||
}
|
||||
}
|
||||
const drops = this.drops;
|
||||
let readersOpen = true;
|
||||
const subscription: BusSubscription = {
|
||||
bus: this, id, startPair: Number(line[2]), startedAt: process.hrtime.bigint(),
|
||||
get dropped() { return drops.get(id) ?? 0; },
|
||||
videoFd: video?.fd ?? null, audioFd: audio?.fd ?? null,
|
||||
droppedBefore: (other) => (other.bus === this ? peers.get(other.id) : undefined),
|
||||
releaseReaders() { if (readersOpen) { readersOpen = false; for (const r of [video, audio]) if (r) closeSync(r.fd); } },
|
||||
adoptReaders() { readersOpen = false; },
|
||||
unsubscribe: () => {
|
||||
subscription.releaseReaders();
|
||||
drops.delete(id);
|
||||
try { this.send(`UNSUB ${id}\n`); } catch { /* helper gone */ }
|
||||
},
|
||||
};
|
||||
return subscription;
|
||||
}
|
||||
|
||||
async stats(): Promise<SourceBusStats> {
|
||||
const line = await this.request("STATS\n", (l) => l[0] === "STATS", 5_000);
|
||||
const kv = new Map(line.slice(1).map((p) => p.split("=") as [string, string]));
|
||||
const n = (k: string): number => Number(kv.get(k) ?? 0);
|
||||
return { pairs: n("pairs"), silenceFilled: n("silence_filled"), audioTrimmed: n("audio_trimmed"), subscriberDrops: n("subscriber_drops"), allocs: n("allocs"),
|
||||
input: { videoWaiting: n("video_waiting"), audioWaiting: n("audio_waiting"), videoEof: n("video_eof") !== 0, audioEof: n("audio_eof") !== 0 },
|
||||
subscribers: (kv.get("subs") ?? "-").split(",").filter((e) => e !== "-").map((e) => {
|
||||
const [id, dropped, queued] = e.split(":").map(Number);
|
||||
return { id: id ?? 0, dropped: dropped ?? 0, queued: queued ?? 0 };
|
||||
}) };
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.stopping = true;
|
||||
const proc = this.proc;
|
||||
this.ingest?.releaseWriters();
|
||||
if (!proc) return;
|
||||
try { this.send("QUIT\n"); } catch { /* already gone */ }
|
||||
const timer = setTimeout(() => proc.kill("SIGKILL"), 2_000);
|
||||
await proc.exited;
|
||||
clearTimeout(timer);
|
||||
this.proc = null;
|
||||
rmSync(join(this.options.scratchDir, `bus-${this.busId}.v.fifo`), { force: true });
|
||||
rmSync(join(this.options.scratchDir, `bus-${this.busId}.a.fifo`), { force: true });
|
||||
}
|
||||
}
|
||||
211
apps/media-worker/src/compositor/testing.ts
Normal file
211
apps/media-worker/src/compositor/testing.ts
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
/**
|
||||
* Test/spike fixtures for the compositor chain (not used in production paths).
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { closeSync, copyFileSync, openSync, readFileSync } from "node:fs";
|
||||
import { Socket } from "node:net";
|
||||
import { join } from "node:path";
|
||||
import { SCENE_LAYOUTS, type SceneLayout } from "../production";
|
||||
import { ensureHelperBinaries } from "../gate/binary";
|
||||
import { layoutRects, type Canvas, type CompositorGraph, type OverlaySpec, type PxRect } from "./command-builder";
|
||||
import type { FrameGate } from "../gate/frame-gate";
|
||||
import type { ComposeRequest } from "./compositor";
|
||||
import { audioFrameBytes, videoFrameBytes, type FrameGeometry } from "../gate/protocol";
|
||||
import type { SinkReport } from "../gate/sink";
|
||||
import { spawnManaged, type ManagedProcess } from "../relay/ffmpeg";
|
||||
import { SourceBus } from "./source-bus";
|
||||
|
||||
export const passthroughRedactor = { redact: (s: string): string => s };
|
||||
|
||||
/**
|
||||
* Luma of source frame n: a frame counter that survives scaling. Solid frames
|
||||
* scale to (almost) the same value; swscale can round ±1 at some ratios
|
||||
* (4x4 tiles), so the counter moves in steps of 4 and the sink rounds.
|
||||
*/
|
||||
export function counterLuma(n: number): number { return 20 + 4 * (n % 50); }
|
||||
|
||||
/**
|
||||
* A live source paced by the wall clock that writes into a bus's ingest FIFOs
|
||||
* (the bytes an ingest process would write): solid frames whose luma is the
|
||||
* frame counter, and a 1500 Hz sine that holds a whole number of periods per
|
||||
* frame at 24/25/30/60 fps — any splice that is not on a frame boundary, and
|
||||
* any inserted silence, shows up at the sink.
|
||||
*/
|
||||
export class CounterSource {
|
||||
readonly bus: SourceBus;
|
||||
private timer: ReturnType<typeof setTimeout> | null = null;
|
||||
private n = 0;
|
||||
private readonly frames: Buffer[];
|
||||
private readonly pcm: Buffer;
|
||||
private out: { video: Socket; audio: Socket } | null = null;
|
||||
|
||||
constructor(private readonly g: FrameGeometry, scratchDir: string, amplitude = 4000) {
|
||||
this.bus = new SourceBus({ geometry: g, scratchDir, redactor: passthroughRedactor, onStderr: (m) => process.stderr.write(m) });
|
||||
const luma = g.width * g.height;
|
||||
this.frames = Array.from({ length: 50 }, (_, i) => {
|
||||
const f = Buffer.alloc(videoFrameBytes(g), 128);
|
||||
f.fill(counterLuma(i), 0, luma);
|
||||
return f;
|
||||
});
|
||||
const samples = audioFrameBytes(g) / 4;
|
||||
this.pcm = Buffer.alloc(audioFrameBytes(g));
|
||||
for (let s = 0; s < samples; s += 1) {
|
||||
const v = Math.round(amplitude * Math.sin((2 * Math.PI * 1500 * s) / 48_000));
|
||||
this.pcm.writeInt16LE(v, s * 4);
|
||||
this.pcm.writeInt16LE(v, s * 4 + 2);
|
||||
}
|
||||
}
|
||||
|
||||
get framesSent(): number { return this.n; }
|
||||
|
||||
async start(): Promise<void> {
|
||||
const fds = await this.bus.start();
|
||||
this.bus.adoptIngestWriters();
|
||||
const open = (fd: number): Socket => new Socket({ fd, readable: false, writable: true }).on("error", () => undefined);
|
||||
const out = { video: open(fds.videoFd), audio: open(fds.audioFd) };
|
||||
this.out = out;
|
||||
const t0 = process.hrtime.bigint();
|
||||
const period = 1_000_000_000n / BigInt(this.g.fps);
|
||||
const step = (): void => {
|
||||
const now = process.hrtime.bigint();
|
||||
while (t0 + BigInt(this.n) * period <= now) {
|
||||
out.video.write(this.frames[this.n % 50]!);
|
||||
out.audio.write(this.pcm);
|
||||
this.n += 1;
|
||||
}
|
||||
const wait = Number((t0 + BigInt(this.n) * period - process.hrtime.bigint()) / 1_000_000n);
|
||||
this.timer = setTimeout(step, Math.max(0, wait));
|
||||
};
|
||||
step();
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
if (this.timer) clearTimeout(this.timer);
|
||||
this.timer = null;
|
||||
this.out?.video.destroy();
|
||||
this.out?.audio.destroy();
|
||||
this.out = null;
|
||||
await this.bus.stop();
|
||||
}
|
||||
}
|
||||
|
||||
/** The verifying null encoder (src/gate/sink-cli.ts) as its own process, reading the gate's encoder FIFOs. */
|
||||
export class SinkProcess {
|
||||
private proc: ManagedProcess | null = null;
|
||||
private out = "";
|
||||
|
||||
async start(gate: FrameGate, g: FrameGeometry): Promise<void> {
|
||||
const cwd = join(__dirname, "../..");
|
||||
const proc = spawnManaged(process.execPath, ["--import", "tsx", join(__dirname, "../gate/sink-cli.ts"),
|
||||
gate.encoderFifos.videoPath, gate.encoderFifos.audioPath, String(g.width), String(g.height), String(g.fps)],
|
||||
["pipe", "pipe", "pipe"], passthroughRedactor, (m) => process.stderr.write(m), cwd);
|
||||
this.proc = proc;
|
||||
proc.child.stdout?.setEncoding("utf8");
|
||||
proc.child.stdout?.on("data", (c: string) => { this.out += c; });
|
||||
const t0 = Date.now();
|
||||
while (!this.out.includes("OPEN\n")) {
|
||||
if (Date.now() - t0 > 15_000) throw new Error("sink did not open");
|
||||
await new Promise((r) => setTimeout(r, 10));
|
||||
}
|
||||
}
|
||||
|
||||
async report(firstActiveTick: number): Promise<SinkReport & { silentAfterFirstActive: number }> {
|
||||
const proc = this.proc;
|
||||
if (!proc) throw new Error("sink not started");
|
||||
proc.child.stdin?.write(`REPORT ${firstActiveTick}\n`);
|
||||
await proc.exited;
|
||||
const line = this.out.split("\n").find((l) => l.startsWith("{"));
|
||||
if (!line) throw new Error("sink produced no report");
|
||||
return JSON.parse(line) as SinkReport & { silentAfterFirstActive: number };
|
||||
}
|
||||
|
||||
/** Also close the sink when a gate assertion fails before REPORT is requested. */
|
||||
async stop(): Promise<void> {
|
||||
await this.proc?.terminate(500);
|
||||
this.proc = null;
|
||||
}
|
||||
}
|
||||
|
||||
/** H.264 + AAC MPEG-TS clip at the canvas size (encoded once, before measuring): the stand-in for a live HLS source. */
|
||||
export function encodeSourceClip(path: string, canvas: Canvas, seconds = 12): void {
|
||||
const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", `testsrc2=s=${canvas.width}x${canvas.height}:r=${canvas.fps}:d=${seconds}`,
|
||||
"-f", "lavfi", "-i", `sine=frequency=500:sample_rate=48000:d=${seconds}`,
|
||||
"-c:v", "libx264", "-preset", "veryfast", "-b:v", "6000k", "-g", String(2 * canvas.fps), "-c:a", "aac", "-b:a", "128k", "-f", "mpegts", path]);
|
||||
if (r.status !== 0) throw new Error("source clip encode failed");
|
||||
}
|
||||
|
||||
/**
|
||||
* Stand-in for the pinned egress relay (WP7): a paced, looping remux of `clip` into a FIFO.
|
||||
* Returns the read end for an `mpegtsPipe` ingest's stdin (close it after spawning the ingest).
|
||||
*/
|
||||
export async function startPacer(clip: string, dir: string, name: string): Promise<{ readFd: number; proc: ManagedProcess }> {
|
||||
const fifo = join(dir, `${name}.fifo`);
|
||||
spawnSync("mkfifo", [fifo]);
|
||||
const readFd = openSync(fifo, "r+");
|
||||
const writeFd = openSync(fifo, "w");
|
||||
const proc = spawnManaged("ffmpeg", ["-hide_banner", "-loglevel", "error", "-re", "-stream_loop", "-1", "-i", clip, "-c", "copy", "-f", "mpegts", "pipe:1"],
|
||||
["ignore", writeFd, "pipe"], passthroughRedactor, () => undefined);
|
||||
await proc.started;
|
||||
closeSync(writeFd);
|
||||
return { readFd, proc };
|
||||
}
|
||||
|
||||
export const ZERO_GAP_CANVAS: Canvas = { width: 320, height: 180, fps: 30 };
|
||||
const canvas = ZERO_GAP_CANVAS;
|
||||
|
||||
/** Both native helpers (frame gate + source bus), built offline if missing or stale (src/gate/binary.ts). */
|
||||
export function ensureGateBinary(): boolean { return ensureHelperBinaries(); }
|
||||
|
||||
export function makeAssets(dir: string): string[] {
|
||||
const out: string[] = [];
|
||||
for (const src of ["color=c=red@0.6:s=160x24", "testsrc2=s=600x16"]) {
|
||||
const tmp = join(dir, `tmp-${out.length}.png`);
|
||||
const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", `${src},format=rgba`, "-frames:v", "1", "-fflags", "+bitexact", "-flags", "+bitexact", tmp]);
|
||||
assert.equal(r.status, 0, "asset render");
|
||||
const path = join(dir, `${createHash("sha256").update(readFileSync(tmp)).digest("hex")}.png`);
|
||||
copyFileSync(tmp, path);
|
||||
out.push(path);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const CUSTOM: PxRect[] = [{ x: 0, y: 0, w: 160, h: 90 }, { x: 160, y: 90, w: 160, h: 90 }, { x: 100, y: 40, w: 120, h: 68 }];
|
||||
|
||||
/** Swap i: cycles layouts × overlay sets × gains. Slot 0 always shows source 0 at the top-left (the counter pixel). */
|
||||
export function scriptedGraph(i: number): CompositorGraph {
|
||||
const layouts = SCENE_LAYOUTS;
|
||||
const layout: SceneLayout = layouts[i % layouts.length]!;
|
||||
const rects = layout === "pip" ? [{ x: 0, y: 0, w: 320, h: 180 }, { x: 200, y: 100, w: 100, h: 56 }]
|
||||
: layout === "custom" ? CUSTOM : layoutRects(layout, canvas);
|
||||
const tiles = rects.map((rect, t) => ({ source: t === 0 ? 0 : (t % 5 === 4 ? null : t % 4), rect, fit: (["contain", "cover", "crop"] as const)[(i + t) % 3]! }));
|
||||
if (tiles[0]) tiles[0].fit = "cover"; // cover/crop keep the counter pixel (0,0) on picture, contain letterboxes it
|
||||
const overlays: OverlaySpec[] = [];
|
||||
if (i % 3 !== 0) overlays.push({ kind: "image", asset: 0, x: 10, y: 130 });
|
||||
if (i % 2 === 0) overlays.push({ kind: "chyron", asset: 1, y: 160, speedPxPerSec: 40 + ((i * 20) % 360) });
|
||||
const gain = -3 * (i % 9);
|
||||
const audible = [...new Set(tiles.map((t) => t.source).filter((s): s is number => s !== null))].slice(0, 1 + (i % 3));
|
||||
return { canvas, layout, tiles, overlays, audio: { inputs: audible.map((source, k) => ({ source, gainDb: k === 0 ? gain : -6 })), masterGainDb: -3 * (i % 3) } };
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Compose request over only the sources the graph uses (renumbered), as
|
||||
* `planScene` does in production: ffmpeg never reads an input that is not wired
|
||||
* into the filtergraph, so an unused bus subscription would just stall.
|
||||
*/
|
||||
export function composeOver(buses: SourceBus[], assets: string[], graph: CompositorGraph): ComposeRequest {
|
||||
const used = [...new Set(graph.tiles.map((t) => t.source).concat(graph.audio.inputs.map((m) => m.source)).filter((s): s is number => s !== null))].sort((a, b) => a - b);
|
||||
const index = (s: number): number => used.indexOf(s);
|
||||
const audible = new Set(graph.audio.inputs.map((m) => m.source));
|
||||
return {
|
||||
inputs: used.map((s) => ({ bus: buses[s]!, audio: audible.has(s) })),
|
||||
assets,
|
||||
graph: {
|
||||
...graph,
|
||||
tiles: graph.tiles.map((t) => ({ ...t, source: t.source === null ? null : index(t.source) })),
|
||||
audio: { ...graph.audio, inputs: graph.audio.inputs.map((m) => ({ ...m, source: index(m.source) })) },
|
||||
},
|
||||
};
|
||||
}
|
||||
492
apps/media-worker/src/compositor/wp12-hardening.test.ts
Normal file
492
apps/media-worker/src/compositor/wp12-hardening.test.ts
Normal file
|
|
@ -0,0 +1,492 @@
|
|||
/**
|
||||
* WP12 hardening tests (flash slice) — apps/media-worker/src/compositor only.
|
||||
*
|
||||
* Two layers of the §4.3 "Injection" defense are exercised here:
|
||||
*
|
||||
* 1. Graph-builder injection at the type/argv layer. Hostile payloads
|
||||
* (`movie=`, `amovie=`, `filename=`, protocol prefixes, shell
|
||||
* metacharacters, newline/graph-chaining) are injected into every
|
||||
* string-typed input surface of the compositor plan. Either the typed
|
||||
* validators reject the plan (CommandBuildError) or the produced argv is
|
||||
* proven inert: the payload never appears, not even escaped.
|
||||
*
|
||||
* 2. Crop-mode full matrix: contain/cover/crop × every scene layout —
|
||||
* per-tile scale/pad/crop chains asserted from the generated filtergraph
|
||||
* and argv only (no ffmpeg execution).
|
||||
*
|
||||
* Vitest suite: `node_modules/.bin/vitest` of a sibling workspace app resolves
|
||||
* this file (the media-worker devDependencies install only tsx/node:test);
|
||||
* see the run note at the bottom of this file.
|
||||
*/
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
buildCompositorArgs,
|
||||
buildFilterGraph,
|
||||
CommandBuildError,
|
||||
layoutRects,
|
||||
rectFromBasisPoints,
|
||||
SYNTHETIC_PATTERNS,
|
||||
type Canvas,
|
||||
type CompositorPlan,
|
||||
type OverlaySpec,
|
||||
} from "./command-builder";
|
||||
|
||||
const canvas: Canvas = { width: 320, height: 180, fps: 30 };
|
||||
const W = canvas.width;
|
||||
const H = canvas.height;
|
||||
|
||||
function synthetic(i: number) {
|
||||
return {
|
||||
video: { kind: "synthetic" as const, pattern: SYNTHETIC_PATTERNS[i % SYNTHETIC_PATTERNS.length]!, seed: 1000 + i },
|
||||
audio: { kind: "synthetic" as const, toneHz: 220 + 110 * i },
|
||||
};
|
||||
}
|
||||
|
||||
const sha = (c: string) => c.repeat(64);
|
||||
/** Valid absolute scratch asset paths (what `assetPath` accepts). */
|
||||
const ASSET_A = `/scratch/assets/${sha("a")}.png`;
|
||||
const ASSET_B = `/scratch/assets/${sha("b")}.png`;
|
||||
|
||||
/** Minimal valid plan: one live full-frame tile, one image overlay, silent mix. */
|
||||
function basePlan(): CompositorPlan {
|
||||
return {
|
||||
sources: [synthetic(0)],
|
||||
assets: [ASSET_A],
|
||||
graph: {
|
||||
canvas,
|
||||
layout: "single",
|
||||
tiles: [{ source: 0, rect: { x: 0, y: 0, w: W, h: H }, fit: "contain" }],
|
||||
overlays: [{ kind: "image", asset: 0, x: 8, y: 8 }],
|
||||
audio: { inputs: [], masterGainDb: 0 },
|
||||
},
|
||||
output: { kind: "gate", videoFd: 1, audioFd: 3 },
|
||||
};
|
||||
}
|
||||
|
||||
/** Input-index map that buildFilterGraph expects for a one-source/one-asset plan. */
|
||||
const IDX1 = { video: [0], audio: [1], assets: [2] };
|
||||
|
||||
function graphOf(plan: CompositorPlan): string {
|
||||
return buildFilterGraph(plan, IDX1);
|
||||
}
|
||||
|
||||
function argvOf(plan: CompositorPlan): string[] {
|
||||
return buildCompositorArgs(plan);
|
||||
}
|
||||
|
||||
/**
|
||||
* A payload is inert when it is absent from every argv element — including the
|
||||
* escaped forms a filtergraph string could need (`\;`, `\'`, `\\\,` …): the
|
||||
* builder either drops or rejects it, it never transcodes it into a graph.
|
||||
*/
|
||||
function expectInert(plan: CompositorPlan, payload: string): void {
|
||||
const argv = argvOf(plan);
|
||||
const escaped = payload.replace(/([[\];,'\\])/g, "\\$1");
|
||||
for (const arg of argv) {
|
||||
assert.ok(!(arg).includes(payload));
|
||||
assert.ok(!(arg).includes(escaped));
|
||||
}
|
||||
assert.ok(!(argv.join(" ")).includes(escaped));
|
||||
}
|
||||
|
||||
/** Every mutation must either throw CommandBuildError or leave the payload out of argv. */
|
||||
function expectRejectedOrInert(mutate: (plan: CompositorPlan) => CompositorPlan, payload: string): void {
|
||||
const plan = mutate(basePlan());
|
||||
try {
|
||||
argvOf(plan);
|
||||
} catch (error) {
|
||||
assert.ok((error) instanceof CommandBuildError);
|
||||
return;
|
||||
}
|
||||
expectInert(plan, payload);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Graph-builder injection: every string-typed input surface
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** [wb] WP12 payload classes, each must be rejected or proven inert. */
|
||||
const PAYLOADS: Array<[string, string]> = [
|
||||
["lavfi movie/amovie", "movie=evil.mp4"],
|
||||
["lavfi amovie", "amovie=evil.mp3"],
|
||||
["lavfi filename", "filename=/etc/passwd"],
|
||||
["lavfi chained filter", "color=c=black:s=320x180,drawtext=text=pwned"],
|
||||
["file protocol", "file:/etc/passwd"],
|
||||
["http url", "http://evil.example/x.mp4"],
|
||||
["https url", "https://evil.example/x.m3u8"],
|
||||
["pipe protocol", "pipe:0"],
|
||||
["subfile protocol", "subfile,,start=0,end=0,,:/etc/passwd"],
|
||||
["concat protocol", "concat:/etc/passwd|/etc/shadow"],
|
||||
["data protocol", "data:text/plain,pwned"],
|
||||
["hls protocol", "hls+http://evil.example/x.m3u8"],
|
||||
["rtsp url", "rtsp://evil.example:8554/live"],
|
||||
["shell separator", ";rm -rf /"],
|
||||
["shell pipe", "|tee /tmp/pwned"],
|
||||
["shell command substitution", "$(id -u)"],
|
||||
["shell backtick substitution", "`id -u`"],
|
||||
["shell redirect", ">/tmp/pwned"],
|
||||
["shell and-chain", "&&rm -rf /"],
|
||||
["shell or-chain", "||rm -rf /"],
|
||||
["graph pad link", "[0:v]null[sink]"],
|
||||
["graph output pad", "[vout]"],
|
||||
["graph chain separator", "null;nullsink"],
|
||||
["filter option comma", "scale=1:1,scale=2:2"],
|
||||
["filter option equals", "rate=999"],
|
||||
["quoted value", "'$(id -u)'"],
|
||||
["newline chaining", "a\nb"],
|
||||
["crlf chaining", "a\r\nb"],
|
||||
["newline + filter", "\nmovie=evil.mp4"],
|
||||
["tab separator", "\tmovie=evil.mp4"],
|
||||
["unicode line separator", "a\u2028b"],
|
||||
["control char", "a\x01b"],
|
||||
["backslash escape", "\\;movie=evil.mp4"],
|
||||
];
|
||||
|
||||
/** One maximal payload spanning every class at once. */
|
||||
const KITCHEN_SINK = "movie=x|amovie=y|filename=z;[0:v]null[v];$(id)|`id`|\nhttp://e/x|file:/e";
|
||||
|
||||
describe("WP12 injection: overlay asset paths (plan.assets[i])", () => {
|
||||
const withAsset = (path: string): CompositorPlan => ({ ...basePlan(), assets: [path] });
|
||||
|
||||
for (const [_name, payload] of PAYLOADS) {
|
||||
it(`rejects or neutralizes: ${_name}`, () => {
|
||||
// Raw path form…
|
||||
expectRejectedOrInert(withAsset(payload), payload);
|
||||
// …and smuggled inside an otherwise-valid-looking scratch path shape.
|
||||
const smuggle = `/scratch/assets/${payload.replace(/[/\s]/g, "-")}-${sha("p")}.png`;
|
||||
expectRejectedOrInert(withAsset(smuggle), payload);
|
||||
});
|
||||
|
||||
it("rejects traversal and traversal-lookalikes and non-scratch shapes", () => {
|
||||
for (const path of [
|
||||
"/scratch/../etc/passwd.png",
|
||||
`/scratch/assets/../${sha("a")}.png`,
|
||||
`/scratch/assets/${sha("a")}.png/../../${sha("b")}.png`,
|
||||
`relative/${sha("a")}.png`,
|
||||
`${sha("a")}.png`,
|
||||
`/scratch/assets/${sha("a")}.PNG`,
|
||||
`/scratch/assets/${sha("a").slice(0, 63)}.png`,
|
||||
`/scratch/assets/${sha("a")} extra.png`,
|
||||
"",
|
||||
]) {
|
||||
assert.throws(() => argvOf(withAsset(path)), path, CommandBuildError);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects every protocol prefix as an asset path", () => {
|
||||
for (const prefix of ["file:", "http:", "https:", "pipe:", "fd:", "concat:", "subfile,", "data:", "hls+", "rtsp:", "movie=", "amovie=", "filename="]) {
|
||||
assert.throws(() => argvOf(withAsset(`${prefix}${sha("a")}.png`)), prefix, CommandBuildError);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects shell and graph metacharacters anywhere in an asset path", () => {
|
||||
for (const ch of [";", "&", "|", "<", ">", "`", "$", "(", ")", "'", '"', "\\", "[", "]", ",", "\n", "\r", "\t", " ", "=", ":"]) {
|
||||
const path = `/scratch/assets/x${ch}${sha("a").slice(0, 60)}.png`;
|
||||
assert.throws(() => argvOf(withAsset(path)), CommandBuildError);
|
||||
}
|
||||
});
|
||||
|
||||
it("argv carries valid assets only as closed file:<sha>.png inputs", () => {
|
||||
const argv = argvOf(basePlan());
|
||||
assert.ok((argv).includes(`file:${ASSET_A}`));
|
||||
const fileArgs = argv.filter((a) => a.startsWith("file:"));
|
||||
assert.deepStrictEqual(fileArgs, [`file:${ASSET_A}`]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("WP12 injection: overlay geometry and asset indexes (typed, no string surface)", () => {
|
||||
const withOverlay = (o: OverlaySpec): CompositorPlan => ({ ...basePlan(), graph: { ...basePlan().graph, overlays: [o] } });
|
||||
|
||||
it("rejects an overlay naming a nonexistent asset index", () => {
|
||||
assert.throws(() => argvOf(withOverlay({ kind: "image", asset: 9, x: 0, y: 0 })), /asset/);
|
||||
assert.throws(() => argvOf(withOverlay({ kind: "image", asset: -1, x: 0, y: 0 })), /asset/);
|
||||
assert.throws(() => argvOf(withOverlay({ kind: "chyron", asset: 2, y: 0, speedPxPerSec: 120 })), /asset/);
|
||||
});
|
||||
|
||||
it("chyron x-expression is built only from a bounded integer speed — payloads cannot ride the expression", () => {
|
||||
const plan = basePlan();
|
||||
plan.graph.overlays = [{ kind: "chyron", asset: 0, y: 160, speedPxPerSec: 120 }];
|
||||
const graph = graphOf(plan);
|
||||
assert.ok(!/overlay=x='W-mod/.test(graph));
|
||||
assert.ok(!(graph).includes("movie="));
|
||||
// Out-of-band speeds are rejected before any expression string is built.
|
||||
for (const speed of [0, 39, 401, 1e9, -120, 12.5, Number.NaN]) {
|
||||
const bad: CompositorPlan = { ...plan, graph: { ...plan.graph, overlays: [{ kind: "chyron", asset: 0, y: 160, speedPxPerSec: speed }] } };
|
||||
assert.throws(() => argvOf(bad), String(speed), CommandBuildError);
|
||||
}
|
||||
});
|
||||
|
||||
it("overlay geometry is integer-bounded; hostile numbers are type errors, not payload carriers", () => {
|
||||
for (const x of [-1, W + 2, 0.5, Number.NaN, Number.POSITIVE_INFINITY]) {
|
||||
assert.throws(() => argvOf(withOverlay({ kind: "image", asset: 0, x, y: 8 })), String(x), CommandBuildError);
|
||||
}
|
||||
for (const y of [-2, H + 4, 0.5, Number.NaN]) {
|
||||
assert.throws(() => argvOf(withOverlay({ kind: "chyron", asset: 0, y, speedPxPerSec: 120 })), String(y), CommandBuildError);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("WP12 injection: labels and source ids never reach the graph or argv", () => {
|
||||
it("hostile labels on a desired-state snapshot leave no trace in compositor argv", async () => {
|
||||
const { parseProductionSpec } = await import("../production");
|
||||
const { planScene, programScene } = await import("./scene-plan");
|
||||
const id = (n: number) => `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`;
|
||||
const payloads = [
|
||||
"movie=evil.mp4",
|
||||
"amovie=evil.mp3",
|
||||
"filename=/etc/passwd",
|
||||
"; rm -rf /",
|
||||
"$(id -u)",
|
||||
"`id`",
|
||||
"[0:v]null[sink]",
|
||||
"http://evil.example/x.m3u8",
|
||||
"file:/etc/passwd",
|
||||
"a\nmovie=evil.mp4",
|
||||
"pipe:0",
|
||||
KITCHEN_SINK,
|
||||
];
|
||||
// parseProductionSpec must reject all of these as labels (strict allowlist).
|
||||
for (const label of payloads) {
|
||||
assert.throws(() => parseProductionSpec(labelSpec(id, label)));
|
||||
}
|
||||
// A spec with clean labels plans/compiles; ids, labels and URLs stay out of argv.
|
||||
const spec = parseProductionSpec(labelSpec(id, "Cam A"));
|
||||
const plan = planScene(spec, programScene(spec), canvas, (sourceId) => sourceId === id(1));
|
||||
const argv = argvOf({
|
||||
sources: plan.sourceIds.map((_, i) => ({ video: { kind: "rawPipe" as const, fd: 4 + 2 * i }, audio: null })),
|
||||
assets: plan.assetIds.map((a) => `/scratch/assets/${a}.png`),
|
||||
graph: plan.graph,
|
||||
output: { kind: "gate", videoFd: 1, audioFd: 3 },
|
||||
});
|
||||
const joined = argv.join(" ");
|
||||
for (const payload of payloads) {
|
||||
assert.ok(!(joined).includes(payload));
|
||||
assert.ok(!(joined).includes(payload.replace(/([[\];,'\\])/g, "\\$1")));
|
||||
}
|
||||
assert.ok(!(joined).includes("Cam"));
|
||||
assert.ok(!(joined).includes("example"));
|
||||
});
|
||||
|
||||
it("source ids are never argv material: synthetic plans reference patterns and integers only", () => {
|
||||
const argv = argvOf(basePlan());
|
||||
const patternArg = argv.find((a) => a.includes(`testsrc2=s=${W}x${H}`));
|
||||
assert.notStrictEqual(patternArg, undefined);
|
||||
assert.ok(!/[;|`$]|\bmovie=|\bamovie=|\bfilename=/.test(patternArg));
|
||||
// Input references are structured (`<index>:v`, `<index>:a`) — never id strings.
|
||||
assert.ok(new RegExp(/\[0:v\]/).test(graphOf(basePlan())));
|
||||
});
|
||||
});
|
||||
|
||||
describe("WP12 injection: filtergraph token provenance (labels, splits, pads)", () => {
|
||||
it("graph labels are builder-owned; a payload cannot introduce new pad labels", () => {
|
||||
const plan = basePlan();
|
||||
plan.graph.tiles.push({ source: 0, rect: rectFromBasisPoints({ x: 6000, y: 6000, w: 3500, h: 3500 }, canvas), fit: "contain" });
|
||||
const graph = graphOf(plan);
|
||||
// split outputs and tile labels follow the fixed grammar [s<src>_<n>] / [t<i>] / [ov<i>] / [base] / [vout] / [aout]
|
||||
assert.ok(new RegExp(/\[0:v\]split=2\[s0_0\]\[s0_1\]/).test(graph));
|
||||
assert.ok(new RegExp(/\[s0_1\]scale=96:96/).test(graph));
|
||||
assert.ok(new RegExp(/\[t1\]overlay=x=192:y=108/).test(graph));
|
||||
const labels = graph.match(/\[[a-z0-9_:]+\]/g) ?? [];
|
||||
assert.ok((labels.length) > (0));
|
||||
assert.strictEqual(labels.every((m) => /^(?:\d+:v|\d+:a|s\d+_\d+|t\d+|ov\d+|c\d+|m\d+|bg|base|vout|aout)$/.test(m.slice(1, -1))), true);
|
||||
assert.ok(!(graph).includes("[v]"));
|
||||
assert.ok(!(graph).includes("[sink]"));
|
||||
});
|
||||
|
||||
it("the full filtergraph never contains a payload in raw or escaped form", () => {
|
||||
const plan = basePlan();
|
||||
plan.graph.tiles.push({ source: 0, rect: rectFromBasisPoints({ x: 6000, y: 6000, w: 3500, h: 3500 }, canvas), fit: "cover" });
|
||||
plan.graph.overlays = [
|
||||
{ kind: "image", asset: 0, x: 8, y: 8 },
|
||||
{ kind: "chyron", asset: 0, y: 160, speedPxPerSec: 200 },
|
||||
];
|
||||
plan.graph.audio = { inputs: [{ source: 0, gainDb: -6 }], masterGainDb: -3 };
|
||||
const graph = graphOf(plan);
|
||||
for (const [, payload] of PAYLOADS) {
|
||||
assert.ok(!(graph).includes(payload));
|
||||
assert.ok(!(graph).includes(payload.replace(/([\[\];,'\\])/g, "\\$1")));
|
||||
}
|
||||
assert.ok(!(graph).includes(KITCHEN_SINK));
|
||||
});
|
||||
|
||||
it("kitchen-sink payload stays inert end to end (argv layer)", () => {
|
||||
const argv = argvOf(basePlan());
|
||||
const joined = argv.join(" ");
|
||||
assert.ok(!(joined).includes(KITCHEN_SINK));
|
||||
assert.ok(!(joined).includes("movie="));
|
||||
assert.ok(!(joined).includes("amovie="));
|
||||
assert.ok(!(joined).includes("filename="));
|
||||
assert.ok(!(joined).includes("$(id"));
|
||||
assert.ok(!(joined).includes("`id`"));
|
||||
});
|
||||
|
||||
it("single quotes appear only as the chyron x-expression quoting around the builder-owned expression", () => {
|
||||
const single = (plan: CompositorPlan): CompositorPlan => {
|
||||
plan.graph.overlays = [{ kind: "chyron", asset: 0, y: 160, speedPxPerSec: 80 }];
|
||||
return plan;
|
||||
};
|
||||
const graph = graphOf(single(basePlan()));
|
||||
const quoted = graph.match(/'[^']*'/g) ?? [];
|
||||
assert.deepStrictEqual(quoted, ["W-mod(t*80\\,w+W)"]);
|
||||
// No quote-escape sequences exist anywhere in the graph.
|
||||
assert.ok(!(graph).includes("\\'"));
|
||||
assert.ok(!(graph).includes("''"));
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Crop-mode full matrix: contain/cover/crop × layouts
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* `tileFilter` chains per fit (from command-builder.ts):
|
||||
* contain → scale(w,h:force_original_aspect_ratio=decrease:force_divisible_by=2) → pad(w,h,centered,black) → setsar=1
|
||||
* cover → scale(w,h:force_original_aspect_ratio=increase:force_divisible_by=2) → crop(w,h) → setsar=1
|
||||
* crop → crop(w,h) → setsar=1
|
||||
*/
|
||||
const FIT_CHAIN: Record<"contain" | "cover" | "crop", (w: number, h: number) => RegExp> = {
|
||||
contain: (w, h) => new RegExp(`^scale=${w}:${h}:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=${w}:${h}:\\(ow-iw\)/2:\\(oh-ih\)/2:color=black,setsar=1$`),
|
||||
cover: (w, h) => new RegExp(`^scale=${w}:${h}:force_original_aspect_ratio=increase:force_divisible_by=2,crop=${w}:${h},setsar=1$`),
|
||||
crop: (w, h) => new RegExp(`^crop=${w}:${h},setsar=1$`),
|
||||
};
|
||||
|
||||
interface MatrixCase {
|
||||
name: string;
|
||||
layout: "single" | "pip" | "2x1" | "2x2" | "3x3" | "4x4" | "custom";
|
||||
rects: PxRect[];
|
||||
/** Slot indexes rendered black (source=null) — excluded from the live-tile assertions. */
|
||||
empty?: number[];
|
||||
}
|
||||
|
||||
const MATRIX: MatrixCase[] = [
|
||||
{ name: "single", layout: "single", rects: layoutRects("single", canvas) },
|
||||
{ name: "pip", layout: "pip", rects: [{ x: 0, y: 0, w: 320, h: 180 }, rectFromBasisPoints({ x: 6000, y: 6000, w: 3500, h: 3500 }, canvas)] },
|
||||
{ name: "2x1", layout: "2x1", rects: layoutRects("2x1", canvas) },
|
||||
{ name: "2x2", layout: "2x2", rects: layoutRects("2x2", canvas), empty: [3] },
|
||||
{ name: "3x3", layout: "3x3", rects: layoutRects("3x3", canvas), empty: [4, 8] },
|
||||
{ name: "4x4", layout: "4x4", rects: layoutRects("4x4", canvas), empty: [5, 10, 15] },
|
||||
{ name: "custom (non-overlapping, xstack)", layout: "custom", rects: [{ x: 0, y: 0, w: 160, h: 90 }, { x: 160, y: 90, w: 160, h: 90 }] },
|
||||
{ name: "custom (overlapping, overlay chain)", layout: "custom", rects: [{ x: 0, y: 0, w: 200, h: 112 }, { x: 120, y: 68, w: 200, h: 112 }] },
|
||||
];
|
||||
|
||||
/** Every layout × every fit, via buildCompositorArgs (the production entry point). */
|
||||
function matrixPlan(c: MatrixCase, fit: "contain" | "cover" | "crop"): CompositorPlan {
|
||||
const plan = basePlan();
|
||||
plan.sources = [0, 1, 2, 3].map(synthetic);
|
||||
plan.graph.layout = c.layout;
|
||||
plan.graph.tiles = c.rects.map((rect, i) => ({ source: c.empty?.includes(i) ? null : i % 4, rect, fit }));
|
||||
plan.graph.overlays = [];
|
||||
plan.graph.audio = { inputs: [], masterGainDb: 0 };
|
||||
plan.assets = [];
|
||||
return plan;
|
||||
}
|
||||
|
||||
describe("WP12 crop-mode matrix: contain/cover/crop × single/pip/two_up/split grids/custom", () => {
|
||||
it("layout cell rects are even-aligned and tile the canvas exactly", () => {
|
||||
for (const layout of ["2x1", "2x2", "3x3", "4x4"] as const) {
|
||||
const rects = layoutRects(layout, canvas);
|
||||
assert.strictEqual(rects.reduce((a, r) => a + r.w * r.h, 0), W * H);
|
||||
for (const r of rects) for (const v of [r.x, r.y, r.w, r.h]) assert.strictEqual(v % 2, 0);
|
||||
}
|
||||
});
|
||||
|
||||
for (const { c, fit } of MATRIX.flatMap((c) => (["contain", "cover", "crop"] as const).map((fit) => ({ c, fit })))) {
|
||||
const plan = matrixPlan(c, fit);
|
||||
const argv = argvOf(plan);
|
||||
const graph = argv[argv.indexOf("-filter_complex") + 1]!;
|
||||
|
||||
const live = c.rects.filter((_, i) => !c.empty?.includes(i));
|
||||
live.forEach((rect, i) => {
|
||||
const chain = new RegExp(`\\[t${i}\\]`.replace("[", "\\[") + `\\]`); // placeholder, replaced below
|
||||
void chain;
|
||||
const tileChain = new RegExp(`\\[s?\\d*(?::v)?\\][^;]*\\[t${i}\\]`);
|
||||
const match = graph.match(tileChain);
|
||||
assert.notStrictEqual(match, null);
|
||||
assert.strictEqual(FIT_CHAIN[fit](rect.w, rect.h).test(match![0].slice(match![0].indexOf("]") + 1)), true);
|
||||
// Geometry is exactly the layout cell: no source- or fit-dependent drift.
|
||||
assert.ok((graph).includes(`[t${i}]`));
|
||||
});
|
||||
|
||||
// The assembly stage per layout family:
|
||||
if (c.layout === "pip" || c.name.includes("overlapping")) {
|
||||
// PIP and overlapping customs are overlay chains; xstack must not appear.
|
||||
assert.ok(!(graph).includes("xstack"));
|
||||
assert.ok(new RegExp(/overlay=x=\d+:y=\d+:eof_action=repeat/).test(graph));
|
||||
} else if (live.length >= 2) {
|
||||
const layout = live.map((r) => `${r.x}_${r.y}`).join("|");
|
||||
assert.ok(graph.includes(`xstack=inputs=${live.length}:layout=${layout.replace(/([|_])/g, "\\$1")}:fill=black`));
|
||||
} else {
|
||||
assert.ok(!(graph).includes("xstack"));
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("argv structure per matrix row: -i ordering, -filter_complex once, gate outputs on pipe:1/pipe:3", () => {
|
||||
for (const c of MATRIX) {
|
||||
for (const fit of ["contain", "cover", "crop"] as const) {
|
||||
const argv = argvOf(matrixPlan(c, fit));
|
||||
assert.strictEqual((argv.filter((a) => a === "-filter_complex")).length, 1);
|
||||
assert.strictEqual((argv.filter((a) => a === "-i")).length, c.rects.filter((_, i) => !c.empty?.includes(i)).length || 1);
|
||||
assert.ok(!/[&<>]/.test(argv[argv.indexOf("-filter_complex") + 1]));
|
||||
assert.ok(!/[;&|`$<>]/.test(argv[argv.indexOf("-filter_complex") + 1]));
|
||||
assert.strictEqual(argv.at(-1), "pipe:3");
|
||||
assert.ok((argv).includes("pipe:1"));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("split_horizontal/split_vertical equivalents (2x1 / custom row-split) keep per-half fit chains", () => {
|
||||
// Horizontal split = 2x1 grid: left/right halves, each independently cropped.
|
||||
const row = matrixPlan({ name: "2x1", layout: "2x1", rects: layoutRects("2x1", canvas) }, "crop");
|
||||
const rowGraph = argvOf(row)[argvOf(row).indexOf("-filter_complex") + 1]!;
|
||||
assert.ok(new RegExp(/\[t0\]$/m).test(rowGraph));
|
||||
assert.ok((rowGraph).includes("crop=160:180,setsar=1[t0]"));
|
||||
assert.ok((rowGraph).includes("crop=160:180,setsar=1[t1]"));
|
||||
// Vertical split = a custom top/bottom split: same halves stacked, xstack fills black.
|
||||
const col = matrixPlan({ name: "custom vsplit", layout: "custom", rects: [{ x: 0, y: 0, w: 320, h: 90 }, { x: 0, y: 90, w: 320, h: 90 }] }, "cover");
|
||||
const colGraph = argvOf(col)[argvOf(col).indexOf("-filter_complex") + 1]!;
|
||||
assert.ok((colGraph).includes("scale=320:90:force_original_aspect_ratio=increase:force_divisible_by=2,crop=320:90,setsar=1[t0]"));
|
||||
assert.ok((colGraph).includes("scale=320:90:force_original_aspect_ratio=increase:force_divisible_by=2,crop=320:90,setsar=1[t1]"));
|
||||
assert.ok(new RegExp(/xstack=inputs=2:layout=0_0\|0_90:fill=black/).test(colGraph));
|
||||
});
|
||||
|
||||
it("mixed fits inside one layout keep each tile's own chain (cover base + contain PIP inset)", () => {
|
||||
const plan = matrixPlan({ name: "pip", layout: "pip", rects: [{ x: 0, y: 0, w: 320, h: 180 }, rectFromBasisPoints({ x: 6000, y: 6000, w: 3500, h: 3500 }, canvas)] }, "cover");
|
||||
plan.graph.tiles[1]!.fit = "contain";
|
||||
const graph = argvOf(plan)[argvOf(plan).indexOf("-filter_complex") + 1]!;
|
||||
assert.ok((graph).includes("scale=320:180:force_original_aspect_ratio=increase:force_divisible_by=2,crop=320:180,setsar=1[t0]"));
|
||||
assert.ok((graph).includes("scale=96:96:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=96:96:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1[t1]"));
|
||||
assert.ok(new RegExp(/overlay=x=192:y=108:eof_action=repeat/).test(graph));
|
||||
});
|
||||
|
||||
it("contain letterboxes black and never crops; crop never pads; cover never letterboxes", () => {
|
||||
const rect = { x: 0, y: 0, w: 160, h: 180 };
|
||||
const graph = (fit: "contain" | "cover" | "crop") => graphOf({ ...basePlan(), graph: { ...basePlan().graph, tiles: [{ source: 0, rect, fit }] } });
|
||||
const contain = graph("contain");
|
||||
assert.ok((contain).includes("pad=160:180:(ow-iw)/2:(oh-ih)/2:color=black"));
|
||||
assert.ok(!(contain).includes(`crop=${rect.w}:${rect.h}`));
|
||||
const cover = graph("cover");
|
||||
assert.ok((cover).includes(`crop=${rect.w}:${rect.h},setsar=1`));
|
||||
assert.ok(!(cover).includes("pad="));
|
||||
const cropOnly = graph("crop");
|
||||
assert.ok((cropOnly).includes(`crop=${rect.w}:${rect.h},setsar=1`));
|
||||
assert.ok(!(cropOnly).includes("scale="));
|
||||
assert.ok(!(cropOnly).includes("pad="));
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Run note
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// This suite runs under the repo's vitest 2.1.9 (installed with the api app;
|
||||
// the media-worker package itself has no vitest dependency):
|
||||
//
|
||||
// cd apps/api && ./node_modules/.bin/vitest run \
|
||||
// --root ../media-worker src/compositor/wp12-hardening.test.ts
|
||||
//
|
||||
// The media-worker `npm test` glob (node --test) also picks this file up as a
|
||||
// bare `describe/it` import failure — it is authored for vitest only, per the
|
||||
// WP12 deliverable.
|
||||
116
apps/media-worker/src/compositor/zero-gap.test.ts
Normal file
116
apps/media-worker/src/compositor/zero-gap.test.ts
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
/**
|
||||
* S2 gate: zero gap at the encoder input across 1 000 scripted scene, overlay
|
||||
* and gain swaps (streaming-architecture.md §6 S2), plus kill -9 of the
|
||||
* compositor mid-swap leaving the encoder input up.
|
||||
*
|
||||
* Real compositor ffmpeg processes built by MediaCommandBuilder, the Rust frame
|
||||
* gate, and a verifying null encoder in its own process. Sources are
|
||||
* CounterSources (luma = frame counter, phase-locked 1500 Hz sine).
|
||||
* The regular suite runs 50 swaps. The standalone acceptance gate runs
|
||||
* ZERO_GAP_SWAPS=1000, keeping the extended media workload separate from other live-process tests.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import type { ComposeRequest } from "./compositor";
|
||||
import { ProgramChain } from "./program";
|
||||
import { composeOver, CounterSource, ensureGateBinary, makeAssets, passthroughRedactor, scriptedGraph, SinkProcess, ZERO_GAP_CANVAS as canvas } from "./testing";
|
||||
|
||||
const SWAPS = Number(process.env.ZERO_GAP_SWAPS ?? 50);
|
||||
|
||||
test("zero gap at the encoder input across scripted scene/overlay/gain swaps", { timeout: 1_800_000 }, async (t) => {
|
||||
if (!ensureGateBinary()) { t.skip("frame-gate binary unavailable (cargo build failed)"); return; }
|
||||
const scratch = mkdtempSync(join(tmpdir(), "zero-gap-"));
|
||||
const assets = makeAssets(scratch);
|
||||
const sources = Array.from({ length: 4 }, () => new CounterSource(canvas, scratch));
|
||||
for (const s of sources) await s.start();
|
||||
const lost: string[] = [];
|
||||
const chain = new ProgramChain({ canvas, scratchDir: scratch, videoBitrateKbps: 1000, output: { kind: "external" },
|
||||
redactor: passthroughRedactor, onStderr: () => undefined, onFatal: (r) => lost.push(`fatal ${r}`), onCompositorLost: (r) => lost.push(r) });
|
||||
const sink = new SinkProcess();
|
||||
await sink.start(chain.gate, canvas);
|
||||
const request = (i: number): ComposeRequest => composeOver(sources.map((s) => s.bus), assets, scriptedGraph(i));
|
||||
try {
|
||||
await chain.start(request(0));
|
||||
const first = await chain.stats();
|
||||
const firstTick = first.ticks;
|
||||
const swapTicks: number[] = [];
|
||||
for (let i = 1; i <= SWAPS; i += 1) swapTicks.push(await chain.swap(request(i)));
|
||||
// Let the last compositor run, then read the counters.
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
const stats = await chain.stats();
|
||||
// Report before teardown: stopping the chain makes the gate hold its last frame.
|
||||
const report = await sink.report(firstTick);
|
||||
await chain.stop();
|
||||
|
||||
assert.equal(chain.swaps, SWAPS + 1);
|
||||
assert.equal(stats.swaps, SWAPS + 1);
|
||||
assert.deepEqual(lost.filter((l) => !l.startsWith("fatal frame gate exited")), [], "no compositor lost on air");
|
||||
assert.ok(swapTicks.every((tick, k) => k === 0 || tick > (swapTicks[k - 1] ?? 0)), "swap ticks strictly increase");
|
||||
// Encoder-side continuity: every tick delivered exactly one video and one audio frame.
|
||||
assert.ok(Math.abs(report.videoFrames - report.audioFrames) <= 1, "video/audio lockstep at the encoder input");
|
||||
assert.ok(report.videoFrames >= stats.ticks - 2, "encoder received every tick");
|
||||
assert.equal(stats.outDrops, 0, "gate never dropped toward the encoder");
|
||||
// Every delivered frame advances by exactly one, including across every swap.
|
||||
assert.deepEqual(Object.keys(report.tagDeltas), ["1"], `frame counter all +1 (deltas ${JSON.stringify(report.tagDeltas)}, gate lagged=${stats.lagged} behindMax=${stats.behindMax} drops=${stats.drops} vrep=${stats.vrep})`);
|
||||
// PCM continuity: no inserted silence after the first compositor and no step larger than the sine allows.
|
||||
const bound = Math.ceil(report.maxPcmAbs * Math.sin((2 * Math.PI * 1500) / 48_000)) + 4;
|
||||
assert.ok(report.maxPcmStep <= bound, `PCM step ${report.maxPcmStep} <= ${bound}`);
|
||||
const fills = stats.vrep + stats.asil;
|
||||
// On an overloaded host a fill can come from the gate itself not being scheduled; it must stay rare.
|
||||
assert.ok(fills <= Math.ceil(stats.ticks / 1000), `repeat/silence fills ${fills} over ${stats.ticks} ticks`);
|
||||
t.diagnostic(JSON.stringify({ swaps: SWAPS, lagRetries: chain.lagRetries, lagged: stats.lagged, behindMax: stats.behindMax, ticks: stats.ticks, vrep: stats.vrep, asil: stats.asil, lateMaxUs: stats.lateMaxUs, lateOverFrame: stats.lateOverFrame,
|
||||
tagDeltas: report.tagDeltas, maxPcmStep: report.maxPcmStep, bound, maxGapMs: report.maxGapMs }));
|
||||
} finally {
|
||||
await sink.stop();
|
||||
for (const s of sources) await s.stop();
|
||||
await chain.stop();
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("kill -9 of the compositor mid-swap leaves the encoder input up", { timeout: 120_000 }, async (t) => {
|
||||
if (!ensureGateBinary()) { t.skip("frame-gate binary unavailable"); return; }
|
||||
const scratch = mkdtempSync(join(tmpdir(), "kill9-"));
|
||||
const assets = makeAssets(scratch);
|
||||
const sources = Array.from({ length: 4 }, () => new CounterSource(canvas, scratch));
|
||||
for (const s of sources) await s.start();
|
||||
const lost: string[] = [];
|
||||
const fatal: string[] = [];
|
||||
const chain = new ProgramChain({ canvas, scratchDir: scratch, videoBitrateKbps: 1000, output: { kind: "external" },
|
||||
redactor: passthroughRedactor, onStderr: () => undefined, onFatal: (r) => fatal.push(r), onCompositorLost: (r) => lost.push(r) });
|
||||
const sink = new SinkProcess();
|
||||
await sink.start(chain.gate, canvas);
|
||||
const request = (i: number): ComposeRequest => composeOver(sources.map((s) => s.bus), assets, { ...scriptedGraph(i), audio: { inputs: [{ source: 0, gainDb: 0 }], masterGainDb: 0 } });
|
||||
try {
|
||||
await chain.start(request(0));
|
||||
const firstTick = (await chain.stats()).ticks;
|
||||
// Mid-swap: B is starting while A is SIGKILLed.
|
||||
const a = chain.activeCompositor;
|
||||
const swapping = chain.swap(request(1));
|
||||
a?.kill("SIGKILL");
|
||||
await swapping;
|
||||
// On air: SIGKILL the active one; the chain respawns it behind the gate.
|
||||
chain.activeCompositor?.kill("SIGKILL");
|
||||
await new Promise((r) => setTimeout(r, 2_000));
|
||||
await chain.swap(request(2));
|
||||
const before = await chain.stats();
|
||||
await new Promise((r) => setTimeout(r, 1_000));
|
||||
const after = await chain.stats();
|
||||
const report = await sink.report(firstTick);
|
||||
await chain.stop();
|
||||
assert.deepEqual(fatal.filter((f) => !f.startsWith("frame gate exited")), [], "gate and encoder input never went down");
|
||||
assert.ok(lost.length >= 1, "the on-air kill was detected");
|
||||
assert.ok(after.ticks - before.ticks >= canvas.fps * 0.8, "gate kept ticking");
|
||||
assert.ok(Math.abs(report.videoFrames - report.audioFrames) <= 1);
|
||||
assert.ok(report.videoFrames >= after.ticks - 2, "encoder input never starved");
|
||||
t.diagnostic(JSON.stringify({ lost, vrep: after.vrep, asil: after.asil, ticks: after.ticks }));
|
||||
} finally {
|
||||
await sink.stop();
|
||||
for (const s of sources) await s.stop();
|
||||
await chain.stop();
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
85
apps/media-worker/src/gate/binary.ts
Normal file
85
apps/media-worker/src/gate/binary.ts
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
/**
|
||||
* Native helper binaries (src/gate/rust, std only): `frame-gate` (the channel's
|
||||
* clock master) and `source-bus` (per-source raw frame fan-out).
|
||||
*
|
||||
* Resolution order, first hit wins:
|
||||
* 1. `MEDIA_FRAME_GATE_BIN` / `MEDIA_SOURCE_BUS_BIN`: an explicit path per binary;
|
||||
* 2. `MEDIA_HELPER_DIR/<name>`: the packaged directory (the image builds the crate
|
||||
* with `scripts/build-helpers.sh` and sets this; see runs/wp4/s2b-report.md item 5);
|
||||
* 3. the crate's `target/release/<name>` (development checkouts and tests).
|
||||
*
|
||||
* Nothing builds at runtime in production. `ensureHelperBinaries()` is for tests
|
||||
* and the spike/smoke scripts: it builds the crate offline when a binary is
|
||||
* missing or older than its sources.
|
||||
*/
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { existsSync, readdirSync, statSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
export type HelperBinary = "frame-gate" | "source-bus";
|
||||
|
||||
const ENV: Record<HelperBinary, string> = { "frame-gate": "MEDIA_FRAME_GATE_BIN", "source-bus": "MEDIA_SOURCE_BUS_BIN" };
|
||||
|
||||
/** The crate directory, from `src/gate` (tsx) or `dist/gate` (compiled). */
|
||||
export function helperCrateDir(): string {
|
||||
const local = resolve(__dirname, "rust");
|
||||
return existsSync(join(local, "Cargo.toml")) ? local : resolve(__dirname, "../../src/gate/rust");
|
||||
}
|
||||
|
||||
function candidates(name: HelperBinary): string[] {
|
||||
return [
|
||||
process.env[ENV[name]],
|
||||
process.env.MEDIA_HELPER_DIR ? join(process.env.MEDIA_HELPER_DIR, name) : undefined,
|
||||
join(helperCrateDir(), "target/release", name),
|
||||
].filter((path): path is string => typeof path === "string" && path.length > 0);
|
||||
}
|
||||
|
||||
export function resolveHelperBinary(name: HelperBinary): string {
|
||||
const found = candidates(name).find((path) => existsSync(path));
|
||||
if (!found) throw new Error(`${name} binary not found: set ${ENV[name]} or MEDIA_HELPER_DIR, or run scripts/build-helpers.sh`);
|
||||
return found;
|
||||
}
|
||||
|
||||
function newestSource(crate: string): number {
|
||||
const files = ["Cargo.toml", "Cargo.lock"].map((f) => join(crate, f));
|
||||
const walk = (dir: string): void => {
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
const path = join(dir, entry.name);
|
||||
if (entry.isDirectory()) walk(path);
|
||||
else if (entry.name.endsWith(".rs")) files.push(path);
|
||||
}
|
||||
};
|
||||
walk(join(crate, "src"));
|
||||
return Math.max(...files.filter((f) => existsSync(f)).map((f) => statSync(f).mtimeMs));
|
||||
}
|
||||
|
||||
/** Development binaries that are missing or older than the crate sources. Env/packaged binaries are taken as given. */
|
||||
export function staleHelpers(): HelperBinary[] {
|
||||
const crate = helperCrateDir();
|
||||
if (!existsSync(join(crate, "Cargo.toml"))) return [];
|
||||
const newest = newestSource(crate);
|
||||
return (Object.keys(ENV) as HelperBinary[]).filter((name) => {
|
||||
if (process.env[ENV[name]] || process.env.MEDIA_HELPER_DIR) return false;
|
||||
const path = join(crate, "target/release", name);
|
||||
return !existsSync(path) || statSync(path).mtimeMs < newest;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests/scripts: build the crate if a development binary is missing or stale.
|
||||
* `--offline --locked`: no network, and the (dependency-free) lockfile must not change.
|
||||
* Concurrent callers are serialized by cargo's own build-directory lock.
|
||||
*/
|
||||
export function ensureHelperBinaries(): boolean {
|
||||
if (staleHelpers().length > 0) {
|
||||
const built = spawnSync("cargo", ["build", "--release", "--offline", "--locked", "--manifest-path", join(helperCrateDir(), "Cargo.toml")], { stdio: "ignore" });
|
||||
if (built.status !== 0) return false;
|
||||
}
|
||||
try {
|
||||
resolveHelperBinary("frame-gate");
|
||||
resolveHelperBinary("source-bus");
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
29
apps/media-worker/src/gate/buffers.ts
Normal file
29
apps/media-worker/src/gate/buffers.ts
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
/**
|
||||
* Preallocated, refcounted frame buffers. Raw 1080p frames are 3 MB; allocating
|
||||
* one per frame is ~93 MB/s of external memory per stream and drives GC, so the
|
||||
* gate and the source bus recycle buffers once every in-flight write is done.
|
||||
*/
|
||||
export class BufferPool {
|
||||
private readonly free: Buffer[] = [];
|
||||
private readonly refs = new Map<Buffer, number>();
|
||||
/** Buffers allocated after construction; stays 0 in steady state. */
|
||||
allocs = 0;
|
||||
constructor(readonly size: number, prealloc: number) {
|
||||
for (let i = 0; i < prealloc; i += 1) this.free.push(Buffer.alloc(size));
|
||||
}
|
||||
/** The caller owns one reference; `release()` it when done. */
|
||||
get(): Buffer {
|
||||
let buf = this.free.pop();
|
||||
if (!buf) { this.allocs += 1; buf = Buffer.alloc(this.size); }
|
||||
this.refs.set(buf, 1);
|
||||
return buf;
|
||||
}
|
||||
retain(buf: Buffer): void { this.refs.set(buf, (this.refs.get(buf) ?? 0) + 1); }
|
||||
release(buf: Buffer): void {
|
||||
const n = (this.refs.get(buf) ?? 0) - 1;
|
||||
if (n > 0) { this.refs.set(buf, n); return; }
|
||||
if (n < 0) return; // not pool-owned or already free
|
||||
this.refs.delete(buf);
|
||||
this.free.push(buf);
|
||||
}
|
||||
}
|
||||
79
apps/media-worker/src/gate/fifo.ts
Normal file
79
apps/media-worker/src/gate/fifo.ts
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
import { spawnSync } from "node:child_process";
|
||||
import { closeSync, constants, openSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { mediaEnv } from "../relay/ffmpeg";
|
||||
|
||||
export interface FifoPair {
|
||||
videoPath: string;
|
||||
audioPath: string;
|
||||
/**
|
||||
* O_WRONLY write ends for the producer's stdio. A temporary non-blocking read
|
||||
* end keeps each FIFO open (no ENXIO, no blocking open) until the gate has
|
||||
* attached. After both releases only the producer writes and only the gate
|
||||
* reads: producer death is an EOF at the gate, and a gate-side close is an
|
||||
* EPIPE at the producer (a retired compositor cannot hang on a full pipe).
|
||||
*/
|
||||
videoFd: number;
|
||||
audioFd: number;
|
||||
/** Close the worker's copies of the write ends (after spawning the producer). */
|
||||
releaseWriters(): void;
|
||||
/** Close the temporary read ends (after the gate has opened its own). */
|
||||
releaseHolders(): void;
|
||||
unlink(): void;
|
||||
}
|
||||
|
||||
const SCRATCH = /^\/[A-Za-z0-9._/-]+$/;
|
||||
|
||||
/** mkfifo(1) via argv (Node has no mkfifo binding). */
|
||||
const PREFIX = /^[a-z][a-z0-9-]{0,31}$/;
|
||||
|
||||
/** Creates `<prefix>-<id>.{v,a}.fifo`; `prefix` names the owner (`gate`, `bus`, `sub-3`). Paths are scratch dir + prefix + numeric id only. */
|
||||
export function makeFifoPair(scratchDir: string, id: number, prefix: string): { videoPath: string; audioPath: string } {
|
||||
if (!SCRATCH.test(scratchDir) || scratchDir.includes("..") || !Number.isSafeInteger(id) || id < 0 || !PREFIX.test(prefix)) throw new Error("unsafe fifo location");
|
||||
const videoPath = join(scratchDir, `${prefix}-${id}.v.fifo`);
|
||||
const audioPath = join(scratchDir, `${prefix}-${id}.a.fifo`);
|
||||
const made = spawnSync("mkfifo", ["-m", "600", videoPath, audioPath], { shell: false, env: mediaEnv(), stdio: "ignore" });
|
||||
if (made.status !== 0) throw new Error("mkfifo failed");
|
||||
return { videoPath, audioPath };
|
||||
}
|
||||
|
||||
const NAME = /^[a-z][a-z0-9-]{0,47}\.fifo$/;
|
||||
|
||||
/** One FIFO `<scratchDir>/<name>` (name: lowercase, digits, '-', ending `.fifo`). */
|
||||
export function makeFifo(scratchDir: string, name: string): string {
|
||||
if (!SCRATCH.test(scratchDir) || scratchDir.includes("..") || !NAME.test(name)) throw new Error("unsafe fifo location");
|
||||
const path = join(scratchDir, name);
|
||||
const made = spawnSync("mkfifo", ["-m", "600", path], { shell: false, env: mediaEnv(), stdio: "ignore" });
|
||||
if (made.status !== 0) throw new Error("mkfifo failed");
|
||||
return path;
|
||||
}
|
||||
|
||||
export function createFifoPair(scratchDir: string, id: number, prefix = "gate"): FifoPair {
|
||||
const { videoPath, audioPath } = makeFifoPair(scratchDir, id, prefix);
|
||||
const holders = [openSync(videoPath, constants.O_RDONLY | constants.O_NONBLOCK), openSync(audioPath, constants.O_RDONLY | constants.O_NONBLOCK)];
|
||||
const videoFd = openSync(videoPath, constants.O_WRONLY);
|
||||
const audioFd = openSync(audioPath, constants.O_WRONLY);
|
||||
let writers = true;
|
||||
let held = true;
|
||||
return {
|
||||
videoPath, audioPath, videoFd, audioFd,
|
||||
releaseWriters() { if (writers) { writers = false; closeSync(videoFd); closeSync(audioFd); } },
|
||||
releaseHolders() { if (held) { held = false; for (const fd of holders) closeSync(fd); } },
|
||||
unlink() { rmSync(videoPath, { force: true }); rmSync(audioPath, { force: true }); },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A *blocking* read end for a consumer's stdio (ffmpeg must not get O_NONBLOCK).
|
||||
* A plain O_RDONLY open blocks until a writer exists, so a temporary writer is
|
||||
* opened first (itself made possible by a temporary non-blocking reader).
|
||||
* Call `releaseHolder()` once the real writer has opened the FIFO.
|
||||
*/
|
||||
export function openBlockingReader(path: string): { fd: number; releaseHolder(): void } {
|
||||
const probe = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK);
|
||||
const holder = openSync(path, constants.O_WRONLY);
|
||||
const fd = openSync(path, constants.O_RDONLY);
|
||||
closeSync(probe);
|
||||
let held = true;
|
||||
return { fd, releaseHolder() { if (held) { held = false; closeSync(holder); } } };
|
||||
}
|
||||
244
apps/media-worker/src/gate/frame-gate.ts
Normal file
244
apps/media-worker/src/gate/frame-gate.ts
Normal file
|
|
@ -0,0 +1,244 @@
|
|||
import { spawnSync } from "node:child_process";
|
||||
import { closeSync, constants, mkdirSync, openSync, rmSync } from "node:fs";
|
||||
import { Socket } from "node:net";
|
||||
import { join, resolve } from "node:path";
|
||||
import { mediaEnv, spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg";
|
||||
import { resolveHelperBinary } from "./binary";
|
||||
import { createFifoPair, openBlockingReader, type FifoPair } from "./fifo";
|
||||
import { assertGeometry, attachLine, parseGateEvent, type FrameGeometry, type GateEvent, type GateStats } from "./protocol";
|
||||
|
||||
/**
|
||||
* `rust` is the production gate (spike decision). `node` is the
|
||||
* reference implementation of the pipe contract, kept for the spike comparison and the contract
|
||||
* test only: it lacks A/V pair lockstep and origin alignment, and ProgramChain refuses it.
|
||||
*/
|
||||
export type GateImpl = "rust" | "node";
|
||||
|
||||
export interface FrameGateOptions {
|
||||
geometry: FrameGeometry;
|
||||
scratchDir: string;
|
||||
redactor: StderrRedactor;
|
||||
onStderr(message: string): void;
|
||||
/** Unsolicited events (ACTIVE, RETIRED, EOF, STALL...) and gate exit. */
|
||||
onEvent?(event: GateEvent): void;
|
||||
onExit?(reason: string): void;
|
||||
impl?: GateImpl;
|
||||
/** Frames buffered per compositor stream (default 16). */
|
||||
ringFrames?: number;
|
||||
/**
|
||||
* Frames a pending compositor must buffer before going on air with no aligned predecessor
|
||||
* (default 12, 400 ms at 30 fps). This sets the on-air delay margin, which every later swap
|
||||
* preserves: a swap-in compositor can splice forward only while its pipeline latency stays inside
|
||||
* it. 8 frames was not enough for a CPU-starved spawn on an oversubscribed host (s2a-report: alignment).
|
||||
*/
|
||||
warmFrames?: number;
|
||||
/** Rust gate: write with plain copies instead of vmsplice (spike comparison only). */
|
||||
copyWrites?: boolean;
|
||||
}
|
||||
|
||||
export interface GateSlot { id: number; fifos: FifoPair }
|
||||
|
||||
/**
|
||||
* The gate's output FIFOs. The reader (the persistent encoder via
|
||||
* `encoderInputs()`, or a test sink via `openReaders()`) opens them before
|
||||
* `start()`, so the gate's write-side open never blocks.
|
||||
*/
|
||||
export interface EncoderFifos { videoPath: string; audioPath: string }
|
||||
|
||||
export function resolveGateCommand(impl: GateImpl): { command: string; args: string[]; cwd?: string } {
|
||||
if (impl === "rust") return { command: resolveHelperBinary("frame-gate"), args: [] };
|
||||
if (__filename.endsWith(".js")) return { command: process.execPath, args: [resolve(__dirname, "node-gate.js")] };
|
||||
return { command: process.execPath, args: ["--import", "tsx", resolve(__dirname, "node-gate.ts")], cwd: resolve(__dirname, "../..") };
|
||||
}
|
||||
|
||||
interface Waiter { match(event: GateEvent): boolean; resolve(event: GateEvent): void; reject(error: Error): void; timer: ReturnType<typeof setTimeout> }
|
||||
|
||||
export class CompositorLagError extends Error {
|
||||
constructor(readonly slotId: number, readonly behind: number) {
|
||||
super(`compositor ${slotId} stayed ${behind} frames behind the on-air source position`);
|
||||
this.name = "CompositorLagError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Process manager for one channel's frame gate. */
|
||||
export class FrameGate {
|
||||
private proc: ManagedProcess | null = null;
|
||||
private nextId = 1;
|
||||
private waiters: Waiter[] = [];
|
||||
private lineBuf = "";
|
||||
private stopping = false;
|
||||
readonly encoderFifos: EncoderFifos;
|
||||
readonly ringFrames: number;
|
||||
readonly warmFrames: number;
|
||||
|
||||
constructor(private readonly options: FrameGateOptions) {
|
||||
assertGeometry(options.geometry);
|
||||
this.ringFrames = options.ringFrames ?? 16;
|
||||
this.warmFrames = options.warmFrames ?? 12;
|
||||
mkdirSync(options.scratchDir, { recursive: true, mode: 0o700 });
|
||||
this.encoderFifos = { videoPath: join(options.scratchDir, "encoder.v.fifo"), audioPath: join(options.scratchDir, "encoder.a.fifo") };
|
||||
for (const path of [this.encoderFifos.videoPath, this.encoderFifos.audioPath]) rmSync(path, { force: true });
|
||||
const made = spawnSync("mkfifo", ["-m", "600", this.encoderFifos.videoPath, this.encoderFifos.audioPath], { shell: false, env: mediaEnv(), stdio: "ignore" });
|
||||
if (made.status !== 0) throw new Error("mkfifo failed");
|
||||
}
|
||||
|
||||
private holders: Array<() => void> = [];
|
||||
private inputFds: number[] = [];
|
||||
|
||||
/**
|
||||
* Blocking read ends for the persistent encoder's stdio (video on fd 0, audio
|
||||
* on fd 3: `-i pipe:0`, `-i pipe:3`). Spawn the encoder, then `start()`; the
|
||||
* worker's copies are closed once the gate is up, so the encoder sees EOF
|
||||
* only if the gate itself dies.
|
||||
*/
|
||||
encoderInputs(): { videoFd: number; audioFd: number } {
|
||||
const video = openBlockingReader(this.encoderFifos.videoPath);
|
||||
const audio = openBlockingReader(this.encoderFifos.audioPath);
|
||||
this.holders.push(video.releaseHolder, audio.releaseHolder);
|
||||
this.inputFds.push(video.fd, audio.fd);
|
||||
return { videoFd: video.fd, audioFd: audio.fd };
|
||||
}
|
||||
|
||||
/** Null-encoder side for tests and the spike: non-blocking read ends of the encoder FIFOs. */
|
||||
openReaders(): { video: Socket; audio: Socket } {
|
||||
const open = (path: string) => new Socket({ fd: openSync(path, constants.O_RDONLY | constants.O_NONBLOCK), readable: true, writable: false });
|
||||
return { video: open(this.encoderFifos.videoPath), audio: open(this.encoderFifos.audioPath) };
|
||||
}
|
||||
|
||||
get pid(): number | undefined { return this.proc?.child.pid; }
|
||||
|
||||
async start(): Promise<void> {
|
||||
const { command, args, cwd } = resolveGateCommand(this.options.impl ?? "rust");
|
||||
const g = this.options.geometry;
|
||||
const argv = [...args, String(g.width), String(g.height), String(g.fps), String(this.ringFrames), String(this.warmFrames),
|
||||
this.encoderFifos.videoPath, this.encoderFifos.audioPath, ...(this.options.copyWrites ? ["copy"] : [])];
|
||||
const proc = spawnManaged(command, argv, ["pipe", "pipe", "pipe"], this.options.redactor, this.options.onStderr, cwd);
|
||||
this.proc = proc;
|
||||
proc.child.stdout?.setEncoding("utf8");
|
||||
proc.child.stdout?.on("data", (chunk: string) => this.onData(chunk));
|
||||
// EPIPE on a control line to a gate that just died: its exit is reported below, never as a crash.
|
||||
proc.child.stdin?.on("error", () => undefined);
|
||||
void proc.exited.then(({ code, signal }) => {
|
||||
const reason = `frame gate exited code=${code} signal=${signal}`;
|
||||
for (const w of this.waiters.splice(0)) { clearTimeout(w.timer); w.reject(new Error(reason)); }
|
||||
if (!this.stopping) this.options.onExit?.(reason);
|
||||
});
|
||||
const ready = this.waitFor((e) => e.type === "READY", 10_000);
|
||||
try {
|
||||
await proc.started;
|
||||
await ready;
|
||||
} catch (error) {
|
||||
ready.catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
for (const release of this.holders.splice(0)) release();
|
||||
for (const fd of this.inputFds.splice(0)) closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
private onData(chunk: string): void {
|
||||
this.lineBuf += chunk;
|
||||
let nl = this.lineBuf.indexOf("\n");
|
||||
while (nl >= 0) {
|
||||
const event = parseGateEvent(this.lineBuf.slice(0, nl));
|
||||
this.lineBuf = this.lineBuf.slice(nl + 1);
|
||||
nl = this.lineBuf.indexOf("\n");
|
||||
if (!event) continue;
|
||||
const i = this.waiters.findIndex((w) => w.match(event));
|
||||
if (i >= 0) {
|
||||
const [w] = this.waiters.splice(i, 1);
|
||||
if (w) { clearTimeout(w.timer); w.resolve(event); }
|
||||
}
|
||||
this.options.onEvent?.(event);
|
||||
}
|
||||
}
|
||||
|
||||
waitFor(match: (event: GateEvent) => boolean, timeoutMs: number): Promise<GateEvent> {
|
||||
return new Promise<GateEvent>((resolvePromise, reject) => {
|
||||
const waiter: Waiter = {
|
||||
match, resolve: resolvePromise, reject,
|
||||
timer: setTimeout(() => {
|
||||
this.waiters = this.waiters.filter((w) => w !== waiter);
|
||||
reject(new Error("frame gate event timeout"));
|
||||
}, timeoutMs),
|
||||
};
|
||||
this.waiters.push(waiter);
|
||||
});
|
||||
}
|
||||
|
||||
private send(line: string): void {
|
||||
const stdin = this.proc?.child.stdin;
|
||||
if (!stdin || stdin.destroyed) throw new Error("frame gate is not running");
|
||||
stdin.write(line);
|
||||
}
|
||||
|
||||
/** Sends a control line and waits for its reply; a failed send never leaves a waiter behind. */
|
||||
private request(line: string, match: (event: GateEvent) => boolean, timeoutMs: number): Promise<GateEvent> {
|
||||
const reply = this.waitFor(match, timeoutMs);
|
||||
try {
|
||||
this.send(line);
|
||||
} catch (error) {
|
||||
reply.catch(() => undefined);
|
||||
const waiter = this.waiters.at(-1);
|
||||
if (waiter) { clearTimeout(waiter.timer); this.waiters.pop(); }
|
||||
throw error;
|
||||
}
|
||||
return reply;
|
||||
}
|
||||
|
||||
/** FIFOs for the next compositor: spawn it with `fifos.videoFd`/`fifos.audioFd`, then `attach()`. */
|
||||
allocateSlot(): GateSlot {
|
||||
const id = this.nextId;
|
||||
this.nextId += 1;
|
||||
return { id, fifos: createFifoPair(this.options.scratchDir, id) };
|
||||
}
|
||||
|
||||
/** Hands the slot to the gate as the pending compositor; resolves once both FIFOs are open. */
|
||||
async attach(slot: GateSlot, origin?: number): Promise<void> {
|
||||
slot.fifos.releaseWriters();
|
||||
try {
|
||||
const event = await this.request(attachLine(slot.id, slot.fifos.videoPath, slot.fifos.audioPath, origin),
|
||||
(e) => (e.type === "ATTACHED" || e.type === "ERROR") && e.id === slot.id, 5_000);
|
||||
if (event.type === "ERROR") throw new Error("frame gate could not open compositor fifos");
|
||||
} finally {
|
||||
slot.fifos.releaseHolders();
|
||||
slot.fifos.unlink();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves with the tick at which the slot went on air. Rejects with `CompositorLagError` when
|
||||
* the gate reports the slot cannot reach the on-air source position (it never splices backwards).
|
||||
*/
|
||||
async waitActive(id: number, timeoutMs: number): Promise<number> {
|
||||
const event = await this.waitFor((e) => (e.type === "ACTIVE" || e.type === "EOF" || e.type === "DROPPED" || e.type === "LAG") && "id" in e && e.id === id, timeoutMs);
|
||||
if (event.type === "LAG") throw new CompositorLagError(id, event.behind);
|
||||
if (event.type !== "ACTIVE") throw new Error(`compositor ${id} ended before going on air`);
|
||||
return event.tick;
|
||||
}
|
||||
|
||||
detach(id: number): void { this.send(`DETACH ${id}\n`); }
|
||||
|
||||
async stats(): Promise<GateStats> {
|
||||
const event = await this.request("STATS\n", (e) => e.type === "STATS", 5_000);
|
||||
if (event.type !== "STATS") throw new Error("unexpected gate reply");
|
||||
return event.stats;
|
||||
}
|
||||
|
||||
/** Zero the peak counters (late/write maxima), e.g. after encoder warm-up. */
|
||||
async resetPeaks(): Promise<void> {
|
||||
await this.request("RESET\n", (e) => e.type === "RESET", 5_000);
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.stopping = true;
|
||||
const proc = this.proc;
|
||||
if (!proc) return;
|
||||
try { this.send("QUIT\n"); } catch { /* already gone */ }
|
||||
const timer = setTimeout(() => proc.kill("SIGKILL"), 2_000);
|
||||
await proc.exited;
|
||||
clearTimeout(timer);
|
||||
this.proc = null;
|
||||
for (const path of [this.encoderFifos.videoPath, this.encoderFifos.audioPath]) rmSync(path, { force: true });
|
||||
}
|
||||
}
|
||||
144
apps/media-worker/src/gate/gate.test.ts
Normal file
144
apps/media-worker/src/gate/gate.test.ts
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { ensureGateBinary, passthroughRedactor, SinkProcess } from "../compositor/testing";
|
||||
import { FrameGate, type GateImpl } from "./frame-gate";
|
||||
import { assertGeometry, attachLine, audioFrameBytes, parseGateEvent, videoFrameBytes } from "./protocol";
|
||||
import { startProducer } from "./spike";
|
||||
import { spawnManaged } from "../relay/ffmpeg";
|
||||
|
||||
test("pipe contract: frame sizes, geometry and control-line safety", () => {
|
||||
assert.equal(videoFrameBytes({ width: 1920, height: 1080, fps: 30 }), 3_110_400);
|
||||
assert.equal(audioFrameBytes({ width: 1920, height: 1080, fps: 30 }), 6_400);
|
||||
assert.equal(audioFrameBytes({ width: 1920, height: 1080, fps: 60 }), 3_200);
|
||||
for (const g of [{ width: 1921, height: 1080, fps: 30 }, { width: 1920, height: 1080, fps: 7 }, { width: 0, height: 2, fps: 30 }]) assert.throws(() => assertGeometry(g));
|
||||
assert.equal(attachLine(3, "/tmp/x/gate-3.v.fifo", "/tmp/x/gate-3.a.fifo", 42), "ATTACH 3 /tmp/x/gate-3.v.fifo /tmp/x/gate-3.a.fifo 42\n");
|
||||
for (const bad of ["/tmp/a b", "/tmp/../etc", "relative", "/tmp/x\nQUIT", "/tmp/x;y"]) assert.throws(() => attachLine(1, bad, "/tmp/ok"), bad);
|
||||
assert.throws(() => attachLine(1, "/tmp/a", "/tmp/b", 1.5));
|
||||
assert.deepEqual(parseGateEvent("ACTIVE 4 120"), { type: "ACTIVE", id: 4, tick: 120 });
|
||||
const stats = parseGateEvent("STATS ticks=10 vrep=1 asil=2 active=4:2/2 unknown=9");
|
||||
assert.ok(stats?.type === "STATS" && stats.stats.ticks === 10 && stats.stats.vrep === 1 && stats.stats.active === "4:2/2");
|
||||
assert.equal(parseGateEvent("BOGUS"), null);
|
||||
});
|
||||
|
||||
/** Same contract, both implementations: boot black, swap producers at frame boundaries, survive a dead producer. */
|
||||
for (const impl of ["rust", "node"] as GateImpl[]) {
|
||||
test(`${impl} gate: swaps are frame-continuous and a dead producer never starves the encoder`, { timeout: 120_000 }, async (t) => {
|
||||
if (impl === "rust" && !ensureGateBinary()) { t.skip("frame-gate binary unavailable"); return; }
|
||||
const g = { width: 320, height: 180, fps: 30 };
|
||||
const scratch = mkdtempSync(join(tmpdir(), `gate-${impl}-`));
|
||||
const fatal: string[] = [];
|
||||
const gate = new FrameGate({ geometry: g, scratchDir: scratch, impl, redactor: passthroughRedactor, onStderr: () => undefined, onExit: (r) => fatal.push(r) });
|
||||
const sink = new SinkProcess();
|
||||
await sink.start(gate, g);
|
||||
await gate.start();
|
||||
const producers = [];
|
||||
try {
|
||||
let current = await startProducer(gate, g, 20);
|
||||
producers.push(current.proc);
|
||||
const first = await gate.waitActive(current.id, 10_000);
|
||||
for (let i = 1; i <= 10; i += 1) {
|
||||
const next = await startProducer(gate, g, 20 + i);
|
||||
producers.push(next.proc);
|
||||
await gate.waitActive(next.id, 10_000);
|
||||
void current.proc.terminate();
|
||||
current = next;
|
||||
}
|
||||
const beforeKill = await gate.stats();
|
||||
current.proc.kill("SIGKILL");
|
||||
await new Promise((r) => setTimeout(r, 700));
|
||||
const afterKill = await gate.stats();
|
||||
const report = await sink.report(first);
|
||||
assert.deepEqual(fatal, []);
|
||||
assert.equal(beforeKill.swaps, 11);
|
||||
assert.equal(beforeKill.vrep + beforeKill.asil, 0, "no repeat/silence while producers were live");
|
||||
assert.ok(afterKill.ticks - beforeKill.ticks >= 15, "gate kept ticking after the producer died");
|
||||
assert.ok(Math.abs(report.videoFrames - report.audioFrames) <= 1);
|
||||
// Generation tags 20..30 appear in order, each once (after the boot black frames, tag 16).
|
||||
const tags = report.tagChanges.map(([, tag]) => tag).filter((tag) => tag !== 16);
|
||||
assert.deepEqual(tags, Array.from({ length: 11 }, (_, i) => 20 + i));
|
||||
const bound = Math.ceil(report.maxPcmAbs * Math.sin((2 * Math.PI * 1500) / 48_000)) + 4;
|
||||
assert.ok(report.maxPcmStep <= bound, `PCM step ${report.maxPcmStep} <= ${bound}`);
|
||||
} finally {
|
||||
await sink.stop();
|
||||
await gate.stop();
|
||||
for (const p of producers) await p.terminate(500);
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test("Rust holds an ahead compositor until A reaches its source position instead of skipping frames", { timeout: 60_000 }, async (t) => {
|
||||
if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; }
|
||||
const g = { width: 320, height: 180, fps: 30 };
|
||||
const scratch = mkdtempSync(join(tmpdir(), "gate-ahead-"));
|
||||
const gate = new FrameGate({ geometry: g, scratchDir: scratch, redactor: passthroughRedactor, onStderr: () => undefined });
|
||||
const sink = new SinkProcess();
|
||||
const producers = [];
|
||||
try {
|
||||
await sink.start(gate, g);
|
||||
await gate.start();
|
||||
const a = await startProducer(gate, g, 20, 0, true);
|
||||
producers.push(a.proc);
|
||||
const first = await gate.waitActive(a.id, 10_000);
|
||||
const before = await gate.stats();
|
||||
const origin = before.ticks + 2 * g.fps;
|
||||
const b = await startProducer(gate, g, 21, origin, true);
|
||||
producers.push(b.proc);
|
||||
const promoted = await gate.waitActive(b.id, 10_000);
|
||||
// A's source sequence cannot exceed elapsed gate ticks. B must wait until
|
||||
// the output reaches its origin even if it has already filled its rings.
|
||||
assert.ok(promoted >= origin, `B promoted at tick ${promoted} before source position ${origin}`);
|
||||
const stats = await gate.stats();
|
||||
assert.equal(stats.vrep + stats.asil, 0, "A keeps supplying the encoder while B waits");
|
||||
const report = await sink.report(first);
|
||||
assert.deepEqual(report.tagChanges.map(([, tag]) => tag).filter((tag) => tag !== 16), [20, 21]);
|
||||
} finally {
|
||||
await sink.stop();
|
||||
await gate.stop();
|
||||
for (const proc of producers) await proc.terminate(500);
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("Rust resumes a producer at native rate after a joint A/V stall", { timeout: 60_000 }, async (t) => {
|
||||
if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; }
|
||||
const g = { width: 64, height: 36, fps: 30 };
|
||||
const scratch = mkdtempSync(join(tmpdir(), "gate-resume-"));
|
||||
const gate = new FrameGate({ geometry: g, scratchDir: scratch, redactor: passthroughRedactor, onStderr: () => undefined });
|
||||
const sink = new SinkProcess();
|
||||
let producer: ReturnType<typeof spawnManaged> | null = null;
|
||||
try {
|
||||
await sink.start(gate, g);
|
||||
await gate.start();
|
||||
const slot = gate.allocateSlot();
|
||||
// setInterval deliberately resumes at native rate without replaying paused ticks.
|
||||
producer = spawnManaged(process.execPath, ["-e", `const fs=require('node:fs'); const v=Buffer.alloc(${videoFrameBytes(g)},20),a=Buffer.alloc(${audioFrameBytes(g)}); setInterval(()=>{fs.writeSync(1,v);fs.writeSync(3,a)},33);`],
|
||||
["ignore", slot.fifos.videoFd, "pipe", slot.fifos.audioFd], passthroughRedactor, () => undefined);
|
||||
slot.fifos.releaseWriters();
|
||||
await producer.started;
|
||||
await gate.attach(slot, 0);
|
||||
const first = await gate.waitActive(slot.id, 10_000);
|
||||
const initial = await gate.stats();
|
||||
producer.kill("SIGSTOP");
|
||||
await new Promise((r) => setTimeout(r, 1_000));
|
||||
const stalled = await gate.stats();
|
||||
assert.ok(stalled.vrep > initial.vrep && stalled.asil > initial.asil, "pause exhausted both warm rings");
|
||||
producer.kill("SIGCONT");
|
||||
await new Promise((r) => setTimeout(r, 2_000));
|
||||
const resumed = await gate.stats();
|
||||
await new Promise((r) => setTimeout(r, 1_000));
|
||||
const steady = await gate.stats();
|
||||
assert.equal(steady.vrep - resumed.vrep, 0, "video recovered instead of discarding every future frame");
|
||||
assert.equal(steady.asil - resumed.asil, 0, "PCM recovered in lockstep");
|
||||
await sink.report(first);
|
||||
} finally {
|
||||
producer?.kill("SIGCONT");
|
||||
await sink.stop();
|
||||
await gate.stop();
|
||||
await producer?.terminate(500);
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
255
apps/media-worker/src/gate/node-gate.ts
Normal file
255
apps/media-worker/src/gate/node-gate.ts
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
/**
|
||||
* Node reference implementation of the frame gate (same pipe contract as the
|
||||
* Rust helper; see src/gate/protocol.ts). Built for the §1.4 spike: preallocated
|
||||
* frame rings, `net.Socket` `onread` into one reused buffer, refcounted output
|
||||
* buffers so a frame is never mutated while a write is in flight.
|
||||
*
|
||||
* Run as its own process: `node node-gate.js <w> <h> <fps> <ring> <warm> <encVideoFifo> <encAudioFifo>`.
|
||||
*/
|
||||
import { constants, openSync } from "node:fs";
|
||||
import { Socket } from "node:net";
|
||||
import { PerformanceObserver } from "node:perf_hooks";
|
||||
import { BufferPool } from "./buffers";
|
||||
import { assertGeometry, audioFrameBytes, videoFrameBytes } from "./protocol";
|
||||
|
||||
const READ_CHUNK = 1 << 20;
|
||||
/* Node cannot grow a FIFO past 64 KiB (no F_SETPIPE_SZ), so reads stay event-driven per chunk. */
|
||||
|
||||
class Ring {
|
||||
private readonly frames: Buffer[] = [];
|
||||
head = 0;
|
||||
count = 0;
|
||||
fill = 0;
|
||||
constructor(private readonly cap: number, private readonly size: number, private readonly pool: BufferPool) {
|
||||
for (let i = 0; i <= cap; i += 1) this.frames.push(pool.get());
|
||||
}
|
||||
full(): boolean { return this.count >= this.cap; }
|
||||
/** Copies `src` into the ring; returns bytes consumed (stops when full unless dropping). */
|
||||
push(src: Buffer, start: number, end: number, dropWhenFull: boolean): { used: number; dropped: number } {
|
||||
let at = start;
|
||||
let dropped = 0;
|
||||
while (at < end) {
|
||||
if (this.full()) {
|
||||
if (!dropWhenFull) break;
|
||||
this.dropOldest();
|
||||
dropped += 1;
|
||||
}
|
||||
const slot = this.frames[(this.head + this.count) % this.frames.length]!;
|
||||
const n = Math.min(end - at, this.size - this.fill);
|
||||
src.copy(slot, this.fill, at, at + n);
|
||||
at += n;
|
||||
this.fill += n;
|
||||
if (this.fill === this.size) { this.fill = 0; this.count += 1; }
|
||||
}
|
||||
return { used: at - start, dropped };
|
||||
}
|
||||
dropOldest(): void {
|
||||
if (this.count === 0) return;
|
||||
this.head = (this.head + 1) % this.frames.length;
|
||||
this.count -= 1;
|
||||
}
|
||||
/** Hands the oldest frame to the caller and refills the slot from the pool. */
|
||||
take(): Buffer {
|
||||
const frame = this.frames[this.head]!;
|
||||
this.frames[this.head] = this.pool.get();
|
||||
this.head = (this.head + 1) % this.frames.length;
|
||||
this.count -= 1;
|
||||
return frame;
|
||||
}
|
||||
dispose(): void { for (const f of this.frames) this.pool.release(f); }
|
||||
}
|
||||
|
||||
class InStream {
|
||||
readonly ring: Ring;
|
||||
eof = false;
|
||||
paused = false;
|
||||
private readonly socket: Socket;
|
||||
/** Bytes read past a full active ring, held until the next tick frees a slot. */
|
||||
private carry: { buf: Buffer; start: number } | null = null;
|
||||
constructor(path: string, cap: number, size: number, pool: BufferPool, private readonly gate: { dropping(s: InStream): boolean; drops: number }) {
|
||||
this.ring = new Ring(cap, size, pool);
|
||||
const fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK);
|
||||
const chunk = Buffer.alloc(READ_CHUNK);
|
||||
this.socket = new Socket({
|
||||
fd, readable: true, writable: false,
|
||||
onread: {
|
||||
buffer: chunk,
|
||||
callback: (nread: number, buf: Buffer): boolean => {
|
||||
const r = this.ring.push(buf, 0, nread, this.gate.dropping(this));
|
||||
this.gate.drops += r.dropped;
|
||||
if (r.used < nread) {
|
||||
// Active ring full: keep the remainder in a side copy and apply backpressure.
|
||||
this.carry = { buf: Buffer.from(buf.subarray(r.used, nread)), start: 0 };
|
||||
this.paused = true;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
},
|
||||
},
|
||||
});
|
||||
this.socket.on("end", () => { this.eof = true; });
|
||||
this.socket.on("error", () => { this.eof = true; });
|
||||
this.socket.on("close", () => { this.eof = true; });
|
||||
}
|
||||
maybeResume(dropping: boolean): void {
|
||||
if (!this.paused) return;
|
||||
if (this.carry) {
|
||||
const r = this.ring.push(this.carry.buf, this.carry.start, this.carry.buf.length, dropping);
|
||||
this.carry.start += r.used;
|
||||
if (this.carry.start < this.carry.buf.length) return;
|
||||
this.carry = null;
|
||||
}
|
||||
this.paused = false;
|
||||
this.socket.resume();
|
||||
}
|
||||
close(): void {
|
||||
this.socket.destroy();
|
||||
this.ring.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
interface Slot { id: number; video: InStream; audio: InStream }
|
||||
|
||||
function main(): void {
|
||||
const [w, h, f, r, k] = process.argv.slice(2, 7).map(Number);
|
||||
const [encVideoPath, encAudioPath] = process.argv.slice(7, 9);
|
||||
if (encVideoPath === undefined || encAudioPath === undefined) throw new Error("encoder fifo paths required");
|
||||
const geometry = { width: w ?? 0, height: h ?? 0, fps: f ?? 0 };
|
||||
assertGeometry(geometry);
|
||||
const ringFrames = r ?? 0;
|
||||
const warm = k ?? 0;
|
||||
if (!(ringFrames > 0 && warm > 0 && warm <= ringFrames)) throw new Error("ring/warm invalid");
|
||||
const vBytes = videoFrameBytes(geometry);
|
||||
const aBytes = audioFrameBytes(geometry);
|
||||
const vPool = new BufferPool(vBytes, (ringFrames + 1) * 2 + 8);
|
||||
const aPool = new BufferPool(aBytes, (ringFrames + 1) * 2 + 8);
|
||||
|
||||
// Non-blocking open fails fast (ENXIO) unless the encoder already holds the read ends, like the Rust gate.
|
||||
const encVideo = new Socket({ fd: openSync(encVideoPath, constants.O_WRONLY | constants.O_NONBLOCK), readable: false, writable: true });
|
||||
const encAudio = new Socket({ fd: openSync(encAudioPath, constants.O_WRONLY | constants.O_NONBLOCK), readable: false, writable: true });
|
||||
for (const s of [encVideo, encAudio]) s.on("error", () => { process.stderr.write("frame-gate: encoder input closed\n"); process.exit(3); });
|
||||
|
||||
let last = vPool.get();
|
||||
last.fill(16, 0, geometry.width * geometry.height);
|
||||
last.fill(128, geometry.width * geometry.height);
|
||||
const silence = Buffer.alloc(aBytes);
|
||||
|
||||
let active: Slot | null = null;
|
||||
let pending: Slot | null = null;
|
||||
const stats = { ticks: 0, idle: 0, vrep: 0, asil: 0, drops: 0, swaps: 0, lateMaxUs: 0, lateOverFrame: 0, writeMaxUs: 0, stallRun: 0, gcMaxUs: 0, backlogMax: 0, lateOver2ms: 0, lateOver5ms: 0 };
|
||||
const gate = { dropping: (s: InStream) => pending !== null && (s === pending.video || s === pending.audio), get drops() { return stats.drops; }, set drops(v: number) { stats.drops = v; } };
|
||||
|
||||
new PerformanceObserver((list) => {
|
||||
for (const entry of list.getEntries()) stats.gcMaxUs = Math.max(stats.gcMaxUs, Math.round(entry.duration * 1000));
|
||||
}).observe({ entryTypes: ["gc"] });
|
||||
|
||||
const emit = (line: string): void => { process.stdout.write(`${line}\n`); };
|
||||
const ended = (s: Slot): boolean => (s.video.eof && s.video.ring.count === 0) || (s.audio.eof && s.audio.ring.count === 0);
|
||||
const closeSlot = (s: Slot): void => { s.video.close(); s.audio.close(); };
|
||||
|
||||
const start = process.hrtime.bigint();
|
||||
const cpuStart = process.cpuUsage();
|
||||
const periodNs = 1_000_000_000n / BigInt(geometry.fps);
|
||||
const dueAt = (n: number): bigint => start + (BigInt(n) * 1_000_000_000n) / BigInt(geometry.fps);
|
||||
|
||||
const tick = (): void => {
|
||||
let now = process.hrtime.bigint();
|
||||
while (now >= dueAt(stats.ticks)) {
|
||||
const late = Number((now - dueAt(stats.ticks)) / 1000n);
|
||||
stats.lateMaxUs = Math.max(stats.lateMaxUs, late);
|
||||
if (BigInt(late) * 1000n > periodNs) stats.lateOverFrame += 1;
|
||||
if (late > 2_000) stats.lateOver2ms += 1;
|
||||
if (late > 5_000) stats.lateOver5ms += 1;
|
||||
|
||||
if (pending && ended(pending)) { emit(`EOF ${pending.id}`); closeSlot(pending); pending = null; }
|
||||
if (pending && pending.video.ring.count >= warm && pending.audio.ring.count >= warm) {
|
||||
const p: Slot = pending;
|
||||
pending = null;
|
||||
while (p.video.ring.count > warm) p.video.ring.dropOldest();
|
||||
while (p.audio.ring.count > warm) p.audio.ring.dropOldest();
|
||||
const old = active;
|
||||
active = p;
|
||||
stats.swaps += 1;
|
||||
stats.stallRun = 0;
|
||||
emit(`ACTIVE ${p.id} ${stats.ticks}`);
|
||||
if (old) { closeSlot(old); emit(`RETIRED ${old.id}`); }
|
||||
}
|
||||
|
||||
let audio: Buffer = silence;
|
||||
let audioOwned = false;
|
||||
if (active) {
|
||||
if (active.video.ring.count > 0) {
|
||||
vPool.release(last);
|
||||
last = active.video.ring.take();
|
||||
stats.stallRun = 0;
|
||||
} else {
|
||||
stats.vrep += 1;
|
||||
stats.stallRun += 1;
|
||||
if (stats.stallRun === geometry.fps) emit(`STALL ${active.id} ${stats.ticks}`);
|
||||
}
|
||||
if (active.audio.ring.count > 0) { audio = active.audio.ring.take(); audioOwned = true; } else stats.asil += 1;
|
||||
active.video.maybeResume(false);
|
||||
active.audio.maybeResume(false);
|
||||
if (ended(active)) { emit(`EOF ${active.id}`); closeSlot(active); active = null; }
|
||||
} else stats.idle += 1;
|
||||
|
||||
const w0 = process.hrtime.bigint();
|
||||
const frame = last;
|
||||
vPool.retain(frame);
|
||||
encAudio.write(audio, () => { if (audioOwned) aPool.release(audio); });
|
||||
encVideo.write(frame, () => vPool.release(frame));
|
||||
stats.writeMaxUs = Math.max(stats.writeMaxUs, Number((process.hrtime.bigint() - w0) / 1000n));
|
||||
stats.backlogMax = Math.max(stats.backlogMax, encVideo.writableLength);
|
||||
stats.ticks += 1;
|
||||
now = process.hrtime.bigint();
|
||||
}
|
||||
const waitMs = Number((dueAt(stats.ticks) - process.hrtime.bigint()) / 1_000_000n);
|
||||
setTimeout(tick, Math.max(0, waitMs));
|
||||
};
|
||||
|
||||
let lineBuf = "";
|
||||
process.stdin.setEncoding("utf8");
|
||||
process.stdin.on("end", () => process.exit(0));
|
||||
process.stdin.on("data", (chunk: string) => {
|
||||
lineBuf += chunk;
|
||||
let nl = lineBuf.indexOf("\n");
|
||||
while (nl >= 0) {
|
||||
const parts = lineBuf.slice(0, nl).trim().split(/\s+/);
|
||||
lineBuf = lineBuf.slice(nl + 1);
|
||||
nl = lineBuf.indexOf("\n");
|
||||
const [cmd, a, b, c] = parts;
|
||||
if (cmd === "ATTACH" && a !== undefined && b !== undefined && c !== undefined) {
|
||||
const id = Number(a);
|
||||
try {
|
||||
const slot: Slot = { id, video: new InStream(b, ringFrames, vBytes, vPool, gate), audio: new InStream(c, ringFrames, aBytes, aPool, gate) };
|
||||
if (pending) { closeSlot(pending); emit(`DROPPED ${pending.id}`); }
|
||||
pending = slot;
|
||||
emit(`ATTACHED ${id}`);
|
||||
} catch { emit(`ERROR ${id} open`); }
|
||||
} else if (cmd === "DETACH" && a !== undefined) {
|
||||
const id = Number(a);
|
||||
if (pending?.id === id) { closeSlot(pending); pending = null; emit(`DETACHED ${id}`); }
|
||||
else if (active?.id === id) { closeSlot(active); active = null; emit(`DETACHED ${id}`); }
|
||||
} else if (cmd === "STATS") {
|
||||
const cpu = process.cpuUsage(cpuStart);
|
||||
const ring = (s: Slot | null): string => (s ? `${s.id}:${s.video.ring.count}/${s.audio.ring.count}` : "-");
|
||||
emit(`STATS ticks=${stats.ticks} idle=${stats.idle} vrep=${stats.vrep} asil=${stats.asil} drops=${stats.drops} swaps=${stats.swaps} late_max_us=${stats.lateMaxUs} late_over_frame=${stats.lateOverFrame} write_max_us=${stats.writeMaxUs} cpu_us=${cpu.user + cpu.system} wall_us=${Number((process.hrtime.bigint() - start) / 1000n)} active=${ring(active)} pending=${ring(pending)} gc_max_us=${stats.gcMaxUs} allocs=${vPool.allocs + aPool.allocs} backlog_max=${stats.backlogMax} late_over_2ms=${stats.lateOver2ms} late_over_5ms=${stats.lateOver5ms}`);
|
||||
} else if (cmd === "RESET") {
|
||||
stats.lateMaxUs = 0; stats.lateOverFrame = 0; stats.writeMaxUs = 0; stats.gcMaxUs = 0; stats.backlogMax = 0; stats.lateOver2ms = 0; stats.lateOver5ms = 0;
|
||||
emit("RESET");
|
||||
} else if (cmd === "QUIT") process.exit(0);
|
||||
else if (cmd !== "" && cmd !== undefined) emit("ERROR 0 command");
|
||||
}
|
||||
});
|
||||
|
||||
emit(`READY ${vBytes} ${aBytes} ${ringFrames} ${warm}`);
|
||||
tick();
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
try { main(); } catch (error: unknown) {
|
||||
process.stderr.write(`frame-gate: ${error instanceof Error ? error.message : "startup failed"}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
}
|
||||
|
||||
111
apps/media-worker/src/gate/protocol.ts
Normal file
111
apps/media-worker/src/gate/protocol.ts
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
/**
|
||||
* Frame-gate pipe contract (docs/design/streaming-architecture.md §1.4).
|
||||
*
|
||||
* Two implementations speak it: the Rust helper (src/gate/rust, the production
|
||||
* choice, see runs/wp4/s2a-report.md) and the Node reference gate
|
||||
* (src/gate/node-gate.ts, kept for spike/contract comparison only; ProgramChain refuses it).
|
||||
*
|
||||
* argv <width> <height> <fps> <ringFrames> <warmFrames> <encoderVideoFifo> <encoderAudioFifo> [copy]
|
||||
* fd 0 control lines: ATTACH <id> <videoFifo> <audioFifo> [origin] | DETACH <id> | STATS | RESET (peak counters) | QUIT
|
||||
* fd 1 event lines: READY <videoBytes> <audioBytes> <ring> <warm> | ATTACHED <id>
|
||||
* | ACTIVE <id> <tick> [<depthA> <behind> <have> <age>] (Rust: A's buffered depth, B's backward step, B's depth, ticks B warmed)
|
||||
* | RETIRED <id> | DROPPED <id> | DETACHED <id> | EOF <id> | STALL <id> <tick>
|
||||
* | STATS k=v... | ERROR <id> <what>
|
||||
* | LAG <id> <behind> (Rust: pending slot still behind A's on-air source position after 3 s;
|
||||
* the gate never splices backwards, the worker replaces the compositor)
|
||||
* | GAP <v|a> <id> <tick> <ticksSinceActive> <lateUs> (diagnostic: a repeat/silence fill, first 200)
|
||||
* encoder FIFOs (opened for writing by path; the open is the rendezvous with the encoder):
|
||||
* video = rawvideo yuv420p, fixed frame size; audio = s16le 48 kHz stereo, fixed bytes per frame
|
||||
* compositor FIFOs (ATTACH): same formats, read non-blocking into frame rings. Video frame i and audio
|
||||
* frame i of one compositor are one source pair: the Rust gate drops and skips both streams in lockstep.
|
||||
*
|
||||
* Exactly one video frame and one audio frame are written per tick.
|
||||
*/
|
||||
|
||||
export const GATE_SAMPLE_RATE = 48_000;
|
||||
export const GATE_CHANNELS = 2;
|
||||
const BYTES_PER_SAMPLE = 2 * GATE_CHANNELS;
|
||||
|
||||
export interface FrameGeometry { width: number; height: number; fps: number }
|
||||
|
||||
export function videoFrameBytes(g: FrameGeometry): number { return (g.width * g.height * 3) / 2; }
|
||||
|
||||
export function audioFrameBytes(g: FrameGeometry): number { return (GATE_SAMPLE_RATE / g.fps) * BYTES_PER_SAMPLE; }
|
||||
|
||||
export function assertGeometry(g: FrameGeometry): void {
|
||||
if (!Number.isSafeInteger(g.width) || !Number.isSafeInteger(g.height) || g.width < 2 || g.height < 2 || g.width % 2 !== 0 || g.height % 2 !== 0 || g.width > 7680 || g.height > 4320) {
|
||||
throw new Error("frame geometry must be even and at most 7680x4320");
|
||||
}
|
||||
if (!Number.isSafeInteger(g.fps) || g.fps < 1 || g.fps > 120 || GATE_SAMPLE_RATE % g.fps !== 0) throw new Error("fps must divide 48000");
|
||||
}
|
||||
|
||||
export interface GateStats {
|
||||
ticks: number; idle: number; vrep: number; asil: number; drops: number; swaps: number;
|
||||
lateMaxUs: number; lateOverFrame: number; writeMaxUs: number; cpuUs: number; wallUs: number;
|
||||
active: string; pending: string;
|
||||
/** Node gate only: longest GC pause, pool allocations after warm-up, peak encoder write backlog. */
|
||||
gcMaxUs: number; allocs: number; backlogMax: number;
|
||||
/** Frames dropped because the encoder stopped reading (backlog > 2 s). */
|
||||
outDrops: number;
|
||||
lateOver2ms: number; lateOver5ms: number;
|
||||
/** Pending compositors reported `LAG` (still behind A's on-air source position after 3 s), and the largest gap seen. */
|
||||
lagged: number; behindMax: number;
|
||||
}
|
||||
|
||||
export type GateEvent =
|
||||
| { type: "READY"; videoBytes: number; audioBytes: number }
|
||||
| { type: "RESET" }
|
||||
| { type: "GAP"; stream: "v" | "a"; id: number; tick: number; sinceActive: number; lateUs: number }
|
||||
| { type: "ATTACHED" | "RETIRED" | "DROPPED" | "DETACHED" | "EOF"; id: number }
|
||||
| { type: "ACTIVE" | "STALL"; id: number; tick: number }
|
||||
| { type: "LAG"; id: number; behind: number }
|
||||
| { type: "STATS"; stats: GateStats }
|
||||
| { type: "ERROR"; id: number; what: string };
|
||||
|
||||
const STAT_KEYS: Record<string, keyof GateStats> = {
|
||||
ticks: "ticks", idle: "idle", vrep: "vrep", asil: "asil", drops: "drops", swaps: "swaps",
|
||||
late_max_us: "lateMaxUs", late_over_frame: "lateOverFrame", write_max_us: "writeMaxUs",
|
||||
cpu_us: "cpuUs", wall_us: "wallUs", active: "active", pending: "pending",
|
||||
gc_max_us: "gcMaxUs", allocs: "allocs", backlog_max: "backlogMax", out_drops: "outDrops", late_over_2ms: "lateOver2ms", late_over_5ms: "lateOver5ms", lagged: "lagged", behind_max: "behindMax",
|
||||
};
|
||||
|
||||
export function parseGateEvent(line: string): GateEvent | null {
|
||||
const [type, ...rest] = line.trim().split(/\s+/);
|
||||
const n = (i: number): number => Number(rest[i]);
|
||||
switch (type) {
|
||||
case "READY": return { type, videoBytes: n(0), audioBytes: n(1) };
|
||||
case "RESET": return { type };
|
||||
case "GAP": return { type, stream: rest[0] === "a" ? "a" : "v", id: n(1), tick: n(2), sinceActive: n(3), lateUs: n(4) };
|
||||
case "ATTACHED": case "RETIRED": case "DROPPED": case "DETACHED": case "EOF": return { type, id: n(0) };
|
||||
case "ACTIVE": case "STALL": return { type, id: n(0), tick: n(1) };
|
||||
case "LAG": return { type, id: n(0), behind: n(1) };
|
||||
case "ERROR": return { type, id: n(0), what: rest[1] ?? "" };
|
||||
case "STATS": {
|
||||
const stats: GateStats = { ticks: 0, idle: 0, vrep: 0, asil: 0, drops: 0, swaps: 0, lateMaxUs: 0, lateOverFrame: 0, writeMaxUs: 0, cpuUs: 0, wallUs: 0, active: "-", pending: "-", gcMaxUs: 0, allocs: 0, backlogMax: 0, outDrops: 0, lateOver2ms: 0, lateOver5ms: 0, lagged: 0, behindMax: 0 };
|
||||
for (const pair of rest) {
|
||||
const [k, v] = pair.split("=");
|
||||
const key = k === undefined ? undefined : STAT_KEYS[k];
|
||||
if (key === undefined || v === undefined) continue;
|
||||
if (key === "active" || key === "pending") stats[key] = v;
|
||||
else stats[key] = Number(v);
|
||||
}
|
||||
return { type, stats };
|
||||
}
|
||||
default: return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** FIFO paths are generated by the worker (scratch dir + numeric id); nothing else may reach a control line. */
|
||||
const SAFE_PATH = /^\/[A-Za-z0-9._/-]+$/;
|
||||
|
||||
/**
|
||||
* `origin`: source frame index of the compositor's first output frame, in a
|
||||
* numbering shared by consecutive compositors of one chain. The Rust gate uses
|
||||
* it to splice B in at A's next source frame (no backwards step or forward skip at a swap).
|
||||
*/
|
||||
export function attachLine(id: number, videoFifo: string, audioFifo: string, origin?: number): string {
|
||||
if (!Number.isSafeInteger(id) || id < 0 || id > 0xffffffff) throw new Error("gate slot id out of range");
|
||||
for (const path of [videoFifo, audioFifo]) if (!SAFE_PATH.test(path) || path.includes("..")) throw new Error("unsafe gate fifo path");
|
||||
if (origin !== undefined && !Number.isSafeInteger(origin)) throw new Error("gate origin must be an integer");
|
||||
return `ATTACH ${id} ${videoFifo} ${audioFifo}${origin === undefined ? "" : ` ${origin}`}\n`;
|
||||
}
|
||||
1
apps/media-worker/src/gate/rust/.gitignore
vendored
Normal file
1
apps/media-worker/src/gate/rust/.gitignore
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
target/
|
||||
7
apps/media-worker/src/gate/rust/Cargo.lock
generated
Normal file
7
apps/media-worker/src/gate/rust/Cargo.lock
generated
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "frame-gate"
|
||||
version = "0.1.0"
|
||||
15
apps/media-worker/src/gate/rust/Cargo.toml
Normal file
15
apps/media-worker/src/gate/rust/Cargo.toml
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
[package]
|
||||
name = "frame-gate"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "Clock-master frame gate between swappable compositors and a persistent encoder (streaming-architecture.md §1.4)"
|
||||
publish = false
|
||||
|
||||
# std only: no crates, so the build works offline and the binary has no supply chain.
|
||||
[dependencies]
|
||||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
lto = true
|
||||
codegen-units = 1
|
||||
panic = "abort"
|
||||
503
apps/media-worker/src/gate/rust/src/bin/source-bus.rs
Normal file
503
apps/media-worker/src/gate/rust/src/bin/source-bus.rs
Normal file
|
|
@ -0,0 +1,503 @@
|
|||
//! Source bus (docs/design/streaming-architecture.md §1.4; runs/wp4/s2b-report.md item 1).
|
||||
//!
|
||||
//! One process per source, between the source's ingest and every consumer of
|
||||
//! its raw frames: the A/B compositors of a swap and the preview taps. The
|
||||
//! Node worker used to move these frames itself (S2a); at 4 × 1080p30 that is
|
||||
//! ~750 MB/s through 64 KiB chunks on one event loop, with no headroom left to
|
||||
//! absorb a burst, and the buses dropped ~7.5 % of source pairs. This helper
|
||||
//! moves the bytes; the worker keeps only the control lines.
|
||||
//!
|
||||
//! Semantics are those of the S2a Node bus (src/compositor/source-bus.ts):
|
||||
//!
|
||||
//! * raw yuv420p and s16le are re-framed into fixed-size frames and delivered
|
||||
//! as *pairs* (video n + audio n), so a subscriber that joins mid-stream
|
||||
//! starts at a frame boundary with A/V aligned;
|
||||
//! * a source without audio (or whose audio stalls) gets silence once video
|
||||
//! runs `maxSkew` frames ahead; audio that runs ahead of video is trimmed;
|
||||
//! * a subscriber more than `maxBacklog + history` frames behind loses whole
|
||||
//! pairs, never partial frames (a young subscriber, still starting up, may
|
||||
//! queue one more second of frames for its first two seconds);
|
||||
//! * the last `history` pairs are replayed to a new subscriber, and its first
|
||||
//! pair's index is reported, so the gate can splice a swap-in compositor at
|
||||
//! exactly the on-air compositor's next source frame.
|
||||
//!
|
||||
//! A subscriber may take every `stride`-th pair only (JPEG thumbnails), and may
|
||||
//! take video or audio only (`-`).
|
||||
//!
|
||||
//! fds: 0 control lines, 1 event lines, 2 diagnostics. std only.
|
||||
//! argv: <width> <height> <fps> <maxSkew> <maxBacklog> <history> <ingestVideoFifo> <ingestAudioFifo>
|
||||
//!
|
||||
//! control: SUB <id> <videoFifo|-> <audioFifo|-> [stride] | UNSUB <id> | STATS | QUIT
|
||||
//! events: READY <videoBytes> <audioBytes>
|
||||
//! | SUBSCRIBED <id> <startPair> <otherId>:<dropped>,...|- (pairs the other subscribers had
|
||||
//! dropped at the same instant: the worker's origin arithmetic needs both together)
|
||||
//! | UNSUBSCRIBED <id> | CLOSED <id> (the reader went away) | DROP <id> <total>
|
||||
//! | EOF (the ingest closed both streams) | STATS k=v... | ERROR <id> <what>
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, ErrorKind, Read, Write};
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
use std::os::unix::io::{AsRawFd, FromRawFd};
|
||||
use std::rc::Rc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
#[repr(C)]
|
||||
struct PollFd {
|
||||
fd: i32,
|
||||
events: i16,
|
||||
revents: i16,
|
||||
}
|
||||
|
||||
extern "C" {
|
||||
fn poll(fds: *mut PollFd, nfds: u64, timeout: i32) -> i32;
|
||||
fn fcntl(fd: i32, cmd: i32, ...) -> i32;
|
||||
}
|
||||
|
||||
const POLLIN: i16 = 0x1;
|
||||
const POLLOUT: i16 = 0x4;
|
||||
const O_NONBLOCK: i32 = 0o4000;
|
||||
const F_GETFL: i32 = 3;
|
||||
const F_SETFL: i32 = 4;
|
||||
const F_SETPIPE_SZ: i32 = 1031;
|
||||
const F_GETPIPE_SZ: i32 = 1032;
|
||||
const PIPE_BYTES: i32 = 1 << 20;
|
||||
const SAMPLE_RATE: usize = 48_000;
|
||||
const AUDIO_BYTES_PER_SAMPLE: usize = 4; // s16le stereo
|
||||
const MAX_READS_PER_PUMP: usize = 64;
|
||||
/// Pipes are serviced in bulk at most this often (as in the gate): waking per
|
||||
/// 32 KiB chunk costs far more than moving 1 MiB per wake. 1 MiB / 2 ms =
|
||||
/// 500 MB/s per pipe, above 1080p60 raw (186 MB/s).
|
||||
const PUMP_INTERVAL: Duration = Duration::from_micros(2_000);
|
||||
const YOUNG: Duration = Duration::from_secs(2);
|
||||
|
||||
type Frame = Rc<Vec<u8>>;
|
||||
|
||||
fn set_nonblocking(fd: i32) {
|
||||
unsafe {
|
||||
let flags = fcntl(fd, F_GETFL);
|
||||
if flags >= 0 {
|
||||
fcntl(fd, F_SETFL, flags | O_NONBLOCK);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn emit(line: &str) {
|
||||
let mut out = io::stdout().lock();
|
||||
let _ = out.write_all(line.as_bytes());
|
||||
let _ = out.write_all(b"\n");
|
||||
let _ = out.flush();
|
||||
}
|
||||
|
||||
fn arg(args: &[String], i: usize, name: &str, min: usize) -> usize {
|
||||
match args.get(i).and_then(|v| v.parse::<usize>().ok()) {
|
||||
Some(v) if v >= min => v,
|
||||
_ => {
|
||||
eprintln!("source-bus: invalid {}", name);
|
||||
std::process::exit(2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Recycled frame buffers: a frame returns here when its last reference
|
||||
/// (history, a subscriber queue) lets go, so steady state never allocates.
|
||||
struct Pool {
|
||||
free: Vec<Vec<u8>>,
|
||||
size: usize,
|
||||
keep: usize,
|
||||
allocs: u64,
|
||||
}
|
||||
|
||||
impl Pool {
|
||||
fn get(&mut self) -> Vec<u8> {
|
||||
match self.free.pop() {
|
||||
Some(b) => b,
|
||||
None => {
|
||||
self.allocs += 1;
|
||||
vec![0u8; self.size]
|
||||
}
|
||||
}
|
||||
}
|
||||
fn put(&mut self, frame: Frame) {
|
||||
if let Ok(buf) = Rc::try_unwrap(frame) {
|
||||
if self.free.len() < self.keep {
|
||||
self.free.push(buf);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One ingest stream, re-framed into fixed-size frames.
|
||||
struct Input {
|
||||
file: File,
|
||||
pipe_bytes: usize,
|
||||
cur: Vec<u8>,
|
||||
fill: usize,
|
||||
eof: bool,
|
||||
}
|
||||
|
||||
impl Input {
|
||||
fn open(path: &str, pool: &mut Pool) -> Input {
|
||||
match OpenOptions::new().read(true).custom_flags(O_NONBLOCK).open(path) {
|
||||
Ok(file) => {
|
||||
unsafe { fcntl(file.as_raw_fd(), F_SETPIPE_SZ, PIPE_BYTES) };
|
||||
let pipe_bytes = unsafe { fcntl(file.as_raw_fd(), F_GETPIPE_SZ) }.max(0) as usize;
|
||||
Input { file, pipe_bytes, cur: pool.get(), fill: 0, eof: false }
|
||||
}
|
||||
Err(_) => {
|
||||
eprintln!("source-bus: cannot open ingest fifo");
|
||||
std::process::exit(2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads what the pipe holds; complete frames go to `out`.
|
||||
fn pump(&mut self, pool: &mut Pool, out: &mut VecDeque<Frame>) {
|
||||
for _ in 0..MAX_READS_PER_PUMP {
|
||||
if self.eof {
|
||||
break;
|
||||
}
|
||||
let fill = self.fill;
|
||||
match self.file.read(&mut self.cur[fill..]) {
|
||||
Ok(0) => self.eof = true,
|
||||
Ok(n) => {
|
||||
self.fill += n;
|
||||
if self.fill == pool.size {
|
||||
let next = pool.get();
|
||||
out.push_back(Rc::new(std::mem::replace(&mut self.cur, next)));
|
||||
self.fill = 0;
|
||||
}
|
||||
}
|
||||
Err(e) if e.kind() == ErrorKind::WouldBlock => break,
|
||||
Err(e) if e.kind() == ErrorKind::Interrupted => continue,
|
||||
Err(_) => self.eof = true,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One consumer stream: queued frame references, written non-blocking in order.
|
||||
struct Out {
|
||||
file: File,
|
||||
pipe_bytes: usize,
|
||||
queue: VecDeque<(Frame, usize)>,
|
||||
}
|
||||
|
||||
impl Out {
|
||||
fn open(path: &str) -> io::Result<Out> {
|
||||
let file = OpenOptions::new().write(true).custom_flags(O_NONBLOCK).open(path)?;
|
||||
unsafe { fcntl(file.as_raw_fd(), F_SETPIPE_SZ, PIPE_BYTES) };
|
||||
let pipe_bytes = unsafe { fcntl(file.as_raw_fd(), F_GETPIPE_SZ) }.max(0) as usize;
|
||||
Ok(Out { file, pipe_bytes, queue: VecDeque::new() })
|
||||
}
|
||||
|
||||
/// false = the reader is gone.
|
||||
fn flush(&mut self, pool: &mut Pool) -> bool {
|
||||
while let Some((frame, off)) = self.queue.front_mut() {
|
||||
match self.file.write(&frame[*off..]) {
|
||||
Ok(n) => {
|
||||
*off += n;
|
||||
if *off == frame.len() {
|
||||
let (frame, _) = self.queue.pop_front().unwrap();
|
||||
pool.put(frame);
|
||||
}
|
||||
}
|
||||
Err(e) if e.kind() == ErrorKind::WouldBlock => return true,
|
||||
Err(e) if e.kind() == ErrorKind::Interrupted => continue,
|
||||
Err(_) => return false,
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn dispose(self, pool: &mut Pool) {
|
||||
for (frame, _) in self.queue {
|
||||
pool.put(frame);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Subscriber {
|
||||
id: u32,
|
||||
video: Option<Out>,
|
||||
audio: Option<Out>,
|
||||
stride: u64,
|
||||
start_pair: u64,
|
||||
young_until: Instant,
|
||||
dropped: u64,
|
||||
}
|
||||
|
||||
impl Subscriber {
|
||||
fn queued(&self) -> usize {
|
||||
let v = self.video.as_ref().map_or(0, |o| o.queue.len());
|
||||
let a = self.audio.as_ref().map_or(0, |o| o.queue.len());
|
||||
v.max(a)
|
||||
}
|
||||
}
|
||||
|
||||
struct Bus {
|
||||
vpool: Pool,
|
||||
apool: Pool,
|
||||
vq: VecDeque<Frame>,
|
||||
aq: VecDeque<Frame>,
|
||||
silence: Frame,
|
||||
history: VecDeque<(Frame, Option<Frame>)>,
|
||||
subs: Vec<Subscriber>,
|
||||
max_skew: usize,
|
||||
limit: usize,
|
||||
young_limit: usize,
|
||||
history_frames: usize,
|
||||
pairs: u64,
|
||||
silence_filled: u64,
|
||||
audio_trimmed: u64,
|
||||
subscriber_drops: u64,
|
||||
}
|
||||
|
||||
impl Bus {
|
||||
/// Pairs first, then applies the skew rules to what is left over. A pump reads many frames of
|
||||
/// each stream at once, so trimming before pairing would discard audio its video is about to
|
||||
/// claim (the S2a Node bus drained after every single frame, where the order did not matter).
|
||||
fn drain(&mut self) {
|
||||
while !self.vq.is_empty() && !self.aq.is_empty() {
|
||||
let video = self.vq.pop_front().unwrap();
|
||||
let audio = self.aq.pop_front();
|
||||
self.deliver(video, audio);
|
||||
}
|
||||
// Audio more than `max_skew` frames ahead of video: trim the oldest.
|
||||
while self.aq.len() > self.max_skew {
|
||||
let f = self.aq.pop_front().unwrap();
|
||||
self.apool.put(f);
|
||||
self.audio_trimmed += 1;
|
||||
}
|
||||
// Video more than `max_skew` frames ahead of audio (none, or stalled): pair with silence.
|
||||
while self.vq.len() > self.max_skew {
|
||||
let video = self.vq.pop_front().unwrap();
|
||||
self.silence_filled += 1;
|
||||
self.deliver(video, None);
|
||||
}
|
||||
}
|
||||
|
||||
fn deliver(&mut self, video: Frame, audio: Option<Frame>) {
|
||||
let index = self.pairs;
|
||||
self.pairs += 1;
|
||||
let now = Instant::now();
|
||||
for i in 0..self.subs.len() {
|
||||
if (index - self.subs[i].start_pair) % self.subs[i].stride != 0 {
|
||||
continue;
|
||||
}
|
||||
let sub = &mut self.subs[i];
|
||||
let limit = if now < sub.young_until { self.young_limit } else { self.limit };
|
||||
if sub.queued() > limit {
|
||||
sub.dropped += 1;
|
||||
self.subscriber_drops += 1;
|
||||
emit(&format!("DROP {} {}", sub.id, sub.dropped));
|
||||
continue;
|
||||
}
|
||||
Self::enqueue(&mut self.subs[i], &video, audio.as_ref(), &self.silence);
|
||||
}
|
||||
self.history.push_back((video, audio));
|
||||
while self.history.len() > self.history_frames {
|
||||
let (v, a) = self.history.pop_front().unwrap();
|
||||
self.vpool.put(v);
|
||||
if let Some(a) = a {
|
||||
self.apool.put(a);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn enqueue(sub: &mut Subscriber, video: &Frame, audio: Option<&Frame>, silence: &Frame) {
|
||||
if let Some(v) = sub.video.as_mut() {
|
||||
v.queue.push_back((Rc::clone(video), 0));
|
||||
}
|
||||
if let Some(a) = sub.audio.as_mut() {
|
||||
a.queue.push_back((Rc::clone(audio.unwrap_or(silence)), 0));
|
||||
}
|
||||
}
|
||||
|
||||
/// Replays the history (a strided subscriber gets only the newest pair) and reports where it starts.
|
||||
fn subscribe(&mut self, id: u32, video: Option<Out>, audio: Option<Out>, stride: u64) -> u64 {
|
||||
let replay = if stride > 1 { self.history.len().min(1) } else { self.history.len() };
|
||||
let start_pair = self.pairs - replay as u64;
|
||||
let mut sub = Subscriber { id, video, audio, stride, start_pair, young_until: Instant::now() + YOUNG, dropped: 0 };
|
||||
for (v, a) in self.history.iter().skip(self.history.len() - replay) {
|
||||
Self::enqueue(&mut sub, v, a.as_ref(), &self.silence);
|
||||
}
|
||||
self.subs.push(sub);
|
||||
start_pair
|
||||
}
|
||||
|
||||
fn remove(&mut self, id: u32) -> bool {
|
||||
match self.subs.iter().position(|s| s.id == id) {
|
||||
Some(i) => {
|
||||
let sub = self.subs.swap_remove(i);
|
||||
if let Some(v) = sub.video {
|
||||
v.dispose(&mut self.vpool);
|
||||
}
|
||||
if let Some(a) = sub.audio {
|
||||
a.dispose(&mut self.apool);
|
||||
}
|
||||
true
|
||||
}
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn flush(&mut self) {
|
||||
let mut gone: Vec<u32> = Vec::new();
|
||||
for sub in self.subs.iter_mut() {
|
||||
let v_ok = sub.video.as_mut().map_or(true, |o| o.flush(&mut self.vpool));
|
||||
let a_ok = sub.audio.as_mut().map_or(true, |o| o.flush(&mut self.apool));
|
||||
if !v_ok || !a_ok {
|
||||
gone.push(sub.id);
|
||||
}
|
||||
}
|
||||
for id in gone {
|
||||
self.remove(id);
|
||||
emit(&format!("CLOSED {}", id));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn open_out(path: &str) -> Result<Option<Out>, ()> {
|
||||
if path == "-" {
|
||||
return Ok(None);
|
||||
}
|
||||
if !path.starts_with('/') || path.contains("..") {
|
||||
return Err(());
|
||||
}
|
||||
Out::open(path).map(Some).map_err(|_| ())
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
let width = arg(&args, 1, "width", 2);
|
||||
let height = arg(&args, 2, "height", 2);
|
||||
let fps = arg(&args, 3, "fps", 1);
|
||||
let max_skew = arg(&args, 4, "maxSkew", 1);
|
||||
let max_backlog = arg(&args, 5, "maxBacklog", 1);
|
||||
let history = arg(&args, 6, "history", 0);
|
||||
if width % 2 != 0 || height % 2 != 0 || SAMPLE_RATE % fps != 0 {
|
||||
eprintln!("source-bus: geometry must be even and fps must divide 48000");
|
||||
std::process::exit(2);
|
||||
}
|
||||
let video_bytes = width * height * 3 / 2;
|
||||
let audio_bytes = SAMPLE_RATE / fps * AUDIO_BYTES_PER_SAMPLE;
|
||||
let mut vpool = Pool { free: Vec::new(), size: video_bytes, keep: 0, allocs: 0 };
|
||||
let mut apool = Pool { free: Vec::new(), size: audio_bytes, keep: 0, allocs: 0 };
|
||||
let mut vin = Input::open(args.get(7).map(String::as_str).unwrap_or(""), &mut vpool);
|
||||
let mut ain = Input::open(args.get(8).map(String::as_str).unwrap_or(""), &mut apool);
|
||||
let limit = max_backlog + history;
|
||||
// A young subscriber (process still starting) may queue one more second. Frames are shared
|
||||
// references, so this costs memory only while that subscriber is actually stalled:
|
||||
// at most (limit + fps) frames, 143 MB at 1080p30.
|
||||
let young_limit = limit + fps;
|
||||
// Recycled buffers for history + one backlog; anything beyond is freed when released.
|
||||
vpool.keep = history + limit + 8;
|
||||
apool.keep = 4 * (history + limit + 8) + 4 * fps;
|
||||
let silence: Frame = Rc::new(vec![0u8; audio_bytes]);
|
||||
let mut bus = Bus {
|
||||
vpool, apool, vq: VecDeque::new(), aq: VecDeque::new(), silence, history: VecDeque::new(), subs: Vec::new(),
|
||||
max_skew, limit, young_limit, history_frames: history, pairs: 0, silence_filled: 0, audio_trimmed: 0, subscriber_drops: 0,
|
||||
};
|
||||
|
||||
set_nonblocking(0);
|
||||
let mut control = unsafe { File::from_raw_fd(0) };
|
||||
let mut line_buf: Vec<u8> = Vec::with_capacity(4096);
|
||||
let mut read_buf = [0u8; 4096];
|
||||
let mut last_pump = Instant::now() - PUMP_INTERVAL;
|
||||
let mut eof_sent = false;
|
||||
emit(&format!("READY {} {}", video_bytes, audio_bytes));
|
||||
|
||||
loop {
|
||||
let now = Instant::now();
|
||||
// A shared uid may exceed pipe-user-pages-soft: F_SETPIPE_SZ then fails and new
|
||||
// pipes hold only 8 KiB. Batching those for 2 ms caps raw transport at ~4 MB/s.
|
||||
// Batch only when every video pipe has two pump intervals of headroom.
|
||||
let bulk_min = video_bytes * fps / 250;
|
||||
let bulk = vin.pipe_bytes >= bulk_min && bus.subs.iter().all(|s| s.video.as_ref().map_or(true, |v| v.pipe_bytes >= bulk_min));
|
||||
let pump_due = last_pump + if bulk { PUMP_INTERVAL } else { Duration::ZERO };
|
||||
let mut fds: Vec<PollFd> = Vec::with_capacity(3 + 2 * bus.subs.len());
|
||||
fds.push(PollFd { fd: 0, events: POLLIN, revents: 0 });
|
||||
let timeout_ms = if now >= pump_due {
|
||||
fds.push(PollFd { fd: if vin.eof { -1 } else { vin.file.as_raw_fd() }, events: POLLIN, revents: 0 });
|
||||
fds.push(PollFd { fd: if ain.eof { -1 } else { ain.file.as_raw_fd() }, events: POLLIN, revents: 0 });
|
||||
for sub in &bus.subs {
|
||||
for out in [sub.video.as_ref(), sub.audio.as_ref()].into_iter().flatten() {
|
||||
fds.push(PollFd { fd: if out.queue.is_empty() { -1 } else { out.file.as_raw_fd() }, events: POLLOUT, revents: 0 });
|
||||
}
|
||||
}
|
||||
1000
|
||||
} else {
|
||||
(pump_due.saturating_duration_since(now).as_micros() as i32 + 999) / 1000
|
||||
};
|
||||
unsafe { poll(fds.as_mut_ptr(), fds.len() as u64, timeout_ms) };
|
||||
|
||||
if fds.len() > 1 && fds[1..].iter().any(|f| f.revents != 0) {
|
||||
last_pump = Instant::now();
|
||||
vin.pump(&mut bus.vpool, &mut bus.vq);
|
||||
ain.pump(&mut bus.apool, &mut bus.aq);
|
||||
bus.drain();
|
||||
bus.flush();
|
||||
if vin.eof && ain.eof && !eof_sent {
|
||||
eof_sent = true;
|
||||
emit("EOF");
|
||||
}
|
||||
}
|
||||
|
||||
if fds[0].revents != 0 {
|
||||
loop {
|
||||
match control.read(&mut read_buf) {
|
||||
Ok(0) => std::process::exit(0), // parent gone
|
||||
Ok(n) => line_buf.extend_from_slice(&read_buf[..n]),
|
||||
Err(e) if e.kind() == ErrorKind::WouldBlock => break,
|
||||
Err(e) if e.kind() == ErrorKind::Interrupted => continue,
|
||||
Err(_) => std::process::exit(0),
|
||||
}
|
||||
}
|
||||
while let Some(pos) = line_buf.iter().position(|&b| b == b'\n') {
|
||||
let line: Vec<u8> = line_buf.drain(..=pos).collect();
|
||||
let text = String::from_utf8_lossy(&line[..line.len() - 1]).into_owned();
|
||||
let parts: Vec<&str> = text.split_whitespace().collect();
|
||||
match parts.as_slice() {
|
||||
["SUB", id, vpath, apath, rest @ ..] if rest.len() <= 1 => {
|
||||
let stride = rest.first().map_or(Some(1), |s| s.parse::<u64>().ok().filter(|&s| s >= 1));
|
||||
match (id.parse::<u32>(), stride) {
|
||||
(Ok(id), Some(stride)) if !bus.subs.iter().any(|s| s.id == id) && (*vpath != "-" || *apath != "-") => {
|
||||
match (open_out(vpath), open_out(apath)) {
|
||||
(Ok(video), Ok(audio)) => {
|
||||
let others: Vec<String> = bus.subs.iter().map(|s| format!("{}:{}", s.id, s.dropped)).collect();
|
||||
let start = bus.subscribe(id, video, audio, stride);
|
||||
let others = if others.is_empty() { "-".to_string() } else { others.join(",") };
|
||||
emit(&format!("SUBSCRIBED {} {} {}", id, start, others));
|
||||
bus.flush();
|
||||
}
|
||||
_ => emit(&format!("ERROR {} open", id)),
|
||||
}
|
||||
}
|
||||
(Ok(id), _) => emit(&format!("ERROR {} subscribe", id)),
|
||||
_ => emit("ERROR 0 id"),
|
||||
}
|
||||
}
|
||||
["UNSUB", id] => {
|
||||
let id = id.parse::<u32>().unwrap_or(u32::MAX);
|
||||
if bus.remove(id) {
|
||||
emit(&format!("UNSUBSCRIBED {}", id));
|
||||
}
|
||||
}
|
||||
["STATS"] => {
|
||||
let subs: Vec<String> = bus.subs.iter().map(|s| format!("{}:{}:{}", s.id, s.dropped, s.queued())).collect();
|
||||
emit(&format!(
|
||||
"STATS pairs={} silence_filled={} audio_trimmed={} subscriber_drops={} allocs={} video_waiting={} audio_waiting={} video_eof={} audio_eof={} subs={}",
|
||||
bus.pairs, bus.silence_filled, bus.audio_trimmed, bus.subscriber_drops, bus.vpool.allocs + bus.apool.allocs,
|
||||
bus.vq.len(), bus.aq.len(), vin.eof as u8, ain.eof as u8,
|
||||
if subs.is_empty() { "-".to_string() } else { subs.join(",") }
|
||||
));
|
||||
}
|
||||
["QUIT"] => std::process::exit(0),
|
||||
[] => {}
|
||||
_ => emit("ERROR 0 command"),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
749
apps/media-worker/src/gate/rust/src/main.rs
Normal file
749
apps/media-worker/src/gate/rust/src/main.rs
Normal file
|
|
@ -0,0 +1,749 @@
|
|||
//! Frame gate (docs/design/streaming-architecture.md §1.4).
|
||||
//!
|
||||
//! The gate is the clock master between swappable compositor processes and one
|
||||
//! persistent encoder. It emits exactly one raw video frame and one PCM frame
|
||||
//! per tick of the output frame rate, whatever the compositors do:
|
||||
//!
|
||||
//! * the active compositor's next frame when it has one;
|
||||
//! * otherwise the last video frame again plus silence (a stall never starves
|
||||
//! the encoder, so a dead or SIGKILLed compositor never takes RTMP down);
|
||||
//! * a pending compositor is promoted at a tick boundary once it has buffered
|
||||
//! `warm` frames of both video and audio, and the old one is closed.
|
||||
//!
|
||||
//! Raw yuv420p and s16le have fixed bytes per frame, so there is no demuxing and
|
||||
//! no timestamp parsing: frames are counted. The pipe contract is shared with
|
||||
//! the Node implementation (src/gate/node-gate.ts) and documented in
|
||||
//! src/gate/protocol.ts.
|
||||
//!
|
||||
//! fds: 0 control lines, 1 event lines, 2 diagnostics.
|
||||
//! argv: <width> <height> <fps> <ringFrames> <warmFrames> <encoderVideoFifo> <encoderAudioFifo> [copy]
|
||||
//!
|
||||
//! Single thread, one poll loop. Every pipe is non-blocking and each encoder
|
||||
//! output has its own queue, so the tick clock never waits on the encoder and
|
||||
//! a consumer that reads its inputs in any order (ffmpeg probing one input
|
||||
//! before opening the next) cannot deadlock against the gate.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, ErrorKind, Read, Write};
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
use std::os::unix::io::{AsRawFd, FromRawFd};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
#[repr(C)]
|
||||
struct PollFd {
|
||||
fd: i32,
|
||||
events: i16,
|
||||
revents: i16,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
struct IoVec {
|
||||
base: *const u8,
|
||||
len: usize,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
struct TimeVal {
|
||||
sec: i64,
|
||||
usec: i64,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
struct RUsage {
|
||||
utime: TimeVal,
|
||||
stime: TimeVal,
|
||||
rest: [i64; 14],
|
||||
}
|
||||
|
||||
extern "C" {
|
||||
fn poll(fds: *mut PollFd, nfds: u64, timeout: i32) -> i32;
|
||||
fn fcntl(fd: i32, cmd: i32, ...) -> i32;
|
||||
fn getrusage(who: i32, usage: *mut RUsage) -> i32;
|
||||
fn vmsplice(fd: i32, iov: *const IoVec, nr_segs: u64, flags: u32) -> isize;
|
||||
fn write(fd: i32, buf: *const u8, count: usize) -> isize;
|
||||
}
|
||||
|
||||
const POLLIN: i16 = 0x1;
|
||||
const POLLOUT: i16 = 0x4;
|
||||
const O_NONBLOCK: i32 = 0o4000;
|
||||
const F_GETFL: i32 = 3;
|
||||
const F_SETFL: i32 = 4;
|
||||
const F_SETPIPE_SZ: i32 = 1031;
|
||||
const F_GETPIPE_SZ: i32 = 1032;
|
||||
const SPLICE_F_NONBLOCK: u32 = 0x02;
|
||||
const PIPE_BYTES: i32 = 1 << 20;
|
||||
const SAMPLE_RATE: usize = 48_000;
|
||||
const AUDIO_BYTES_PER_SAMPLE: usize = 4; // s16le stereo
|
||||
const MAX_READS_PER_WAKE: usize = 64;
|
||||
/// Compositor pipes are drained at most this often. Producers write in 32 KiB
|
||||
/// chunks; waking per chunk costs far more CPU than draining a 1 MiB pipe in
|
||||
/// bulk. 1 MiB / 2 ms = 500 MB/s per stream, above 1080p60 raw (186 MB/s).
|
||||
const PUMP_INTERVAL: Duration = Duration::from_micros(2_000);
|
||||
|
||||
fn set_nonblocking(fd: i32) {
|
||||
unsafe {
|
||||
let flags = fcntl(fd, F_GETFL);
|
||||
if flags >= 0 {
|
||||
fcntl(fd, F_SETFL, flags | O_NONBLOCK);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cpu_us() -> u64 {
|
||||
let mut usage = RUsage { utime: TimeVal { sec: 0, usec: 0 }, stime: TimeVal { sec: 0, usec: 0 }, rest: [0; 14] };
|
||||
unsafe { getrusage(0, &mut usage) };
|
||||
(usage.utime.sec * 1_000_000 + usage.utime.usec + usage.stime.sec * 1_000_000 + usage.stime.usec) as u64
|
||||
}
|
||||
|
||||
/// Refcounted, preallocated frame buffers addressed by index.
|
||||
///
|
||||
/// vmsplice hands buffer *pages* to the encoder pipe, so a buffer whose write
|
||||
/// has finished may still be referenced by up to one pipe's worth (1 MiB) of
|
||||
/// unread data. Released buffers therefore sit in quarantine until a later
|
||||
/// frame has been fully written: a whole later frame (>= 1 MiB at the sizes
|
||||
/// that matter) cannot be in the pipe together with any older page.
|
||||
struct Pool {
|
||||
bufs: Vec<Vec<u8>>,
|
||||
refs: Vec<u32>,
|
||||
free: Vec<usize>,
|
||||
quarantine: Vec<(usize, u64)>,
|
||||
size: usize,
|
||||
allocs: u64,
|
||||
}
|
||||
|
||||
impl Pool {
|
||||
fn new(size: usize, prealloc: usize) -> Pool {
|
||||
Pool { bufs: (0..prealloc).map(|_| vec![0u8; size]).collect(), refs: vec![0; prealloc], free: (0..prealloc).rev().collect(), quarantine: Vec::new(), size, allocs: 0 }
|
||||
}
|
||||
fn get(&mut self) -> usize {
|
||||
let idx = match self.free.pop() {
|
||||
Some(i) => i,
|
||||
None => {
|
||||
self.bufs.push(vec![0u8; self.size]);
|
||||
self.refs.push(0);
|
||||
self.allocs += 1;
|
||||
self.bufs.len() - 1
|
||||
}
|
||||
};
|
||||
self.refs[idx] = 1;
|
||||
idx
|
||||
}
|
||||
fn retain(&mut self, idx: usize) {
|
||||
self.refs[idx] += 1;
|
||||
}
|
||||
/// `written` = frames fully written so far; the buffer is reusable once a later one completes.
|
||||
fn release(&mut self, idx: usize, written: u64) {
|
||||
self.refs[idx] -= 1;
|
||||
if self.refs[idx] == 0 {
|
||||
self.quarantine.push((idx, written));
|
||||
}
|
||||
}
|
||||
fn on_frame_written(&mut self, written: u64) {
|
||||
let mut i = 0;
|
||||
while i < self.quarantine.len() {
|
||||
if self.quarantine[i].1 < written {
|
||||
let (idx, _) = self.quarantine.swap_remove(i);
|
||||
self.free.push(idx);
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fixed-size frame ring of pool buffers. One extra slot holds the partial frame being read.
|
||||
struct Ring {
|
||||
slots: Vec<usize>,
|
||||
head: usize,
|
||||
count: usize,
|
||||
fill: usize,
|
||||
cap: usize,
|
||||
/// Frames that have left the ring (taken, dropped or skipped): the source position of the head frame.
|
||||
removed: u64,
|
||||
/// Frames already counted in `removed` that have not arrived yet: discarded on arrival.
|
||||
skip: u64,
|
||||
}
|
||||
|
||||
impl Ring {
|
||||
fn new(cap: usize, pool: &mut Pool) -> Ring {
|
||||
Ring { slots: (0..=cap).map(|_| pool.get()).collect(), head: 0, count: 0, fill: 0, cap, removed: 0, skip: 0 }
|
||||
}
|
||||
fn full(&self) -> bool {
|
||||
self.count >= self.cap
|
||||
}
|
||||
fn write_idx(&self) -> usize {
|
||||
self.slots[(self.head + self.count) % self.slots.len()]
|
||||
}
|
||||
fn advance(&mut self, n: usize, size: usize) {
|
||||
self.fill += n;
|
||||
if self.fill == size {
|
||||
self.fill = 0;
|
||||
if self.skip > 0 {
|
||||
self.skip -= 1; // the slot is overwritten by the next frame
|
||||
} else {
|
||||
self.count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Drops the oldest frame, or the next one to arrive when the ring is empty.
|
||||
fn skip_one(&mut self) {
|
||||
if self.count > 0 {
|
||||
self.drop_oldest();
|
||||
} else {
|
||||
self.skip += 1;
|
||||
self.removed += 1;
|
||||
}
|
||||
}
|
||||
fn drop_oldest(&mut self) {
|
||||
// The partial frame sits at head+count, which this leaves in place.
|
||||
if self.count > 0 {
|
||||
self.head = (self.head + 1) % self.slots.len();
|
||||
self.count -= 1;
|
||||
self.removed += 1;
|
||||
}
|
||||
}
|
||||
/// Hands the oldest frame's buffer to the caller (who owns its reference) and refills the slot.
|
||||
fn take(&mut self, pool: &mut Pool) -> usize {
|
||||
let idx = self.slots[self.head];
|
||||
self.slots[self.head] = pool.get();
|
||||
self.head = (self.head + 1) % self.slots.len();
|
||||
self.count -= 1;
|
||||
self.removed += 1;
|
||||
idx
|
||||
}
|
||||
fn dispose(&mut self, pool: &mut Pool, written: u64) {
|
||||
for &idx in &self.slots {
|
||||
pool.release(idx, written);
|
||||
}
|
||||
self.slots.clear();
|
||||
}
|
||||
}
|
||||
|
||||
struct Stream {
|
||||
file: File,
|
||||
pipe_bytes: usize,
|
||||
ring: Ring,
|
||||
eof: bool,
|
||||
}
|
||||
|
||||
impl Stream {
|
||||
fn open(path: &str, cap: usize, pool: &mut Pool) -> io::Result<Stream> {
|
||||
let file = OpenOptions::new().read(true).custom_flags(O_NONBLOCK).open(path)?;
|
||||
unsafe { fcntl(file.as_raw_fd(), F_SETPIPE_SZ, PIPE_BYTES) };
|
||||
let pipe_bytes = unsafe { fcntl(file.as_raw_fd(), F_GETPIPE_SZ) }.max(0) as usize;
|
||||
Ok(Stream { file, pipe_bytes, ring: Ring::new(cap, pool), eof: false })
|
||||
}
|
||||
|
||||
/// Reads until the pipe is empty or the ring is full (the compositor then sees backpressure).
|
||||
fn pump(&mut self, pool: &mut Pool) {
|
||||
for _ in 0..MAX_READS_PER_WAKE {
|
||||
if self.eof || self.ring.full() {
|
||||
break;
|
||||
}
|
||||
let idx = self.ring.write_idx();
|
||||
let fill = self.ring.fill;
|
||||
match self.file.read(&mut pool.bufs[idx][fill..]) {
|
||||
Ok(0) => self.eof = true,
|
||||
Ok(n) => self.ring.advance(n, pool.size),
|
||||
Err(e) if e.kind() == ErrorKind::WouldBlock => break,
|
||||
Err(e) if e.kind() == ErrorKind::Interrupted => continue,
|
||||
Err(_) => self.eof = true,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Slot {
|
||||
id: u32,
|
||||
video: Stream,
|
||||
audio: Stream,
|
||||
/// Source frame index (in the worker's per-chain numbering) of this compositor's first
|
||||
/// output frame. With origins on both sides, a swap splices B at A's next source frame.
|
||||
origin: Option<i64>,
|
||||
/// Tick at which this pending slot first produced a frame (promotion timeout).
|
||||
warm_since: Option<u64>,
|
||||
/// `LAG` already reported for this pending slot.
|
||||
lagged: bool,
|
||||
}
|
||||
|
||||
impl Slot {
|
||||
fn head_seq(&self) -> Option<i64> {
|
||||
self.origin.map(|o| o + self.video.ring.removed as i64)
|
||||
}
|
||||
/// Video frame i and audio frame i of a compositor are one source pair, so every drop or skip
|
||||
/// applies to both streams and the ring heads always hold the same pair index.
|
||||
///
|
||||
/// Pending slot: keeps complete pairs at A's next position. Only whole pairs are dropped; a stream
|
||||
/// that runs ahead of the other (compositor audio usually leads video) waits in its pipe
|
||||
/// until its partner frames arrive, instead of discarding frames the pairing still needs.
|
||||
fn pump_pending(&mut self, vpool: &mut Pool, apool: &mut Pool, target: Option<i64>) -> u64 {
|
||||
let mut dropped = 0;
|
||||
for _ in 0..4 {
|
||||
self.video.pump(vpool);
|
||||
self.audio.pump(apool);
|
||||
let mut freed = false;
|
||||
while (self.video.ring.full() || self.audio.ring.full()) && self.video.ring.count > 0 && self.audio.ring.count > 0
|
||||
&& target.zip(self.head_seq()).map_or(true, |(t, head)| head < t) {
|
||||
self.video.ring.drop_oldest();
|
||||
self.audio.ring.drop_oldest();
|
||||
dropped += 1;
|
||||
freed = true;
|
||||
}
|
||||
if !freed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
dropped
|
||||
}
|
||||
fn ended(&self) -> bool {
|
||||
(self.video.eof && self.video.ring.count == 0) || (self.audio.eof && self.audio.ring.count == 0)
|
||||
}
|
||||
fn close(mut self, vpool: &mut Pool, apool: &mut Pool, written: u64) {
|
||||
self.video.ring.dispose(vpool, written);
|
||||
self.audio.ring.dispose(apool, written);
|
||||
}
|
||||
}
|
||||
|
||||
/// One encoder output: queued buffer references, written non-blocking in order.
|
||||
struct Out {
|
||||
fd: i32,
|
||||
queue: VecDeque<(usize, usize)>,
|
||||
zero_copy: bool,
|
||||
written_frames: u64,
|
||||
}
|
||||
|
||||
enum Flush {
|
||||
Ok,
|
||||
Closed,
|
||||
}
|
||||
|
||||
impl Out {
|
||||
fn flush(&mut self, pool: &mut Pool) -> Flush {
|
||||
while let Some(&(idx, off)) = self.queue.front() {
|
||||
let buf = &pool.bufs[idx][off..];
|
||||
let n = if self.zero_copy {
|
||||
let iov = IoVec { base: buf.as_ptr(), len: buf.len() };
|
||||
unsafe { vmsplice(self.fd, &iov, 1, SPLICE_F_NONBLOCK) }
|
||||
} else {
|
||||
unsafe { write(self.fd, buf.as_ptr(), buf.len()) }
|
||||
};
|
||||
if n < 0 {
|
||||
let err = io::Error::last_os_error();
|
||||
match err.kind() {
|
||||
ErrorKind::WouldBlock => return Flush::Ok,
|
||||
ErrorKind::Interrupted => continue,
|
||||
_ => return Flush::Closed,
|
||||
}
|
||||
}
|
||||
let n = n as usize;
|
||||
if n == buf.len() {
|
||||
self.queue.pop_front();
|
||||
self.written_frames += 1;
|
||||
pool.release(idx, self.written_frames - 1);
|
||||
pool.on_frame_written(self.written_frames);
|
||||
} else {
|
||||
self.queue.front_mut().unwrap().1 += n;
|
||||
}
|
||||
}
|
||||
Flush::Ok
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Stats {
|
||||
ticks: u64,
|
||||
idle: u64,
|
||||
vrep: u64,
|
||||
asil: u64,
|
||||
drops: u64,
|
||||
swaps: u64,
|
||||
late_max_us: u64,
|
||||
late_over_frame: u64,
|
||||
late_over_2ms: u64,
|
||||
late_over_5ms: u64,
|
||||
backlog_max: u64,
|
||||
out_drops: u64,
|
||||
stall_run: u64,
|
||||
active_since: u64,
|
||||
gap_events: u64,
|
||||
lagged: u64,
|
||||
behind_max: u64,
|
||||
}
|
||||
|
||||
fn emit(line: &str) {
|
||||
let mut out = io::stdout().lock();
|
||||
let _ = out.write_all(line.as_bytes());
|
||||
let _ = out.write_all(b"\n");
|
||||
let _ = out.flush();
|
||||
}
|
||||
|
||||
fn arg(args: &[String], i: usize, name: &str) -> usize {
|
||||
match args.get(i).and_then(|v| v.parse::<usize>().ok()) {
|
||||
Some(v) if v > 0 => v,
|
||||
_ => {
|
||||
eprintln!("frame-gate: invalid {}", name);
|
||||
std::process::exit(2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn open_out(args: &[String], i: usize) -> File {
|
||||
let path = args.get(i).map(String::as_str).unwrap_or("");
|
||||
match OpenOptions::new().write(true).custom_flags(O_NONBLOCK).open(path) {
|
||||
Ok(f) => {
|
||||
unsafe { fcntl(f.as_raw_fd(), F_SETPIPE_SZ, PIPE_BYTES) };
|
||||
f
|
||||
}
|
||||
Err(_) => {
|
||||
eprintln!("frame-gate: cannot open encoder fifo (reader must be open first)");
|
||||
std::process::exit(2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
let width = arg(&args, 1, "width");
|
||||
let height = arg(&args, 2, "height");
|
||||
let fps = arg(&args, 3, "fps");
|
||||
let ring_frames = arg(&args, 4, "ringFrames");
|
||||
let warm = arg(&args, 5, "warmFrames");
|
||||
if width % 2 != 0 || height % 2 != 0 || SAMPLE_RATE % fps != 0 || warm > ring_frames {
|
||||
eprintln!("frame-gate: geometry must be even, fps must divide 48000, warm <= ring");
|
||||
std::process::exit(2);
|
||||
}
|
||||
let video_bytes = width * height * 3 / 2;
|
||||
let audio_bytes = SAMPLE_RATE / fps * AUDIO_BYTES_PER_SAMPLE;
|
||||
// vmsplice is only safe when a frame is at least a pipe's worth (see Pool); `copy` disables it.
|
||||
let zero_copy = args.get(8).map_or(true, |v| v != "copy") && video_bytes >= PIPE_BYTES as usize;
|
||||
// Encoder backlog beyond 0.5 s is an encoder that stopped reading: drop the oldest unstarted frames.
|
||||
let max_backlog = (fps / 2).max(4);
|
||||
|
||||
let enc_video_file = open_out(&args, 6);
|
||||
let enc_audio_file = open_out(&args, 7);
|
||||
set_nonblocking(0);
|
||||
let mut control = unsafe { File::from_raw_fd(0) };
|
||||
|
||||
// Two slots' rings + last + a few in flight; backlog beyond that allocates (counted in `allocs`).
|
||||
let pool_frames = 2 * (ring_frames + 1) + 6;
|
||||
// Compositor audio leads video by the video graph's extra latency. PCM frames are small, so the
|
||||
// audio rings hold 4 s: the leading stream never backs up into the compositor (whose blocked
|
||||
// audio write would stall the very video frames its pairs are waiting for).
|
||||
let audio_ring_frames = ring_frames.max(4 * fps);
|
||||
let mut vpool = Pool::new(video_bytes, pool_frames);
|
||||
let mut apool = Pool::new(audio_bytes, 2 * (audio_ring_frames + 1) + 6);
|
||||
let mut vout = Out { fd: enc_video_file.as_raw_fd(), queue: VecDeque::new(), zero_copy, written_frames: 0 };
|
||||
let mut aout = Out { fd: enc_audio_file.as_raw_fd(), queue: VecDeque::new(), zero_copy: false, written_frames: 0 };
|
||||
|
||||
// Black (limited-range yuv420p) until the first compositor is promoted.
|
||||
let mut last_video = vpool.get();
|
||||
{
|
||||
let b = &mut vpool.bufs[last_video];
|
||||
b[..width * height].fill(16);
|
||||
b[width * height..].fill(128);
|
||||
}
|
||||
let silence = apool.get(); // zeroed; never released
|
||||
|
||||
let mut active: Option<Slot> = None;
|
||||
let mut pending: Option<Slot> = None;
|
||||
let mut stats = Stats::default();
|
||||
let mut line_buf: Vec<u8> = Vec::with_capacity(4096);
|
||||
let mut read_buf = [0u8; 4096];
|
||||
let mut last_pump = Instant::now();
|
||||
|
||||
let start = Instant::now();
|
||||
let period = Duration::from_nanos(1_000_000_000 / fps as u64);
|
||||
// Exact due instant for tick n (no per-frame rounding drift, no overflow).
|
||||
let due_at = |n: u64| start + Duration::from_nanos(n * 1_000_000_000 / fps as u64);
|
||||
let cpu_start = cpu_us();
|
||||
emit(&format!("READY {} {} {} {}", video_bytes, audio_bytes, ring_frames, warm));
|
||||
|
||||
loop {
|
||||
// 1. Emit every tick that is due (catch-up keeps the frame count locked to wall time).
|
||||
let now = Instant::now();
|
||||
while now >= due_at(stats.ticks) {
|
||||
let late = (now - due_at(stats.ticks)).as_micros() as u64;
|
||||
stats.late_max_us = stats.late_max_us.max(late);
|
||||
if late > period.as_micros() as u64 {
|
||||
stats.late_over_frame += 1;
|
||||
}
|
||||
if late > 2_000 {
|
||||
stats.late_over_2ms += 1;
|
||||
}
|
||||
if late > 5_000 {
|
||||
stats.late_over_5ms += 1;
|
||||
}
|
||||
let written = vout.written_frames;
|
||||
|
||||
if pending.as_ref().map_or(false, |p| p.ended()) {
|
||||
let p = pending.take().unwrap();
|
||||
emit(&format!("EOF {}", p.id));
|
||||
p.close(&mut vpool, &mut apool, written);
|
||||
}
|
||||
// Promotion. With origins on A and B: drop B's frames older than A's next source
|
||||
// frame and splice there (B inherits A's latency; the source never runs backwards).
|
||||
// If B is ahead, hold its frames until A reaches them; never skip on promotion.
|
||||
// Without origins (or no active): newest `warm` frames.
|
||||
let target = active.as_ref().and_then(|a| a.head_seq());
|
||||
// A's current buffer depth: B must match it (capped at `warm`) to keep the same jitter margin.
|
||||
let active_depth = active.as_ref().map_or(warm, |a| a.video.ring.count.min(a.audio.ring.count));
|
||||
let mut ready = false;
|
||||
let mut aligned = false;
|
||||
// Diagnostics for the ACTIVE line: A's depth, B's backward step, B's depth, B's age.
|
||||
let mut promo = (active_depth, 0i64, 0usize, 0u64);
|
||||
if let Some(p) = pending.as_mut() {
|
||||
if p.warm_since.is_none() && p.video.ring.count > 0 {
|
||||
p.warm_since = Some(stats.ticks);
|
||||
}
|
||||
let need_aligned = active_depth.min(warm).saturating_sub(1).max(1);
|
||||
let mut behind = 0i64;
|
||||
if let (Some(t), Some(head)) = (target, p.head_seq()) {
|
||||
aligned = true;
|
||||
let count = p.video.ring.count.min(p.audio.ring.count);
|
||||
let newest = head + count as i64 - 1;
|
||||
if count > 0 && newest >= t {
|
||||
// B holds A's next source frame: splice exactly there.
|
||||
while p.video.ring.count > 0 && p.audio.ring.count > 0 && p.head_seq().unwrap_or(t) < t {
|
||||
p.video.ring.drop_oldest();
|
||||
p.audio.ring.drop_oldest();
|
||||
}
|
||||
behind = t - p.head_seq().unwrap_or(t);
|
||||
} else if count > 0 {
|
||||
// B's pipeline is slower than A's on-air position: take its newest frames
|
||||
// (smallest backward step) rather than hold the swap.
|
||||
while p.video.ring.count > need_aligned && p.audio.ring.count > need_aligned {
|
||||
p.video.ring.drop_oldest();
|
||||
p.audio.ring.drop_oldest();
|
||||
}
|
||||
behind = t - p.head_seq().unwrap_or(t);
|
||||
}
|
||||
}
|
||||
let have = p.video.ring.count.min(p.audio.ring.count);
|
||||
let need = if aligned { need_aligned } else { warm };
|
||||
let age = p.warm_since.map_or(0, |w| stats.ticks - w);
|
||||
// Aligned but shallower than A (B's pipeline slightly slower): splicing costs no
|
||||
// artifact, only jitter margin, so accept it after a second of production.
|
||||
let in_step = behind == 0 && (have >= need || (have >= 1 && age >= fps as u64));
|
||||
// Unaligned (first compositor, or no shared numbering): go with what there is after 5 s.
|
||||
let timed_out = !aligned && age >= 5 * fps as u64 && have >= 1;
|
||||
ready = in_step || timed_out;
|
||||
promo = (active_depth, behind, have, age);
|
||||
// B behind A's on-air source position: never splice backwards (that replays picture
|
||||
// and audio). A stays on air; B is usually a CPU-starved spawn still draining its
|
||||
// startup backlog. After 3 s report it so the worker can replace it.
|
||||
if behind > 0 {
|
||||
stats.behind_max = stats.behind_max.max(behind as u64);
|
||||
if age >= 3 * fps as u64 && !p.lagged {
|
||||
p.lagged = true;
|
||||
stats.lagged += 1;
|
||||
emit(&format!("LAG {} {}", p.id, behind));
|
||||
}
|
||||
}
|
||||
}
|
||||
if ready {
|
||||
let mut p = pending.take().unwrap();
|
||||
if !aligned {
|
||||
while p.video.ring.count > warm || p.audio.ring.count > warm {
|
||||
p.video.ring.skip_one();
|
||||
p.audio.ring.skip_one();
|
||||
}
|
||||
}
|
||||
let id = p.id;
|
||||
let old = active.replace(p);
|
||||
stats.swaps += 1;
|
||||
stats.stall_run = 0;
|
||||
stats.active_since = stats.ticks;
|
||||
emit(&format!("ACTIVE {} {} {} {} {} {}", id, stats.ticks, promo.0, promo.1, promo.2, promo.3));
|
||||
if let Some(o) = old {
|
||||
emit(&format!("RETIRED {}", o.id));
|
||||
o.close(&mut vpool, &mut apool, written);
|
||||
}
|
||||
}
|
||||
|
||||
let mut audio_idx = silence;
|
||||
apool.retain(silence);
|
||||
match active.as_mut() {
|
||||
Some(a) => {
|
||||
// A joint stall repeats/silences without consuming a source position.
|
||||
// Skipping both future frames on every empty tick can keep a producer
|
||||
// running at exactly fps permanently behind the skip counters after a pause.
|
||||
// One-sided stalls still skip their late partner to preserve A/V lockstep.
|
||||
let joint_stall = a.video.ring.count == 0 && a.audio.ring.count == 0;
|
||||
if a.video.ring.count > 0 {
|
||||
let idx = a.video.ring.take(&mut vpool);
|
||||
vpool.release(last_video, written);
|
||||
last_video = idx;
|
||||
stats.stall_run = 0;
|
||||
} else {
|
||||
// Repeat now, discard the late frame when it arrives (keeps A/V index-locked).
|
||||
if !joint_stall { a.video.ring.skip_one(); }
|
||||
stats.vrep += 1;
|
||||
if stats.gap_events < 200 {
|
||||
stats.gap_events += 1;
|
||||
emit(&format!("GAP v {} {} {} {}", a.id, stats.ticks, stats.ticks - stats.active_since, late));
|
||||
}
|
||||
stats.stall_run += 1;
|
||||
if stats.stall_run == fps as u64 {
|
||||
emit(&format!("STALL {} {}", a.id, stats.ticks));
|
||||
}
|
||||
}
|
||||
if a.audio.ring.count > 0 {
|
||||
apool.release(silence, aout.written_frames);
|
||||
audio_idx = a.audio.ring.take(&mut apool);
|
||||
} else {
|
||||
if !joint_stall { a.audio.ring.skip_one(); }
|
||||
stats.asil += 1;
|
||||
if stats.gap_events < 200 {
|
||||
stats.gap_events += 1;
|
||||
emit(&format!("GAP a {} {} {} {}", a.id, stats.ticks, stats.ticks - stats.active_since, late));
|
||||
}
|
||||
}
|
||||
}
|
||||
None => stats.idle += 1,
|
||||
}
|
||||
if active.as_ref().map_or(false, |a| a.ended()) {
|
||||
let a = active.take().unwrap();
|
||||
emit(&format!("EOF {}", a.id));
|
||||
a.close(&mut vpool, &mut apool, written);
|
||||
}
|
||||
|
||||
vpool.retain(last_video);
|
||||
vout.queue.push_back((last_video, 0));
|
||||
aout.queue.push_back((audio_idx, 0));
|
||||
for (out, pool) in [(&mut vout, &mut vpool), (&mut aout, &mut apool)] {
|
||||
// Never drop the frame in flight (index 0 may be partially written).
|
||||
while out.queue.len() > max_backlog && out.queue.len() > 1 {
|
||||
let (idx, _) = out.queue.remove(1).unwrap();
|
||||
pool.release(idx, out.written_frames);
|
||||
stats.out_drops += 1;
|
||||
}
|
||||
}
|
||||
stats.backlog_max = stats.backlog_max.max(vout.queue.len() as u64);
|
||||
stats.ticks += 1;
|
||||
}
|
||||
for (out, pool) in [(&mut vout, &mut vpool), (&mut aout, &mut apool)] {
|
||||
if let Flush::Closed = out.flush(pool) {
|
||||
eprintln!("frame-gate: encoder input closed");
|
||||
std::process::exit(3);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Wait for input, encoder writability, or the next tick.
|
||||
let due = due_at(stats.ticks);
|
||||
let now = Instant::now();
|
||||
// F_SETPIPE_SZ can fail under a shared uid's pipe quota. An 8 KiB video pipe
|
||||
// must be drained as soon as readable; a 2 ms hold would cap it at ~4 MB/s.
|
||||
let bulk_min = video_bytes * fps / 250;
|
||||
let bulk = [active.as_ref(), pending.as_ref()].into_iter().flatten().all(|s| s.video.pipe_bytes >= bulk_min);
|
||||
let pump_due = last_pump + if bulk { PUMP_INTERVAL } else { Duration::ZERO };
|
||||
let pump_now = now >= pump_due;
|
||||
let until = if pump_now { due } else { due.min(pump_due) };
|
||||
let timeout_ms = (until.saturating_duration_since(now).as_micros() as i32 + 999) / 1000;
|
||||
let mut fds: Vec<PollFd> = Vec::with_capacity(7);
|
||||
fds.push(PollFd { fd: 0, events: POLLIN, revents: 0 });
|
||||
fds.push(PollFd { fd: if vout.queue.is_empty() { -1 } else { vout.fd }, events: POLLOUT, revents: 0 });
|
||||
fds.push(PollFd { fd: if aout.queue.is_empty() { -1 } else { aout.fd }, events: POLLOUT, revents: 0 });
|
||||
if pump_now {
|
||||
for slot in [active.as_ref(), pending.as_ref()] {
|
||||
if let Some(s) = slot {
|
||||
for st in [&s.video, &s.audio] {
|
||||
let want = !st.eof && !st.ring.full();
|
||||
fds.push(PollFd { fd: if want { st.file.as_raw_fd() } else { -1 }, events: POLLIN, revents: 0 });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
unsafe { poll(fds.as_mut_ptr(), fds.len() as u64, timeout_ms) };
|
||||
|
||||
// 3. Pump compositor pipes, then hold off for PUMP_INTERVAL so they refill in bulk.
|
||||
if fds.len() > 3 && fds[3..].iter().any(|f| f.revents != 0) {
|
||||
last_pump = Instant::now();
|
||||
if let Some(a) = active.as_mut() {
|
||||
a.video.pump(&mut vpool);
|
||||
a.audio.pump(&mut apool);
|
||||
}
|
||||
if let Some(p) = pending.as_mut() {
|
||||
let target = active.as_ref().and_then(|a| a.head_seq());
|
||||
stats.drops += p.pump_pending(&mut vpool, &mut apool, target);
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Control lines.
|
||||
if fds[0].revents != 0 {
|
||||
loop {
|
||||
match control.read(&mut read_buf) {
|
||||
Ok(0) => std::process::exit(0), // parent gone
|
||||
Ok(n) => line_buf.extend_from_slice(&read_buf[..n]),
|
||||
Err(e) if e.kind() == ErrorKind::WouldBlock => break,
|
||||
Err(e) if e.kind() == ErrorKind::Interrupted => continue,
|
||||
Err(_) => std::process::exit(0),
|
||||
}
|
||||
}
|
||||
while let Some(pos) = line_buf.iter().position(|&b| b == b'\n') {
|
||||
let line: Vec<u8> = line_buf.drain(..=pos).collect();
|
||||
let text = String::from_utf8_lossy(&line[..line.len() - 1]).into_owned();
|
||||
let parts: Vec<&str> = text.split_whitespace().collect();
|
||||
let written = vout.written_frames;
|
||||
match parts.as_slice() {
|
||||
["ATTACH", id, vpath, apath, rest @ ..] if rest.len() <= 1 => match id.parse::<u32>() {
|
||||
Ok(id) => {
|
||||
let origin = rest.first().and_then(|v| v.parse::<i64>().ok());
|
||||
let video = Stream::open(vpath, ring_frames, &mut vpool);
|
||||
let audio = Stream::open(apath, audio_ring_frames, &mut apool);
|
||||
match (video, audio) {
|
||||
(Ok(video), Ok(audio)) => {
|
||||
if let Some(old) = pending.replace(Slot { id, video, audio, origin, warm_since: None, lagged: false }) {
|
||||
emit(&format!("DROPPED {}", old.id));
|
||||
old.close(&mut vpool, &mut apool, written);
|
||||
}
|
||||
emit(&format!("ATTACHED {}", id));
|
||||
}
|
||||
(v, a) => {
|
||||
if let Ok(mut s) = v {
|
||||
s.ring.dispose(&mut vpool, written);
|
||||
}
|
||||
if let Ok(mut s) = a {
|
||||
s.ring.dispose(&mut apool, written);
|
||||
}
|
||||
emit(&format!("ERROR {} open", id));
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(_) => emit("ERROR 0 id"),
|
||||
},
|
||||
["DETACH", id] => {
|
||||
let id = id.parse::<u32>().unwrap_or(u32::MAX);
|
||||
if pending.as_ref().map_or(false, |p| p.id == id) {
|
||||
pending.take().unwrap().close(&mut vpool, &mut apool, written);
|
||||
emit(&format!("DETACHED {}", id));
|
||||
} else if active.as_ref().map_or(false, |a| a.id == id) {
|
||||
active.take().unwrap().close(&mut vpool, &mut apool, written);
|
||||
emit(&format!("DETACHED {}", id));
|
||||
}
|
||||
}
|
||||
["STATS"] => {
|
||||
let ring = |s: Option<&Slot>| s.map_or("-".to_string(), |s| format!("{}:{}/{}", s.id, s.video.ring.count, s.audio.ring.count));
|
||||
emit(&format!(
|
||||
"STATS ticks={} idle={} vrep={} asil={} drops={} swaps={} late_max_us={} late_over_frame={} write_max_us=0 cpu_us={} wall_us={} active={} pending={} gc_max_us=0 allocs={} backlog_max={} out_drops={} late_over_2ms={} late_over_5ms={} lagged={} behind_max={}",
|
||||
stats.ticks, stats.idle, stats.vrep, stats.asil, stats.drops, stats.swaps, stats.late_max_us,
|
||||
stats.late_over_frame, cpu_us() - cpu_start, start.elapsed().as_micros(),
|
||||
ring(active.as_ref()), ring(pending.as_ref()), vpool.allocs + apool.allocs, stats.backlog_max, stats.out_drops, stats.late_over_2ms, stats.late_over_5ms, stats.lagged, stats.behind_max
|
||||
));
|
||||
}
|
||||
["RESET"] => {
|
||||
stats.late_max_us = 0;
|
||||
stats.late_over_frame = 0;
|
||||
stats.late_over_2ms = 0;
|
||||
stats.late_over_5ms = 0;
|
||||
stats.backlog_max = 0;
|
||||
emit("RESET");
|
||||
}
|
||||
["QUIT"] => std::process::exit(0),
|
||||
[] => {}
|
||||
_ => emit("ERROR 0 command"),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
30
apps/media-worker/src/gate/sink-cli.ts
Normal file
30
apps/media-worker/src/gate/sink-cli.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
/**
|
||||
* Verifying null encoder as its own process (so its timing is not polluted by
|
||||
* the harness that spawns compositors): `sink-cli.ts <videoFifo> <audioFifo> <w> <h> <fps>`.
|
||||
* Opens the gate's encoder FIFOs non-blocking, prints `OPEN`; on a
|
||||
* `REPORT <firstActiveTick>` line prints the JSON report and exits.
|
||||
*/
|
||||
import { constants, openSync } from "node:fs";
|
||||
import { Socket } from "node:net";
|
||||
import { VerifyingSink } from "./sink";
|
||||
|
||||
function main(): void {
|
||||
const [videoPath, audioPath, w, h, fps] = process.argv.slice(2);
|
||||
if (!videoPath || !audioPath) throw new Error("usage: sink-cli <videoFifo> <audioFifo> <w> <h> <fps>");
|
||||
const sink = new VerifyingSink({ width: Number(w), height: Number(h), fps: Number(fps) });
|
||||
const open = (path: string) => new Socket({ fd: openSync(path, constants.O_RDONLY | constants.O_NONBLOCK), readable: true, writable: false });
|
||||
sink.attach(open(videoPath), open(audioPath));
|
||||
process.stdout.write("OPEN\n");
|
||||
let buf = "";
|
||||
process.stdin.setEncoding("utf8");
|
||||
process.stdin.on("data", (chunk: string) => {
|
||||
buf += chunk;
|
||||
const m = /REPORT (\d+)\n/.exec(buf);
|
||||
if (!m) return;
|
||||
const first = Number(m[1]);
|
||||
const r = sink.report;
|
||||
process.stdout.write(`${JSON.stringify({ ...r, silentAt: undefined, tagChanges: r.tagChanges.slice(0, 20_000), silentAfterFirstActive: sink.silentFramesFrom(first) })}\n`, () => process.exit(0));
|
||||
});
|
||||
}
|
||||
|
||||
if (require.main === module) main();
|
||||
139
apps/media-worker/src/gate/sink.ts
Normal file
139
apps/media-worker/src/gate/sink.ts
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
import type { Readable } from "node:stream";
|
||||
import { audioFrameBytes, videoFrameBytes, type FrameGeometry } from "./protocol";
|
||||
|
||||
/**
|
||||
* Null encoder that verifies what an encoder would receive from the gate:
|
||||
* frame counts in lockstep, arrival gaps, a luma tag per frame (Y at byte 0,
|
||||
* which tests use as a monotonic frame/scene counter), and PCM continuity
|
||||
* (largest sample-to-sample step, all-zero frames).
|
||||
*/
|
||||
export interface SinkReport {
|
||||
videoFrames: number;
|
||||
audioFrames: number;
|
||||
/** Largest gap between two consecutive completed video frames. */
|
||||
maxGapMs: number;
|
||||
/** Video frames whose arrival gap exceeded two frame intervals. */
|
||||
gapsOverTwoFrames: number;
|
||||
/** Tag of each video frame is recorded only when it changes: [frameIndex, tag]. */
|
||||
tagChanges: Array<[number, number]>;
|
||||
/**
|
||||
* Histogram of per-frame counter steps for counter tags (luma 20 + 4·(n mod 50), decoded
|
||||
* with ±1 tolerance): "1" is a normal frame, "0" a repeat, "2+" a skip, negative backwards.
|
||||
*/
|
||||
tagDeltas: Record<string, number>;
|
||||
maxPcmStep: number;
|
||||
maxPcmAbs: number;
|
||||
silentAudioFrames: number;
|
||||
/** Frame index of each all-zero audio frame (first 100 000). */
|
||||
silentAt: number[];
|
||||
}
|
||||
|
||||
export class VerifyingSink {
|
||||
private readonly vBytes: number;
|
||||
private readonly aBytes: number;
|
||||
private vPos = 0;
|
||||
private aPos = 0;
|
||||
private lastFrameAt = 0n;
|
||||
private prev: [number, number] | null = null;
|
||||
private frameNonZero = false;
|
||||
private lastTag = -1;
|
||||
private readonly intervalNs: bigint;
|
||||
readonly report: SinkReport = {
|
||||
videoFrames: 0, audioFrames: 0, maxGapMs: 0, gapsOverTwoFrames: 0, tagChanges: [], tagDeltas: {},
|
||||
maxPcmStep: 0, maxPcmAbs: 0, silentAudioFrames: 0, silentAt: [],
|
||||
};
|
||||
/** PCM steps are not measured before this frame (gate boot silence before the first compositor). */
|
||||
skipSilenceBefore = 0;
|
||||
|
||||
/** All-zero audio frames at or after `frame` (e.g. the tick the first compositor went on air). */
|
||||
silentFramesFrom(frame: number): number { return this.report.silentAt.filter((i) => i >= frame).length; }
|
||||
|
||||
constructor(geometry: FrameGeometry) {
|
||||
this.vBytes = videoFrameBytes(geometry);
|
||||
this.aBytes = audioFrameBytes(geometry);
|
||||
this.intervalNs = 1_000_000_000n / BigInt(geometry.fps);
|
||||
}
|
||||
|
||||
attach(video: Readable, audio: Readable): void {
|
||||
video.on("data", (chunk: Buffer) => this.onVideo(chunk));
|
||||
audio.on("data", (chunk: Buffer) => this.onAudio(chunk));
|
||||
}
|
||||
|
||||
private onVideo(chunk: Buffer): void {
|
||||
let at = 0;
|
||||
while (at < chunk.length) {
|
||||
if (this.vPos === 0) {
|
||||
const tag = chunk[at] ?? 0;
|
||||
if (this.lastTag >= 18 && this.lastTag < 220 && tag >= 18 && tag < 220) {
|
||||
const decode = (y: number): number => Math.round((y - 20) / 4);
|
||||
let d = (((decode(tag) - decode(this.lastTag)) % 50) + 50) % 50;
|
||||
if (d > 25) d -= 50;
|
||||
const key = String(d);
|
||||
this.report.tagDeltas[key] = (this.report.tagDeltas[key] ?? 0) + 1;
|
||||
}
|
||||
if (tag !== this.lastTag) {
|
||||
if (this.report.tagChanges.length < 100_000) this.report.tagChanges.push([this.report.videoFrames, tag]);
|
||||
this.lastTag = tag;
|
||||
}
|
||||
}
|
||||
const n = Math.min(chunk.length - at, this.vBytes - this.vPos);
|
||||
at += n;
|
||||
this.vPos += n;
|
||||
if (this.vPos === this.vBytes) {
|
||||
this.vPos = 0;
|
||||
const now = process.hrtime.bigint();
|
||||
if (this.lastFrameAt !== 0n) {
|
||||
const gap = now - this.lastFrameAt;
|
||||
this.report.maxGapMs = Math.max(this.report.maxGapMs, Number(gap) / 1e6);
|
||||
if (gap > 2n * this.intervalNs) this.report.gapsOverTwoFrames += 1;
|
||||
}
|
||||
this.lastFrameAt = now;
|
||||
this.report.videoFrames += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private onAudio(chunk: Buffer): void {
|
||||
// Chunks from a stream socket are not guaranteed to be 4-byte aligned.
|
||||
let at = 0;
|
||||
while (at < chunk.length) {
|
||||
const sampleStart = this.aPos % 4 === 0;
|
||||
if (sampleStart && at + 4 <= chunk.length) {
|
||||
const l = chunk.readInt16LE(at);
|
||||
const r = chunk.readInt16LE(at + 2);
|
||||
this.sample(l, r);
|
||||
at += 4;
|
||||
this.aPos += 4;
|
||||
} else {
|
||||
// Rare split sample: fall back to byte stepping with a carry.
|
||||
this.carry.push(chunk[at] ?? 0);
|
||||
at += 1;
|
||||
this.aPos += 1;
|
||||
if (this.carry.length === 4) {
|
||||
const b = Buffer.from(this.carry);
|
||||
this.carry = [];
|
||||
this.sample(b.readInt16LE(0), b.readInt16LE(2));
|
||||
}
|
||||
}
|
||||
if (this.aPos === this.aBytes) {
|
||||
this.aPos = 0;
|
||||
if (!this.frameNonZero) {
|
||||
this.report.silentAudioFrames += 1;
|
||||
if (this.report.silentAt.length < 100_000) this.report.silentAt.push(this.report.audioFrames);
|
||||
}
|
||||
this.frameNonZero = false;
|
||||
this.report.audioFrames += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
private carry: number[] = [];
|
||||
|
||||
private sample(l: number, r: number): void {
|
||||
if (l !== 0 || r !== 0) this.frameNonZero = true;
|
||||
this.report.maxPcmAbs = Math.max(this.report.maxPcmAbs, Math.abs(l), Math.abs(r));
|
||||
if (this.prev && this.report.audioFrames >= this.skipSilenceBefore) {
|
||||
this.report.maxPcmStep = Math.max(this.report.maxPcmStep, Math.abs(l - this.prev[0]), Math.abs(r - this.prev[1]));
|
||||
}
|
||||
this.prev = [l, r];
|
||||
}
|
||||
}
|
||||
150
apps/media-worker/src/gate/spike.ts
Normal file
150
apps/media-worker/src/gate/spike.ts
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
/**
|
||||
* §1.4 frame-gate spike harness (not part of the test suite; numbers go to
|
||||
* runs/wp4/s2a-report.md).
|
||||
*
|
||||
* node --import tsx src/gate/spike.ts steady --impl rust --size 1920x1080 --fps 60 --seconds 20
|
||||
* node --import tsx src/gate/spike.ts swaps --impl node --size 1920x1080 --fps 30 --swaps 1000
|
||||
*
|
||||
* Producers are compositor stand-ins (ffmpeg lavfi: solid luma tag = generation,
|
||||
* 1500 Hz sine, so every frame holds a whole number of periods and any
|
||||
* mis-spliced or inserted PCM shows up as a sample step). The sink is the
|
||||
* verifying null encoder (src/gate/sink.ts).
|
||||
*/
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { spawnManaged, type ManagedProcess } from "../relay/ffmpeg";
|
||||
import { FrameGate, type GateImpl } from "./frame-gate";
|
||||
import type { FrameGeometry } from "./protocol";
|
||||
import type { SinkReport } from "./sink";
|
||||
|
||||
async function waitFor(cond: () => boolean, timeoutMs = 15_000): Promise<void> {
|
||||
const t0 = Date.now();
|
||||
while (!cond()) {
|
||||
if (Date.now() - t0 > timeoutMs) throw new Error("timeout");
|
||||
await new Promise((r) => setTimeout(r, 10));
|
||||
}
|
||||
}
|
||||
|
||||
const passthrough = { redact: (s: string) => s };
|
||||
|
||||
export function producerArgs(g: FrameGeometry, tag: number, paced = false): string[] {
|
||||
return [
|
||||
"-hide_banner", "-nostdin", "-loglevel", "error",
|
||||
...(paced ? ["-re"] : []), "-f", "lavfi", "-i", `color=c=black:s=${g.width}x${g.height}:r=${g.fps},lutyuv=y=${tag}:u=128:v=128`,
|
||||
...(paced ? ["-re"] : []), "-f", "lavfi", "-i", "sine=frequency=1500:sample_rate=48000",
|
||||
"-map", "0:v", "-f", "rawvideo", "-pix_fmt", "yuv420p", "-fps_mode", "passthrough", "pipe:1",
|
||||
"-map", "1:a", "-ac", "2", "-f", "s16le", "-flush_packets", "1", "pipe:3",
|
||||
];
|
||||
}
|
||||
|
||||
export async function startProducer(gate: FrameGate, g: FrameGeometry, tag: number, origin?: number, paced = false): Promise<{ id: number; proc: ManagedProcess }> {
|
||||
const slot = gate.allocateSlot();
|
||||
const proc = spawnManaged("ffmpeg", producerArgs(g, tag, paced), ["ignore", slot.fifos.videoFd, "pipe", slot.fifos.audioFd], passthrough, () => undefined);
|
||||
slot.fifos.releaseWriters();
|
||||
await proc.started;
|
||||
await gate.attach(slot, origin);
|
||||
return { id: slot.id, proc };
|
||||
}
|
||||
|
||||
function arg(name: string, fallback: string): string {
|
||||
const i = process.argv.indexOf(`--${name}`);
|
||||
return i >= 0 ? (process.argv[i + 1] ?? fallback) : fallback;
|
||||
}
|
||||
|
||||
function percentile(values: number[], p: number): number {
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
return sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))] ?? 0;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const mode = process.argv[2] ?? "steady";
|
||||
const impl = arg("impl", "rust") as GateImpl;
|
||||
const [w, h] = arg("size", "1920x1080").split("x").map(Number);
|
||||
const g: FrameGeometry = { width: w ?? 1920, height: h ?? 1080, fps: Number(arg("fps", "30")) };
|
||||
const scratch = mkdtempSync(join(tmpdir(), "gate-spike-"));
|
||||
const events: string[] = [];
|
||||
const gate = new FrameGate({
|
||||
geometry: g, scratchDir: scratch, impl, copyWrites: process.argv.includes("--copy"),
|
||||
redactor: passthrough, onStderr: (m) => process.stderr.write(m),
|
||||
onEvent: (e) => { if (process.env.SPIKE_DEBUG) process.stderr.write(`${JSON.stringify(e)}\n`); if (e.type === "STALL" || e.type === "EOF" || e.type === "GAP") events.push(JSON.stringify(e)); },
|
||||
});
|
||||
// --sink ffmpeg: a real ffmpeg demuxer reads the encoder FIFOs (-f null), as the persistent encoder would.
|
||||
const ffmpegSink = process.argv.includes("ffmpeg");
|
||||
// Default sink: the verifier in its own process (src/gate/sink-cli.ts).
|
||||
const verifier = ffmpegSink ? null : spawnManaged(process.execPath, ["--import", "tsx", join(__dirname, "sink-cli.ts"),
|
||||
gate.encoderFifos.videoPath, gate.encoderFifos.audioPath, String(g.width), String(g.height), String(g.fps)],
|
||||
["pipe", "pipe", "pipe"], passthrough, (m) => process.stderr.write(m), join(__dirname, "../.."));
|
||||
let verifierOut = "";
|
||||
verifier?.child.stdout?.setEncoding("utf8");
|
||||
verifier?.child.stdout?.on("data", (c: string) => { verifierOut += c; });
|
||||
if (verifier) await waitFor(() => verifierOut.includes("OPEN\n"));
|
||||
const inputs = ffmpegSink ? gate.encoderInputs() : null;
|
||||
const nullEncoder = inputs ? spawnManaged("ffmpeg", ["-hide_banner", "-nostdin", "-loglevel", "error",
|
||||
"-probesize", "32", "-analyzeduration", "0", "-f", "rawvideo", "-pix_fmt", "yuv420p", "-s", `${g.width}x${g.height}`, "-framerate", String(g.fps), "-i", "pipe:0",
|
||||
"-probesize", "32", "-analyzeduration", "0", "-f", "s16le", "-ar", "48000", "-ac", "2", "-i", "pipe:3", "-map", "0:v", "-map", "1:a", "-f", "null", "-"],
|
||||
[inputs.videoFd, "ignore", "pipe", inputs.audioFd], passthrough, (m) => process.stderr.write(m)) : null;
|
||||
await nullEncoder?.started;
|
||||
await gate.start();
|
||||
const t0 = Date.now();
|
||||
let current = await startProducer(gate, g, 20);
|
||||
const firstTick = await gate.waitActive(current.id, 10_000);
|
||||
await new Promise((r) => setTimeout(r, 1_000)); // encoder/producer warm-up excluded from peaks
|
||||
await gate.resetPeaks();
|
||||
const base = await gate.stats();
|
||||
const swapMs: number[] = [];
|
||||
|
||||
if (mode === "steady") {
|
||||
await new Promise((r) => setTimeout(r, Number(arg("seconds", "20")) * 1000));
|
||||
} else {
|
||||
const swaps = Number(arg("swaps", "1000"));
|
||||
for (let i = 1; i <= swaps; i += 1) {
|
||||
const s0 = performance.now();
|
||||
const next = await startProducer(gate, g, 20 + (i % 200));
|
||||
await gate.waitActive(next.id, 10_000);
|
||||
swapMs.push(performance.now() - s0);
|
||||
void current.proc.terminate();
|
||||
current = next;
|
||||
}
|
||||
}
|
||||
const end = await gate.stats();
|
||||
await gate.stop();
|
||||
await current.proc.terminate();
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
await nullEncoder?.terminate();
|
||||
let report: SinkReport | null = null;
|
||||
if (verifier) {
|
||||
verifier.child.stdin?.write(`REPORT ${firstTick}\n`);
|
||||
await verifier.exited;
|
||||
const line = verifierOut.split("\n").find((l) => l.startsWith("{"));
|
||||
report = line ? (JSON.parse(line) as SinkReport) : null;
|
||||
}
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
|
||||
const ticks = end.ticks - base.ticks;
|
||||
const wallUs = end.wallUs - base.wallUs;
|
||||
// Producer generation g has luma tag 20 + (g % 200); frames before the first producer are gate black (16).
|
||||
const tags = (report?.tagChanges ?? []).map(([, t]) => t).filter((t, i) => !(i === 0 && t === 16));
|
||||
let nonMonotonic = 0;
|
||||
for (let i = 1; i < tags.length; i += 1) {
|
||||
const prev = tags[i - 1] ?? 0;
|
||||
if (tags[i] !== (prev === 219 ? 20 : prev + 1)) nonMonotonic += 1;
|
||||
}
|
||||
const result = {
|
||||
mode, impl, sink: ffmpegSink ? "ffmpeg-null" : "node-verifier", copyWrites: process.argv.includes("--copy"), geometry: g, wallS: (Date.now() - t0) / 1000,
|
||||
gateCpuPctOfCore: Number((((end.cpuUs - base.cpuUs) / wallUs) * 100).toFixed(2)),
|
||||
ticks, swaps: end.swaps - base.swaps,
|
||||
repeatsAfterFirstActive: end.vrep - base.vrep, silenceAfterFirstActive: end.asil - base.asil,
|
||||
drops: end.drops, lateMaxUs: end.lateMaxUs, lateOverFrame: end.lateOverFrame, lateOver2ms: end.lateOver2ms, lateOver5ms: end.lateOver5ms, writeMaxUs: end.writeMaxUs,
|
||||
gcMaxUs: end.gcMaxUs, poolAllocs: end.allocs, encoderBacklogMax: end.backlogMax, encoderOutDrops: end.outDrops,
|
||||
verifier: report ? { ...report, tagChanges: report.tagChanges.length } : null,
|
||||
sinkLockstep: report ? Math.abs(report.videoFrames - report.audioFrames) <= 1 : null,
|
||||
tagSequenceViolations: nonMonotonic, distinctGenerationsSeen: tags.length,
|
||||
swapLatencyMs: swapMs.length ? { p50: percentile(swapMs, 50), p99: percentile(swapMs, 99), max: Math.max(...swapMs) } : null,
|
||||
events: events.slice(0, 40),
|
||||
};
|
||||
process.stdout.write(`${JSON.stringify(result)}\n`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (require.main === module) void main().catch((error: unknown) => { process.stderr.write(`${String(error)}\n`); process.exit(1); });
|
||||
101
apps/media-worker/src/main.ts
Normal file
101
apps/media-worker/src/main.ts
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
import * as http from "node:http";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { createHash, createPublicKey } from "node:crypto";
|
||||
import { resolve } from "node:path";
|
||||
import pino from "pino";
|
||||
import { DurableFence } from "./bus/fence";
|
||||
import { RedisBus } from "./bus/redis";
|
||||
import { FfmpegRelayJob, type EgressRelayFactory, type VaultHandleBroker } from "./relay/ffmpeg";
|
||||
import type { EndpointValidator } from "./relay/endpoints";
|
||||
import { MediaSupervisor, type RelayJob } from "./supervisor";
|
||||
import { ProductionJob, type AssetStore, type ProductionEgress } from "./compositor/production-job";
|
||||
import { resolveHelperBinary } from "./gate/binary";
|
||||
import { isDesiredChannel, type DesiredChannel } from "./types";
|
||||
|
||||
const logger = pino({ level: process.env.LOG_LEVEL ?? "info" });
|
||||
const port = Number(process.env.PORT ?? 3100);
|
||||
const workerId = process.env.MEDIA_WORKER_ID ?? "media-worker-dev";
|
||||
let supervisor: MediaSupervisor | null = null;
|
||||
let bus: RedisBus | null = null;
|
||||
let ready = false;
|
||||
|
||||
const server = http.createServer((req, res) => {
|
||||
if (req.url === "/healthz") {
|
||||
res.writeHead(ready ? 200 : 503, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({ status: ready ? "alive" : "starting", worker: workerId, channels: supervisor?.status() ?? [] }));
|
||||
return;
|
||||
}
|
||||
res.writeHead(404).end();
|
||||
});
|
||||
|
||||
function readDesired(): DesiredChannel[] {
|
||||
const path = process.env.MEDIA_DESIRED_CONFIG;
|
||||
if (!path) return [];
|
||||
const value: unknown = JSON.parse(readFileSync(path, "utf8"));
|
||||
if (!Array.isArray(value) || !value.every(isDesiredChannel)) throw new Error("invalid desired channel config");
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Deliberately fails closed until WP7 injects an egress relay and vault broker. */
|
||||
const unavailableEgress: EgressRelayFactory = { async prepare() { throw new Error("egress relay is not configured"); } };
|
||||
const unavailableValidator: EndpointValidator = { async validate() { throw new Error("endpoint validator is not configured"); } };
|
||||
const unavailableVault: VaultHandleBroker = { async deliverPushTarget() { throw new Error("vault broker is not configured"); } };
|
||||
const unavailableProductionEgress: ProductionEgress = { async prepare() { throw new Error("production egress is not configured"); } };
|
||||
const unavailableAssets: AssetStore = { async localPath() { throw new Error("overlay asset store is not configured"); } };
|
||||
const redactor = { redact: (_chunk: string) => "[media stderr redacted until a job secret-aware redactor is injected]" };
|
||||
|
||||
/** Per-channel scratch for gate FIFOs; the name is a hash so channel ids never reach a path. */
|
||||
function scratchFor(channel: DesiredChannel): string {
|
||||
const id = createHash("sha256").update(`${channel.workspaceId}:${channel.channelId}`).digest("hex").slice(0, 32);
|
||||
return resolve(process.env.MEDIA_SCRATCH_DIR ?? ".media-worker-state/scratch", id);
|
||||
}
|
||||
|
||||
/** v2 (production) snapshots run the compositor chain; v1 snapshots keep the single relay. */
|
||||
function createRelay(channel: DesiredChannel, onExit: (reason: string) => void): RelayJob {
|
||||
const onStderr = (message: string): void => logger.warn({ message }, "ffmpeg stderr");
|
||||
if (channel.production) {
|
||||
return new ProductionJob({
|
||||
validator: unavailableValidator, egress: unavailableProductionEgress, vault: unavailableVault, assets: unavailableAssets,
|
||||
redactor, onStderr, onExit, scratchDir: scratchFor(channel),
|
||||
});
|
||||
}
|
||||
return new FfmpegRelayJob(unavailableValidator, unavailableEgress, unavailableVault, redactor, onStderr, onExit);
|
||||
}
|
||||
|
||||
/** Production channels need the packaged native helpers; say at boot which ones resolve (src/gate/binary.ts). */
|
||||
function checkHelpers(): void {
|
||||
for (const name of ["frame-gate", "source-bus"] as const) {
|
||||
try { logger.info({ helper: name, path: resolveHelperBinary(name) }, "native helper"); }
|
||||
catch (error) { logger.warn({ helper: name, err: String(error) }, "native helper missing: production channels will fail to start"); }
|
||||
}
|
||||
}
|
||||
|
||||
async function start(): Promise<void> {
|
||||
checkHelpers();
|
||||
const desired = readDesired();
|
||||
const keyPath = process.env.MEDIA_CONTROL_PUBLIC_KEY;
|
||||
const keys = new Map<string, ReturnType<typeof createPublicKey>>();
|
||||
if (keyPath) keys.set(process.env.MEDIA_CONTROL_KID ?? "control-v1", createPublicKey(readFileSync(keyPath)));
|
||||
const fence = new DurableFence(process.env.MEDIA_FENCE_PATH ?? ".media-worker-state/fence.json");
|
||||
bus = await RedisBus.connect(process.env.REDIS_URL ?? `redis://${process.env.REDIS_HOST ?? "localhost"}:6379`);
|
||||
supervisor = new MediaSupervisor(workerId, desired, bus, fence, keys, createRelay,
|
||||
{ transition: (event) => logger.info(event, "media state transition") });
|
||||
await supervisor.start();
|
||||
const workspaces = [...new Set(desired.map((item) => item.workspaceId))];
|
||||
for (const workspaceId of workspaces) {
|
||||
// ACLs should grant only this worker's assigned command channels.
|
||||
await bus.subscribe(workerId, [workspaceId], (message) => {
|
||||
let command: unknown;
|
||||
try { command = JSON.parse(message); } catch { logger.warn("malformed command rejected"); return; }
|
||||
void supervisor?.handle(command).catch(() => logger.warn("command rejected"));
|
||||
});
|
||||
}
|
||||
ready = true;
|
||||
logger.info({ port, workerId, channels: desired.length }, "media worker listening");
|
||||
}
|
||||
|
||||
server.listen(port, () => { void start().catch((error: unknown) => { logger.error({ err: String(error) }, "media worker startup failed"); process.exitCode = 1; server.close(); }); });
|
||||
for (const signal of ["SIGINT", "SIGTERM"] as const) process.on(signal, () => {
|
||||
ready = false;
|
||||
void supervisor?.stop().finally(() => { bus?.close(); server.close(); });
|
||||
});
|
||||
103
apps/media-worker/src/preview/argv.ts
Normal file
103
apps/media-worker/src/preview/argv.ts
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
/**
|
||||
* Preview tap argv (streaming-architecture.md §2.1). Typed params in, argv out,
|
||||
* like MediaCommandBuilder: every tap reads raw frames from a source-bus
|
||||
* subscription on inherited pipes (video fd 0, PCM fd 3) and never touches a
|
||||
* remote; HLS taps write only into their own scratch directory.
|
||||
*
|
||||
* - JPEG: one still per `everySeconds` (the bus delivers every n-th pair), 360p,
|
||||
* as an mjpeg stream on stdout (the worker splits frames and publishes).
|
||||
* - video: 360p ≤ 500 kb/s H.264 + AAC 64 kb/s, fMP4 HLS with 0.5 s parts and a
|
||||
* keyframe every second (each 1 s segment = 2 parts, the first independent).
|
||||
* - audio: AAC 64 kb/s only, same part/segment grid.
|
||||
*
|
||||
* FFmpeg 5.1's hls muxer has no EXT-X-PART support, so ffmpeg writes the parts
|
||||
* as its own 0.5 s "segments" (split_by_time) and `ll-hls.ts` publishes them as
|
||||
* LL-HLS parts, concatenates them into segments and writes the media playlists.
|
||||
*/
|
||||
import { assertGeometry, GATE_CHANNELS, GATE_SAMPLE_RATE, type FrameGeometry } from "../gate/protocol";
|
||||
|
||||
export const TAP_HEIGHT = 360;
|
||||
export const PART_SECONDS = 0.5;
|
||||
export const VIDEO_TAP_KBPS = 400;
|
||||
export const AUDIO_TAP_KBPS = 64;
|
||||
/** The 500 kb/s rendition budget includes its AAC audio. */
|
||||
export const VIDEO_TAP_MAX_KBPS = 500 - AUDIO_TAP_KBPS;
|
||||
/** H.264 Main@3.1 + AAC-LC (also supports the 360p60 production tap). */
|
||||
export const VIDEO_TAP_CODECS = "avc1.4d401f,mp4a.40.2";
|
||||
export const AUDIO_TAP_CODECS = "mp4a.40.2";
|
||||
|
||||
const SCRATCH = /^\/[A-Za-z0-9._/-]+$/;
|
||||
|
||||
function scratch(dir: string): string {
|
||||
if (!SCRATCH.test(dir) || dir.split("/").includes("..")) throw new Error("tap output must be an absolute scratch path");
|
||||
return dir;
|
||||
}
|
||||
|
||||
function even(n: number): number { return Math.max(2, Math.round(n / 2) * 2); }
|
||||
|
||||
/** 360p (or the source height, if smaller) at the canvas aspect, even dimensions. */
|
||||
export function tapSize(g: FrameGeometry): { width: number; height: number } {
|
||||
const height = Math.min(TAP_HEIGHT, g.height);
|
||||
return { width: even((height * g.width) / g.height), height: even(height) };
|
||||
}
|
||||
|
||||
/** Geometry of the program tap bus (the persistent encoder downscales once, §2.1 "program tap"). */
|
||||
export function programTapGeometry(canvas: FrameGeometry): FrameGeometry {
|
||||
return { ...tapSize(canvas), fps: canvas.fps };
|
||||
}
|
||||
|
||||
function rawVideoIn(g: FrameGeometry, rate: string): string[] {
|
||||
return ["-thread_queue_size", "64", "-probesize", "32", "-analyzeduration", "0",
|
||||
"-f", "rawvideo", "-pix_fmt", "yuv420p", "-s", `${g.width}x${g.height}`, "-framerate", rate, "-i", "pipe:0"];
|
||||
}
|
||||
|
||||
function rawAudioIn(): string[] {
|
||||
return ["-thread_queue_size", "64", "-probesize", "32", "-analyzeduration", "0",
|
||||
"-f", "s16le", "-ar", String(GATE_SAMPLE_RATE), "-ac", String(GATE_CHANNELS), "-i", "pipe:3"];
|
||||
}
|
||||
|
||||
function scale(g: FrameGeometry): string {
|
||||
const { width, height } = tapSize(g);
|
||||
return width === g.width && height === g.height ? "format=yuv420p" : `scale=${width}:${height}:flags=bilinear,format=yuv420p`;
|
||||
}
|
||||
|
||||
export interface JpegTapPlan { geometry: FrameGeometry; everySeconds: number }
|
||||
|
||||
/** Pairs the bus skips between stills (subscription `stride`). */
|
||||
export function jpegStride(plan: JpegTapPlan): number { return plan.everySeconds * plan.geometry.fps; }
|
||||
|
||||
export function buildJpegTapArgs(plan: JpegTapPlan): string[] {
|
||||
assertGeometry(plan.geometry);
|
||||
if (!Number.isSafeInteger(plan.everySeconds) || plan.everySeconds < 1 || plan.everySeconds > 10) throw new Error("thumbnail cadence must be 1-10 s");
|
||||
// The input runs at one frame per `everySeconds`, so every input frame becomes one still.
|
||||
return ["-hide_banner", "-nostdin", "-loglevel", "error", ...rawVideoIn(plan.geometry, `1/${plan.everySeconds}`),
|
||||
"-vf", scale(plan.geometry), "-c:v", "mjpeg", "-q:v", "5", "-f", "image2pipe", "pipe:1"];
|
||||
}
|
||||
|
||||
export type HlsRendition = "video" | "audio";
|
||||
|
||||
export interface HlsTapPlan { geometry: FrameGeometry; rendition: HlsRendition; outDir: string }
|
||||
|
||||
/** ffmpeg's own playlist of parts (read by the packager, never published). */
|
||||
export const PARTS_PLAYLIST = "parts.m3u8";
|
||||
|
||||
export function buildHlsTapArgs(plan: HlsTapPlan): string[] {
|
||||
const g = plan.geometry;
|
||||
assertGeometry(g);
|
||||
const dir = scratch(plan.outDir);
|
||||
const args = ["-hide_banner", "-nostdin", "-loglevel", "error"];
|
||||
if (plan.rendition === "video") {
|
||||
args.push(...rawVideoIn(g, String(g.fps)), ...rawAudioIn(), "-map", "0:v", "-map", "1:a", "-vf", scale(g),
|
||||
"-c:v", "libx264", "-preset", "veryfast", "-tune", "zerolatency", "-profile:v", "main", "-level:v", "3.1",
|
||||
"-b:v", `${VIDEO_TAP_KBPS}k`, "-maxrate", `${VIDEO_TAP_MAX_KBPS}k`, "-bufsize", `${VIDEO_TAP_MAX_KBPS}k`,
|
||||
// Keyframe exactly every second, never on scene cuts: parts alternate independent/dependent.
|
||||
"-g", String(g.fps), "-keyint_min", String(g.fps), "-sc_threshold", "0");
|
||||
} else {
|
||||
args.push(...rawAudioIn(), "-map", "0:a");
|
||||
}
|
||||
args.push("-c:a", "aac", "-b:a", `${AUDIO_TAP_KBPS}k`, "-ac", "2", "-ar", String(GATE_SAMPLE_RATE),
|
||||
"-f", "hls", "-hls_time", String(PART_SECONDS), "-hls_list_size", "12", "-hls_segment_type", "fmp4",
|
||||
"-hls_fmp4_init_filename", "init.mp4", "-hls_flags", "split_by_time+temp_file+delete_segments+omit_endlist",
|
||||
"-hls_segment_filename", `${dir}/p%d.m4s`, "-protocol_whitelist", "file", `${dir}/${PARTS_PLAYLIST}`);
|
||||
return args;
|
||||
}
|
||||
85
apps/media-worker/src/preview/contract.ts
Normal file
85
apps/media-worker/src/preview/contract.ts
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
/**
|
||||
* Worker side of the preview contract (streaming-architecture.md §2.1).
|
||||
*
|
||||
* The API owns serving: `apps/api/src/production/preview.ts` mints signed,
|
||||
* path-bound capability URLs (60 s TTL), verifies them, single-flights fills,
|
||||
* keeps a 6-object LRU per tap, sets `ETag` from the body hash, rewrites HLS
|
||||
* URIs to signed same-origin paths, and never redirects to S3. The worker does
|
||||
* none of that. It only *publishes*: each preview file is stored as a
|
||||
* content-addressed object and announced by a Redis manifest
|
||||
*
|
||||
* preview:<workspaceId>:<channelId>:<tap>:<file> → {"key":"sha256/<hex>.<ext>","contentType":...}
|
||||
*
|
||||
* which is exactly what the API's `fill()` reads. `preview.contract.test.ts`
|
||||
* checks the constants below against the API source so the two cannot drift.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
/** The API's preview file-name rule (also the only names its HLS rewriter accepts). */
|
||||
export const PREVIEW_FILE = /^(?:[A-Za-z0-9_-]+\.(?:m3u8|m4s|ts|aac|jpg|jpeg|mp4))$/;
|
||||
/** The API's manifest object-key rule. */
|
||||
export const PREVIEW_OBJECT_KEY = /^sha256\/[0-9a-f]{64}\.[a-z0-9]+$/;
|
||||
/** Content types the API accepts in a manifest (subset the worker produces). */
|
||||
export type PreviewContentType = "image/jpeg" | "application/vnd.apple.mpegurl" | "video/mp4" | "audio/mp4";
|
||||
export const PREVIEW_CONTENT_TYPES: readonly PreviewContentType[] = ["image/jpeg", "application/vnd.apple.mpegurl", "video/mp4", "audio/mp4"];
|
||||
|
||||
/** Files the API hands out capability URLs for (`PreviewService.urls`). */
|
||||
export const THUMBNAIL_FILE = "thumbnail.jpg";
|
||||
export const VIDEO_PLAYLIST = "video.m3u8";
|
||||
export const AUDIO_PLAYLIST = "audio.m3u8";
|
||||
/** Multivariant playlist over the two renditions (servable by the same route; not in `urls()` yet). */
|
||||
export const MASTER_PLAYLIST = "master.m3u8";
|
||||
|
||||
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
||||
|
||||
/** A tap is the channel's program, or one of its sources (virtual_channel_sources.id), as the API's controller accepts. */
|
||||
export function assertTapId(tap: string): void {
|
||||
if (tap !== "program" && !UUID.test(tap)) throw new Error("preview tap must be 'program' or a source UUID");
|
||||
}
|
||||
|
||||
export interface PreviewChannel { workspaceId: string; channelId: string }
|
||||
|
||||
export function previewManifestKey(channel: PreviewChannel, tap: string, file: string): string {
|
||||
if (!UUID.test(channel.workspaceId) || !UUID.test(channel.channelId)) throw new Error("preview channel ids must be UUIDs");
|
||||
assertTapId(tap);
|
||||
if (!PREVIEW_FILE.test(file)) throw new Error("invalid preview file name");
|
||||
return `preview:${channel.workspaceId}:${channel.channelId}:${tap}:${file}`;
|
||||
}
|
||||
|
||||
/** Content-addressed object storage (the API's S3 storage with envelope encryption, or a test double). */
|
||||
export interface PreviewObjectStore { putContent(body: Buffer, contentType: PreviewContentType): Promise<{ key: string }> }
|
||||
/** The manifest store the API reads (Redis `SET key value EX ttl`). */
|
||||
export interface PreviewManifestStore { set(key: string, value: string, ttlSeconds: number): Promise<void> }
|
||||
|
||||
export interface PreviewPublisher {
|
||||
/** Store `body` and point the tap's `file` at it for `ttlSeconds`. */
|
||||
publish(tap: string, file: string, body: Buffer, contentType: PreviewContentType, ttlSeconds: number): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Publisher over the API's storage + manifest contract. Re-publishing an
|
||||
* unchanged body (an init segment kept alive, a still thumbnail) only renews
|
||||
* the manifest; the object is already stored under its hash.
|
||||
*/
|
||||
export class ManifestPreviewPublisher implements PreviewPublisher {
|
||||
private readonly stored = new Map<string, string>();
|
||||
|
||||
constructor(private readonly channel: PreviewChannel, private readonly objects: PreviewObjectStore, private readonly manifests: PreviewManifestStore) {}
|
||||
|
||||
async publish(tap: string, file: string, body: Buffer, contentType: PreviewContentType, ttlSeconds: number): Promise<void> {
|
||||
const manifestKey = previewManifestKey(this.channel, tap, file);
|
||||
if (!PREVIEW_CONTENT_TYPES.includes(contentType)) throw new Error("unsupported preview content type");
|
||||
if (body.length === 0 || body.length > 4_000_000) throw new Error("preview object size out of range");
|
||||
if (!Number.isSafeInteger(ttlSeconds) || ttlSeconds < 1 || ttlSeconds > 86_400) throw new Error("preview ttl out of range");
|
||||
const hash = createHash("sha256").update(body).update(contentType).digest("hex");
|
||||
let key = this.stored.get(hash);
|
||||
if (!key) {
|
||||
key = (await this.objects.putContent(body, contentType)).key;
|
||||
if (!PREVIEW_OBJECT_KEY.test(key)) throw new Error("object store returned an invalid key");
|
||||
this.stored.set(hash, key);
|
||||
// Hash → key only matters for recent bodies (renewals); bound the map.
|
||||
if (this.stored.size > 512) this.stored.delete(this.stored.keys().next().value ?? "");
|
||||
}
|
||||
await this.manifests.set(manifestKey, JSON.stringify({ key, contentType }), ttlSeconds);
|
||||
}
|
||||
}
|
||||
207
apps/media-worker/src/preview/ll-hls.ts
Normal file
207
apps/media-worker/src/preview/ll-hls.ts
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
/**
|
||||
* LL-HLS packaging for preview taps (streaming-architecture.md §2.1, §2.2).
|
||||
*
|
||||
* ffmpeg (argv.ts) writes 0.5 s fMP4 fragments and lists them in its own
|
||||
* playlist. This module turns them into an LL-HLS media playlist:
|
||||
*
|
||||
* - every fragment is published as an `EXT-X-PART` (`INDEPENDENT=YES` when its
|
||||
* first video sample is a sync sample, read from the `moof`);
|
||||
* - consecutive parts from one independent part to the next form a segment,
|
||||
* published as the parts' concatenation (fMP4 fragments concatenate);
|
||||
* - media sequence numbers start at the wall-clock millisecond the tap started, so
|
||||
* they keep increasing across tap restarts (on-demand taps stop when idle)
|
||||
* and file names never collide with a previous run's still-cached manifests.
|
||||
*
|
||||
* There is no blocking playlist reload (`CAN-BLOCK-RELOAD`): the API serves
|
||||
* stored objects with a 1 s cache, so players poll; hls.js `lowLatencyMode`
|
||||
* still uses the parts and `PART-HOLD-BACK`.
|
||||
*/
|
||||
import { PART_SECONDS, type HlsRendition } from "./argv";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// fMP4 fragment inspection
|
||||
|
||||
function boxes(buf: Buffer, start: number, end: number): Array<{ type: string; at: number; size: number }> {
|
||||
const out: Array<{ type: string; at: number; size: number }> = [];
|
||||
let at = start;
|
||||
while (at + 8 <= end) {
|
||||
let size = buf.readUInt32BE(at);
|
||||
const type = buf.toString("latin1", at + 4, at + 8);
|
||||
let header = 8;
|
||||
if (size === 1) {
|
||||
if (at + 16 > end) break;
|
||||
size = Number(buf.readBigUInt64BE(at + 8));
|
||||
header = 16;
|
||||
} else if (size === 0) {
|
||||
size = end - at;
|
||||
}
|
||||
if (size < header || at + size > end) break;
|
||||
out.push({ type, at, size });
|
||||
at += size;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const NON_SYNC = 0x10000;
|
||||
|
||||
/** First-sample flags of one `traf`: trun first_sample_flags, else trun sample[0] flags, else tfhd default flags. */
|
||||
function firstSampleFlags(buf: Buffer, traf: { at: number; size: number }): number | null {
|
||||
let defaults: number | null = null;
|
||||
for (const child of boxes(buf, traf.at + 8, traf.at + traf.size)) {
|
||||
const flags = buf.readUInt32BE(child.at + 8) & 0xffffff;
|
||||
if (child.type === "tfhd") {
|
||||
let p = child.at + 16; // version/flags, track_ID
|
||||
if (flags & 0x1) p += 8; // base_data_offset
|
||||
if (flags & 0x2) p += 4; // sample_description_index
|
||||
if (flags & 0x8) p += 4; // default_sample_duration
|
||||
if (flags & 0x10) p += 4; // default_sample_size
|
||||
if (flags & 0x20 && p + 4 <= child.at + child.size) defaults = buf.readUInt32BE(p);
|
||||
} else if (child.type === "trun") {
|
||||
let p = child.at + 16; // version/flags, sample_count
|
||||
if (flags & 0x1) p += 4; // data_offset
|
||||
if (flags & 0x4) return buf.readUInt32BE(p);
|
||||
if (flags & 0x400) {
|
||||
if (flags & 0x100) p += 4; // sample_duration
|
||||
if (flags & 0x200) p += 4; // sample_size
|
||||
return buf.readUInt32BE(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
return defaults;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a fragment can start playback: every track's first sample is a sync
|
||||
* sample. Unknown flags (no trun/tfhd information) count as not independent.
|
||||
*/
|
||||
export function fragmentIndependent(buf: Buffer): boolean {
|
||||
let trafs = 0;
|
||||
for (const top of boxes(buf, 0, buf.length)) {
|
||||
if (top.type !== "moof") continue;
|
||||
for (const traf of boxes(buf, top.at + 8, top.at + top.size)) {
|
||||
if (traf.type !== "traf") continue;
|
||||
trafs += 1;
|
||||
const flags = firstSampleFlags(buf, traf);
|
||||
if (flags === null || (flags & NON_SYNC) !== 0) return false;
|
||||
}
|
||||
}
|
||||
return trafs > 0;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ffmpeg's part playlist
|
||||
|
||||
export interface FfmpegPart { file: string; duration: number }
|
||||
|
||||
/** Parts listed in ffmpeg's playlist (`#EXTINF:<d>,` then the file name). */
|
||||
export function parseFfmpegPlaylist(text: string): FfmpegPart[] {
|
||||
const parts: FfmpegPart[] = [];
|
||||
let duration: number | null = null;
|
||||
for (const raw of text.split(/\r?\n/)) {
|
||||
const line = raw.trim();
|
||||
if (line.startsWith("#EXTINF:")) duration = Number.parseFloat(line.slice(8));
|
||||
else if (line !== "" && !line.startsWith("#") && duration !== null) {
|
||||
if (/^p\d+\.m4s$/.test(line) && Number.isFinite(duration) && duration > 0) parts.push({ file: line, duration });
|
||||
duration = null;
|
||||
}
|
||||
}
|
||||
return parts;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// LL-HLS media playlist
|
||||
|
||||
export interface PublishedPart { name: string; duration: number; independent: boolean; bytes: Buffer }
|
||||
export interface ClosedSegment { msn: number; name: string; duration: number; parts: PublishedPart[] }
|
||||
|
||||
/** Segments listed in the playlist; parts are listed for the last `PART_SEGMENTS` of them plus the open one. */
|
||||
const WINDOW_SEGMENTS = 4;
|
||||
const PART_SEGMENTS = 2;
|
||||
|
||||
function fmt(n: number): string { return n.toFixed(5); }
|
||||
|
||||
/**
|
||||
* File names: `<v|a>init<base>.mp4`, segments `<v|a>s<msn>.m4s`, parts `<v|a>p<msn>_<i>.m4s`
|
||||
* (all within the API's preview file-name rule).
|
||||
*/
|
||||
export class LlHlsPackager {
|
||||
readonly prefix: string;
|
||||
readonly initName: string;
|
||||
private msn: number;
|
||||
private open: PublishedPart[] = [];
|
||||
private segments: ClosedSegment[] = [];
|
||||
|
||||
constructor(rendition: HlsRendition, readonly base: number) {
|
||||
if (!Number.isSafeInteger(base) || base < 0) throw new Error("media sequence base must be a non-negative integer");
|
||||
this.prefix = rendition === "video" ? "v" : "a";
|
||||
this.initName = `${this.prefix}init${base}.mp4`;
|
||||
this.msn = base;
|
||||
}
|
||||
|
||||
/** Name the next part will be published under (it belongs to the open segment). */
|
||||
private partName(): string { return `${this.prefix}p${this.msn}_${this.open.length}.m4s`; }
|
||||
|
||||
/**
|
||||
* Adds the next fragment. Returns the part to publish (null for a dependent fragment before the
|
||||
* first keyframe, which is dropped: a stream starts at an independent part) and, when this part
|
||||
* starts a new segment, the segment it closed. Publish both before the playlist.
|
||||
*/
|
||||
add(fragment: { duration: number; independent: boolean; bytes: Buffer }): { part: PublishedPart | null; closed: ClosedSegment | null } {
|
||||
let closed: ClosedSegment | null = null;
|
||||
if (fragment.independent && this.open.length > 0) {
|
||||
closed = {
|
||||
msn: this.msn, name: `${this.prefix}s${this.msn}.m4s`, parts: this.open,
|
||||
duration: this.open.reduce((a, p) => a + p.duration, 0),
|
||||
};
|
||||
this.segments.push(closed);
|
||||
while (this.segments.length > WINDOW_SEGMENTS) this.segments.shift();
|
||||
this.msn += 1;
|
||||
this.open = [];
|
||||
}
|
||||
if (this.open.length === 0 && !fragment.independent) return { part: null, closed };
|
||||
const part: PublishedPart = { name: this.partName(), ...fragment };
|
||||
this.open.push(part);
|
||||
return { part, closed };
|
||||
}
|
||||
|
||||
static segmentBytes(segment: ClosedSegment): Buffer { return Buffer.concat(segment.parts.map((p) => p.bytes)); }
|
||||
|
||||
/** null until the first segment has closed (a player needs at least one full segment). */
|
||||
playlist(): string | null {
|
||||
if (this.segments.length === 0) return null;
|
||||
const first = this.segments[0]!;
|
||||
const target = Math.max(1, ...this.segments.map((s) => Math.ceil(s.duration)));
|
||||
const partTarget = Math.max(PART_SECONDS, ...this.segments.flatMap((s) => s.parts.map((p) => p.duration)), ...this.open.map((p) => p.duration));
|
||||
const lines = [
|
||||
"#EXTM3U",
|
||||
"#EXT-X-VERSION:7",
|
||||
`#EXT-X-TARGETDURATION:${target}`,
|
||||
`#EXT-X-SERVER-CONTROL:PART-HOLD-BACK=${fmt(3 * partTarget)}`,
|
||||
`#EXT-X-PART-INF:PART-TARGET=${fmt(partTarget)}`,
|
||||
`#EXT-X-MEDIA-SEQUENCE:${first.msn}`,
|
||||
"#EXT-X-INDEPENDENT-SEGMENTS",
|
||||
`#EXT-X-MAP:URI="${this.initName}"`,
|
||||
];
|
||||
const partLine = (p: PublishedPart): string => `#EXT-X-PART:DURATION=${fmt(p.duration)},URI="${p.name}"${p.independent ? ",INDEPENDENT=YES" : ""}`;
|
||||
this.segments.forEach((s, i) => {
|
||||
if (i >= this.segments.length - PART_SEGMENTS) for (const p of s.parts) lines.push(partLine(p));
|
||||
lines.push(`#EXTINF:${fmt(s.duration)},`, s.name);
|
||||
});
|
||||
for (const p of this.open) lines.push(partLine(p));
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
}
|
||||
|
||||
/** Multivariant playlist over a tap's two renditions (§2.1: 360p video+audio, audio-only). */
|
||||
export function masterPlaylist(video: { width: number; height: number; bandwidth: number; codecs: string }, audio: { bandwidth: number; codecs: string }): string {
|
||||
return [
|
||||
"#EXTM3U",
|
||||
"#EXT-X-VERSION:7",
|
||||
"#EXT-X-INDEPENDENT-SEGMENTS",
|
||||
`#EXT-X-STREAM-INF:BANDWIDTH=${video.bandwidth},RESOLUTION=${video.width}x${video.height},CODECS="${video.codecs}"`,
|
||||
"video.m3u8",
|
||||
`#EXT-X-STREAM-INF:BANDWIDTH=${audio.bandwidth},CODECS="${audio.codecs}"`,
|
||||
"audio.m3u8",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
305
apps/media-worker/src/preview/preview.test.ts
Normal file
305
apps/media-worker/src/preview/preview.test.ts
Normal file
|
|
@ -0,0 +1,305 @@
|
|||
/**
|
||||
* Preview taps (streaming-architecture.md §2.1): the API contract mirror, argv
|
||||
* goldens, fMP4/LL-HLS packaging, and the JPEG / video / audio tap lifecycle on
|
||||
* a real source bus, plus the program tap's isolation from the encoder.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import test from "node:test";
|
||||
import { buildEncoderArgs } from "../compositor/command-builder";
|
||||
import { ProgramChain } from "../compositor/program";
|
||||
import { CounterSource, ensureGateBinary, passthroughRedactor } from "../compositor/testing";
|
||||
import { buildHlsTapArgs, buildJpegTapArgs, jpegStride, programTapGeometry, tapSize } from "./argv";
|
||||
import {
|
||||
ManifestPreviewPublisher, PREVIEW_CONTENT_TYPES, PREVIEW_FILE, PREVIEW_OBJECT_KEY, previewManifestKey,
|
||||
type PreviewContentType, type PreviewPublisher,
|
||||
} from "./contract";
|
||||
import { fragmentIndependent, LlHlsPackager, masterPlaylist, parseFfmpegPlaylist } from "./ll-hls";
|
||||
import { JpegSplitter, PreviewTaps, TTL } from "./taps";
|
||||
|
||||
const WS = "8f0c5a8e-3b1d-4c3e-9a5b-0c1d2e3f4a5b";
|
||||
const CH = "1a2b3c4d-5e6f-4a1b-8c2d-3e4f5a6b7c8d";
|
||||
const SRC = "9e8d7c6b-5a4f-4e3d-a2c1-b0a9f8e7d6c5";
|
||||
const small = { width: 320, height: 180, fps: 30 };
|
||||
|
||||
test("worker preview contract mirrors the API preview service (file rule, manifest key, object key, types)", () => {
|
||||
const api = readFileSync(resolve(__dirname, "../../../api/src/production/preview.ts"), "utf8");
|
||||
const file = /const FILE = \/(.+)\/;/.exec(api)?.[1];
|
||||
assert.equal(file, PREVIEW_FILE.source, "file-name rule");
|
||||
assert.ok(api.includes("`preview:${ws}:${ch}:${tap}:${file}`"), "manifest key layout");
|
||||
assert.ok(api.includes(PREVIEW_OBJECT_KEY.source.replace(/\\\//g, "\\/")), "object key rule");
|
||||
const types = /const TYPES = new Set\(\[(.+)\]\);/.exec(api)?.[1] ?? "";
|
||||
for (const t of PREVIEW_CONTENT_TYPES) assert.ok(types.includes(`"${t}"`), `API accepts ${t}`);
|
||||
// Every name the worker publishes passes the API's rule (the HLS rewriter rejects anything else).
|
||||
const video = new LlHlsPackager("video", 1_700_000_000);
|
||||
for (const name of ["thumbnail.jpg", "video.m3u8", "audio.m3u8", "master.m3u8", video.initName, "vs1700000000.m4s", "vp1700000000_1.m4s", "ainit1700000000.mp4"]) {
|
||||
assert.ok(PREVIEW_FILE.test(name), name);
|
||||
}
|
||||
assert.equal(previewManifestKey({ workspaceId: WS, channelId: CH }, "program", "thumbnail.jpg"), `preview:${WS}:${CH}:program:thumbnail.jpg`);
|
||||
assert.throws(() => previewManifestKey({ workspaceId: WS, channelId: CH }, "../x", "thumbnail.jpg"));
|
||||
assert.throws(() => previewManifestKey({ workspaceId: WS, channelId: CH }, SRC, "../../etc.jpg"));
|
||||
assert.throws(() => previewManifestKey({ workspaceId: "workspace-1", channelId: CH }, SRC, "thumbnail.jpg"));
|
||||
});
|
||||
|
||||
test("ManifestPreviewPublisher stores content-addressed objects and renews manifests without re-uploading", async () => {
|
||||
const puts: string[] = [];
|
||||
const manifests = new Map<string, { value: string; ttl: number }>();
|
||||
const publisher = new ManifestPreviewPublisher({ workspaceId: WS, channelId: CH },
|
||||
{ async putContent(body, contentType) { puts.push(contentType); return { key: `sha256/${"a".repeat(63)}${puts.length}.jpg` }; } },
|
||||
{ async set(key, value, ttl) { manifests.set(key, { value, ttl }); } });
|
||||
await publisher.publish("program", "thumbnail.jpg", Buffer.from("jpeg-1"), "image/jpeg", 30);
|
||||
await publisher.publish("program", "thumbnail.jpg", Buffer.from("jpeg-1"), "image/jpeg", 30);
|
||||
assert.equal(puts.length, 1, "an unchanged body is stored once");
|
||||
await publisher.publish("program", "thumbnail.jpg", Buffer.from("jpeg-2"), "image/jpeg", 30);
|
||||
assert.equal(puts.length, 2);
|
||||
const m = manifests.get(`preview:${WS}:${CH}:program:thumbnail.jpg`);
|
||||
assert.deepEqual(JSON.parse(m?.value ?? "{}"), { key: `sha256/${"a".repeat(63)}2.jpg`, contentType: "image/jpeg" });
|
||||
assert.equal(m?.ttl, 30);
|
||||
await assert.rejects(publisher.publish("program", "x.exe", Buffer.from("x"), "image/jpeg", 30));
|
||||
await assert.rejects(publisher.publish("program", "a.jpg", Buffer.alloc(0), "image/jpeg", 30));
|
||||
const bad = new ManifestPreviewPublisher({ workspaceId: WS, channelId: CH }, { async putContent() { return { key: "https://evil/x" }; } }, { async set() {} });
|
||||
await assert.rejects(bad.publish("program", "a.jpg", Buffer.from("x"), "image/jpeg", 30), /invalid key/);
|
||||
});
|
||||
|
||||
test("tap argv goldens: JPEG, video and audio LL-HLS read only inherited pipes and their scratch dir", () => {
|
||||
const hd = { width: 1920, height: 1080, fps: 30 };
|
||||
assert.deepEqual(tapSize(hd), { width: 640, height: 360 });
|
||||
assert.deepEqual(tapSize({ width: 1280, height: 720, fps: 30 }), { width: 640, height: 360 });
|
||||
assert.deepEqual(programTapGeometry(hd), { width: 640, height: 360, fps: 30 });
|
||||
assert.equal(jpegStride({ geometry: hd, everySeconds: 2 }), 60);
|
||||
const raw = ["-thread_queue_size", "64", "-probesize", "32", "-analyzeduration", "0"];
|
||||
assert.deepEqual(buildJpegTapArgs({ geometry: hd, everySeconds: 2 }), ["-hide_banner", "-nostdin", "-loglevel", "error",
|
||||
...raw, "-f", "rawvideo", "-pix_fmt", "yuv420p", "-s", "1920x1080", "-framerate", "1/2", "-i", "pipe:0",
|
||||
"-vf", "scale=640:360:flags=bilinear,format=yuv420p", "-c:v", "mjpeg", "-q:v", "5", "-f", "image2pipe", "pipe:1"]);
|
||||
const hlsTail = (dir: string) => ["-c:a", "aac", "-b:a", "64k", "-ac", "2", "-ar", "48000",
|
||||
"-f", "hls", "-hls_time", "0.5", "-hls_list_size", "12", "-hls_segment_type", "fmp4", "-hls_fmp4_init_filename", "init.mp4",
|
||||
"-hls_flags", "split_by_time+temp_file+delete_segments+omit_endlist", "-hls_segment_filename", `${dir}/p%d.m4s`, "-protocol_whitelist", "file", `${dir}/parts.m3u8`];
|
||||
assert.deepEqual(buildHlsTapArgs({ geometry: hd, rendition: "video", outDir: "/scratch/t/v" }), ["-hide_banner", "-nostdin", "-loglevel", "error",
|
||||
...raw, "-f", "rawvideo", "-pix_fmt", "yuv420p", "-s", "1920x1080", "-framerate", "30", "-i", "pipe:0",
|
||||
...raw, "-f", "s16le", "-ar", "48000", "-ac", "2", "-i", "pipe:3", "-map", "0:v", "-map", "1:a",
|
||||
"-vf", "scale=640:360:flags=bilinear,format=yuv420p", "-c:v", "libx264", "-preset", "veryfast", "-tune", "zerolatency", "-profile:v", "main", "-level:v", "3.1",
|
||||
"-b:v", "400k", "-maxrate", "436k", "-bufsize", "436k", "-g", "30", "-keyint_min", "30", "-sc_threshold", "0", ...hlsTail("/scratch/t/v")]);
|
||||
assert.deepEqual(buildHlsTapArgs({ geometry: programTapGeometry(hd), rendition: "audio", outDir: "/scratch/t/a" }), ["-hide_banner", "-nostdin", "-loglevel", "error",
|
||||
...raw, "-f", "s16le", "-ar", "48000", "-ac", "2", "-i", "pipe:3", "-map", "0:a", ...hlsTail("/scratch/t/a")]);
|
||||
// The 360p program bus is not scaled again.
|
||||
assert.ok(buildJpegTapArgs({ geometry: programTapGeometry(hd), everySeconds: 1 }).includes("format=yuv420p"));
|
||||
for (const bad of ["relative/dir", "/scratch/../etc", "/scratch/t;rm"]) assert.throws(() => buildHlsTapArgs({ geometry: hd, rendition: "audio", outDir: bad }), bad);
|
||||
for (const s of [0, 11, 1.5]) assert.throws(() => buildJpegTapArgs({ geometry: hd, everySeconds: s }));
|
||||
const argv = [...buildJpegTapArgs({ geometry: hd, everySeconds: 2 }), ...buildHlsTapArgs({ geometry: hd, rendition: "video", outDir: "/s/v" })].join(" ");
|
||||
assert.ok(!/https?:|rtmp|tcp:|udp:|concat|movie=/.test(argv), "no network or concat protocols");
|
||||
});
|
||||
|
||||
test("encoder program tap: split + 360p tee outputs with a null anchor slave, main path unchanged", () => {
|
||||
const canvas = { width: 1920, height: 1080, fps: 30 };
|
||||
const plain = buildEncoderArgs({ canvas, videoBitrateKbps: 6000, audioBitrateKbps: 128, output: { kind: "flvPipe", fd: 4 } });
|
||||
const tapped = buildEncoderArgs({ canvas, videoBitrateKbps: 6000, audioBitrateKbps: 128, output: { kind: "flvPipe", fd: 4 }, programTap: { videoFd: 5, audioFd: 6, size: { width: 640, height: 360 } } });
|
||||
assert.ok(!plain.includes("-filter_complex") && plain.includes("0:v"));
|
||||
assert.equal(tapped[tapped.indexOf("-filter_complex") + 1], "[0:v]split=2[enc][tap];[tap]scale=640:360:flags=bilinear,format=yuv420p[tapv];[1:a]asplit=2[enca][tapa]");
|
||||
assert.deepEqual(tapped.slice(-16), ["-f", "flv", "pipe:4",
|
||||
"-map", "[tapv]", "-c:v", "rawvideo", "-f", "tee", "[f=rawvideo:onfail=ignore]pipe:5|[f=null]-",
|
||||
"-map", "[tapa]", "-c:a", "pcm_s16le", "-f", "tee", "[f=s16le:onfail=ignore]pipe:6|[f=null]-"].slice(-16));
|
||||
// Encoding options of the main output are identical with and without the tap.
|
||||
const opts = (a: string[]) => a.slice(a.indexOf("-c:v"), a.indexOf("-f", a.indexOf("-c:v")));
|
||||
assert.deepEqual(opts(tapped), opts(plain));
|
||||
assert.throws(() => buildEncoderArgs({ canvas, videoBitrateKbps: 6000, audioBitrateKbps: 128, output: { kind: "null" }, programTap: { videoFd: 5, audioFd: 6, size: { width: 641, height: 360 } } }));
|
||||
});
|
||||
|
||||
test("fMP4 independence from moof sample flags on real ffmpeg parts (keyframe every 2nd part)", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "fmp4-"));
|
||||
try {
|
||||
const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=s=320x180:r=30:d=3", "-f", "lavfi", "-i", "sine=d=3",
|
||||
// Same GOP shape as the video tap (zerolatency: no B-frames, so 0.5 s splits are exact).
|
||||
"-c:v", "libx264", "-preset", "veryfast", "-tune", "zerolatency", "-g", "30", "-keyint_min", "30", "-sc_threshold", "0", "-c:a", "aac", "-b:a", "64k",
|
||||
"-f", "hls", "-hls_time", "0.5", "-hls_list_size", "0", "-hls_segment_type", "fmp4", "-hls_flags", "split_by_time+temp_file",
|
||||
"-hls_segment_filename", `${dir}/p%d.m4s`, `${dir}/parts.m3u8`]);
|
||||
assert.equal(r.status, 0);
|
||||
const parts = parseFfmpegPlaylist(readFileSync(join(dir, "parts.m3u8"), "utf8"));
|
||||
assert.ok(parts.length >= 5, `parts ${parts.length}`);
|
||||
assert.ok(parts.slice(0, -1).every((p) => Math.abs(p.duration - 0.5) < 0.01), "0.5 s parts");
|
||||
const flags = parts.map((p) => fragmentIndependent(readFileSync(join(dir, p.file))));
|
||||
assert.deepEqual(flags.slice(0, 6), [true, false, true, false, true, false]);
|
||||
assert.equal(fragmentIndependent(Buffer.from("not an mp4")), false);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("LL-HLS packager: parts, segments at independent parts, monotonic sequence, bounded window", () => {
|
||||
const p = new LlHlsPackager("video", 100);
|
||||
const frag = (independent: boolean) => ({ duration: 0.5, independent, bytes: Buffer.from(independent ? "K" : "d") });
|
||||
assert.equal(p.add(frag(false)).part, null, "a stream starts at an independent part");
|
||||
assert.equal(p.playlist(), null, "no playlist before the first full segment");
|
||||
const first = p.add(frag(true));
|
||||
assert.equal(first.part?.name, "vp100_0.m4s");
|
||||
assert.equal(p.add(frag(false)).part?.name, "vp100_1.m4s");
|
||||
const third = p.add(frag(true));
|
||||
assert.equal(third.closed?.name, "vs100.m4s");
|
||||
assert.equal(third.part?.name, "vp101_0.m4s");
|
||||
assert.equal(LlHlsPackager.segmentBytes(third.closed!).toString(), "Kd");
|
||||
assert.equal(p.playlist(), ["#EXTM3U", "#EXT-X-VERSION:7", "#EXT-X-TARGETDURATION:1", "#EXT-X-SERVER-CONTROL:PART-HOLD-BACK=1.50000",
|
||||
"#EXT-X-PART-INF:PART-TARGET=0.50000", "#EXT-X-MEDIA-SEQUENCE:100", "#EXT-X-INDEPENDENT-SEGMENTS", '#EXT-X-MAP:URI="vinit100.mp4"',
|
||||
'#EXT-X-PART:DURATION=0.50000,URI="vp100_0.m4s",INDEPENDENT=YES', '#EXT-X-PART:DURATION=0.50000,URI="vp100_1.m4s"',
|
||||
"#EXTINF:1.00000,", "vs100.m4s", '#EXT-X-PART:DURATION=0.50000,URI="vp101_0.m4s",INDEPENDENT=YES', ""].join("\n"));
|
||||
for (let i = 0; i < 20; i += 1) p.add(frag(i % 2 === 1));
|
||||
const text = p.playlist() ?? "";
|
||||
assert.equal((text.match(/#EXTINF/g) ?? []).length, 4, "four segments in the window");
|
||||
assert.match(text, /#EXT-X-MEDIA-SEQUENCE:107/);
|
||||
assert.equal((text.match(/#EXT-X-PART:/g) ?? []).length, 2 * 2 + 1, "parts only for the last two segments + the open one");
|
||||
assert.match(new LlHlsPackager("audio", 5).initName, /^ainit5\.mp4$/);
|
||||
assert.equal(masterPlaylist({ width: 640, height: 360, bandwidth: 564000, codecs: "avc1.4d401e,mp4a.40.2" }, { bandwidth: 64000, codecs: "mp4a.40.2" }),
|
||||
'#EXTM3U\n#EXT-X-VERSION:7\n#EXT-X-INDEPENDENT-SEGMENTS\n#EXT-X-STREAM-INF:BANDWIDTH=564000,RESOLUTION=640x360,CODECS="avc1.4d401e,mp4a.40.2"\nvideo.m3u8\n#EXT-X-STREAM-INF:BANDWIDTH=64000,CODECS="mp4a.40.2"\naudio.m3u8\n');
|
||||
});
|
||||
|
||||
test("JPEG splitter yields whole stills across chunk boundaries", () => {
|
||||
const a = Buffer.from([0xff, 0xd8, 1, 2, 0xff, 0x00, 3, 0xff, 0xd9]);
|
||||
const b = Buffer.from([0xff, 0xd8, 9, 0xff, 0xd9]);
|
||||
const s = new JpegSplitter();
|
||||
const all = Buffer.concat([Buffer.from([7]), a, b]);
|
||||
const got = [...s.push(all.subarray(0, 4)), ...s.push(all.subarray(4, 11)), ...s.push(all.subarray(11))];
|
||||
assert.deepEqual(got, [a, b]);
|
||||
});
|
||||
|
||||
class MemoryPublisher implements PreviewPublisher {
|
||||
readonly files = new Map<string, { body: Buffer; contentType: PreviewContentType; ttl: number; at: number }>();
|
||||
async publish(tap: string, file: string, body: Buffer, contentType: PreviewContentType, ttlSeconds: number): Promise<void> {
|
||||
assert.ok(PREVIEW_FILE.test(file), file);
|
||||
this.files.set(`${tap}/${file}`, { body, contentType, ttl: ttlSeconds, at: Date.now() });
|
||||
}
|
||||
}
|
||||
|
||||
async function until(cond: () => boolean, ms: number, what: string): Promise<void> {
|
||||
const t0 = Date.now();
|
||||
while (!cond()) {
|
||||
if (Date.now() - t0 > ms) throw new Error(`timeout: ${what}`);
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
}
|
||||
}
|
||||
|
||||
test("tap lifecycle on a live bus: JPEG always on, video/audio LL-HLS on demand with lease expiry", { timeout: 120_000 }, async (t) => {
|
||||
if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; }
|
||||
const scratch = mkdtempSync(join(tmpdir(), "taps-"));
|
||||
const source = new CounterSource(small, scratch);
|
||||
const publisher = new MemoryPublisher();
|
||||
const errors: string[] = [];
|
||||
const taps = new PreviewTaps({ publisher, scratchDir: scratch, redactor: passthroughRedactor, onStderr: () => undefined, onError: (tap, r) => errors.push(`${tap}: ${r}`) }, 5_000);
|
||||
try {
|
||||
await source.start();
|
||||
await taps.add(SRC, source.bus);
|
||||
await until(() => publisher.files.has(`${SRC}/thumbnail.jpg`), 15_000, "thumbnail");
|
||||
const thumb = publisher.files.get(`${SRC}/thumbnail.jpg`)!;
|
||||
assert.equal(thumb.contentType, "image/jpeg");
|
||||
assert.equal(thumb.ttl, TTL.thumbnail);
|
||||
assert.deepEqual([...thumb.body.subarray(0, 2)], [0xff, 0xd8]);
|
||||
// Nothing else runs until demanded.
|
||||
assert.equal(taps.hls(SRC, "video"), undefined);
|
||||
|
||||
await Promise.all([taps.demand(SRC, "video"), taps.demand(SRC, "video")]);
|
||||
assert.equal((await source.bus.stats()).subscribers.length, 2, "concurrent demands share one video process plus JPEG");
|
||||
await taps.demand(SRC, "audio");
|
||||
await until(() => publisher.files.has(`${SRC}/video.m3u8`) && publisher.files.has(`${SRC}/audio.m3u8`), 30_000, "playlists");
|
||||
await until(() => (taps.hls(SRC, "video")?.parts ?? 0) >= 4, 30_000, "video parts");
|
||||
const video = publisher.files.get(`${SRC}/video.m3u8`)!.body.toString();
|
||||
assert.match(video, /#EXT-X-PART-INF:PART-TARGET=0\.50000/);
|
||||
assert.match(video, /INDEPENDENT=YES/);
|
||||
// Every URI the playlist names has been published before it.
|
||||
for (const uri of [...video.matchAll(/URI="([^"]+)"/g)].map((m) => m[1]!).concat(video.split("\n").filter((l) => l.endsWith(".m4s")))) {
|
||||
assert.ok(publisher.files.has(`${SRC}/${uri}`), `published ${uri}`);
|
||||
}
|
||||
const audio = publisher.files.get(`${SRC}/audio.m3u8`)!.body.toString();
|
||||
assert.match(audio, /ainit\d+\.mp4/);
|
||||
assert.ok(publisher.files.has(`${SRC}/master.m3u8`));
|
||||
// A video tap reads both streams of the bus and decodes to 360p (here: source size).
|
||||
const seg = [...publisher.files.keys()].find((k) => /\/vs\d+\.m4s$/.test(k));
|
||||
assert.ok(seg, "a closed video segment");
|
||||
|
||||
// A dead HLS rendition is replaced without retaining its watcher or scratch directory.
|
||||
const dead = taps.hls(SRC, "video")!;
|
||||
const oldInit = /URI="(vinit\d+\.mp4)"/.exec(video)?.[1];
|
||||
process.kill(dead.pid ?? 0, "SIGKILL");
|
||||
await until(() => !dead.running, 5_000, "video exit");
|
||||
await Promise.all([taps.demand(SRC, "video"), taps.demand(SRC, "video")]);
|
||||
await until(() => (taps.hls(SRC, "video")?.parts ?? 0) >= 4, 30_000, "restarted video parts");
|
||||
assert.equal(readdirSync(join(scratch, "preview")).length, 2, "only current video and audio scratch directories");
|
||||
const newVideo = publisher.files.get(`${SRC}/video.m3u8`)!.body.toString();
|
||||
assert.notEqual(/URI="(vinit\d+\.mp4)"/.exec(newVideo)?.[1], oldInit, "restart never aliases a cached init");
|
||||
|
||||
// Re-demand extends the lease; expiry stops the process and removes its scratch files.
|
||||
const vtap = taps.hls(SRC, "video")!;
|
||||
const pid = vtap.pid;
|
||||
await taps.demand(SRC, "video");
|
||||
assert.equal(taps.hls(SRC, "video")?.pid, pid, "a live rendition is reused");
|
||||
await taps.sweep(Date.now() + 10_000);
|
||||
assert.equal(taps.hls(SRC, "video"), undefined);
|
||||
assert.equal(taps.hls(SRC, "audio"), undefined);
|
||||
assert.ok(!vtap.running);
|
||||
assert.deepEqual(existsSync(join(scratch, "preview")) ? readdirSync(join(scratch, "preview")) : [], [], "rendition scratch removed");
|
||||
|
||||
// A killed JPEG tap only loses its own pairs; the bus keeps serving, and the sweep restarts it.
|
||||
const jpeg = taps.jpeg(SRC)!;
|
||||
const before = jpeg.stills;
|
||||
process.kill(jpeg.pid ?? 0, "SIGKILL");
|
||||
await until(() => !jpeg.running, 5_000, "jpeg exit");
|
||||
const pairs = (await source.bus.stats()).pairs;
|
||||
await taps.sweep();
|
||||
await until(() => jpeg.stills > before, 15_000, "jpeg restarted");
|
||||
assert.ok((await source.bus.stats()).pairs > pairs, "bus kept running");
|
||||
assert.ok(errors.some((e) => e.includes("tap exited")), "the kill was reported");
|
||||
|
||||
await taps.setThumbnailCadence(SRC, 1);
|
||||
assert.equal(taps.jpeg(SRC)?.cadence, 1);
|
||||
await assert.rejects(taps.setThumbnailCadence(SRC, 0));
|
||||
assert.ok(taps.jpeg(SRC)?.running, "invalid cadence leaves the live thumbnail running");
|
||||
|
||||
await taps.remove(SRC);
|
||||
await taps.add(SRC, source.bus, { alwaysOn: ["video"] });
|
||||
const masterAt = publisher.files.get(`${SRC}/master.m3u8`)!.at;
|
||||
await new Promise((r) => setTimeout(r, 10));
|
||||
await taps.sweep(Date.now() + TTL.init * 1000);
|
||||
assert.ok(publisher.files.get(`${SRC}/master.m3u8`)!.at > masterAt, "active master manifest renewed before expiry");
|
||||
assert.ok(taps.hls(SRC, "video")?.running, "always-on rendition never expires");
|
||||
} finally {
|
||||
await taps.stop();
|
||||
await source.stop();
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("program tap: the encoder feeds a 360p bus, and losing that bus never takes the encoder down", { timeout: 120_000 }, async (t) => {
|
||||
if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; }
|
||||
const { composeOver, makeAssets, scriptedGraph, ZERO_GAP_CANVAS: canvas } = await import("../compositor/testing");
|
||||
const scratch = mkdtempSync(join(tmpdir(), "pgm-tap-"));
|
||||
const sources = Array.from({ length: 4 }, () => new CounterSource(canvas, scratch));
|
||||
const fatal: string[] = [];
|
||||
const chain = new ProgramChain({ canvas, scratchDir: scratch, videoBitrateKbps: 500, output: { kind: "null" }, programTap: true,
|
||||
redactor: passthroughRedactor, onStderr: () => undefined, onFatal: (r) => fatal.push(r) });
|
||||
const publisher = new MemoryPublisher();
|
||||
const taps = new PreviewTaps({ publisher, scratchDir: scratch, redactor: passthroughRedactor, onStderr: () => undefined });
|
||||
try {
|
||||
for (const s of sources) await s.start();
|
||||
const assets = makeAssets(scratch);
|
||||
await chain.start(composeOver(sources.map((s) => s.bus), assets, scriptedGraph(0)));
|
||||
const bus = chain.programBus!;
|
||||
assert.deepEqual(bus.geometry, programTapGeometry(canvas));
|
||||
await taps.add("program", bus);
|
||||
await until(() => publisher.files.has("program/thumbnail.jpg"), 15_000, "program thumbnail");
|
||||
const p0 = (await bus.stats()).pairs;
|
||||
await new Promise((r) => setTimeout(r, 1_000));
|
||||
const p1 = (await bus.stats()).pairs;
|
||||
assert.ok(p1 - p0 >= canvas.fps * 0.7, `program bus at frame rate (${p1 - p0}/s)`);
|
||||
// Kill the tap bus helper: the encoder's tee drops that slave and keeps encoding.
|
||||
process.kill(bus.pid ?? 0, "SIGKILL");
|
||||
const s0 = await chain.stats();
|
||||
await new Promise((r) => setTimeout(r, 2_000));
|
||||
const s1 = await chain.stats();
|
||||
assert.ok(chain.encoderPid !== undefined && spawnSync("kill", ["-0", String(chain.encoderPid)]).status === 0, "encoder alive");
|
||||
assert.ok(s1.ticks - s0.ticks >= canvas.fps, "gate kept ticking");
|
||||
assert.equal(s1.outDrops - s0.outDrops, 0, "encoder kept reading the gate");
|
||||
assert.deepEqual(fatal, []);
|
||||
} finally {
|
||||
await taps.stop();
|
||||
await chain.stop();
|
||||
for (const s of sources) await s.stop();
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
352
apps/media-worker/src/preview/taps.ts
Normal file
352
apps/media-worker/src/preview/taps.ts
Normal file
|
|
@ -0,0 +1,352 @@
|
|||
/**
|
||||
* Preview tap lifecycle (streaming-architecture.md §2.1).
|
||||
*
|
||||
* Every tap is a source-bus subscriber, so taps never sit in the gate path: a
|
||||
* slow or dead tap loses only its own pairs (the bus drops whole pairs for a
|
||||
* subscriber that falls behind; its EPIPE removes it), and the compositors and
|
||||
* the encoder never wait for it. The program tap's bus is fed by the persistent
|
||||
* encoder (360p, `tee` with a null anchor), so the PGM tile shows exactly what
|
||||
* the encoder receives.
|
||||
*
|
||||
* - JPEG: always on per tap, one still every 2 s by default (1–10 s; 1 s for the
|
||||
* PST tile), published as `thumbnail.jpg`.
|
||||
* - video / audio LL-HLS: on demand. `demand()` starts a rendition and extends
|
||||
* its lease; an expired lease stops it. The program tap may be made always-on.
|
||||
*/
|
||||
import { mkdirSync, readFileSync, rmSync, watch, type FSWatcher } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { Readable } from "node:stream";
|
||||
import type { BusSubscription, SourceBus } from "../compositor/source-bus";
|
||||
import { spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg";
|
||||
import {
|
||||
AUDIO_TAP_CODECS, AUDIO_TAP_KBPS, buildHlsTapArgs, buildJpegTapArgs, jpegStride, PARTS_PLAYLIST, tapSize,
|
||||
VIDEO_TAP_CODECS, VIDEO_TAP_MAX_KBPS, type HlsRendition,
|
||||
} from "./argv";
|
||||
import { assertTapId, AUDIO_PLAYLIST, MASTER_PLAYLIST, THUMBNAIL_FILE, VIDEO_PLAYLIST, type PreviewPublisher } from "./contract";
|
||||
import { fragmentIndependent, LlHlsPackager, masterPlaylist, parseFfmpegPlaylist } from "./ll-hls";
|
||||
|
||||
/** Manifest TTLs: a stopped tap's files expire on their own. */
|
||||
export const TTL = { thumbnail: 30, playlist: 10, media: 60, init: 600 } as const;
|
||||
|
||||
let lastSequenceBase = 0;
|
||||
function nextSequenceBase(): number {
|
||||
lastSequenceBase = Math.max(Date.now(), lastSequenceBase + 1);
|
||||
return lastSequenceBase;
|
||||
}
|
||||
|
||||
export interface TapDeps {
|
||||
publisher: PreviewPublisher;
|
||||
scratchDir: string;
|
||||
redactor: StderrRedactor;
|
||||
onStderr(message: string): void;
|
||||
/** A tap process failed (it is restarted on the next demand/cadence change; on air is unaffected). */
|
||||
onError?(tap: string, reason: string): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits an mjpeg byte stream into whole JPEGs (SOI … EOI). Inside entropy-coded
|
||||
* data 0xFF is always stuffed (FF 00), so FF D9 occurs only as the end marker.
|
||||
*/
|
||||
export class JpegSplitter {
|
||||
private buf: Buffer = Buffer.alloc(0);
|
||||
|
||||
push(chunk: Buffer): Buffer[] {
|
||||
this.buf = this.buf.length === 0 ? chunk : Buffer.concat([this.buf, chunk]);
|
||||
const out: Buffer[] = [];
|
||||
for (;;) {
|
||||
const start = this.buf.indexOf(Buffer.from([0xff, 0xd8]));
|
||||
if (start < 0) {
|
||||
// Keep a trailing 0xFF: it may be the first byte of the next SOI.
|
||||
this.buf = this.buf.length > 0 && this.buf[this.buf.length - 1] === 0xff ? this.buf.subarray(-1) : Buffer.alloc(0);
|
||||
break;
|
||||
}
|
||||
const end = this.buf.indexOf(Buffer.from([0xff, 0xd9]), start + 2);
|
||||
if (end < 0) { this.buf = this.buf.subarray(start); break; }
|
||||
out.push(Buffer.from(this.buf.subarray(start, end + 2)));
|
||||
this.buf = this.buf.subarray(end + 2);
|
||||
}
|
||||
if (this.buf.length > 8_000_000) this.buf = Buffer.alloc(0); // never a valid preview still
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
/** One tap process reading a bus subscription. */
|
||||
abstract class BusTap {
|
||||
protected proc: ManagedProcess | null = null;
|
||||
protected sub: BusSubscription | null = null;
|
||||
protected stopping = false;
|
||||
|
||||
constructor(readonly tap: string, protected readonly bus: SourceBus, protected readonly deps: TapDeps) { assertTapId(tap); }
|
||||
|
||||
get running(): boolean { return this.proc !== null; }
|
||||
get pid(): number | undefined { return this.proc?.child.pid; }
|
||||
|
||||
protected async spawn(args: string[], opts: { video: boolean; audio: boolean; stride?: number }, stdout: "pipe" | "ignore"): Promise<ManagedProcess> {
|
||||
this.stopping = false;
|
||||
const sub = await this.bus.subscribe(opts);
|
||||
this.sub = sub;
|
||||
let proc: ManagedProcess;
|
||||
try {
|
||||
proc = spawnManaged("ffmpeg", args, [sub.videoFd ?? "ignore", stdout, "pipe", sub.audioFd ?? "ignore"], this.deps.redactor, this.deps.onStderr);
|
||||
} finally {
|
||||
sub.releaseReaders();
|
||||
}
|
||||
this.proc = proc;
|
||||
void proc.exited.then(({ code, signal }) => {
|
||||
if (this.proc === proc) { this.proc = null; this.sub?.unsubscribe(); this.sub = null; }
|
||||
if (!this.stopping) this.deps.onError?.(this.tap, `tap exited code=${code} signal=${signal}`);
|
||||
});
|
||||
await proc.started;
|
||||
return proc;
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.stopping = true;
|
||||
const proc = this.proc;
|
||||
this.proc = null;
|
||||
this.sub?.unsubscribe();
|
||||
this.sub = null;
|
||||
await proc?.terminate(1_000);
|
||||
}
|
||||
}
|
||||
|
||||
export class JpegTap extends BusTap {
|
||||
private everySeconds = 2;
|
||||
stills = 0;
|
||||
|
||||
get cadence(): number { return this.everySeconds; }
|
||||
|
||||
async start(everySeconds = this.everySeconds): Promise<void> {
|
||||
this.everySeconds = everySeconds;
|
||||
const plan = { geometry: this.bus.geometry, everySeconds };
|
||||
const proc = await this.spawn(buildJpegTapArgs(plan), { video: true, audio: false, stride: jpegStride(plan) }, "pipe");
|
||||
const splitter = new JpegSplitter();
|
||||
(proc.child.stdout as Readable | null)?.on("data", (chunk: Buffer) => {
|
||||
for (const jpeg of splitter.push(chunk)) {
|
||||
this.stills += 1;
|
||||
void this.deps.publisher.publish(this.tap, THUMBNAIL_FILE, jpeg, "image/jpeg", TTL.thumbnail)
|
||||
.catch((error: unknown) => this.deps.onError?.(this.tap, `thumbnail publish failed: ${String(error)}`));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** 1–10 s (§2.2: 2 s default, 1 s for the PST tile). Restarts the tap process. */
|
||||
async setCadence(everySeconds: number): Promise<void> {
|
||||
if (everySeconds === this.everySeconds && this.running) return;
|
||||
await this.stop();
|
||||
await this.start(everySeconds);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One LL-HLS rendition. ffmpeg writes 0.5 s fMP4 parts into a private scratch
|
||||
* dir; each time its playlist is renamed into place, new parts are published
|
||||
* (parts, closed segments, then the playlist), and the local files are removed.
|
||||
*/
|
||||
export class HlsTap extends BusTap {
|
||||
private dir = "";
|
||||
private watcher: FSWatcher | null = null;
|
||||
private packager: LlHlsPackager | null = null;
|
||||
private seen = new Set<string>();
|
||||
private initPublished = "";
|
||||
private initRenewedAt = 0;
|
||||
private chain: Promise<void> = Promise.resolve();
|
||||
parts = 0;
|
||||
|
||||
constructor(tap: string, bus: SourceBus, deps: TapDeps, readonly rendition: HlsRendition) { super(tap, bus, deps); }
|
||||
|
||||
get playlistFile(): string { return this.rendition === "video" ? VIDEO_PLAYLIST : AUDIO_PLAYLIST; }
|
||||
|
||||
async start(): Promise<void> {
|
||||
const name = `${this.tap === "program" ? "program" : this.tap.slice(0, 8)}-${this.rendition}-${Date.now()}`;
|
||||
this.dir = join(this.deps.scratchDir, "preview", name);
|
||||
mkdirSync(this.dir, { recursive: true, mode: 0o700 });
|
||||
this.packager = new LlHlsPackager(this.rendition, nextSequenceBase());
|
||||
this.seen = new Set();
|
||||
this.initPublished = "";
|
||||
this.watcher = watch(this.dir, (_event, file) => { if (file === PARTS_PLAYLIST) this.schedule(); });
|
||||
await this.spawn(buildHlsTapArgs({ geometry: this.bus.geometry, rendition: this.rendition, outDir: this.dir }),
|
||||
{ video: this.rendition === "video", audio: true }, "ignore");
|
||||
}
|
||||
|
||||
private schedule(): void {
|
||||
this.chain = this.chain.then(() => this.collect()).catch((error: unknown) => this.deps.onError?.(this.tap, `hls publish failed: ${String(error)}`));
|
||||
}
|
||||
|
||||
/** Waits for in-flight publishing (tests). */
|
||||
settled(): Promise<void> { return this.chain; }
|
||||
|
||||
private async collect(): Promise<void> {
|
||||
const packager = this.packager;
|
||||
if (!packager) return;
|
||||
let text: string;
|
||||
try { text = readFileSync(join(this.dir, PARTS_PLAYLIST), "utf8"); } catch { return; }
|
||||
const contentType = this.rendition === "video" ? "video/mp4" : "audio/mp4";
|
||||
for (const entry of parseFfmpegPlaylist(text)) {
|
||||
if (this.seen.has(entry.file)) continue;
|
||||
let bytes: Buffer;
|
||||
try { bytes = readFileSync(join(this.dir, entry.file)); } catch { continue; }
|
||||
this.seen.add(entry.file);
|
||||
if (this.seen.size > 64) this.seen.delete(this.seen.values().next().value ?? "");
|
||||
rmSync(join(this.dir, entry.file), { force: true });
|
||||
await this.publishInit(packager, contentType);
|
||||
const { part, closed } = packager.add({ duration: entry.duration, independent: fragmentIndependent(bytes), bytes });
|
||||
if (closed) await this.deps.publisher.publish(this.tap, closed.name, LlHlsPackager.segmentBytes(closed), contentType, TTL.media);
|
||||
if (!part) continue;
|
||||
await this.deps.publisher.publish(this.tap, part.name, part.bytes, contentType, TTL.media);
|
||||
this.parts += 1;
|
||||
const playlist = packager.playlist();
|
||||
if (playlist) await this.deps.publisher.publish(this.tap, this.playlistFile, Buffer.from(playlist), "application/vnd.apple.mpegurl", TTL.playlist);
|
||||
}
|
||||
}
|
||||
|
||||
/** The init segment is published once and its manifest renewed well inside its TTL. */
|
||||
private async publishInit(packager: LlHlsPackager, contentType: "video/mp4" | "audio/mp4"): Promise<void> {
|
||||
const due = Date.now() - this.initRenewedAt > (TTL.init / 2) * 1000;
|
||||
if (this.initPublished === packager.initName && !due) return;
|
||||
let init: Buffer;
|
||||
try { init = readFileSync(join(this.dir, "init.mp4")); } catch { return; }
|
||||
await this.deps.publisher.publish(this.tap, packager.initName, init, contentType, TTL.init);
|
||||
this.initPublished = packager.initName;
|
||||
this.initRenewedAt = Date.now();
|
||||
}
|
||||
|
||||
override async stop(): Promise<void> {
|
||||
this.watcher?.close();
|
||||
this.watcher = null;
|
||||
await super.stop();
|
||||
await this.chain;
|
||||
this.packager = null;
|
||||
if (this.dir) rmSync(this.dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
interface TapSet { bus: SourceBus; jpeg: JpegTap; hls: Partial<Record<HlsRendition, { tap: HlsTap; until: number }>>; masterRenewedAt: number }
|
||||
|
||||
/**
|
||||
* All preview taps of one channel: a JPEG tap per bus (always on), LL-HLS
|
||||
* renditions on demand with a lease. `alwaysOn` renditions (the program tap's,
|
||||
* per §2.2 "PGM always live") never expire.
|
||||
*/
|
||||
export class PreviewTaps {
|
||||
private readonly taps = new Map<string, TapSet>();
|
||||
private readonly alwaysOn = new Set<string>();
|
||||
private sweeper: ReturnType<typeof setInterval> | null = null;
|
||||
private operations: Promise<unknown> = Promise.resolve();
|
||||
|
||||
constructor(private readonly deps: TapDeps, private readonly leaseMs = 30_000) {}
|
||||
|
||||
get tapIds(): string[] { return [...this.taps.keys()]; }
|
||||
jpeg(tap: string): JpegTap | undefined { return this.taps.get(tap)?.jpeg; }
|
||||
hls(tap: string, rendition: HlsRendition): HlsTap | undefined { return this.taps.get(tap)?.hls[rendition]?.tap; }
|
||||
|
||||
private serialize<T>(operation: () => Promise<T>): Promise<T> {
|
||||
const next = this.operations.catch(() => undefined).then(operation);
|
||||
this.operations = next;
|
||||
return next;
|
||||
}
|
||||
|
||||
add(tap: string, bus: SourceBus, opts: { alwaysOn?: HlsRendition[] } = {}): Promise<void> {
|
||||
return this.serialize(() => this.addNow(tap, bus, opts));
|
||||
}
|
||||
|
||||
private async addNow(tap: string, bus: SourceBus, opts: { alwaysOn?: HlsRendition[] }): Promise<void> {
|
||||
assertTapId(tap);
|
||||
if (this.taps.has(tap)) throw new Error(`preview tap ${tap} already exists`);
|
||||
const jpeg = new JpegTap(tap, bus, this.deps);
|
||||
this.taps.set(tap, { bus, jpeg, hls: {}, masterRenewedAt: 0 });
|
||||
try {
|
||||
await jpeg.start();
|
||||
for (const r of opts.alwaysOn ?? []) {
|
||||
this.alwaysOn.add(`${tap}:${r}`);
|
||||
await this.demandNow(tap, r);
|
||||
}
|
||||
} catch (error) {
|
||||
await this.removeNow(tap);
|
||||
throw error;
|
||||
}
|
||||
this.sweeper ??= setInterval(() => {
|
||||
void this.sweep().catch((error: unknown) => this.deps.onError?.("program", `preview sweep failed: ${String(error)}`));
|
||||
}, 1_000);
|
||||
}
|
||||
|
||||
/** Starts (or keeps) a rendition for `leaseMs` more. */
|
||||
demand(tap: string, rendition: HlsRendition): Promise<void> {
|
||||
return this.serialize(() => this.demandNow(tap, rendition));
|
||||
}
|
||||
|
||||
private async demandNow(tap: string, rendition: HlsRendition): Promise<void> {
|
||||
const set = this.taps.get(tap);
|
||||
if (!set) throw new Error(`no preview tap ${tap}`);
|
||||
const until = this.alwaysOn.has(`${tap}:${rendition}`) ? Infinity : Date.now() + this.leaseMs;
|
||||
const current = set.hls[rendition];
|
||||
if (current?.tap.running) { current.until = Math.max(current.until, until); return; }
|
||||
await current?.tap.stop();
|
||||
const hls = new HlsTap(tap, set.bus, this.deps, rendition);
|
||||
set.hls[rendition] = { tap: hls, until };
|
||||
try { await hls.start(); } catch (error) {
|
||||
delete set.hls[rendition];
|
||||
await hls.stop();
|
||||
throw error;
|
||||
}
|
||||
await this.publishMaster(tap, set.bus);
|
||||
}
|
||||
|
||||
setThumbnailCadence(tap: string, everySeconds: number): Promise<void> {
|
||||
return this.serialize(async () => {
|
||||
const set = this.taps.get(tap);
|
||||
if (!set) throw new Error(`no preview tap ${tap}`);
|
||||
buildJpegTapArgs({ geometry: set.bus.geometry, everySeconds });
|
||||
await set.jpeg.setCadence(everySeconds);
|
||||
});
|
||||
}
|
||||
|
||||
private async publishMaster(tap: string, bus: SourceBus): Promise<void> {
|
||||
const size = tapSize(bus.geometry);
|
||||
const body = masterPlaylist({ ...size, bandwidth: (VIDEO_TAP_MAX_KBPS + AUDIO_TAP_KBPS) * 1000, codecs: VIDEO_TAP_CODECS }, { bandwidth: AUDIO_TAP_KBPS * 1000, codecs: AUDIO_TAP_CODECS });
|
||||
await this.deps.publisher.publish(tap, MASTER_PLAYLIST, Buffer.from(body), "application/vnd.apple.mpegurl", TTL.init);
|
||||
const set = this.taps.get(tap);
|
||||
if (set) set.masterRenewedAt = Date.now();
|
||||
}
|
||||
|
||||
/** Stops expired renditions; restarts a JPEG tap whose process died. */
|
||||
sweep(now = Date.now()): Promise<void> {
|
||||
return this.serialize(() => this.sweepNow(now));
|
||||
}
|
||||
|
||||
private async sweepNow(now: number): Promise<void> {
|
||||
for (const set of this.taps.values()) {
|
||||
for (const r of ["video", "audio"] as const) {
|
||||
const entry = set.hls[r];
|
||||
if (entry && entry.until <= now) { delete set.hls[r]; await entry.tap.stop(); }
|
||||
else if (entry && !entry.tap.running) {
|
||||
await this.demandNow(set.jpeg.tap, r).catch((error: unknown) => this.deps.onError?.(set.jpeg.tap, `hls restart failed: ${String(error)}`));
|
||||
}
|
||||
}
|
||||
if (!set.jpeg.running) await set.jpeg.start().catch((error: unknown) => this.deps.onError?.(set.jpeg.tap, `thumbnail restart failed: ${String(error)}`));
|
||||
if (Object.keys(set.hls).length && now - set.masterRenewedAt >= TTL.init * 500) await this.publishMaster(set.jpeg.tap, set.bus);
|
||||
}
|
||||
}
|
||||
|
||||
remove(tap: string): Promise<void> { return this.serialize(() => this.removeNow(tap)); }
|
||||
|
||||
private async removeNow(tap: string): Promise<void> {
|
||||
const set = this.taps.get(tap);
|
||||
if (!set) return;
|
||||
this.taps.delete(tap);
|
||||
this.alwaysOn.delete(`${tap}:video`);
|
||||
this.alwaysOn.delete(`${tap}:audio`);
|
||||
await set.jpeg.stop();
|
||||
for (const entry of Object.values(set.hls)) await entry?.tap.stop();
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
if (this.sweeper) clearInterval(this.sweeper);
|
||||
this.sweeper = null;
|
||||
await this.serialize(async () => {
|
||||
if (this.sweeper) clearInterval(this.sweeper);
|
||||
this.sweeper = null;
|
||||
for (const tap of [...this.taps.keys()]) await this.removeNow(tap);
|
||||
});
|
||||
}
|
||||
}
|
||||
358
apps/media-worker/src/production.ts
Normal file
358
apps/media-worker/src/production.ts
Normal file
|
|
@ -0,0 +1,358 @@
|
|||
/**
|
||||
* Desired-state v2: the production part of a channel snapshot
|
||||
* (docs/design/streaming-architecture.md §2, §4, §6 S1).
|
||||
*
|
||||
* This file is mirrored byte-for-byte at apps/media-worker/src/production.ts
|
||||
* (a test asserts it). It has no imports so both builds compile it as-is.
|
||||
*
|
||||
* Everything that can reach a filtergraph or argv is typed: ids are UUIDs,
|
||||
* assets are sha256 content ids, geometry is integer basis points, gains are
|
||||
* 3 dB steps, enums are closed. Labels are display text for operators and logs
|
||||
* only; they are still held to a strict allowlist so a filtergraph payload
|
||||
* (`;`, `[`, `,`, `'`, `movie=`) can never ride along in a snapshot.
|
||||
*/
|
||||
|
||||
export type SceneLayout = "single" | "pip" | "2x1" | "2x2" | "3x3" | "4x4" | "custom";
|
||||
export type SlotFit = "contain" | "cover" | "crop";
|
||||
export type AudioMode = "off" | "on" | "primary";
|
||||
export type OverlayKind = "lower_third" | "chyron" | "logo" | "clock";
|
||||
export type ClockFormat = "HH:MM" | "HH:MM:SS";
|
||||
export type FadeMs = 300 | 500 | 1000 | 2000;
|
||||
|
||||
export const SCENE_LAYOUTS: readonly SceneLayout[] = ["single", "pip", "2x1", "2x2", "3x3", "4x4", "custom"];
|
||||
export const SLOT_FITS: readonly SlotFit[] = ["contain", "cover", "crop"];
|
||||
export const AUDIO_MODES: readonly AudioMode[] = ["off", "on", "primary"];
|
||||
export const OVERLAY_KINDS: readonly OverlayKind[] = ["lower_third", "chyron", "logo", "clock"];
|
||||
export const CLOCK_FORMATS: readonly ClockFormat[] = ["HH:MM", "HH:MM:SS"];
|
||||
export const FADE_MS: readonly FadeMs[] = [300, 500, 1000, 2000];
|
||||
|
||||
/** Slot capacity per layout. `custom` places up to 16 explicit rectangles. */
|
||||
export const LAYOUT_CAPACITY: Readonly<Record<SceneLayout, number>> = {
|
||||
single: 1, pip: 2, "2x1": 2, "2x2": 4, "3x3": 9, "4x4": 16, custom: 16,
|
||||
};
|
||||
|
||||
/** Geometry is integer basis points of the output frame (0..10000), resolution independent. */
|
||||
export const GEOMETRY_SCALE = 10_000;
|
||||
export const GAIN_DB_MIN = -24;
|
||||
export const GAIN_DB_MAX = 0;
|
||||
export const GAIN_DB_STEP = 3;
|
||||
export const CHYRON_SPEED_MIN = 40;
|
||||
export const CHYRON_SPEED_MAX = 400;
|
||||
export const CHYRON_STRIP_MAX_PX = 16_384;
|
||||
export const MAX_SOURCES = 32;
|
||||
export const MAX_SCENES = 32;
|
||||
export const MAX_OVERLAYS = 64;
|
||||
export const MAX_PLAYLIST_ITEMS = 500;
|
||||
export const MAX_OVERLAY_LAYER = 7;
|
||||
|
||||
export interface Rect { x: number; y: number; w: number; h: number }
|
||||
export interface Point { x: number; y: number }
|
||||
|
||||
export interface HlsProductionSource { id: string; label: string; kind: "hls"; url: string }
|
||||
/** Ordered, pre-normalized video asset ids (video_assets.id), fed by the worker's asset feeder. */
|
||||
export interface PlaylistProductionSource { id: string; label: string; kind: "playlist"; assetIds: string[]; loop: boolean }
|
||||
export type ProductionSource = HlsProductionSource | PlaylistProductionSource;
|
||||
|
||||
export interface SceneSlot {
|
||||
index: number;
|
||||
/** null = empty slot (renders the fallback). */
|
||||
sourceId: string | null;
|
||||
/** Required for `custom` and `pip` layouts; null means the layout's own grid cell. */
|
||||
rect: Rect | null;
|
||||
fit: SlotFit;
|
||||
audioMode: AudioMode;
|
||||
gainDb: number;
|
||||
}
|
||||
|
||||
export interface ProductionScene { id: string; label: string; layout: SceneLayout; slots: SceneSlot[] }
|
||||
|
||||
export interface ProductionBuses { preview: string | null; program: string | null }
|
||||
|
||||
export interface ProductionOverlay {
|
||||
/** overlay_slots.id */
|
||||
id: string;
|
||||
sceneId: string;
|
||||
kind: OverlayKind;
|
||||
enabled: boolean;
|
||||
layer: number;
|
||||
position: Point;
|
||||
/** overlay_revisions.id; null only for the server-generated clock. */
|
||||
revisionId: string | null;
|
||||
/** sha256 content id of the rendered PNG; null only for the clock. */
|
||||
assetId: string | null;
|
||||
chyron: { speedPxPerSec: number; stripWidthPx: number } | null;
|
||||
clock: { format: ClockFormat } | null;
|
||||
}
|
||||
|
||||
export type Transition = { type: "cut" } | { type: "fade"; ms: FadeMs };
|
||||
|
||||
export interface ProgramAudio { masterGainDb: number }
|
||||
|
||||
export interface ProductionSpec {
|
||||
scenes: ProductionScene[];
|
||||
buses: ProductionBuses;
|
||||
sources: ProductionSource[];
|
||||
overlays: ProductionOverlay[];
|
||||
transition: Transition;
|
||||
audio: ProgramAudio;
|
||||
}
|
||||
|
||||
export class ProductionSpecError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "ProductionSpecError";
|
||||
}
|
||||
}
|
||||
|
||||
const SECRETISH = /key|secret|token|password|passwd|credential(?!HandleId)|auth/i;
|
||||
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
||||
const SHA256 = /^[0-9a-f]{64}$/;
|
||||
/** Letters, digits, space and `_ . -` only: no filtergraph or shell metacharacters, no `=`. */
|
||||
const LABEL = /^[\p{L}\p{N}][\p{L}\p{N} _.-]{0,63}$/u;
|
||||
|
||||
function fail(message: string): never { throw new ProductionSpecError(message); }
|
||||
|
||||
function record(value: unknown, field: string, allowed: readonly string[]): Record<string, unknown> {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${field} must be an object`);
|
||||
for (const key of Object.keys(value)) {
|
||||
if (allowed.includes(key)) continue;
|
||||
if (SECRETISH.test(key)) fail(`${field}.${key}: secrets are never accepted; reference a vault credentialHandleId`);
|
||||
fail(`${field}.${key} is not allowed`);
|
||||
}
|
||||
for (const key of allowed) if (!(key in value)) fail(`${field}.${key} is required`);
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function list(value: unknown, field: string, max: number): unknown[] {
|
||||
if (!Array.isArray(value)) fail(`${field} must be an array`);
|
||||
if (value.length > max) fail(`${field} allows at most ${max} entries`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function uuid(value: unknown, field: string): string {
|
||||
if (typeof value !== "string" || !UUID.test(value)) fail(`${field} must be a lowercase UUID`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function nullableUuid(value: unknown, field: string): string | null {
|
||||
return value === null ? null : uuid(value, field);
|
||||
}
|
||||
|
||||
function label(value: unknown, field: string): string {
|
||||
if (typeof value !== "string" || !LABEL.test(value)) {
|
||||
fail(`${field} must be 1-64 letters, digits, spaces, '_', '.', '-'`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function int(value: unknown, field: string, min: number, max: number): number {
|
||||
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < min || value > max) {
|
||||
fail(`${field} must be an integer in ${min}..${max}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function bool(value: unknown, field: string): boolean {
|
||||
if (typeof value !== "boolean") fail(`${field} must be a boolean`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function oneOf<T extends string | number>(value: unknown, field: string, options: readonly T[]): T {
|
||||
const found = options.find((option) => option === value);
|
||||
if (found === undefined) fail(`${field} must be one of ${options.join(", ")}`);
|
||||
return found;
|
||||
}
|
||||
|
||||
export function gainDb(value: unknown, field: string): number {
|
||||
const gain = int(value, field, GAIN_DB_MIN, GAIN_DB_MAX);
|
||||
if (gain % GAIN_DB_STEP !== 0) fail(`${field} must be a multiple of ${GAIN_DB_STEP} dB`);
|
||||
return gain === 0 ? 0 : gain;
|
||||
}
|
||||
|
||||
function rect(value: unknown, field: string): Rect {
|
||||
const r = record(value, field, ["x", "y", "w", "h"]);
|
||||
const x = int(r.x, `${field}.x`, 0, GEOMETRY_SCALE - 1);
|
||||
const y = int(r.y, `${field}.y`, 0, GEOMETRY_SCALE - 1);
|
||||
const w = int(r.w, `${field}.w`, 1, GEOMETRY_SCALE);
|
||||
const h = int(r.h, `${field}.h`, 1, GEOMETRY_SCALE);
|
||||
if (x + w > GEOMETRY_SCALE || y + h > GEOMETRY_SCALE) fail(`${field} must lie inside the frame`);
|
||||
return { x, y, w, h };
|
||||
}
|
||||
|
||||
function point(value: unknown, field: string): Point {
|
||||
const p = record(value, field, ["x", "y"]);
|
||||
return { x: int(p.x, `${field}.x`, 0, GEOMETRY_SCALE), y: int(p.y, `${field}.y`, 0, GEOMETRY_SCALE) };
|
||||
}
|
||||
|
||||
function unique(ids: string[], field: string): void {
|
||||
if (new Set(ids).size !== ids.length) fail(`${field} ids must be unique`);
|
||||
}
|
||||
|
||||
/**
|
||||
* HLS URLs are shape-checked by the caller with its own endpoint validator
|
||||
* (API and worker differ on loopback strictness); here only bounds are checked.
|
||||
*/
|
||||
function source(value: unknown, field: string): ProductionSource {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${field} must be an object`);
|
||||
const kind = (value as Record<string, unknown>).kind;
|
||||
if (kind === "hls") {
|
||||
const s = record(value, field, ["id", "label", "kind", "url"]);
|
||||
if (typeof s.url !== "string" || s.url.trim() === "" || s.url.length > 2048) fail(`${field}.url must be a non-empty string`);
|
||||
return { id: uuid(s.id, `${field}.id`), label: label(s.label, `${field}.label`), kind: "hls", url: s.url };
|
||||
}
|
||||
if (kind === "playlist") {
|
||||
const s = record(value, field, ["id", "label", "kind", "assetIds", "loop"]);
|
||||
const assetIds = list(s.assetIds, `${field}.assetIds`, MAX_PLAYLIST_ITEMS).map((item, i) => uuid(item, `${field}.assetIds[${i}]`));
|
||||
if (assetIds.length === 0) fail(`${field}.assetIds must not be empty`);
|
||||
return { id: uuid(s.id, `${field}.id`), label: label(s.label, `${field}.label`), kind: "playlist", assetIds, loop: bool(s.loop, `${field}.loop`) };
|
||||
}
|
||||
return fail(`${field}.kind must be hls or playlist`);
|
||||
}
|
||||
|
||||
function slot(value: unknown, field: string, layout: SceneLayout, sources: ReadonlySet<string>): SceneSlot {
|
||||
const s = record(value, field, ["index", "sourceId", "rect", "fit", "audioMode", "gainDb"]);
|
||||
const index = int(s.index, `${field}.index`, 0, LAYOUT_CAPACITY[layout] - 1);
|
||||
const sourceId = nullableUuid(s.sourceId, `${field}.sourceId`);
|
||||
if (sourceId !== null && !sources.has(sourceId)) fail(`${field}.sourceId must name a source of this channel`);
|
||||
const needsRect = layout === "custom" || (layout === "pip" && index > 0);
|
||||
const slotRect = s.rect === null ? null : rect(s.rect, `${field}.rect`);
|
||||
if (needsRect && slotRect === null) fail(`${field}.rect is required for the ${layout} layout`);
|
||||
if (!needsRect && slotRect !== null) fail(`${field}.rect is only allowed for custom layouts and PIP insets`);
|
||||
return {
|
||||
index, sourceId, rect: slotRect,
|
||||
fit: oneOf(s.fit, `${field}.fit`, SLOT_FITS),
|
||||
audioMode: oneOf(s.audioMode, `${field}.audioMode`, AUDIO_MODES),
|
||||
gainDb: gainDb(s.gainDb, `${field}.gainDb`),
|
||||
};
|
||||
}
|
||||
|
||||
function scene(value: unknown, field: string, sources: ReadonlySet<string>): ProductionScene {
|
||||
const s = record(value, field, ["id", "label", "layout", "slots"]);
|
||||
const layout = oneOf(s.layout, `${field}.layout`, SCENE_LAYOUTS);
|
||||
const slots = list(s.slots, `${field}.slots`, LAYOUT_CAPACITY[layout]).map((item, i) => slot(item, `${field}.slots[${i}]`, layout, sources));
|
||||
if (new Set(slots.map((x) => x.index)).size !== slots.length) fail(`${field}.slots indexes must be unique`);
|
||||
if (slots.filter((x) => x.audioMode === "primary").length > 1) fail(`${field}.slots: at most one primary audio slot per scene`);
|
||||
return { id: uuid(s.id, `${field}.id`), label: label(s.label, `${field}.label`), layout, slots };
|
||||
}
|
||||
|
||||
function overlay(value: unknown, field: string, scenes: ReadonlySet<string>): ProductionOverlay {
|
||||
const o = record(value, field, ["id", "sceneId", "kind", "enabled", "layer", "position", "revisionId", "assetId", "chyron", "clock"]);
|
||||
const sceneId = uuid(o.sceneId, `${field}.sceneId`);
|
||||
if (!scenes.has(sceneId)) fail(`${field}.sceneId must name a scene of this channel`);
|
||||
const kind = oneOf(o.kind, `${field}.kind`, OVERLAY_KINDS);
|
||||
const rendered = kind !== "clock";
|
||||
let revisionId: string | null = null;
|
||||
let assetId: string | null = null;
|
||||
if (rendered) {
|
||||
revisionId = uuid(o.revisionId, `${field}.revisionId`);
|
||||
if (typeof o.assetId !== "string" || !SHA256.test(o.assetId)) fail(`${field}.assetId must be a sha256 content id`);
|
||||
assetId = o.assetId;
|
||||
} else if (o.revisionId !== null || o.assetId !== null) fail(`${field}: a clock overlay has no rendered asset`);
|
||||
|
||||
let chyron: ProductionOverlay["chyron"] = null;
|
||||
if (kind === "chyron") {
|
||||
const c = record(o.chyron, `${field}.chyron`, ["speedPxPerSec", "stripWidthPx"]);
|
||||
chyron = {
|
||||
speedPxPerSec: int(c.speedPxPerSec, `${field}.chyron.speedPxPerSec`, CHYRON_SPEED_MIN, CHYRON_SPEED_MAX),
|
||||
stripWidthPx: int(c.stripWidthPx, `${field}.chyron.stripWidthPx`, 1, CHYRON_STRIP_MAX_PX),
|
||||
};
|
||||
} else if (o.chyron !== null) fail(`${field}.chyron is only allowed for chyron overlays`);
|
||||
|
||||
let clock: ProductionOverlay["clock"] = null;
|
||||
if (kind === "clock") {
|
||||
const c = record(o.clock, `${field}.clock`, ["format"]);
|
||||
clock = { format: oneOf(c.format, `${field}.clock.format`, CLOCK_FORMATS) };
|
||||
} else if (o.clock !== null) fail(`${field}.clock is only allowed for clock overlays`);
|
||||
|
||||
return {
|
||||
id: uuid(o.id, `${field}.id`), sceneId, kind,
|
||||
enabled: bool(o.enabled, `${field}.enabled`),
|
||||
layer: int(o.layer, `${field}.layer`, 0, MAX_OVERLAY_LAYER),
|
||||
position: point(o.position, `${field}.position`),
|
||||
revisionId, assetId, chyron, clock,
|
||||
};
|
||||
}
|
||||
|
||||
function transition(value: unknown, field: string): Transition {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${field} must be an object`);
|
||||
const type = (value as Record<string, unknown>).type;
|
||||
if (type === "cut") {
|
||||
record(value, field, ["type"]);
|
||||
return { type: "cut" };
|
||||
}
|
||||
if (type === "fade") {
|
||||
const t = record(value, field, ["type", "ms"]);
|
||||
return { type: "fade", ms: oneOf(t.ms, `${field}.ms`, FADE_MS) };
|
||||
}
|
||||
return fail(`${field}.type must be cut or fade`);
|
||||
}
|
||||
|
||||
/** Parse and normalize; throws ProductionSpecError with a field path on the first problem. */
|
||||
export function parseProductionSpec(value: unknown, field = "production"): ProductionSpec {
|
||||
const p = record(value, field, ["scenes", "buses", "sources", "overlays", "transition", "audio"]);
|
||||
const sources = list(p.sources, `${field}.sources`, MAX_SOURCES).map((item, i) => source(item, `${field}.sources[${i}]`));
|
||||
unique(sources.map((s) => s.id), `${field}.sources`);
|
||||
const sourceIds = new Set(sources.map((s) => s.id));
|
||||
const scenes = list(p.scenes, `${field}.scenes`, MAX_SCENES).map((item, i) => scene(item, `${field}.scenes[${i}]`, sourceIds));
|
||||
unique(scenes.map((s) => s.id), `${field}.scenes`);
|
||||
const sceneIds = new Set(scenes.map((s) => s.id));
|
||||
const overlays = list(p.overlays, `${field}.overlays`, MAX_OVERLAYS).map((item, i) => overlay(item, `${field}.overlays[${i}]`, sceneIds));
|
||||
unique(overlays.map((o) => o.id), `${field}.overlays`);
|
||||
|
||||
const b = record(p.buses, `${field}.buses`, ["preview", "program"]);
|
||||
const buses: ProductionBuses = {
|
||||
preview: nullableUuid(b.preview, `${field}.buses.preview`),
|
||||
program: nullableUuid(b.program, `${field}.buses.program`),
|
||||
};
|
||||
for (const bus of ["preview", "program"] as const) {
|
||||
const id = buses[bus];
|
||||
if (id !== null && !sceneIds.has(id)) fail(`${field}.buses.${bus} must name a scene of this channel`);
|
||||
}
|
||||
const a = record(p.audio, `${field}.audio`, ["masterGainDb"]);
|
||||
return {
|
||||
scenes, buses, sources, overlays,
|
||||
transition: transition(p.transition, `${field}.transition`),
|
||||
audio: { masterGainDb: gainDb(a.masterGainDb, `${field}.audio.masterGainDb`) },
|
||||
};
|
||||
}
|
||||
|
||||
export function isProductionSpec(value: unknown): value is ProductionSpec {
|
||||
try {
|
||||
parseProductionSpec(value);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Every remote URL a production snapshot can make the worker fetch. */
|
||||
export function productionHlsUrls(spec: ProductionSpec): string[] {
|
||||
return spec.sources.flatMap((s) => (s.kind === "hls" ? [s.url] : []));
|
||||
}
|
||||
|
||||
/** Declared key order for signing: insertion order of the input can never drift the signed bytes. */
|
||||
export function canonicalProduction(spec: ProductionSpec): ProductionSpec {
|
||||
return {
|
||||
scenes: spec.scenes.map((s) => ({
|
||||
id: s.id, label: s.label, layout: s.layout,
|
||||
slots: s.slots.map((x) => ({
|
||||
index: x.index, sourceId: x.sourceId,
|
||||
rect: x.rect === null ? null : { x: x.rect.x, y: x.rect.y, w: x.rect.w, h: x.rect.h },
|
||||
fit: x.fit, audioMode: x.audioMode, gainDb: x.gainDb,
|
||||
})),
|
||||
})),
|
||||
buses: { preview: spec.buses.preview, program: spec.buses.program },
|
||||
sources: spec.sources.map((s): ProductionSource => (s.kind === "hls"
|
||||
? { id: s.id, label: s.label, kind: "hls", url: s.url }
|
||||
: { id: s.id, label: s.label, kind: "playlist", assetIds: [...s.assetIds], loop: s.loop })),
|
||||
overlays: spec.overlays.map((o) => ({
|
||||
id: o.id, sceneId: o.sceneId, kind: o.kind, enabled: o.enabled, layer: o.layer,
|
||||
position: { x: o.position.x, y: o.position.y },
|
||||
revisionId: o.revisionId, assetId: o.assetId,
|
||||
chyron: o.chyron === null ? null : { speedPxPerSec: o.chyron.speedPxPerSec, stripWidthPx: o.chyron.stripWidthPx },
|
||||
clock: o.clock === null ? null : { format: o.clock.format },
|
||||
})),
|
||||
transition: spec.transition.type === "cut" ? { type: "cut" } : { type: "fade", ms: spec.transition.ms },
|
||||
audio: { masterGainDb: spec.audio.masterGainDb },
|
||||
};
|
||||
}
|
||||
39
apps/media-worker/src/relay/endpoints.ts
Normal file
39
apps/media-worker/src/relay/endpoints.ts
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
import { BlockList, isIP } from "node:net";
|
||||
|
||||
export type EndpointUse = "hls" | "output";
|
||||
export interface ValidatedEndpoint { url: URL; pinnedAddress: string }
|
||||
|
||||
/** WP7 supplies DNS pinning, redirect and HLS child-URI validation. */
|
||||
export interface EndpointValidator {
|
||||
validate(raw: string, use: EndpointUse): Promise<ValidatedEndpoint>;
|
||||
}
|
||||
|
||||
const blocked = new BlockList();
|
||||
for (const [subnet, prefix] of [
|
||||
["0.0.0.0", 8], ["10.0.0.0", 8], ["100.64.0.0", 10], ["127.0.0.0", 8],
|
||||
["169.254.0.0", 16], ["172.16.0.0", 12], ["192.0.0.0", 24], ["192.168.0.0", 16],
|
||||
["192.0.2.0", 24], ["198.18.0.0", 15], ["198.51.100.0", 24],
|
||||
["203.0.113.0", 24], ["224.0.0.0", 4], ["240.0.0.0", 4],
|
||||
] as const) blocked.addSubnet(subnet, prefix, "ipv4");
|
||||
for (const [subnet, prefix] of [
|
||||
["::", 128], ["::1", 128], ["fc00::", 7], ["fe80::", 10], ["ff00::", 8],
|
||||
["2001::", 32], ["2001:db8::", 32], ["2002::", 16], ["64:ff9b::", 96],
|
||||
] as const) blocked.addSubnet(subnet, prefix, "ipv6");
|
||||
|
||||
export function assertPublicAddress(address: string): void {
|
||||
const normalized = address.replace(/^\[|\]$/g, "").toLowerCase();
|
||||
const family = isIP(normalized);
|
||||
if (family === 0) throw new Error("unresolved endpoint address");
|
||||
// Node's BlockList normalizes IPv4 to mapped IPv6 internally; reject mapped literals outright.
|
||||
if (normalized.startsWith("::ffff:")) throw new Error("mapped endpoint rejected");
|
||||
if (blocked.check(normalized, family === 4 ? "ipv4" : "ipv6")) throw new Error("private endpoint rejected");
|
||||
}
|
||||
|
||||
export function validateEndpointShape(raw: string, use: EndpointUse): URL {
|
||||
let url: URL;
|
||||
try { url = new URL(raw); } catch { throw new Error("invalid endpoint URL"); }
|
||||
const schemes = use === "hls" ? ["https:"] : ["rtmp:", "rtmps:"];
|
||||
if (!schemes.includes(url.protocol) || !url.hostname || url.username || url.password || url.search || url.hash) throw new Error("endpoint scheme or credentials rejected");
|
||||
if (isIP(url.hostname.replace(/^\[|\]$/g, ""))) assertPublicAddress(url.hostname);
|
||||
return url;
|
||||
}
|
||||
122
apps/media-worker/src/relay/ffmpeg.ts
Normal file
122
apps/media-worker/src/relay/ffmpeg.ts
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
import { spawn, type ChildProcess, type StdioOptions } from "node:child_process";
|
||||
import { OUTPUT_PROFILES, type DesiredChannel } from "../types";
|
||||
import { assertPublicAddress, validateEndpointShape, type EndpointValidator } from "./endpoints";
|
||||
|
||||
/** The vault broker delivers a key directly to the push relay's inherited fd. */
|
||||
export interface VaultHandleBroker {
|
||||
deliverPushTarget(handleId: string, context: { workspaceId: string; channelId: string; assignmentEpoch: number }, relayFd: number): Promise<void>;
|
||||
}
|
||||
export interface LocalRelayPipes { ingestFd: number; pushFd: number; close(): Promise<void> }
|
||||
/** WP7 implementation fetches HLS through pinned egress and pushes RTMP with the vault key. */
|
||||
export interface EgressRelayFactory {
|
||||
prepare(config: DesiredChannel, validator: EndpointValidator, vault: VaultHandleBroker): Promise<LocalRelayPipes>;
|
||||
}
|
||||
export interface StderrRedactor { redact(chunk: string): string }
|
||||
|
||||
/** Media children get a minimal environment: no worker secrets, no inherited config. */
|
||||
export function mediaEnv(): NodeJS.ProcessEnv { return { PATH: process.env.PATH ?? "/usr/bin:/bin" }; }
|
||||
|
||||
export interface ManagedProcess {
|
||||
readonly child: ChildProcess;
|
||||
/** Resolves on spawn, rejects if the binary cannot start. */
|
||||
readonly started: Promise<void>;
|
||||
/** Resolves once the process has exited (never rejects). */
|
||||
readonly exited: Promise<{ code: number | null; signal: NodeJS.Signals | null }>;
|
||||
kill(signal?: NodeJS.Signals): void;
|
||||
/** SIGTERM, then SIGKILL after `graceMs`; resolves on exit. */
|
||||
terminate(graceMs?: number): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared spawn path for every media child (relay, ingest, compositor, encoder,
|
||||
* frame gate): argv only, never a shell; minimal env; stderr is redacted and
|
||||
* truncated before it reaches the logger.
|
||||
*/
|
||||
export function spawnManaged(
|
||||
command: string, args: readonly string[], stdio: StdioOptions,
|
||||
redactor: StderrRedactor, onStderr: (message: string) => void, cwd?: string,
|
||||
): ManagedProcess {
|
||||
const child = spawn(command, args, { stdio, shell: false, env: mediaEnv(), ...(cwd === undefined ? {} : { cwd }) });
|
||||
child.stderr?.on("data", (chunk: Buffer) => onStderr(redactor.redact(chunk.toString("utf8").slice(0, 4096))));
|
||||
const started = new Promise<void>((resolve, reject) => {
|
||||
child.once("spawn", () => resolve());
|
||||
child.once("error", reject);
|
||||
});
|
||||
started.catch(() => undefined);
|
||||
const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => {
|
||||
child.once("exit", (code, signal) => resolve({ code, signal }));
|
||||
child.once("error", () => { if (child.pid === undefined) resolve({ code: null, signal: null }); });
|
||||
});
|
||||
const kill = (signal: NodeJS.Signals = "SIGTERM"): void => { if (child.exitCode === null && child.signalCode === null) child.kill(signal); };
|
||||
return {
|
||||
child, started, exited, kill,
|
||||
async terminate(graceMs = 2_000) {
|
||||
kill("SIGTERM");
|
||||
const timer = setTimeout(() => kill("SIGKILL"), graceMs);
|
||||
await exited;
|
||||
clearTimeout(timer);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function ffmpegArgs(config: DesiredChannel): string[] {
|
||||
const args = ["-hide_banner", "-nostdin", "-loglevel", "warning", "-f", "mpegts", "-i", "pipe:3"];
|
||||
if (config.mode === "passthrough") args.push("-c", "copy");
|
||||
else {
|
||||
const profile = OUTPUT_PROFILES[config.profile];
|
||||
args.push("-vf", `scale=${profile.width}:${profile.height},fps=${profile.fps}`,
|
||||
"-c:v", "libx264", "-preset", "veryfast", "-b:v", profile.videoBitrate,
|
||||
"-c:a", "aac", "-b:a", "128k");
|
||||
}
|
||||
args.push("-f", "flv", "pipe:4");
|
||||
return args;
|
||||
}
|
||||
|
||||
export class FfmpegRelayJob {
|
||||
private child: ChildProcess | null = null;
|
||||
private pipes: LocalRelayPipes | null = null;
|
||||
private stopping = false;
|
||||
constructor(
|
||||
private readonly validator: EndpointValidator,
|
||||
private readonly egress: EgressRelayFactory,
|
||||
private readonly vault: VaultHandleBroker,
|
||||
private readonly redactor: StderrRedactor,
|
||||
private readonly onStderr: (message: string) => void,
|
||||
private readonly onExit: (reason: string) => void,
|
||||
) {}
|
||||
|
||||
async start(config: DesiredChannel): Promise<void> {
|
||||
if (!config.source || !config.output) throw new Error("running channel needs source and output");
|
||||
validateEndpointShape(config.source.url, "hls");
|
||||
validateEndpointShape(config.output.endpointUrl, "output");
|
||||
const source = await this.validator.validate(config.source.url, "hls");
|
||||
const output = await this.validator.validate(config.output.endpointUrl, "output");
|
||||
assertPublicAddress(source.pinnedAddress);
|
||||
assertPublicAddress(output.pinnedAddress);
|
||||
this.pipes = await this.egress.prepare(config, this.validator, this.vault);
|
||||
this.stopping = false;
|
||||
const managed = spawnManaged("ffmpeg", ffmpegArgs(config),
|
||||
["ignore", "ignore", "pipe", this.pipes.ingestFd, this.pipes.pushFd], this.redactor, this.onStderr);
|
||||
const child = managed.child;
|
||||
this.child = child;
|
||||
let spawned = false;
|
||||
void managed.started.then(() => { spawned = true; }, () => undefined);
|
||||
child.on("error", (error) => { if (spawned) this.onExit(this.redactor.redact(error.message)); });
|
||||
const started = managed.started;
|
||||
child.once("exit", (code, signal) => {
|
||||
void this.pipes?.close();
|
||||
this.pipes = null;
|
||||
this.child = null;
|
||||
if (!this.stopping) this.onExit(`ffmpeg exited code=${code} signal=${signal}`);
|
||||
});
|
||||
await started;
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.stopping = true;
|
||||
this.child?.kill("SIGTERM");
|
||||
await this.pipes?.close();
|
||||
this.pipes = null;
|
||||
this.child = null;
|
||||
}
|
||||
}
|
||||
56
apps/media-worker/src/relay/relay.test.ts
Normal file
56
apps/media-worker/src/relay/relay.test.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { resolve } from "node:path";
|
||||
import test from "node:test";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { assertPublicAddress, validateEndpointShape } from "./endpoints";
|
||||
import { ffmpegArgs } from "./ffmpeg";
|
||||
import type { DesiredChannel } from "../types";
|
||||
|
||||
const channel: DesiredChannel = {
|
||||
workspaceId: "workspace-1", channelId: "channel-1", workerId: "worker-1",
|
||||
assignmentEpoch: 1, desiredVersion: 1, desiredStatus: "RUNNING",
|
||||
source: { kind: "hls", url: "https://mock.local/hls/live-1/master.m3u8" },
|
||||
output: { endpointUrl: "rtmp://mock.local/live", credentialHandleId: "vault-handle" },
|
||||
mode: "passthrough", profile: "720p30",
|
||||
};
|
||||
|
||||
test("relay argv uses only local inherited pipes and mode profile", () => {
|
||||
const passthrough = ffmpegArgs(channel);
|
||||
assert.deepEqual(passthrough.slice(-3), ["-f", "flv", "pipe:4"]);
|
||||
assert.ok(passthrough.includes("copy"));
|
||||
assert.ok(!passthrough.join(" ").includes("mock.local"));
|
||||
assert.ok(!passthrough.join(" ").includes("vault-handle"));
|
||||
const normalized = ffmpegArgs({ ...channel, mode: "normalize", profile: "1080p60" });
|
||||
assert.ok(normalized.includes("scale=1920:1080,fps=60"));
|
||||
});
|
||||
|
||||
test("endpoint shape and literal/private pin denial", () => {
|
||||
assert.equal(validateEndpointShape(channel.source?.url ?? "", "hls").protocol, "https:");
|
||||
assert.throws(() => validateEndpointShape("file:///etc/passwd", "hls"));
|
||||
assert.throws(() => validateEndpointShape("https://127.0.0.1/live.m3u8", "hls"));
|
||||
assert.throws(() => validateEndpointShape("rtmp://169.254.169.254/live", "output"));
|
||||
assert.throws(() => validateEndpointShape("rtmp://user:secret@example.com/live", "output"));
|
||||
for (const address of ["10.0.0.1", "172.20.1.2", "192.168.1.1", "100.64.0.1", "::1", "fc00::1", "::ffff:127.0.0.1", "::ffff:7f00:1"]) {
|
||||
assert.throws(() => assertPublicAddress(address), address);
|
||||
}
|
||||
assert.doesNotThrow(() => assertPublicAddress("8.8.8.8"));
|
||||
});
|
||||
|
||||
test("mock livestream source and single-live simulator fixture feed the relay contract", async () => {
|
||||
const mockPath = pathToFileURL(resolve(__dirname, "../../../../packages/connector-mock-livestream/src/index.ts")).href;
|
||||
const fixturePath = pathToFileURL(resolve(__dirname, "../../../../packages/testing/src/scenarios.ts")).href;
|
||||
const mockModule: unknown = await import(mockPath);
|
||||
const fixtures: unknown = await import(fixturePath);
|
||||
const mock = mockModule as { createMockLivestreamConnector(): {
|
||||
createLiveStream(connection: null, request: { channelId: string; title: string; permanent: boolean; saveReplay: boolean }): Promise<{ remoteId: string }>;
|
||||
getStreamPlaybackSources(connection: null, request: { remoteId: string }): Promise<Array<{ url: string; kind: string }>>;
|
||||
} };
|
||||
const scenarios = fixtures as { SCENARIOS: Record<string, { steps: Array<{ source: string; action: string }> }> };
|
||||
assert.deepEqual(scenarios.SCENARIOS["single-live"]?.steps.map((step) => [step.source, step.action]), [["A", "go-live"]]);
|
||||
const connector = mock.createMockLivestreamConnector();
|
||||
const live = await connector.createLiveStream(null, { channelId: "A", title: "test", permanent: true, saveReplay: false });
|
||||
const sources = await connector.getStreamPlaybackSources(null, { remoteId: live.remoteId });
|
||||
assert.equal(sources[0]?.kind, "hls");
|
||||
assert.equal(validateEndpointShape(sources[0]?.url ?? "", "hls").hostname, "mock.local");
|
||||
assert.ok(!ffmpegArgs({ ...channel, source: { kind: "hls", url: sources[0]?.url ?? "" } }).join(" ").includes("mock.local"));
|
||||
});
|
||||
249
apps/media-worker/src/supervisor/index.ts
Normal file
249
apps/media-worker/src/supervisor/index.ts
Normal file
|
|
@ -0,0 +1,249 @@
|
|||
import type { KeyObject } from "node:crypto";
|
||||
import { sandboxRelay, supervisorSandboxEnabled } from "./sandbox";
|
||||
import { verifyCommand } from "../bus/envelope";
|
||||
import { DurableFence } from "../bus/fence";
|
||||
import type { ObservedStore } from "../bus/redis";
|
||||
import type { ChannelState, DesiredChannel, ObservedChannel } from "../types";
|
||||
|
||||
export interface TransitionLogger {
|
||||
transition(event: { workspaceId: string; channelId: string; from: ChannelState; to: ChannelState; reason: string }): void;
|
||||
}
|
||||
interface ChannelRuntime {
|
||||
desired: DesiredChannel;
|
||||
state: ChannelState;
|
||||
reason: string;
|
||||
realizedVersion: number | null;
|
||||
relay: RelayJob | null;
|
||||
retries: number;
|
||||
retryTimer: { cancel(): void } | null;
|
||||
/** Consecutive failed swaps for the current desired version (the previous version stays on air). */
|
||||
swapFailures: number;
|
||||
}
|
||||
export interface RelayJob {
|
||||
start(channel: DesiredChannel): Promise<void>;
|
||||
stop(): Promise<void>;
|
||||
/**
|
||||
* Optional in-place change (production jobs: an A/B compositor swap behind the
|
||||
* frame gate). Resolves true once the new version is on air; false means the
|
||||
* change needs a restart (sources, output or profile changed).
|
||||
*/
|
||||
reconfigure?(channel: DesiredChannel): Promise<boolean>;
|
||||
}
|
||||
export interface RetryScheduler { schedule(action: () => Promise<void>, delayMs: number): { cancel(): void } }
|
||||
|
||||
/**
|
||||
* A swap that fails (B never went on air; A is still on air) is retried this many times, 1 s then
|
||||
* 2 s later, before the supervisor falls back to a full restart. The full restart converges too,
|
||||
* but it takes the encoder and the push session down (WP9 "output that does not stop").
|
||||
*/
|
||||
export const SWAP_RETRIES = 2;
|
||||
const defaultRetryScheduler: RetryScheduler = {
|
||||
schedule(action, delayMs) {
|
||||
const timer = setTimeout(() => { void action(); }, delayMs);
|
||||
return { cancel: () => clearTimeout(timer) };
|
||||
},
|
||||
};
|
||||
|
||||
/** Accepted versions are durable; realized versions describe only this process lifetime. */
|
||||
export class MediaSupervisor {
|
||||
private readonly channels = new Map<string, ChannelRuntime>();
|
||||
private readonly operations = new Map<string, Promise<unknown>>();
|
||||
private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
|
||||
/** Set by `stop()`: late retries and relaunches become no-ops. */
|
||||
private stopped = false;
|
||||
constructor(
|
||||
private readonly workerId: string,
|
||||
desired: DesiredChannel[],
|
||||
private readonly store: ObservedStore,
|
||||
private readonly fence: DurableFence,
|
||||
private readonly controlKeys: ReadonlyMap<string, KeyObject>,
|
||||
private readonly createRelay: (channel: DesiredChannel, onExit: (reason: string) => void) => RelayJob,
|
||||
private readonly logger: TransitionLogger,
|
||||
private readonly retryScheduler: RetryScheduler = defaultRetryScheduler,
|
||||
) {
|
||||
supervisorSandboxEnabled(); // Production cannot silently opt out of confinement.
|
||||
for (const channel of desired) {
|
||||
if (channel.workerId !== workerId) throw new Error("config worker mismatch");
|
||||
this.channels.set(this.key(channel.workspaceId, channel.channelId), {
|
||||
desired: channel, state: "STOPPED", reason: "worker boot", realizedVersion: null,
|
||||
relay: null, retries: 0, retryTimer: null, swapFailures: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private key(workspaceId: string, channelId: string): string { return `${workspaceId}:${channelId}`; }
|
||||
status(): ObservedChannel[] {
|
||||
return [...this.channels.values()].map((runtime) => ({
|
||||
workspaceId: runtime.desired.workspaceId, channelId: runtime.desired.channelId,
|
||||
workerId: this.workerId, assignmentEpoch: runtime.desired.assignmentEpoch,
|
||||
state: runtime.state, realizedVersion: runtime.realizedVersion,
|
||||
updatedAt: new Date().toISOString(), reason: runtime.reason,
|
||||
}));
|
||||
}
|
||||
private async transition(runtime: ChannelRuntime, to: ChannelState, reason: string): Promise<void> {
|
||||
const from = runtime.state;
|
||||
runtime.state = to;
|
||||
runtime.reason = reason;
|
||||
this.logger.transition({ workspaceId: runtime.desired.workspaceId, channelId: runtime.desired.channelId, from, to, reason });
|
||||
await this.store.put(this.status().find((item) => item.workspaceId === runtime.desired.workspaceId && item.channelId === runtime.desired.channelId)!);
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
for (const runtime of this.channels.values()) {
|
||||
const old = await this.store.get(runtime.desired.workspaceId, runtime.desired.channelId);
|
||||
await this.transition(runtime, "STOPPED", old
|
||||
? `restart reconcile: prior ${old.state}; awaiting fresh signed snapshot`
|
||||
: "start reconcile: awaiting signed snapshot");
|
||||
}
|
||||
await this.beat();
|
||||
this.heartbeatTimer = setInterval(() => { void this.beat().catch((error: unknown) => this.logger.transition({ workspaceId: "", channelId: "", from: "DEGRADED", to: "DEGRADED", reason: `heartbeat failed: ${String(error)}` })); }, 5_000);
|
||||
}
|
||||
|
||||
async beat(): Promise<void> { await this.store.heartbeat(this.workerId, this.status()); }
|
||||
|
||||
async handle(raw: unknown, now = Date.now()): Promise<"accepted" | "replay" | "stale"> {
|
||||
const command = verifyCommand(raw, this.controlKeys, this.workerId, now);
|
||||
const body = command.body;
|
||||
return this.serialize(this.key(body.workspaceId, body.channelId), () => this.apply(body, now));
|
||||
}
|
||||
|
||||
/** One operation at a time per channel (commands and swap retries). */
|
||||
private serialize<T>(key: string, operation: () => Promise<T>): Promise<T> {
|
||||
const previous = this.operations.get(key) ?? Promise.resolve();
|
||||
const task = previous.catch(() => undefined).then(operation);
|
||||
this.operations.set(key, task);
|
||||
void task.then(() => { if (this.operations.get(key) === task) this.operations.delete(key); },
|
||||
() => { if (this.operations.get(key) === task) this.operations.delete(key); });
|
||||
return task;
|
||||
}
|
||||
|
||||
private async apply(body: ReturnType<typeof verifyCommand>["body"], now: number): Promise<"accepted" | "replay" | "stale"> {
|
||||
const runtime = this.channels.get(this.key(body.workspaceId, body.channelId));
|
||||
if (!runtime || runtime.desired.assignmentEpoch !== body.assignmentEpoch) throw new Error("unassigned channel or epoch");
|
||||
const result = this.fence.accept(body, now);
|
||||
if (result !== "accepted") return result;
|
||||
// RUNNING → RUNNING on a job that can swap: the encoder and push session stay up,
|
||||
// and the version is realized only once the new compositor is on air.
|
||||
const relay = runtime.relay;
|
||||
if (runtime.state === "RUNNING" && body.snapshot.desiredStatus === "RUNNING" && relay?.reconfigure) {
|
||||
runtime.desired = body.snapshot;
|
||||
runtime.retryTimer?.cancel();
|
||||
runtime.retryTimer = null;
|
||||
runtime.swapFailures = 0;
|
||||
if (await this.swap(runtime, relay) !== "restart") return result;
|
||||
}
|
||||
runtime.desired = body.snapshot;
|
||||
runtime.swapFailures = 0;
|
||||
runtime.realizedVersion = null;
|
||||
runtime.retryTimer?.cancel();
|
||||
runtime.retryTimer = null;
|
||||
await runtime.relay?.stop();
|
||||
runtime.relay = null;
|
||||
if (body.snapshot.desiredStatus === "STOPPED") {
|
||||
await this.transition(runtime, "STOPPED", "signed desired state stopped");
|
||||
runtime.realizedVersion = body.desiredVersion;
|
||||
await this.beat();
|
||||
return result;
|
||||
}
|
||||
runtime.retries = 0;
|
||||
await this.launch(runtime);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* One swap attempt for `runtime.desired`. "done": on air (or the job is gone and its own exit
|
||||
* path owns the channel); "retrying": B failed, A stays on air (`realizedVersion` keeps naming
|
||||
* what viewers see) and a retry is scheduled; "restart": not swappable, or out of retries.
|
||||
*/
|
||||
private async swap(runtime: ChannelRuntime, relay: RelayJob): Promise<"done" | "retrying" | "restart"> {
|
||||
const version = runtime.desired.desiredVersion;
|
||||
try {
|
||||
if (!(await relay.reconfigure?.(runtime.desired))) return "restart";
|
||||
if (runtime.relay !== relay) return "done";
|
||||
runtime.swapFailures = 0;
|
||||
runtime.realizedVersion = version;
|
||||
await this.transition(runtime, "RUNNING", "compositor swapped at a frame boundary");
|
||||
await this.beat();
|
||||
return "done";
|
||||
} catch (error: unknown) {
|
||||
if (runtime.relay !== relay) return "done";
|
||||
const name = error instanceof Error ? error.name : "unknown error";
|
||||
runtime.swapFailures += 1;
|
||||
if (runtime.swapFailures > SWAP_RETRIES) {
|
||||
runtime.reason = `swap failed: ${name}`;
|
||||
return "restart";
|
||||
}
|
||||
const delay = 1_000 * 2 ** (runtime.swapFailures - 1);
|
||||
await this.transition(runtime, "RUNNING", `swap to v${version} failed (${name}); v${runtime.realizedVersion ?? "?"} stays on air; retry ${runtime.swapFailures}/${SWAP_RETRIES} in ${delay}ms`);
|
||||
const key = this.key(runtime.desired.workspaceId, runtime.desired.channelId);
|
||||
runtime.retryTimer = this.retryScheduler.schedule(() => this.serialize(key, async () => {
|
||||
runtime.retryTimer = null;
|
||||
if (this.stopped) return;
|
||||
// A newer command or an exit has taken over since.
|
||||
if (runtime.desired.desiredVersion !== version || runtime.relay !== relay || runtime.state !== "RUNNING") return;
|
||||
if (await this.swap(runtime, relay) === "restart") await this.restart(runtime);
|
||||
}), delay);
|
||||
return "retrying";
|
||||
}
|
||||
}
|
||||
|
||||
/** Full restart for the current desired snapshot (the swap path gave up). */
|
||||
private async restart(runtime: ChannelRuntime): Promise<void> {
|
||||
runtime.realizedVersion = null;
|
||||
runtime.swapFailures = 0;
|
||||
await runtime.relay?.stop();
|
||||
runtime.relay = null;
|
||||
runtime.retries = 0;
|
||||
await this.launch(runtime);
|
||||
}
|
||||
|
||||
private async launch(runtime: ChannelRuntime): Promise<void> {
|
||||
await this.transition(runtime, runtime.retries ? "RESTARTING" : "STARTING", runtime.retries ? "relay retry" : "signed desired state running");
|
||||
try {
|
||||
const relay = sandboxRelay(this.createRelay(runtime.desired, (reason) => {
|
||||
if (runtime.relay === relay) void this.relayExited(runtime, reason);
|
||||
}), runtime.desired);
|
||||
runtime.relay = relay;
|
||||
await relay.start(runtime.desired);
|
||||
if (runtime.relay !== relay) return;
|
||||
runtime.realizedVersion = runtime.desired.desiredVersion;
|
||||
await this.transition(runtime, "RUNNING", "ffmpeg spawned with local relay pipes");
|
||||
await this.beat();
|
||||
} catch (error: unknown) {
|
||||
runtime.relay = null;
|
||||
await this.relayExited(runtime, `start failed: ${error instanceof Error ? error.name : "unknown error"}`);
|
||||
}
|
||||
}
|
||||
|
||||
private async relayExited(runtime: ChannelRuntime, reason: string): Promise<void> {
|
||||
if (this.stopped || runtime.desired.desiredStatus !== "RUNNING") return;
|
||||
runtime.realizedVersion = null;
|
||||
runtime.relay = null;
|
||||
await this.transition(runtime, "DEGRADED", reason);
|
||||
if (runtime.retries >= 5) { await this.transition(runtime, "FAILED", "relay retry budget exhausted"); return; }
|
||||
const delay = Math.min(30_000, 1_000 * 2 ** runtime.retries);
|
||||
runtime.retries += 1;
|
||||
await this.transition(runtime, "RESTARTING", `retry in ${delay}ms`);
|
||||
runtime.retryTimer = this.retryScheduler.schedule(() => { runtime.retryTimer = null; return this.stopped ? Promise.resolve() : this.launch(runtime); }, delay);
|
||||
}
|
||||
|
||||
/**
|
||||
* Worker shutdown. In-flight per-channel operations (a command, a swap retry, a relaunch) finish
|
||||
* first, so none of them can start a job after the channels were stopped.
|
||||
*/
|
||||
async stop(): Promise<void> {
|
||||
this.stopped = true;
|
||||
if (this.heartbeatTimer) clearInterval(this.heartbeatTimer);
|
||||
this.heartbeatTimer = null;
|
||||
for (const runtime of this.channels.values()) runtime.retryTimer?.cancel();
|
||||
await Promise.allSettled([...this.operations.values()]);
|
||||
for (const runtime of this.channels.values()) {
|
||||
runtime.retryTimer?.cancel();
|
||||
runtime.retryTimer = null;
|
||||
runtime.desired = { ...runtime.desired, desiredStatus: "STOPPED" };
|
||||
await runtime.relay?.stop();
|
||||
runtime.realizedVersion = null;
|
||||
await this.transition(runtime, "STOPPED", "worker shutdown");
|
||||
}
|
||||
}
|
||||
}
|
||||
215
apps/media-worker/src/supervisor/production.e2e.test.ts
Normal file
215
apps/media-worker/src/supervisor/production.e2e.test.ts
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
/**
|
||||
* A/B lifecycle end to end (streaming-architecture.md §6 S2): signed snapshots
|
||||
* through the real MediaSupervisor into a real ProductionJob (synthetic
|
||||
* ingests standing in for the WP7 relay, a playlist source through the asset
|
||||
* feeder, the Rust gate, an x264 encoder into `-f null`, preview taps).
|
||||
*
|
||||
* - a TAKE (program scene change) is a swap: the encoder process is the same
|
||||
* one, and the version is realized only once the gate reports B on air;
|
||||
* - a playlist edit is adopted by its feeder in place (still the same encoder);
|
||||
* - a changed source URL is a restart (a new encoder);
|
||||
* - STOPPED tears everything down.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { generateKeyPairSync, randomUUID } from "node:crypto";
|
||||
import { createReadStream, mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { signCommand, type CommandBody } from "../bus/envelope";
|
||||
import { DurableFence } from "../bus/fence";
|
||||
import type { ObservedStore } from "../bus/redis";
|
||||
import { ProductionJob } from "../compositor/production-job";
|
||||
import { ensureGateBinary, makeAssets, passthroughRedactor } from "../compositor/testing";
|
||||
import { parseProductionSpec, type ProductionSpec } from "../production";
|
||||
import type { PreviewContentType } from "../preview/contract";
|
||||
import type { DesiredChannel, ObservedChannel } from "../types";
|
||||
import { MediaSupervisor, type RelayJob } from "./index";
|
||||
|
||||
const id = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`;
|
||||
const WS = "8f0c5a8e-3b1d-4c3e-9a5b-0c1d2e3f4a5b";
|
||||
const CH = "1a2b3c4d-5e6f-4a1b-8c2d-3e4f5a6b7c8d";
|
||||
|
||||
function spec(opts: { take: 10 | 11; playlist: string[]; urlB?: string }): ProductionSpec {
|
||||
return parseProductionSpec({
|
||||
sources: [
|
||||
{ id: id(1), label: "Cam A", kind: "hls", url: "https://a.example/live.m3u8" },
|
||||
{ id: id(2), label: "Cam B", kind: "hls", url: opts.urlB ?? "https://b.example/live.m3u8" },
|
||||
{ id: id(3), label: "Loop", kind: "playlist", assetIds: opts.playlist, loop: true },
|
||||
],
|
||||
scenes: [
|
||||
{ id: id(10), label: "Two up", layout: "2x1", slots: [
|
||||
{ index: 0, sourceId: id(1), rect: null, fit: "cover", audioMode: "primary", gainDb: 0 },
|
||||
{ index: 1, sourceId: id(2), rect: null, fit: "cover", audioMode: "on", gainDb: -6 },
|
||||
] },
|
||||
{ id: id(11), label: "Playlist", layout: "pip", slots: [
|
||||
{ index: 0, sourceId: id(3), rect: null, fit: "contain", audioMode: "primary", gainDb: 0 },
|
||||
{ index: 1, sourceId: id(1), rect: { x: 6500, y: 6500, w: 3000, h: 3000 }, fit: "cover", audioMode: "off", gainDb: 0 },
|
||||
] },
|
||||
],
|
||||
buses: { preview: id(10), program: id(opts.take) },
|
||||
overlays: [
|
||||
{ id: id(20), sceneId: id(10), kind: "lower_third", enabled: true, layer: 1, position: { x: 500, y: 7500 }, revisionId: id(30), assetId: "a".repeat(64), chyron: null, clock: null },
|
||||
],
|
||||
transition: { type: "cut" },
|
||||
audio: { masterGainDb: 0 },
|
||||
});
|
||||
}
|
||||
|
||||
class MemoryObserved implements ObservedStore {
|
||||
rows = new Map<string, ObservedChannel>();
|
||||
async get(workspaceId: string, channelId: string): Promise<ObservedChannel | null> { return this.rows.get(`${workspaceId}:${channelId}`) ?? null; }
|
||||
async put(row: ObservedChannel): Promise<void> { this.rows.set(`${row.workspaceId}:${row.channelId}`, row); }
|
||||
async heartbeat(_workerId: string, channels: ObservedChannel[]): Promise<void> { for (const row of channels) await this.put(row); }
|
||||
}
|
||||
|
||||
const alive = (pid: number | undefined): boolean => pid !== undefined && spawnSync("kill", ["-0", String(pid)]).status === 0;
|
||||
|
||||
test("signed TAKE / playlist edit / source change / stop through the supervisor into a real production job", { timeout: 300_000 }, async (t) => {
|
||||
if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; }
|
||||
const dir = mkdtempSync(join(tmpdir(), "prod-e2e-"));
|
||||
const [overlay] = makeAssets(dir);
|
||||
// Two 2 s playlist assets (pre-normalized MP4, faststart), served as byte streams like a presigned GET.
|
||||
const clips = new Map<string, string>();
|
||||
for (const [n, hz] of [[40, 300], [41, 600], [42, 900]] as const) {
|
||||
const path = join(dir, `asset-${n}.mp4`);
|
||||
const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=s=1280x720:r=30:d=2", "-f", "lavfi", "-i", `sine=frequency=${hz}:d=2`,
|
||||
"-c:v", "libx264", "-preset", "ultrafast", "-g", "30", "-c:a", "aac", "-movflags", "+faststart", path]);
|
||||
assert.equal(r.status, 0, "asset encode");
|
||||
clips.set(id(n), path);
|
||||
}
|
||||
const opened: string[] = [];
|
||||
const store = new MemoryObserved();
|
||||
const keys = generateKeyPairSync("ed25519");
|
||||
const published = new Map<string, number>();
|
||||
const jobs: ProductionJob[] = [];
|
||||
const exits: string[] = [];
|
||||
const createRelay = (_channel: DesiredChannel, onExit: (reason: string) => void): RelayJob => {
|
||||
const job = new ProductionJob({
|
||||
validator: { async validate() { throw new Error("not used: synthetic egress"); } },
|
||||
vault: { async deliverPushTarget() { throw new Error("not used: null output"); } },
|
||||
egress: {
|
||||
// Stand-in for the WP7 relay: each HLS source becomes a synthetic, real-time ingest.
|
||||
async prepare(config) {
|
||||
const sources = new Map((config.production?.sources ?? []).filter((s) => s.kind === "hls")
|
||||
.map((s, i) => [s.id, { input: { kind: "synthetic" as const, pattern: "smptehdbars" as const, seed: i + 1, toneHz: 440 + 220 * i }, fd: null }]));
|
||||
return { sources, output: { kind: "null" as const }, async close() {} };
|
||||
},
|
||||
},
|
||||
assets: { async localPath() { return overlay!; } },
|
||||
playlists: {
|
||||
async open(assetId) {
|
||||
opened.push(assetId);
|
||||
const path = clips.get(assetId);
|
||||
if (!path) throw new Error("unknown asset");
|
||||
return createReadStream(path);
|
||||
},
|
||||
},
|
||||
preview: {
|
||||
objects: { async putContent(body: Buffer, contentType: PreviewContentType) { return { key: `sha256/${"0".repeat(64)}.${contentType === "image/jpeg" ? "jpg" : "bin"}` }; } },
|
||||
manifests: { async set(key: string) { published.set(key, (published.get(key) ?? 0) + 1); } },
|
||||
},
|
||||
redactor: passthroughRedactor, onStderr: () => undefined,
|
||||
onExit: (reason) => { exits.push(reason); onExit(reason); },
|
||||
scratchDir: join(dir, `scratch-${jobs.length}`),
|
||||
});
|
||||
jobs.push(job);
|
||||
return job;
|
||||
};
|
||||
const base: DesiredChannel = {
|
||||
workspaceId: WS, channelId: CH, workerId: "worker-1", assignmentEpoch: 1, desiredVersion: 0, desiredStatus: "RUNNING",
|
||||
source: null, output: { endpointUrl: "rtmp://push.example/live", credentialHandleId: "keyref-1" }, profile: "720p30", mode: "normalize",
|
||||
production: spec({ take: 10, playlist: [id(40), id(41)] }),
|
||||
};
|
||||
const worker = new MediaSupervisor("worker-1", [base], store, new DurableFence(join(dir, "fence.json")),
|
||||
new Map([["control-v1", keys.publicKey]]), createRelay, { transition() {} });
|
||||
const send = (version: number, production: ProductionSpec, status: "RUNNING" | "STOPPED" = "RUNNING") => {
|
||||
const body: CommandBody = {
|
||||
workspaceId: WS, channelId: CH, workerId: "worker-1", assignmentEpoch: 1, commandId: randomUUID(), desiredVersion: version,
|
||||
issuedAt: new Date(Date.now() - 1_000).toISOString(), expiresAt: new Date(Date.now() + 120_000).toISOString(),
|
||||
snapshot: { ...base, desiredVersion: version, desiredStatus: status, production },
|
||||
};
|
||||
return worker.handle(signCommand(body, "control-v1", keys.privateKey));
|
||||
};
|
||||
const status = () => worker.status()[0]!;
|
||||
try {
|
||||
await worker.start();
|
||||
assert.equal(await send(1, spec({ take: 10, playlist: [id(40), id(41)] })), "accepted");
|
||||
assert.equal(status().state, "RUNNING");
|
||||
assert.equal(status().realizedVersion, 1);
|
||||
const job1 = jobs[0]!;
|
||||
const encoder = job1.program?.encoderPid;
|
||||
assert.ok(alive(encoder), "encoder up");
|
||||
// Baseline after the startup transient (compositor and x264 warming up repeat a few frames on a
|
||||
// loaded host, as cpu-smoke reports): wait until the gate has not repeated for 1 s (at most 15 s).
|
||||
let s1 = await job1.program!.stats();
|
||||
for (let calm = 0, waited = 0; calm < 2 && waited < 30; waited += 1) {
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
const now = await job1.program!.stats();
|
||||
calm = now.vrep + now.asil === s1.vrep + s1.asil ? calm + 1 : 0;
|
||||
s1 = now;
|
||||
}
|
||||
|
||||
// TAKE: program scene 10 → 11 (playlist PIP). Same encoder; realized once B is on air.
|
||||
let realizedDuringSwap: number | null = -1;
|
||||
const swapping = send(2, spec({ take: 11, playlist: [id(40), id(41)] }));
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
realizedDuringSwap = status().realizedVersion;
|
||||
await swapping;
|
||||
// On an oversubscribed host B can lag the on-air source (CompositorLagError): A stays on air,
|
||||
// v1 stays realized, and the supervisor retries the swap (1 s, then 2 s). That is the contract too.
|
||||
const retried = status().realizedVersion === 1 && /stays on air; retry/.test(status().reason);
|
||||
const tSwap = Date.now();
|
||||
while (status().realizedVersion !== 2 && jobs.length === 1 && Date.now() - tSwap < 30_000) await new Promise((r) => setTimeout(r, 100));
|
||||
if (retried) t.diagnostic("TAKE went on air through a supervisor swap retry (B lagged on the first attempt)");
|
||||
assert.equal(realizedDuringSwap, 1, "not realized before the new compositor is on air");
|
||||
assert.equal(jobs.length, 1, `no restart (${status().reason})`);
|
||||
assert.equal(status().realizedVersion, 2, `TAKE realized (${status().reason})`);
|
||||
assert.match(status().reason, /frame boundary/);
|
||||
const s2 = await job1.program!.stats();
|
||||
assert.equal(job1.program?.encoderPid, encoder, "same encoder process: the push session never dropped");
|
||||
assert.equal(s2.swaps, s1.swaps + 1);
|
||||
// A fill here would be the gate itself starved on an oversubscribed host, never the splice; keep it rare.
|
||||
const fills = s2.vrep + s2.asil - (s1.vrep + s1.asil);
|
||||
assert.ok(fills <= 2, `no repeat or silence fill across the swap (${fills})`);
|
||||
assert.deepEqual(job1.lastSkipped, []);
|
||||
|
||||
// Playlist edit: adopted in place by the feeder (no restart), and the program keeps running.
|
||||
await send(3, spec({ take: 11, playlist: [id(42), id(40)] }));
|
||||
assert.equal(status().realizedVersion, 3);
|
||||
assert.equal(jobs.length, 1);
|
||||
assert.equal(job1.program?.encoderPid, encoder);
|
||||
const feeder = job1.feeder(id(3))!;
|
||||
const t0 = Date.now();
|
||||
while (!opened.includes(id(42)) && Date.now() - t0 < 15_000) await new Promise((r) => setTimeout(r, 100));
|
||||
assert.ok(opened.includes(id(42)), `the edited list is played (opened ${opened.join(",")})`);
|
||||
assert.equal(feeder.stats.failures, 0);
|
||||
|
||||
// Previews: the program tap's thumbnail and always-on LL-HLS reached the manifest store.
|
||||
const key = (tap: string, file: string): string => `preview:${WS}:${CH}:${tap}:${file}`;
|
||||
const t1 = Date.now();
|
||||
while (!published.has(key("program", "video.m3u8")) && Date.now() - t1 < 30_000) await new Promise((r) => setTimeout(r, 200));
|
||||
assert.ok(published.has(key("program", "thumbnail.jpg")), "program thumbnail");
|
||||
assert.ok(published.has(key("program", "video.m3u8")), "program LL-HLS");
|
||||
assert.ok(published.has(key(id(1), "thumbnail.jpg")) || published.has(key(id(2), "thumbnail.jpg")), "source thumbnails");
|
||||
|
||||
// A changed source URL cannot be a swap: full restart, new encoder.
|
||||
await send(4, spec({ take: 11, playlist: [id(42), id(40)], urlB: "https://b2.example/live.m3u8" }));
|
||||
assert.equal(status().realizedVersion, 4);
|
||||
assert.equal(jobs.length, 2, "restarted");
|
||||
assert.ok(!alive(encoder), "the old encoder is gone");
|
||||
assert.ok(alive(jobs[1]!.program?.encoderPid));
|
||||
|
||||
const last = jobs[1]!.program?.encoderPid;
|
||||
await send(5, spec({ take: 11, playlist: [id(42)] }), "STOPPED");
|
||||
assert.equal(status().state, "STOPPED");
|
||||
assert.ok(!alive(last), "stopped");
|
||||
assert.deepEqual(exits, [], "no job exited on its own");
|
||||
} finally {
|
||||
// Also on failure: the heartbeat interval and any pending swap retry would keep the process alive.
|
||||
await worker.stop();
|
||||
for (const job of jobs) await job.stop();
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
155
apps/media-worker/src/supervisor/reconfigure.test.ts
Normal file
155
apps/media-worker/src/supervisor/reconfigure.test.ts
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { generateKeyPairSync, randomUUID } from "node:crypto";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { signCommand, type CommandBody } from "../bus/envelope";
|
||||
import { DurableFence } from "../bus/fence";
|
||||
import type { ObservedStore } from "../bus/redis";
|
||||
import { MediaSupervisor, type RelayJob, type RetryScheduler } from "./index";
|
||||
import type { DesiredChannel, ObservedChannel } from "../types";
|
||||
|
||||
const base: DesiredChannel = {
|
||||
workspaceId: "workspace-1", channelId: "channel-1", workerId: "worker-1",
|
||||
assignmentEpoch: 2, desiredVersion: 1, desiredStatus: "RUNNING",
|
||||
source: { kind: "hls", url: "https://mock.local/hls/live-1/master.m3u8" },
|
||||
output: { endpointUrl: "rtmp://mock.local/live", credentialHandleId: "keyref-1" },
|
||||
profile: "720p30", mode: "normalize",
|
||||
};
|
||||
|
||||
const store: ObservedStore = {
|
||||
async get() { return null; },
|
||||
async put(_row: ObservedChannel) {},
|
||||
async heartbeat() {},
|
||||
};
|
||||
|
||||
test("RUNNING → RUNNING on a swappable job is an in-place reconfigure, realized only when on air", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "media-reconf-"));
|
||||
const keys = generateKeyPairSync("ed25519");
|
||||
const calls: string[] = [];
|
||||
let accept = true;
|
||||
let failures = 0;
|
||||
const job = (): RelayJob => ({
|
||||
async start(c) { calls.push(`start v${c.desiredVersion}`); },
|
||||
async stop() { calls.push("stop"); },
|
||||
async reconfigure(c) {
|
||||
calls.push(`swap v${c.desiredVersion}`);
|
||||
if (failures > 0) { failures -= 1; throw new Error("compositor B never went on air"); }
|
||||
return accept;
|
||||
},
|
||||
});
|
||||
// Manual retry scheduler: the test fires retries itself.
|
||||
const pending: Array<{ action: () => Promise<void>; delayMs: number; cancelled: boolean }> = [];
|
||||
const scheduler: RetryScheduler = {
|
||||
schedule(action, delayMs) { const entry = { action, delayMs, cancelled: false }; pending.push(entry); return { cancel: () => { entry.cancelled = true; } }; },
|
||||
};
|
||||
const fire = async (): Promise<number> => { const entry = pending.shift(); assert.ok(entry && !entry.cancelled, "a retry is scheduled"); await entry.action(); return entry.delayMs; };
|
||||
const worker = new MediaSupervisor("worker-1", [base], store, new DurableFence(join(dir, "fence.json")),
|
||||
new Map([["control-v1", keys.publicKey]]), job, { transition() {} }, scheduler);
|
||||
const send = (version: number, status: "RUNNING" | "STOPPED" = "RUNNING") => {
|
||||
const body: CommandBody = {
|
||||
workspaceId: base.workspaceId, channelId: base.channelId, workerId: base.workerId, assignmentEpoch: base.assignmentEpoch,
|
||||
commandId: randomUUID(), desiredVersion: version, issuedAt: new Date(Date.now() - 1_000).toISOString(),
|
||||
expiresAt: new Date(Date.now() + 60_000).toISOString(), snapshot: { ...base, desiredVersion: version, desiredStatus: status },
|
||||
};
|
||||
return worker.handle(signCommand(body, "control-v1", keys.privateKey));
|
||||
};
|
||||
const status = () => worker.status()[0]!;
|
||||
try {
|
||||
await worker.start();
|
||||
await send(1);
|
||||
assert.deepEqual(calls, ["start v1"]);
|
||||
await send(2);
|
||||
assert.deepEqual(calls, ["start v1", "swap v2"], "a TAKE never stops the job (encoder/RTMP stay up)");
|
||||
assert.equal(status().state, "RUNNING");
|
||||
assert.equal(status().realizedVersion, 2);
|
||||
assert.match(status().reason, /frame boundary/);
|
||||
accept = false;
|
||||
await send(3);
|
||||
assert.deepEqual(calls.slice(2), ["swap v3", "stop", "start v3"], "an unswappable change falls back to restart");
|
||||
accept = true;
|
||||
|
||||
// A failed swap keeps the old compositor on air: still RUNNING, realized = what viewers see, retried.
|
||||
failures = 1;
|
||||
await send(4);
|
||||
assert.deepEqual(calls.slice(5), ["swap v4"]);
|
||||
assert.equal(status().state, "RUNNING");
|
||||
assert.equal(status().realizedVersion, 3, "v3 is still on air");
|
||||
assert.match(status().reason, /swap to v4 failed .*v3 stays on air; retry 1\/2/);
|
||||
assert.equal(await fire(), 1_000);
|
||||
assert.deepEqual(calls.slice(5), ["swap v4", "swap v4"], "the retry swaps; nothing restarts");
|
||||
assert.equal(status().realizedVersion, 4);
|
||||
|
||||
// A newer command supersedes a pending retry.
|
||||
failures = 1;
|
||||
await send(5);
|
||||
await send(6);
|
||||
assert.deepEqual(calls.slice(7), ["swap v5", "swap v6"]);
|
||||
assert.equal(pending.shift()?.cancelled, true, "the v5 retry was cancelled by v6");
|
||||
assert.equal(status().realizedVersion, 6);
|
||||
|
||||
// Out of retries: the full restart converges (and is the only path that drops the push session).
|
||||
failures = 3;
|
||||
await send(7);
|
||||
assert.equal(await fire(), 1_000);
|
||||
assert.equal(await fire(), 2_000);
|
||||
assert.deepEqual(calls.slice(9), ["swap v7", "swap v7", "swap v7", "stop", "start v7"]);
|
||||
assert.equal(status().realizedVersion, 7);
|
||||
assert.equal(status().state, "RUNNING");
|
||||
|
||||
await send(8, "STOPPED");
|
||||
assert.deepEqual(calls.slice(14), ["stop"]);
|
||||
assert.equal(pending.length, 0);
|
||||
await worker.stop();
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("worker stop waits for an in-flight swap and no retry or relaunch runs after it", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "media-reconf-stop-"));
|
||||
const keys = generateKeyPairSync("ed25519");
|
||||
const calls: string[] = [];
|
||||
let releaseSwap: (() => void) | null = null;
|
||||
const job = (): RelayJob => ({
|
||||
async start(c) { calls.push(`start v${c.desiredVersion}`); },
|
||||
async stop() { calls.push("stop"); },
|
||||
async reconfigure(c) {
|
||||
calls.push(`swap v${c.desiredVersion}`);
|
||||
// B is still trying to go on air when the worker shuts down; then it lags.
|
||||
await new Promise<void>((resolve) => { releaseSwap = resolve; });
|
||||
throw new Error("compositor B never went on air");
|
||||
},
|
||||
});
|
||||
const pending: Array<{ action: () => Promise<void>; cancelled: boolean }> = [];
|
||||
const scheduler: RetryScheduler = {
|
||||
schedule(action) { const entry = { action, cancelled: false }; pending.push(entry); return { cancel: () => { entry.cancelled = true; } }; },
|
||||
};
|
||||
const worker = new MediaSupervisor("worker-1", [base], store, new DurableFence(join(dir, "fence.json")),
|
||||
new Map([["control-v1", keys.publicKey]]), job, { transition() {} }, scheduler);
|
||||
const send = (version: number) => {
|
||||
const body: CommandBody = {
|
||||
workspaceId: base.workspaceId, channelId: base.channelId, workerId: base.workerId, assignmentEpoch: base.assignmentEpoch,
|
||||
commandId: randomUUID(), desiredVersion: version, issuedAt: new Date(Date.now() - 1_000).toISOString(),
|
||||
expiresAt: new Date(Date.now() + 60_000).toISOString(), snapshot: { ...base, desiredVersion: version },
|
||||
};
|
||||
return worker.handle(signCommand(body, "control-v1", keys.privateKey));
|
||||
};
|
||||
try {
|
||||
await worker.start();
|
||||
await send(1);
|
||||
const swapping = send(2);
|
||||
while (!releaseSwap) await new Promise((r) => setImmediate(r));
|
||||
let stopped = false;
|
||||
const stopping = worker.stop().then(() => { stopped = true; });
|
||||
await new Promise((r) => setImmediate(r));
|
||||
assert.equal(stopped, false, "stop waits for the in-flight swap");
|
||||
(releaseSwap as () => void)();
|
||||
await swapping;
|
||||
await stopping;
|
||||
assert.deepEqual(calls, ["start v1", "swap v2", "stop"]);
|
||||
assert.equal(worker.status()[0]?.state, "STOPPED");
|
||||
// The retry the failed swap scheduled is cancelled, and running it anyway does nothing.
|
||||
for (const entry of pending) { assert.equal(entry.cancelled, true); await entry.action(); }
|
||||
assert.deepEqual(calls, ["start v1", "swap v2", "stop"]);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
109
apps/media-worker/src/supervisor/sandbox.test.ts
Normal file
109
apps/media-worker/src/supervisor/sandbox.test.ts
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { generateKeyPairSync, randomUUID } from "node:crypto";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import test from "node:test";
|
||||
import { signCommand } from "../bus/envelope";
|
||||
import { DurableFence } from "../bus/fence";
|
||||
import type { ObservedStore } from "../bus/redis";
|
||||
import { buildCompositorArgs, buildEncoderArgs, buildIngestArgs, buildPlaylistRemuxArgs, buildSlateArgs } from "../compositor/command-builder";
|
||||
import { buildHlsTapArgs } from "../preview/argv";
|
||||
import { ffmpegArgs, spawnManaged, type ManagedProcess } from "../relay/ffmpeg";
|
||||
import type { DesiredChannel } from "../types";
|
||||
import { MediaSupervisor, type RelayJob } from "./index";
|
||||
import { supervisorSandboxEnabled } from "./sandbox";
|
||||
const pkg = (): {
|
||||
sandboxCapability(options: { rootfs: string; cgroupRoot: string }): { available: boolean; reason?: string };
|
||||
MediaCommandBuilder: new () => { supervisor(binary: "ffmpeg", args: readonly string[], scratch: string): { argv: readonly string[] } };
|
||||
} => require(resolve(__dirname, "../../../../packages/media-sandbox/dist"));
|
||||
const channel: DesiredChannel = {
|
||||
workspaceId: "sandbox-workspace", channelId: "sandbox-channel", workerId: "sandbox-worker", assignmentEpoch: 1,
|
||||
desiredVersion: 1, desiredStatus: "RUNNING", profile: "720p30", mode: "passthrough",
|
||||
source: { kind: "hls", url: "https://source.example/live.m3u8" },
|
||||
output: { endpointUrl: "rtmp://output.example/live", credentialHandleId: "opaque-handle" },
|
||||
};
|
||||
const store: ObservedStore = { async get() { return null; }, async put() {}, async heartbeat() {} };
|
||||
async function env<T>(values: Record<string, string | undefined>, action: () => Promise<T>): Promise<T> {
|
||||
const previous = Object.fromEntries(Object.keys(values).map(key => [key, process.env[key]]));
|
||||
for (const [key, value] of Object.entries(values)) { if (value === undefined) delete process.env[key]; else process.env[key] = value; }
|
||||
try { return await action(); }
|
||||
finally { for (const [key, value] of Object.entries(previous)) { if (value === undefined) delete process.env[key]; else process.env[key] = value; } }
|
||||
}
|
||||
function command(keys: ReturnType<typeof generateKeyPairSync>) {
|
||||
return signCommand({ workspaceId: channel.workspaceId, channelId: channel.channelId, workerId: channel.workerId,
|
||||
assignmentEpoch: 1, commandId: randomUUID(), desiredVersion: 1,
|
||||
issuedAt: new Date(Date.now() - 1000).toISOString(), expiresAt: new Date(Date.now() + 60_000).toISOString(), snapshot: channel }, "control", keys.privateKey);
|
||||
}
|
||||
test("sandbox flag defaults locally off and on in production; production off is rejected", async () => {
|
||||
await env({ NODE_ENV: "development", MEDIA_SANDBOX: undefined }, async () => assert.equal(supervisorSandboxEnabled(), false));
|
||||
await env({ NODE_ENV: "production", MEDIA_SANDBOX: undefined }, async () => assert.equal(supervisorSandboxEnabled(), true));
|
||||
await env({ NODE_ENV: "production", MEDIA_SANDBOX: "off" }, async () => assert.throws(supervisorSandboxEnabled, /requires/));
|
||||
});
|
||||
test("sandbox builder accepts existing relay/ingest/compositor/encoder/playlist/preview plans", () => {
|
||||
const builder = new (pkg().MediaCommandBuilder)(); const canvas = { width: 1280, height: 720, fps: 30 };
|
||||
const plans = [ffmpegArgs(channel), buildPlaylistRemuxArgs(0),
|
||||
buildIngestArgs({ canvas, input: { kind: "mpegtsPipe" }, audio: "source" }),
|
||||
buildIngestArgs({ canvas, input: { kind: "synthetic", pattern: "testsrc2", seed: 1, toneHz: 440 }, audio: "source" }),
|
||||
buildEncoderArgs({ canvas, videoBitrateKbps: 2800, audioBitrateKbps: 128, output: { kind: "null" }, programTap: { videoFd: 5, audioFd: 6, size: { width: 320, height: 180 } } }),
|
||||
buildSlateArgs({ canvas, seconds: 5, image: `/job/${"a".repeat(64)}.png`, offsetUs: 0 }),
|
||||
buildHlsTapArgs({ geometry: canvas, rendition: "video", outDir: "/job/taps/video" }),
|
||||
buildCompositorArgs({ sources: [{ video: { kind: "rawPipe", fd: 3 }, audio: { kind: "rawPipe", fd: 4 } }], assets: [`/job/${"a".repeat(64)}.png`],
|
||||
graph: { canvas, layout: "single", tiles: [{ source: 0, rect: { x: 0, y: 0, w: 1280, h: 720 }, fit: "cover" }],
|
||||
overlays: [{ kind: "chyron", asset: 0, y: 10, speedPxPerSec: 40 }], audio: { inputs: [{ source: 0, gainDb: 0 }], masterGainDb: 0 } }, output: { kind: "gate", videoFd: 1, audioFd: 3 } })];
|
||||
for (const args of plans) assert.ok(builder.supervisor("ffmpeg", args, "/job").argv.includes("-protocol_whitelist"));
|
||||
});
|
||||
test("supervisor fails closed if sandbox is enabled without configuration", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "media-sandbox-closed-")); const keys = generateKeyPairSync("ed25519");
|
||||
const lines: string[] = [];
|
||||
await env({ NODE_ENV: "development", MEDIA_SANDBOX: "on", MEDIA_SANDBOX_ROOTFS: undefined, MEDIA_SANDBOX_CGROUP_ROOT: undefined, MEDIA_SCRATCH_DIR: join(dir, "scratch") }, async () => {
|
||||
const worker = new MediaSupervisor(channel.workerId, [channel], store, new DurableFence(join(dir, "fence")), new Map([["control", keys.publicKey]]),
|
||||
(): RelayJob => ({ async start() { spawnManaged("ffmpeg", ffmpegArgs(channel), ["ignore", "ignore", "pipe"], { redact: v => v }, line => lines.push(line)); }, async stop() {} }),
|
||||
{ transition: event => lines.push(JSON.stringify(event)) }, { schedule() { return { cancel() {} }; } });
|
||||
try { await worker.start(); await worker.handle(command(keys)); assert.equal(worker.status()[0]?.state, "RESTARTING"); assert.ok(lines.join("\n").includes("SandboxUnavailable")); }
|
||||
finally { await worker.stop(); }
|
||||
});
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
for (const enabled of [false, true]) test(`secret-leak canaries through signed supervisor command (sandbox ${enabled ? "on" : "off"})`, { timeout: 20_000 }, async t => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "media-supervisor-canary-"));
|
||||
const rootfs = join(dir, "rootfs");
|
||||
for (const d of ["upload", "scratch", "tmp", "proc", "dev", "usr/bin"]) mkdirSync(join(rootfs, d), { recursive: true });
|
||||
writeFileSync(join(rootfs, "dev/null"), "");
|
||||
try {
|
||||
if (enabled) {
|
||||
const compiled = spawnSync("cc", ["-static", "-O0", resolve(__dirname, "../../../../packages/media-sandbox/native/test-child.c"), "-o", join(rootfs, "usr/bin/ffmpeg")]);
|
||||
const cap = compiled.status === 0 ? pkg().sandboxCapability({ rootfs, cgroupRoot: process.env.MEDIA_SANDBOX_CGROUP_ROOT ?? "/sys/fs/cgroup" }) : { available: false, reason: "static compiler unavailable" };
|
||||
if (!cap.available) { t.skip(`rootless sandbox canary skipped: ${cap.reason}`); return; }
|
||||
}
|
||||
const canaries = ["stream-key-canary-supervisor-531", "oauth-token-canary-supervisor-794", "srt-passphrase-canary-supervisor-652"];
|
||||
await env({ NODE_ENV: "development", MEDIA_SANDBOX: enabled ? "on" : "off", MEDIA_SANDBOX_ROOTFS: rootfs,
|
||||
MEDIA_SANDBOX_CGROUP_ROOT: process.env.MEDIA_SANDBOX_CGROUP_ROOT ?? "/sys/fs/cgroup", MEDIA_SCRATCH_DIR: join(dir, "scratch"),
|
||||
CANARY_STREAM_KEY: canaries[0], CANARY_OAUTH_TOKEN: canaries[1], CANARY_SRT_PASSPHRASE: canaries[2] }, async () => {
|
||||
const keys = generateKeyPairSync("ed25519"); const lines: string[] = []; let proc: ManagedProcess | undefined;
|
||||
const worker = new MediaSupervisor(channel.workerId, [channel], store, new DurableFence(join(dir, "fence")), new Map([["control", keys.publicKey]]),
|
||||
(_channel, onExit): RelayJob => ({
|
||||
async start() {
|
||||
proc = spawnManaged("ffmpeg", ["-hide_banner", "-nostdin", "-loglevel", "error", "-f", "mpegts", "-i", "pipe:0", "-f", "null", "pipe:1"],
|
||||
["pipe", "ignore", "pipe"], { redact: value => value }, line => lines.push(line));
|
||||
void proc.exited.then(({ code }) => onExit(`ffmpeg exited code=${code}`)); await proc.started;
|
||||
},
|
||||
async stop() { await proc?.terminate(100); },
|
||||
}), { transition: event => lines.push(JSON.stringify(event)) }, { schedule() { return { cancel() {} }; } });
|
||||
try {
|
||||
await worker.start(); await worker.handle(command(keys));
|
||||
assert.equal(worker.status()[0]?.state, "RUNNING"); assert.ok(proc?.child.pid);
|
||||
assert.equal(proc.child.spawnfile.endsWith("media-confine"), enabled);
|
||||
const cmdline = readFileSync(`/proc/${proc.child.pid}/cmdline`, "utf8");
|
||||
const environment = readFileSync(`/proc/${proc.child.pid}/environ`, "utf8");
|
||||
proc.child.stdin!.end("invalid-media-to-force-ffmpeg-error");
|
||||
const result = await proc.exited; assert.notEqual(result.code, 0);
|
||||
await worker.stop();
|
||||
assert.ok(lines.some(line => enabled ? line.includes("stderr suppressed") : /Invalid data|could not find codec|End of file/i.test(line)));
|
||||
for (const canary of canaries) for (const output of [cmdline, environment, ...lines]) assert.ok(!output.includes(canary));
|
||||
} finally { await worker.stop(); }
|
||||
});
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
99
apps/media-worker/src/supervisor/sandbox.ts
Normal file
99
apps/media-worker/src/supervisor/sandbox.ts
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
import { createHash } from "node:crypto";
|
||||
import type * as ChildProcessApi from "node:child_process";
|
||||
import { chmodSync, mkdirSync, constants, openSync, closeSync, fchmodSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { Transform } from "node:stream";
|
||||
import type { DesiredChannel } from "../types";
|
||||
import type { RelayJob } from "./index";
|
||||
|
||||
const childProcess: { spawn: typeof ChildProcessApi.spawn } = require("node:child_process");
|
||||
|
||||
// Resolve the CommonJS package from either src/supervisor or dist/supervisor.
|
||||
// A small structural boundary keeps worker typechecking independent of package build order.
|
||||
interface MediaCommand { binary: "ffmpeg" | "ffprobe"; argv: readonly string[] }
|
||||
interface SandboxModule {
|
||||
MediaCommandBuilder: new () => { supervisor(binary: MediaCommand["binary"], argv: readonly string[], scratch: string): MediaCommand };
|
||||
SandboxUnavailable: new () => Error;
|
||||
spawnSandbox(command: MediaCommand, options: { rootfs: string; cgroupRoot: string; scratchDir: string; limits: { wallTimeMs: number } }, spawnOptions: ChildProcessApi.SpawnOptions | undefined, spawner: typeof ChildProcessApi.spawn): { child: ChildProcessApi.ChildProcess };
|
||||
}
|
||||
const sandbox = (): SandboxModule => require(resolve(__dirname, "../../../../packages/media-sandbox/dist")) as SandboxModule;
|
||||
interface Context { scratch: string }
|
||||
const context = new AsyncLocalStorage<Context>();
|
||||
let installed = false;
|
||||
|
||||
/**
|
||||
* The only common spawn path lives in relay/, outside this change's boundary.
|
||||
* A scoped Node spawn adapter covers existing typed FFmpeg builders, including
|
||||
* asynchronous swaps/playlist producers, without altering their process API.
|
||||
* Non-media children and calls outside a supervisor job retain normal spawning.
|
||||
*/
|
||||
function installAdapter(): void {
|
||||
if (installed) return;
|
||||
const original = childProcess.spawn;
|
||||
childProcess.spawn = ((command: string, args?: readonly string[] | ChildProcessApi.SpawnOptions, options?: ChildProcessApi.SpawnOptions) => {
|
||||
const job = context.getStore();
|
||||
if (!job || !/(?:^|\/)(?:ffmpeg|ffprobe)$/.test(command)) {
|
||||
return typeof args === "object" && !Array.isArray(args)
|
||||
? original(command, args as ChildProcessApi.SpawnOptions)
|
||||
: original(command, (args as readonly string[] | undefined) ?? [], options ?? {});
|
||||
}
|
||||
if (!Array.isArray(args)) throw new Error("media argv required");
|
||||
const runtime = sandbox();
|
||||
const rootfs = process.env.MEDIA_SANDBOX_ROOTFS;
|
||||
const cgroupRoot = process.env.MEDIA_SANDBOX_CGROUP_ROOT;
|
||||
if (!rootfs || !cgroupRoot) throw new runtime.SandboxUnavailable();
|
||||
const media = new runtime.MediaCommandBuilder().supervisor(command.endsWith("ffprobe") ? "ffprobe" : "ffmpeg", args, job.scratch);
|
||||
// Make only generated output directories writable by the mapped UID.
|
||||
// Traverse through directory fds: a compromised sibling cannot race a
|
||||
// symlink replacement into chmod of a host file.
|
||||
for (const path of media.argv.filter(value => value.startsWith("file:/scratch/"))) {
|
||||
const parts = path.slice("file:/scratch/".length).split("/").slice(0, -1).filter(part => part && part !== ".");
|
||||
if (parts.length > 32 || parts.some(part => part === "..")) throw new runtime.SandboxUnavailable();
|
||||
let fd = openSync(job.scratch, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
||||
try {
|
||||
fchmodSync(fd, 0o777);
|
||||
for (const part of parts) {
|
||||
const next = openSync(`/proc/self/fd/${fd}/${part}`, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
||||
closeSync(fd); fd = next; fchmodSync(fd, 0o777);
|
||||
}
|
||||
} finally { closeSync(fd); }
|
||||
}
|
||||
const child = runtime.spawnSandbox(media, {
|
||||
rootfs, cgroupRoot, scratchDir: job.scratch,
|
||||
// Long-running channels still have a finite job lease; uploads use the package defaults.
|
||||
limits: { wallTimeMs: Number(process.env.MEDIA_SANDBOX_WALL_TIME_MS ?? 86_400_000) },
|
||||
}, options, original).child;
|
||||
// Replace the public stderr stream BEFORE spawnManaged attaches its logger.
|
||||
// Arbitrary media bytes cannot bypass a caller-supplied passthrough redactor.
|
||||
if (child.stderr) {
|
||||
const safe = new Transform({ transform(_chunk, _encoding, done) { done(null, Buffer.from("[media stderr suppressed]\n")); } });
|
||||
child.stderr.pipe(safe); child.stderr = safe; child.stdio[2] = safe;
|
||||
}
|
||||
return child;
|
||||
}) as typeof childProcess.spawn;
|
||||
installed = true;
|
||||
}
|
||||
function enabled(): boolean {
|
||||
const flag = process.env.MEDIA_SANDBOX;
|
||||
if (flag !== undefined && flag !== "on" && flag !== "off") throw new Error("MEDIA_SANDBOX must be on or off");
|
||||
if (process.env.NODE_ENV === "production" && flag === "off") throw new Error("production requires media sandbox");
|
||||
return flag === "on" || process.env.NODE_ENV === "production";
|
||||
}
|
||||
export function supervisorSandboxEnabled(): boolean { return enabled(); }
|
||||
|
||||
export function sandboxRelay(job: RelayJob, channel: DesiredChannel): RelayJob {
|
||||
if (!enabled()) return job;
|
||||
installAdapter();
|
||||
const id = createHash("sha256").update(`${channel.workspaceId}:${channel.channelId}`).digest("hex").slice(0, 32);
|
||||
const parent = resolve(process.env.MEDIA_SCRATCH_DIR ?? ".media-worker-state/scratch");
|
||||
const scratch = resolve(parent, id);
|
||||
mkdirSync(parent, { recursive: true }); chmodSync(parent, 0o700);
|
||||
mkdirSync(scratch, { recursive: true }); chmodSync(scratch, 0o777);
|
||||
const run = <T>(action: () => T): T => context.run({ scratch }, action);
|
||||
return {
|
||||
start: config => run(() => job.start(config)),
|
||||
stop: () => run(() => job.stop()),
|
||||
...(job.reconfigure ? { reconfigure: (config: DesiredChannel) => run(() => job.reconfigure!(config)) } : {}),
|
||||
};
|
||||
}
|
||||
135
apps/media-worker/src/supervisor/supervisor.test.ts
Normal file
135
apps/media-worker/src/supervisor/supervisor.test.ts
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { generateKeyPairSync, randomUUID } from "node:crypto";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { signCommand, type CommandBody } from "../bus/envelope";
|
||||
import { DurableFence } from "../bus/fence";
|
||||
import type { ObservedStore } from "../bus/redis";
|
||||
import { MediaSupervisor, type RelayJob, type RetryScheduler } from "./index";
|
||||
import type { DesiredChannel, ObservedChannel } from "../types";
|
||||
|
||||
const source: DesiredChannel = {
|
||||
workspaceId: "workspace-1", channelId: "channel-1", workerId: "worker-1",
|
||||
assignmentEpoch: 2, desiredVersion: 4, desiredStatus: "RUNNING",
|
||||
// Same HLS/RTMP endpoint shape supplied by connector-mock-livestream and the single-live simulator.
|
||||
source: { kind: "hls", url: "https://mock.local/hls/live-1/master.m3u8" },
|
||||
output: { endpointUrl: "rtmp://mock.local/live", credentialHandleId: "keyref-1" },
|
||||
profile: "720p30", mode: "passthrough",
|
||||
};
|
||||
|
||||
class MemoryObserved implements ObservedStore {
|
||||
rows = new Map<string, ObservedChannel>();
|
||||
heartbeats: Array<{ workerId: string; channels: ObservedChannel[] }> = [];
|
||||
async get(workspaceId: string, channelId: string): Promise<ObservedChannel | null> { return this.rows.get(`${workspaceId}:${channelId}`) ?? null; }
|
||||
async put(row: ObservedChannel): Promise<void> { this.rows.set(`${row.workspaceId}:${row.channelId}`, row); }
|
||||
async heartbeat(workerId: string, channels: ObservedChannel[]): Promise<void> {
|
||||
this.heartbeats.push({ workerId, channels });
|
||||
for (const row of channels) await this.put(row);
|
||||
}
|
||||
}
|
||||
|
||||
test("durable replay fence, stale version, and restart reconcile", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "media-fence-"));
|
||||
const store = new MemoryObserved();
|
||||
const keys = generateKeyPairSync("ed25519");
|
||||
const pinned = new Map([["control-v1", keys.publicKey]]);
|
||||
let starts = 0;
|
||||
const makeSupervisor = () => new MediaSupervisor("worker-1", [source], store,
|
||||
new DurableFence(join(dir, "fence.json")), pinned,
|
||||
(): RelayJob => ({ async start() { starts += 1; }, async stop() {} }),
|
||||
{ transition() {} });
|
||||
const makeBody = (commandId: string, version: number): CommandBody => ({
|
||||
workspaceId: source.workspaceId, channelId: source.channelId, workerId: source.workerId,
|
||||
assignmentEpoch: source.assignmentEpoch, commandId, desiredVersion: version,
|
||||
issuedAt: new Date(Date.now() - 1_000).toISOString(),
|
||||
expiresAt: new Date(Date.now() + 60_000).toISOString(),
|
||||
snapshot: { ...source, desiredVersion: version },
|
||||
});
|
||||
try {
|
||||
const first = signCommand(makeBody(randomUUID(), 4), "control-v1", keys.privateKey);
|
||||
const worker = makeSupervisor();
|
||||
await worker.start();
|
||||
assert.equal(worker.status()[0]?.realizedVersion, null);
|
||||
assert.equal(await worker.handle(first), "accepted");
|
||||
assert.equal(worker.status()[0]?.state, "RUNNING");
|
||||
assert.equal(worker.status()[0]?.realizedVersion, 4);
|
||||
assert.equal(starts, 1);
|
||||
await worker.stop();
|
||||
|
||||
const restarted = makeSupervisor();
|
||||
await restarted.start();
|
||||
assert.equal(restarted.status()[0]?.realizedVersion, null);
|
||||
assert.match(restarted.status()[0]?.reason ?? "", /restart reconcile/);
|
||||
assert.equal(await restarted.handle(first), "replay");
|
||||
assert.equal(starts, 1);
|
||||
const stale = signCommand(makeBody(randomUUID(), 3), "control-v1", keys.privateKey);
|
||||
assert.equal(await restarted.handle(stale), "stale");
|
||||
const freshSnapshot = signCommand(makeBody(randomUUID(), 4), "control-v1", keys.privateKey);
|
||||
assert.equal(await restarted.handle(freshSnapshot), "accepted");
|
||||
assert.equal(restarted.status()[0]?.realizedVersion, 4);
|
||||
assert.equal(starts, 2);
|
||||
await restarted.stop();
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("unsigned, expired, wrong target and tampered snapshots are rejected", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "media-command-"));
|
||||
const keys = generateKeyPairSync("ed25519");
|
||||
const worker = new MediaSupervisor("worker-1", [source], new MemoryObserved(),
|
||||
new DurableFence(join(dir, "fence.json")), new Map([["control-v1", keys.publicKey]]),
|
||||
(): RelayJob => ({ async start() {}, async stop() {} }), { transition() {} });
|
||||
const body: CommandBody = {
|
||||
workspaceId: source.workspaceId, channelId: source.channelId, workerId: source.workerId,
|
||||
assignmentEpoch: source.assignmentEpoch, commandId: randomUUID(), desiredVersion: 4,
|
||||
issuedAt: new Date(Date.now() - 2_000).toISOString(), expiresAt: new Date(Date.now() + 30_000).toISOString(), snapshot: source,
|
||||
};
|
||||
try {
|
||||
await worker.start();
|
||||
await assert.rejects(worker.handle({ kid: "control-v1", body, signature: "" }), /signature/);
|
||||
await assert.rejects(worker.handle(signCommand({ ...body, workerId: "other" }, "control-v1", keys.privateKey)), /fields/);
|
||||
const expired = { ...body, expiresAt: new Date(Date.now() - 1).toISOString() };
|
||||
await assert.rejects(worker.handle(signCommand(expired, "control-v1", keys.privateKey)), /expired/);
|
||||
const tampered = signCommand(body, "control-v1", keys.privateKey);
|
||||
tampered.body.snapshot = { ...source, desiredStatus: "STOPPED" };
|
||||
await assert.rejects(worker.handle(tampered), /signature/);
|
||||
await worker.stop();
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("relay failure follows DEGRADED, RESTARTING and bounded FAILED states", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "media-retry-"));
|
||||
const keys = generateKeyPairSync("ed25519");
|
||||
const queued: Array<{ action: () => Promise<void>; delay: number }> = [];
|
||||
const scheduler: RetryScheduler = {
|
||||
schedule(action, delay) { queued.push({ action, delay }); return { cancel() {} }; },
|
||||
};
|
||||
const transitions: string[] = [];
|
||||
const worker = new MediaSupervisor("worker-1", [source], new MemoryObserved(),
|
||||
new DurableFence(join(dir, "fence.json")), new Map([["control-v1", keys.publicKey]]),
|
||||
(): RelayJob => ({ async start() { throw new Error("ingest unavailable"); }, async stop() {} }),
|
||||
{ transition(event) { transitions.push(event.to); } }, scheduler);
|
||||
try {
|
||||
await worker.start();
|
||||
const body: CommandBody = {
|
||||
workspaceId: source.workspaceId, channelId: source.channelId, workerId: source.workerId,
|
||||
assignmentEpoch: source.assignmentEpoch, commandId: randomUUID(), desiredVersion: 4,
|
||||
issuedAt: new Date(Date.now() - 1_000).toISOString(), expiresAt: new Date(Date.now() + 60_000).toISOString(), snapshot: source,
|
||||
};
|
||||
assert.equal(await worker.handle(signCommand(body, "control-v1", keys.privateKey)), "accepted");
|
||||
assert.equal(worker.status()[0]?.state, "RESTARTING");
|
||||
const delays: number[] = [];
|
||||
while (queued.length) {
|
||||
const next = queued.shift();
|
||||
assert.ok(next);
|
||||
delays.push(next.delay);
|
||||
await next.action();
|
||||
}
|
||||
assert.deepEqual(delays, [1_000, 2_000, 4_000, 8_000, 16_000]);
|
||||
assert.equal(worker.status()[0]?.state, "FAILED");
|
||||
assert.equal(worker.status()[0]?.realizedVersion, null);
|
||||
assert.ok(transitions.includes("DEGRADED"));
|
||||
await worker.stop();
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
69
apps/media-worker/src/types.ts
Normal file
69
apps/media-worker/src/types.ts
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
import { isProductionSpec, productionHlsUrls, type ProductionSpec } from "./production";
|
||||
|
||||
export type ChannelState = "STOPPED" | "STARTING" | "RUNNING" | "DEGRADED" | "RESTARTING" | "FAILED";
|
||||
export type DesiredState = "STOPPED" | "RUNNING";
|
||||
export type OutputPreset = "720p30" | "1080p30" | "1080p60";
|
||||
export type OutputMode = "passthrough" | "normalize";
|
||||
|
||||
export const OUTPUT_PROFILES: Record<OutputPreset, { width: number; height: number; fps: number; videoBitrate: string }> = {
|
||||
"720p30": { width: 1280, height: 720, fps: 30, videoBitrate: "3000k" },
|
||||
"1080p30": { width: 1920, height: 1080, fps: 30, videoBitrate: "6000k" },
|
||||
"1080p60": { width: 1920, height: 1080, fps: 60, videoBitrate: "9000k" },
|
||||
};
|
||||
|
||||
/** A complete, secret-free channel snapshot. The control plane owns this record. */
|
||||
export interface DesiredChannel {
|
||||
workspaceId: string;
|
||||
channelId: string;
|
||||
workerId: string;
|
||||
assignmentEpoch: number;
|
||||
desiredVersion: number;
|
||||
desiredStatus: DesiredState;
|
||||
source: { kind: "hls"; url: string } | null;
|
||||
output: { endpointUrl: string; credentialHandleId: string } | null;
|
||||
profile: OutputPreset;
|
||||
mode: OutputMode;
|
||||
/**
|
||||
* Desired-state v2 (production: scenes, buses, sources, overlays,
|
||||
* transition, audio). Absent on v1 snapshots; when present, `source` is null.
|
||||
*/
|
||||
production?: ProductionSpec;
|
||||
}
|
||||
|
||||
export interface ObservedChannel {
|
||||
workspaceId: string;
|
||||
channelId: string;
|
||||
workerId: string;
|
||||
assignmentEpoch: number;
|
||||
state: ChannelState;
|
||||
realizedVersion: number | null;
|
||||
updatedAt: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export function isDesiredChannel(value: unknown): value is DesiredChannel {
|
||||
if (typeof value !== "object" || value === null) return false;
|
||||
const x = value as Partial<DesiredChannel>;
|
||||
if (!onlyKeys(value, ["workspaceId", "channelId", "workerId", "assignmentEpoch", "desiredVersion", "desiredStatus", "source", "output", "profile", "mode", "production"])) return false;
|
||||
if ("production" in value && (x.source !== null || !isProductionSpec(x.production))) return false;
|
||||
return typeof x.workspaceId === "string" && x.workspaceId.length > 0 &&
|
||||
typeof x.channelId === "string" && x.channelId.length > 0 &&
|
||||
typeof x.workerId === "string" && x.workerId.length > 0 &&
|
||||
Number.isSafeInteger(x.assignmentEpoch) && (x.assignmentEpoch ?? -1) >= 0 &&
|
||||
Number.isSafeInteger(x.desiredVersion) && (x.desiredVersion ?? -1) >= 0 &&
|
||||
(x.desiredStatus === "STOPPED" || x.desiredStatus === "RUNNING") &&
|
||||
(x.mode === "passthrough" || x.mode === "normalize") &&
|
||||
(x.profile === "720p30" || x.profile === "1080p30" || x.profile === "1080p60") &&
|
||||
(x.source === null || (typeof x.source === "object" && x.source?.kind === "hls" && typeof x.source.url === "string" && x.source.url.length > 0 && onlyKeys(x.source, ["kind", "url"]))) &&
|
||||
(x.output === null || (typeof x.output === "object" && typeof x.output?.endpointUrl === "string" && x.output.endpointUrl.length > 0 && typeof x.output.credentialHandleId === "string" && x.output.credentialHandleId.length > 0 && onlyKeys(x.output, ["endpointUrl", "credentialHandleId"]))) &&
|
||||
(x.desiredStatus !== "RUNNING" || (x.output !== null && (x.production === undefined ? x.source !== null : x.production.buses.program !== null)));
|
||||
}
|
||||
|
||||
/** Every remote HLS URL a snapshot can make the worker fetch (v1 source or v2 production sources). */
|
||||
export function snapshotHlsUrls(s: DesiredChannel): string[] {
|
||||
return [...(s.source === null ? [] : [s.source.url]), ...(s.production === undefined ? [] : productionHlsUrls(s.production))];
|
||||
}
|
||||
|
||||
function onlyKeys(value: object, allowed: string[]): boolean {
|
||||
return Object.keys(value).every((key) => allowed.includes(key));
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue