Tactical Media Broadcast Hub — initial release
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
This commit is contained in:
commit
aa5e262f3a
880 changed files with 111042 additions and 0 deletions
249
apps/media-worker/src/supervisor/index.ts
Normal file
249
apps/media-worker/src/supervisor/index.ts
Normal file
|
|
@ -0,0 +1,249 @@
|
|||
import type { KeyObject } from "node:crypto";
|
||||
import { sandboxRelay, supervisorSandboxEnabled } from "./sandbox";
|
||||
import { verifyCommand } from "../bus/envelope";
|
||||
import { DurableFence } from "../bus/fence";
|
||||
import type { ObservedStore } from "../bus/redis";
|
||||
import type { ChannelState, DesiredChannel, ObservedChannel } from "../types";
|
||||
|
||||
export interface TransitionLogger {
|
||||
transition(event: { workspaceId: string; channelId: string; from: ChannelState; to: ChannelState; reason: string }): void;
|
||||
}
|
||||
interface ChannelRuntime {
|
||||
desired: DesiredChannel;
|
||||
state: ChannelState;
|
||||
reason: string;
|
||||
realizedVersion: number | null;
|
||||
relay: RelayJob | null;
|
||||
retries: number;
|
||||
retryTimer: { cancel(): void } | null;
|
||||
/** Consecutive failed swaps for the current desired version (the previous version stays on air). */
|
||||
swapFailures: number;
|
||||
}
|
||||
export interface RelayJob {
|
||||
start(channel: DesiredChannel): Promise<void>;
|
||||
stop(): Promise<void>;
|
||||
/**
|
||||
* Optional in-place change (production jobs: an A/B compositor swap behind the
|
||||
* frame gate). Resolves true once the new version is on air; false means the
|
||||
* change needs a restart (sources, output or profile changed).
|
||||
*/
|
||||
reconfigure?(channel: DesiredChannel): Promise<boolean>;
|
||||
}
|
||||
export interface RetryScheduler { schedule(action: () => Promise<void>, delayMs: number): { cancel(): void } }
|
||||
|
||||
/**
|
||||
* A swap that fails (B never went on air; A is still on air) is retried this many times, 1 s then
|
||||
* 2 s later, before the supervisor falls back to a full restart. The full restart converges too,
|
||||
* but it takes the encoder and the push session down (WP9 "output that does not stop").
|
||||
*/
|
||||
export const SWAP_RETRIES = 2;
|
||||
const defaultRetryScheduler: RetryScheduler = {
|
||||
schedule(action, delayMs) {
|
||||
const timer = setTimeout(() => { void action(); }, delayMs);
|
||||
return { cancel: () => clearTimeout(timer) };
|
||||
},
|
||||
};
|
||||
|
||||
/** Accepted versions are durable; realized versions describe only this process lifetime. */
|
||||
export class MediaSupervisor {
|
||||
private readonly channels = new Map<string, ChannelRuntime>();
|
||||
private readonly operations = new Map<string, Promise<unknown>>();
|
||||
private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
|
||||
/** Set by `stop()`: late retries and relaunches become no-ops. */
|
||||
private stopped = false;
|
||||
constructor(
|
||||
private readonly workerId: string,
|
||||
desired: DesiredChannel[],
|
||||
private readonly store: ObservedStore,
|
||||
private readonly fence: DurableFence,
|
||||
private readonly controlKeys: ReadonlyMap<string, KeyObject>,
|
||||
private readonly createRelay: (channel: DesiredChannel, onExit: (reason: string) => void) => RelayJob,
|
||||
private readonly logger: TransitionLogger,
|
||||
private readonly retryScheduler: RetryScheduler = defaultRetryScheduler,
|
||||
) {
|
||||
supervisorSandboxEnabled(); // Production cannot silently opt out of confinement.
|
||||
for (const channel of desired) {
|
||||
if (channel.workerId !== workerId) throw new Error("config worker mismatch");
|
||||
this.channels.set(this.key(channel.workspaceId, channel.channelId), {
|
||||
desired: channel, state: "STOPPED", reason: "worker boot", realizedVersion: null,
|
||||
relay: null, retries: 0, retryTimer: null, swapFailures: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private key(workspaceId: string, channelId: string): string { return `${workspaceId}:${channelId}`; }
|
||||
status(): ObservedChannel[] {
|
||||
return [...this.channels.values()].map((runtime) => ({
|
||||
workspaceId: runtime.desired.workspaceId, channelId: runtime.desired.channelId,
|
||||
workerId: this.workerId, assignmentEpoch: runtime.desired.assignmentEpoch,
|
||||
state: runtime.state, realizedVersion: runtime.realizedVersion,
|
||||
updatedAt: new Date().toISOString(), reason: runtime.reason,
|
||||
}));
|
||||
}
|
||||
private async transition(runtime: ChannelRuntime, to: ChannelState, reason: string): Promise<void> {
|
||||
const from = runtime.state;
|
||||
runtime.state = to;
|
||||
runtime.reason = reason;
|
||||
this.logger.transition({ workspaceId: runtime.desired.workspaceId, channelId: runtime.desired.channelId, from, to, reason });
|
||||
await this.store.put(this.status().find((item) => item.workspaceId === runtime.desired.workspaceId && item.channelId === runtime.desired.channelId)!);
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
for (const runtime of this.channels.values()) {
|
||||
const old = await this.store.get(runtime.desired.workspaceId, runtime.desired.channelId);
|
||||
await this.transition(runtime, "STOPPED", old
|
||||
? `restart reconcile: prior ${old.state}; awaiting fresh signed snapshot`
|
||||
: "start reconcile: awaiting signed snapshot");
|
||||
}
|
||||
await this.beat();
|
||||
this.heartbeatTimer = setInterval(() => { void this.beat().catch((error: unknown) => this.logger.transition({ workspaceId: "", channelId: "", from: "DEGRADED", to: "DEGRADED", reason: `heartbeat failed: ${String(error)}` })); }, 5_000);
|
||||
}
|
||||
|
||||
async beat(): Promise<void> { await this.store.heartbeat(this.workerId, this.status()); }
|
||||
|
||||
async handle(raw: unknown, now = Date.now()): Promise<"accepted" | "replay" | "stale"> {
|
||||
const command = verifyCommand(raw, this.controlKeys, this.workerId, now);
|
||||
const body = command.body;
|
||||
return this.serialize(this.key(body.workspaceId, body.channelId), () => this.apply(body, now));
|
||||
}
|
||||
|
||||
/** One operation at a time per channel (commands and swap retries). */
|
||||
private serialize<T>(key: string, operation: () => Promise<T>): Promise<T> {
|
||||
const previous = this.operations.get(key) ?? Promise.resolve();
|
||||
const task = previous.catch(() => undefined).then(operation);
|
||||
this.operations.set(key, task);
|
||||
void task.then(() => { if (this.operations.get(key) === task) this.operations.delete(key); },
|
||||
() => { if (this.operations.get(key) === task) this.operations.delete(key); });
|
||||
return task;
|
||||
}
|
||||
|
||||
private async apply(body: ReturnType<typeof verifyCommand>["body"], now: number): Promise<"accepted" | "replay" | "stale"> {
|
||||
const runtime = this.channels.get(this.key(body.workspaceId, body.channelId));
|
||||
if (!runtime || runtime.desired.assignmentEpoch !== body.assignmentEpoch) throw new Error("unassigned channel or epoch");
|
||||
const result = this.fence.accept(body, now);
|
||||
if (result !== "accepted") return result;
|
||||
// RUNNING → RUNNING on a job that can swap: the encoder and push session stay up,
|
||||
// and the version is realized only once the new compositor is on air.
|
||||
const relay = runtime.relay;
|
||||
if (runtime.state === "RUNNING" && body.snapshot.desiredStatus === "RUNNING" && relay?.reconfigure) {
|
||||
runtime.desired = body.snapshot;
|
||||
runtime.retryTimer?.cancel();
|
||||
runtime.retryTimer = null;
|
||||
runtime.swapFailures = 0;
|
||||
if (await this.swap(runtime, relay) !== "restart") return result;
|
||||
}
|
||||
runtime.desired = body.snapshot;
|
||||
runtime.swapFailures = 0;
|
||||
runtime.realizedVersion = null;
|
||||
runtime.retryTimer?.cancel();
|
||||
runtime.retryTimer = null;
|
||||
await runtime.relay?.stop();
|
||||
runtime.relay = null;
|
||||
if (body.snapshot.desiredStatus === "STOPPED") {
|
||||
await this.transition(runtime, "STOPPED", "signed desired state stopped");
|
||||
runtime.realizedVersion = body.desiredVersion;
|
||||
await this.beat();
|
||||
return result;
|
||||
}
|
||||
runtime.retries = 0;
|
||||
await this.launch(runtime);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* One swap attempt for `runtime.desired`. "done": on air (or the job is gone and its own exit
|
||||
* path owns the channel); "retrying": B failed, A stays on air (`realizedVersion` keeps naming
|
||||
* what viewers see) and a retry is scheduled; "restart": not swappable, or out of retries.
|
||||
*/
|
||||
private async swap(runtime: ChannelRuntime, relay: RelayJob): Promise<"done" | "retrying" | "restart"> {
|
||||
const version = runtime.desired.desiredVersion;
|
||||
try {
|
||||
if (!(await relay.reconfigure?.(runtime.desired))) return "restart";
|
||||
if (runtime.relay !== relay) return "done";
|
||||
runtime.swapFailures = 0;
|
||||
runtime.realizedVersion = version;
|
||||
await this.transition(runtime, "RUNNING", "compositor swapped at a frame boundary");
|
||||
await this.beat();
|
||||
return "done";
|
||||
} catch (error: unknown) {
|
||||
if (runtime.relay !== relay) return "done";
|
||||
const name = error instanceof Error ? error.name : "unknown error";
|
||||
runtime.swapFailures += 1;
|
||||
if (runtime.swapFailures > SWAP_RETRIES) {
|
||||
runtime.reason = `swap failed: ${name}`;
|
||||
return "restart";
|
||||
}
|
||||
const delay = 1_000 * 2 ** (runtime.swapFailures - 1);
|
||||
await this.transition(runtime, "RUNNING", `swap to v${version} failed (${name}); v${runtime.realizedVersion ?? "?"} stays on air; retry ${runtime.swapFailures}/${SWAP_RETRIES} in ${delay}ms`);
|
||||
const key = this.key(runtime.desired.workspaceId, runtime.desired.channelId);
|
||||
runtime.retryTimer = this.retryScheduler.schedule(() => this.serialize(key, async () => {
|
||||
runtime.retryTimer = null;
|
||||
if (this.stopped) return;
|
||||
// A newer command or an exit has taken over since.
|
||||
if (runtime.desired.desiredVersion !== version || runtime.relay !== relay || runtime.state !== "RUNNING") return;
|
||||
if (await this.swap(runtime, relay) === "restart") await this.restart(runtime);
|
||||
}), delay);
|
||||
return "retrying";
|
||||
}
|
||||
}
|
||||
|
||||
/** Full restart for the current desired snapshot (the swap path gave up). */
|
||||
private async restart(runtime: ChannelRuntime): Promise<void> {
|
||||
runtime.realizedVersion = null;
|
||||
runtime.swapFailures = 0;
|
||||
await runtime.relay?.stop();
|
||||
runtime.relay = null;
|
||||
runtime.retries = 0;
|
||||
await this.launch(runtime);
|
||||
}
|
||||
|
||||
private async launch(runtime: ChannelRuntime): Promise<void> {
|
||||
await this.transition(runtime, runtime.retries ? "RESTARTING" : "STARTING", runtime.retries ? "relay retry" : "signed desired state running");
|
||||
try {
|
||||
const relay = sandboxRelay(this.createRelay(runtime.desired, (reason) => {
|
||||
if (runtime.relay === relay) void this.relayExited(runtime, reason);
|
||||
}), runtime.desired);
|
||||
runtime.relay = relay;
|
||||
await relay.start(runtime.desired);
|
||||
if (runtime.relay !== relay) return;
|
||||
runtime.realizedVersion = runtime.desired.desiredVersion;
|
||||
await this.transition(runtime, "RUNNING", "ffmpeg spawned with local relay pipes");
|
||||
await this.beat();
|
||||
} catch (error: unknown) {
|
||||
runtime.relay = null;
|
||||
await this.relayExited(runtime, `start failed: ${error instanceof Error ? error.name : "unknown error"}`);
|
||||
}
|
||||
}
|
||||
|
||||
private async relayExited(runtime: ChannelRuntime, reason: string): Promise<void> {
|
||||
if (this.stopped || runtime.desired.desiredStatus !== "RUNNING") return;
|
||||
runtime.realizedVersion = null;
|
||||
runtime.relay = null;
|
||||
await this.transition(runtime, "DEGRADED", reason);
|
||||
if (runtime.retries >= 5) { await this.transition(runtime, "FAILED", "relay retry budget exhausted"); return; }
|
||||
const delay = Math.min(30_000, 1_000 * 2 ** runtime.retries);
|
||||
runtime.retries += 1;
|
||||
await this.transition(runtime, "RESTARTING", `retry in ${delay}ms`);
|
||||
runtime.retryTimer = this.retryScheduler.schedule(() => { runtime.retryTimer = null; return this.stopped ? Promise.resolve() : this.launch(runtime); }, delay);
|
||||
}
|
||||
|
||||
/**
|
||||
* Worker shutdown. In-flight per-channel operations (a command, a swap retry, a relaunch) finish
|
||||
* first, so none of them can start a job after the channels were stopped.
|
||||
*/
|
||||
async stop(): Promise<void> {
|
||||
this.stopped = true;
|
||||
if (this.heartbeatTimer) clearInterval(this.heartbeatTimer);
|
||||
this.heartbeatTimer = null;
|
||||
for (const runtime of this.channels.values()) runtime.retryTimer?.cancel();
|
||||
await Promise.allSettled([...this.operations.values()]);
|
||||
for (const runtime of this.channels.values()) {
|
||||
runtime.retryTimer?.cancel();
|
||||
runtime.retryTimer = null;
|
||||
runtime.desired = { ...runtime.desired, desiredStatus: "STOPPED" };
|
||||
await runtime.relay?.stop();
|
||||
runtime.realizedVersion = null;
|
||||
await this.transition(runtime, "STOPPED", "worker shutdown");
|
||||
}
|
||||
}
|
||||
}
|
||||
215
apps/media-worker/src/supervisor/production.e2e.test.ts
Normal file
215
apps/media-worker/src/supervisor/production.e2e.test.ts
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
/**
|
||||
* A/B lifecycle end to end (streaming-architecture.md §6 S2): signed snapshots
|
||||
* through the real MediaSupervisor into a real ProductionJob (synthetic
|
||||
* ingests standing in for the WP7 relay, a playlist source through the asset
|
||||
* feeder, the Rust gate, an x264 encoder into `-f null`, preview taps).
|
||||
*
|
||||
* - a TAKE (program scene change) is a swap: the encoder process is the same
|
||||
* one, and the version is realized only once the gate reports B on air;
|
||||
* - a playlist edit is adopted by its feeder in place (still the same encoder);
|
||||
* - a changed source URL is a restart (a new encoder);
|
||||
* - STOPPED tears everything down.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { generateKeyPairSync, randomUUID } from "node:crypto";
|
||||
import { createReadStream, mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { signCommand, type CommandBody } from "../bus/envelope";
|
||||
import { DurableFence } from "../bus/fence";
|
||||
import type { ObservedStore } from "../bus/redis";
|
||||
import { ProductionJob } from "../compositor/production-job";
|
||||
import { ensureGateBinary, makeAssets, passthroughRedactor } from "../compositor/testing";
|
||||
import { parseProductionSpec, type ProductionSpec } from "../production";
|
||||
import type { PreviewContentType } from "../preview/contract";
|
||||
import type { DesiredChannel, ObservedChannel } from "../types";
|
||||
import { MediaSupervisor, type RelayJob } from "./index";
|
||||
|
||||
const id = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`;
|
||||
const WS = "8f0c5a8e-3b1d-4c3e-9a5b-0c1d2e3f4a5b";
|
||||
const CH = "1a2b3c4d-5e6f-4a1b-8c2d-3e4f5a6b7c8d";
|
||||
|
||||
function spec(opts: { take: 10 | 11; playlist: string[]; urlB?: string }): ProductionSpec {
|
||||
return parseProductionSpec({
|
||||
sources: [
|
||||
{ id: id(1), label: "Cam A", kind: "hls", url: "https://a.example/live.m3u8" },
|
||||
{ id: id(2), label: "Cam B", kind: "hls", url: opts.urlB ?? "https://b.example/live.m3u8" },
|
||||
{ id: id(3), label: "Loop", kind: "playlist", assetIds: opts.playlist, loop: true },
|
||||
],
|
||||
scenes: [
|
||||
{ id: id(10), label: "Two up", layout: "2x1", slots: [
|
||||
{ index: 0, sourceId: id(1), rect: null, fit: "cover", audioMode: "primary", gainDb: 0 },
|
||||
{ index: 1, sourceId: id(2), rect: null, fit: "cover", audioMode: "on", gainDb: -6 },
|
||||
] },
|
||||
{ id: id(11), label: "Playlist", layout: "pip", slots: [
|
||||
{ index: 0, sourceId: id(3), rect: null, fit: "contain", audioMode: "primary", gainDb: 0 },
|
||||
{ index: 1, sourceId: id(1), rect: { x: 6500, y: 6500, w: 3000, h: 3000 }, fit: "cover", audioMode: "off", gainDb: 0 },
|
||||
] },
|
||||
],
|
||||
buses: { preview: id(10), program: id(opts.take) },
|
||||
overlays: [
|
||||
{ id: id(20), sceneId: id(10), kind: "lower_third", enabled: true, layer: 1, position: { x: 500, y: 7500 }, revisionId: id(30), assetId: "a".repeat(64), chyron: null, clock: null },
|
||||
],
|
||||
transition: { type: "cut" },
|
||||
audio: { masterGainDb: 0 },
|
||||
});
|
||||
}
|
||||
|
||||
class MemoryObserved implements ObservedStore {
|
||||
rows = new Map<string, ObservedChannel>();
|
||||
async get(workspaceId: string, channelId: string): Promise<ObservedChannel | null> { return this.rows.get(`${workspaceId}:${channelId}`) ?? null; }
|
||||
async put(row: ObservedChannel): Promise<void> { this.rows.set(`${row.workspaceId}:${row.channelId}`, row); }
|
||||
async heartbeat(_workerId: string, channels: ObservedChannel[]): Promise<void> { for (const row of channels) await this.put(row); }
|
||||
}
|
||||
|
||||
const alive = (pid: number | undefined): boolean => pid !== undefined && spawnSync("kill", ["-0", String(pid)]).status === 0;
|
||||
|
||||
test("signed TAKE / playlist edit / source change / stop through the supervisor into a real production job", { timeout: 300_000 }, async (t) => {
|
||||
if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; }
|
||||
const dir = mkdtempSync(join(tmpdir(), "prod-e2e-"));
|
||||
const [overlay] = makeAssets(dir);
|
||||
// Two 2 s playlist assets (pre-normalized MP4, faststart), served as byte streams like a presigned GET.
|
||||
const clips = new Map<string, string>();
|
||||
for (const [n, hz] of [[40, 300], [41, 600], [42, 900]] as const) {
|
||||
const path = join(dir, `asset-${n}.mp4`);
|
||||
const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=s=1280x720:r=30:d=2", "-f", "lavfi", "-i", `sine=frequency=${hz}:d=2`,
|
||||
"-c:v", "libx264", "-preset", "ultrafast", "-g", "30", "-c:a", "aac", "-movflags", "+faststart", path]);
|
||||
assert.equal(r.status, 0, "asset encode");
|
||||
clips.set(id(n), path);
|
||||
}
|
||||
const opened: string[] = [];
|
||||
const store = new MemoryObserved();
|
||||
const keys = generateKeyPairSync("ed25519");
|
||||
const published = new Map<string, number>();
|
||||
const jobs: ProductionJob[] = [];
|
||||
const exits: string[] = [];
|
||||
const createRelay = (_channel: DesiredChannel, onExit: (reason: string) => void): RelayJob => {
|
||||
const job = new ProductionJob({
|
||||
validator: { async validate() { throw new Error("not used: synthetic egress"); } },
|
||||
vault: { async deliverPushTarget() { throw new Error("not used: null output"); } },
|
||||
egress: {
|
||||
// Stand-in for the WP7 relay: each HLS source becomes a synthetic, real-time ingest.
|
||||
async prepare(config) {
|
||||
const sources = new Map((config.production?.sources ?? []).filter((s) => s.kind === "hls")
|
||||
.map((s, i) => [s.id, { input: { kind: "synthetic" as const, pattern: "smptehdbars" as const, seed: i + 1, toneHz: 440 + 220 * i }, fd: null }]));
|
||||
return { sources, output: { kind: "null" as const }, async close() {} };
|
||||
},
|
||||
},
|
||||
assets: { async localPath() { return overlay!; } },
|
||||
playlists: {
|
||||
async open(assetId) {
|
||||
opened.push(assetId);
|
||||
const path = clips.get(assetId);
|
||||
if (!path) throw new Error("unknown asset");
|
||||
return createReadStream(path);
|
||||
},
|
||||
},
|
||||
preview: {
|
||||
objects: { async putContent(body: Buffer, contentType: PreviewContentType) { return { key: `sha256/${"0".repeat(64)}.${contentType === "image/jpeg" ? "jpg" : "bin"}` }; } },
|
||||
manifests: { async set(key: string) { published.set(key, (published.get(key) ?? 0) + 1); } },
|
||||
},
|
||||
redactor: passthroughRedactor, onStderr: () => undefined,
|
||||
onExit: (reason) => { exits.push(reason); onExit(reason); },
|
||||
scratchDir: join(dir, `scratch-${jobs.length}`),
|
||||
});
|
||||
jobs.push(job);
|
||||
return job;
|
||||
};
|
||||
const base: DesiredChannel = {
|
||||
workspaceId: WS, channelId: CH, workerId: "worker-1", assignmentEpoch: 1, desiredVersion: 0, desiredStatus: "RUNNING",
|
||||
source: null, output: { endpointUrl: "rtmp://push.example/live", credentialHandleId: "keyref-1" }, profile: "720p30", mode: "normalize",
|
||||
production: spec({ take: 10, playlist: [id(40), id(41)] }),
|
||||
};
|
||||
const worker = new MediaSupervisor("worker-1", [base], store, new DurableFence(join(dir, "fence.json")),
|
||||
new Map([["control-v1", keys.publicKey]]), createRelay, { transition() {} });
|
||||
const send = (version: number, production: ProductionSpec, status: "RUNNING" | "STOPPED" = "RUNNING") => {
|
||||
const body: CommandBody = {
|
||||
workspaceId: WS, channelId: CH, workerId: "worker-1", assignmentEpoch: 1, commandId: randomUUID(), desiredVersion: version,
|
||||
issuedAt: new Date(Date.now() - 1_000).toISOString(), expiresAt: new Date(Date.now() + 120_000).toISOString(),
|
||||
snapshot: { ...base, desiredVersion: version, desiredStatus: status, production },
|
||||
};
|
||||
return worker.handle(signCommand(body, "control-v1", keys.privateKey));
|
||||
};
|
||||
const status = () => worker.status()[0]!;
|
||||
try {
|
||||
await worker.start();
|
||||
assert.equal(await send(1, spec({ take: 10, playlist: [id(40), id(41)] })), "accepted");
|
||||
assert.equal(status().state, "RUNNING");
|
||||
assert.equal(status().realizedVersion, 1);
|
||||
const job1 = jobs[0]!;
|
||||
const encoder = job1.program?.encoderPid;
|
||||
assert.ok(alive(encoder), "encoder up");
|
||||
// Baseline after the startup transient (compositor and x264 warming up repeat a few frames on a
|
||||
// loaded host, as cpu-smoke reports): wait until the gate has not repeated for 1 s (at most 15 s).
|
||||
let s1 = await job1.program!.stats();
|
||||
for (let calm = 0, waited = 0; calm < 2 && waited < 30; waited += 1) {
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
const now = await job1.program!.stats();
|
||||
calm = now.vrep + now.asil === s1.vrep + s1.asil ? calm + 1 : 0;
|
||||
s1 = now;
|
||||
}
|
||||
|
||||
// TAKE: program scene 10 → 11 (playlist PIP). Same encoder; realized once B is on air.
|
||||
let realizedDuringSwap: number | null = -1;
|
||||
const swapping = send(2, spec({ take: 11, playlist: [id(40), id(41)] }));
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
realizedDuringSwap = status().realizedVersion;
|
||||
await swapping;
|
||||
// On an oversubscribed host B can lag the on-air source (CompositorLagError): A stays on air,
|
||||
// v1 stays realized, and the supervisor retries the swap (1 s, then 2 s). That is the contract too.
|
||||
const retried = status().realizedVersion === 1 && /stays on air; retry/.test(status().reason);
|
||||
const tSwap = Date.now();
|
||||
while (status().realizedVersion !== 2 && jobs.length === 1 && Date.now() - tSwap < 30_000) await new Promise((r) => setTimeout(r, 100));
|
||||
if (retried) t.diagnostic("TAKE went on air through a supervisor swap retry (B lagged on the first attempt)");
|
||||
assert.equal(realizedDuringSwap, 1, "not realized before the new compositor is on air");
|
||||
assert.equal(jobs.length, 1, `no restart (${status().reason})`);
|
||||
assert.equal(status().realizedVersion, 2, `TAKE realized (${status().reason})`);
|
||||
assert.match(status().reason, /frame boundary/);
|
||||
const s2 = await job1.program!.stats();
|
||||
assert.equal(job1.program?.encoderPid, encoder, "same encoder process: the push session never dropped");
|
||||
assert.equal(s2.swaps, s1.swaps + 1);
|
||||
// A fill here would be the gate itself starved on an oversubscribed host, never the splice; keep it rare.
|
||||
const fills = s2.vrep + s2.asil - (s1.vrep + s1.asil);
|
||||
assert.ok(fills <= 2, `no repeat or silence fill across the swap (${fills})`);
|
||||
assert.deepEqual(job1.lastSkipped, []);
|
||||
|
||||
// Playlist edit: adopted in place by the feeder (no restart), and the program keeps running.
|
||||
await send(3, spec({ take: 11, playlist: [id(42), id(40)] }));
|
||||
assert.equal(status().realizedVersion, 3);
|
||||
assert.equal(jobs.length, 1);
|
||||
assert.equal(job1.program?.encoderPid, encoder);
|
||||
const feeder = job1.feeder(id(3))!;
|
||||
const t0 = Date.now();
|
||||
while (!opened.includes(id(42)) && Date.now() - t0 < 15_000) await new Promise((r) => setTimeout(r, 100));
|
||||
assert.ok(opened.includes(id(42)), `the edited list is played (opened ${opened.join(",")})`);
|
||||
assert.equal(feeder.stats.failures, 0);
|
||||
|
||||
// Previews: the program tap's thumbnail and always-on LL-HLS reached the manifest store.
|
||||
const key = (tap: string, file: string): string => `preview:${WS}:${CH}:${tap}:${file}`;
|
||||
const t1 = Date.now();
|
||||
while (!published.has(key("program", "video.m3u8")) && Date.now() - t1 < 30_000) await new Promise((r) => setTimeout(r, 200));
|
||||
assert.ok(published.has(key("program", "thumbnail.jpg")), "program thumbnail");
|
||||
assert.ok(published.has(key("program", "video.m3u8")), "program LL-HLS");
|
||||
assert.ok(published.has(key(id(1), "thumbnail.jpg")) || published.has(key(id(2), "thumbnail.jpg")), "source thumbnails");
|
||||
|
||||
// A changed source URL cannot be a swap: full restart, new encoder.
|
||||
await send(4, spec({ take: 11, playlist: [id(42), id(40)], urlB: "https://b2.example/live.m3u8" }));
|
||||
assert.equal(status().realizedVersion, 4);
|
||||
assert.equal(jobs.length, 2, "restarted");
|
||||
assert.ok(!alive(encoder), "the old encoder is gone");
|
||||
assert.ok(alive(jobs[1]!.program?.encoderPid));
|
||||
|
||||
const last = jobs[1]!.program?.encoderPid;
|
||||
await send(5, spec({ take: 11, playlist: [id(42)] }), "STOPPED");
|
||||
assert.equal(status().state, "STOPPED");
|
||||
assert.ok(!alive(last), "stopped");
|
||||
assert.deepEqual(exits, [], "no job exited on its own");
|
||||
} finally {
|
||||
// Also on failure: the heartbeat interval and any pending swap retry would keep the process alive.
|
||||
await worker.stop();
|
||||
for (const job of jobs) await job.stop();
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
155
apps/media-worker/src/supervisor/reconfigure.test.ts
Normal file
155
apps/media-worker/src/supervisor/reconfigure.test.ts
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { generateKeyPairSync, randomUUID } from "node:crypto";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { signCommand, type CommandBody } from "../bus/envelope";
|
||||
import { DurableFence } from "../bus/fence";
|
||||
import type { ObservedStore } from "../bus/redis";
|
||||
import { MediaSupervisor, type RelayJob, type RetryScheduler } from "./index";
|
||||
import type { DesiredChannel, ObservedChannel } from "../types";
|
||||
|
||||
const base: DesiredChannel = {
|
||||
workspaceId: "workspace-1", channelId: "channel-1", workerId: "worker-1",
|
||||
assignmentEpoch: 2, desiredVersion: 1, desiredStatus: "RUNNING",
|
||||
source: { kind: "hls", url: "https://mock.local/hls/live-1/master.m3u8" },
|
||||
output: { endpointUrl: "rtmp://mock.local/live", credentialHandleId: "keyref-1" },
|
||||
profile: "720p30", mode: "normalize",
|
||||
};
|
||||
|
||||
const store: ObservedStore = {
|
||||
async get() { return null; },
|
||||
async put(_row: ObservedChannel) {},
|
||||
async heartbeat() {},
|
||||
};
|
||||
|
||||
test("RUNNING → RUNNING on a swappable job is an in-place reconfigure, realized only when on air", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "media-reconf-"));
|
||||
const keys = generateKeyPairSync("ed25519");
|
||||
const calls: string[] = [];
|
||||
let accept = true;
|
||||
let failures = 0;
|
||||
const job = (): RelayJob => ({
|
||||
async start(c) { calls.push(`start v${c.desiredVersion}`); },
|
||||
async stop() { calls.push("stop"); },
|
||||
async reconfigure(c) {
|
||||
calls.push(`swap v${c.desiredVersion}`);
|
||||
if (failures > 0) { failures -= 1; throw new Error("compositor B never went on air"); }
|
||||
return accept;
|
||||
},
|
||||
});
|
||||
// Manual retry scheduler: the test fires retries itself.
|
||||
const pending: Array<{ action: () => Promise<void>; delayMs: number; cancelled: boolean }> = [];
|
||||
const scheduler: RetryScheduler = {
|
||||
schedule(action, delayMs) { const entry = { action, delayMs, cancelled: false }; pending.push(entry); return { cancel: () => { entry.cancelled = true; } }; },
|
||||
};
|
||||
const fire = async (): Promise<number> => { const entry = pending.shift(); assert.ok(entry && !entry.cancelled, "a retry is scheduled"); await entry.action(); return entry.delayMs; };
|
||||
const worker = new MediaSupervisor("worker-1", [base], store, new DurableFence(join(dir, "fence.json")),
|
||||
new Map([["control-v1", keys.publicKey]]), job, { transition() {} }, scheduler);
|
||||
const send = (version: number, status: "RUNNING" | "STOPPED" = "RUNNING") => {
|
||||
const body: CommandBody = {
|
||||
workspaceId: base.workspaceId, channelId: base.channelId, workerId: base.workerId, assignmentEpoch: base.assignmentEpoch,
|
||||
commandId: randomUUID(), desiredVersion: version, issuedAt: new Date(Date.now() - 1_000).toISOString(),
|
||||
expiresAt: new Date(Date.now() + 60_000).toISOString(), snapshot: { ...base, desiredVersion: version, desiredStatus: status },
|
||||
};
|
||||
return worker.handle(signCommand(body, "control-v1", keys.privateKey));
|
||||
};
|
||||
const status = () => worker.status()[0]!;
|
||||
try {
|
||||
await worker.start();
|
||||
await send(1);
|
||||
assert.deepEqual(calls, ["start v1"]);
|
||||
await send(2);
|
||||
assert.deepEqual(calls, ["start v1", "swap v2"], "a TAKE never stops the job (encoder/RTMP stay up)");
|
||||
assert.equal(status().state, "RUNNING");
|
||||
assert.equal(status().realizedVersion, 2);
|
||||
assert.match(status().reason, /frame boundary/);
|
||||
accept = false;
|
||||
await send(3);
|
||||
assert.deepEqual(calls.slice(2), ["swap v3", "stop", "start v3"], "an unswappable change falls back to restart");
|
||||
accept = true;
|
||||
|
||||
// A failed swap keeps the old compositor on air: still RUNNING, realized = what viewers see, retried.
|
||||
failures = 1;
|
||||
await send(4);
|
||||
assert.deepEqual(calls.slice(5), ["swap v4"]);
|
||||
assert.equal(status().state, "RUNNING");
|
||||
assert.equal(status().realizedVersion, 3, "v3 is still on air");
|
||||
assert.match(status().reason, /swap to v4 failed .*v3 stays on air; retry 1\/2/);
|
||||
assert.equal(await fire(), 1_000);
|
||||
assert.deepEqual(calls.slice(5), ["swap v4", "swap v4"], "the retry swaps; nothing restarts");
|
||||
assert.equal(status().realizedVersion, 4);
|
||||
|
||||
// A newer command supersedes a pending retry.
|
||||
failures = 1;
|
||||
await send(5);
|
||||
await send(6);
|
||||
assert.deepEqual(calls.slice(7), ["swap v5", "swap v6"]);
|
||||
assert.equal(pending.shift()?.cancelled, true, "the v5 retry was cancelled by v6");
|
||||
assert.equal(status().realizedVersion, 6);
|
||||
|
||||
// Out of retries: the full restart converges (and is the only path that drops the push session).
|
||||
failures = 3;
|
||||
await send(7);
|
||||
assert.equal(await fire(), 1_000);
|
||||
assert.equal(await fire(), 2_000);
|
||||
assert.deepEqual(calls.slice(9), ["swap v7", "swap v7", "swap v7", "stop", "start v7"]);
|
||||
assert.equal(status().realizedVersion, 7);
|
||||
assert.equal(status().state, "RUNNING");
|
||||
|
||||
await send(8, "STOPPED");
|
||||
assert.deepEqual(calls.slice(14), ["stop"]);
|
||||
assert.equal(pending.length, 0);
|
||||
await worker.stop();
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("worker stop waits for an in-flight swap and no retry or relaunch runs after it", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "media-reconf-stop-"));
|
||||
const keys = generateKeyPairSync("ed25519");
|
||||
const calls: string[] = [];
|
||||
let releaseSwap: (() => void) | null = null;
|
||||
const job = (): RelayJob => ({
|
||||
async start(c) { calls.push(`start v${c.desiredVersion}`); },
|
||||
async stop() { calls.push("stop"); },
|
||||
async reconfigure(c) {
|
||||
calls.push(`swap v${c.desiredVersion}`);
|
||||
// B is still trying to go on air when the worker shuts down; then it lags.
|
||||
await new Promise<void>((resolve) => { releaseSwap = resolve; });
|
||||
throw new Error("compositor B never went on air");
|
||||
},
|
||||
});
|
||||
const pending: Array<{ action: () => Promise<void>; cancelled: boolean }> = [];
|
||||
const scheduler: RetryScheduler = {
|
||||
schedule(action) { const entry = { action, cancelled: false }; pending.push(entry); return { cancel: () => { entry.cancelled = true; } }; },
|
||||
};
|
||||
const worker = new MediaSupervisor("worker-1", [base], store, new DurableFence(join(dir, "fence.json")),
|
||||
new Map([["control-v1", keys.publicKey]]), job, { transition() {} }, scheduler);
|
||||
const send = (version: number) => {
|
||||
const body: CommandBody = {
|
||||
workspaceId: base.workspaceId, channelId: base.channelId, workerId: base.workerId, assignmentEpoch: base.assignmentEpoch,
|
||||
commandId: randomUUID(), desiredVersion: version, issuedAt: new Date(Date.now() - 1_000).toISOString(),
|
||||
expiresAt: new Date(Date.now() + 60_000).toISOString(), snapshot: { ...base, desiredVersion: version },
|
||||
};
|
||||
return worker.handle(signCommand(body, "control-v1", keys.privateKey));
|
||||
};
|
||||
try {
|
||||
await worker.start();
|
||||
await send(1);
|
||||
const swapping = send(2);
|
||||
while (!releaseSwap) await new Promise((r) => setImmediate(r));
|
||||
let stopped = false;
|
||||
const stopping = worker.stop().then(() => { stopped = true; });
|
||||
await new Promise((r) => setImmediate(r));
|
||||
assert.equal(stopped, false, "stop waits for the in-flight swap");
|
||||
(releaseSwap as () => void)();
|
||||
await swapping;
|
||||
await stopping;
|
||||
assert.deepEqual(calls, ["start v1", "swap v2", "stop"]);
|
||||
assert.equal(worker.status()[0]?.state, "STOPPED");
|
||||
// The retry the failed swap scheduled is cancelled, and running it anyway does nothing.
|
||||
for (const entry of pending) { assert.equal(entry.cancelled, true); await entry.action(); }
|
||||
assert.deepEqual(calls, ["start v1", "swap v2", "stop"]);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
109
apps/media-worker/src/supervisor/sandbox.test.ts
Normal file
109
apps/media-worker/src/supervisor/sandbox.test.ts
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { generateKeyPairSync, randomUUID } from "node:crypto";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import test from "node:test";
|
||||
import { signCommand } from "../bus/envelope";
|
||||
import { DurableFence } from "../bus/fence";
|
||||
import type { ObservedStore } from "../bus/redis";
|
||||
import { buildCompositorArgs, buildEncoderArgs, buildIngestArgs, buildPlaylistRemuxArgs, buildSlateArgs } from "../compositor/command-builder";
|
||||
import { buildHlsTapArgs } from "../preview/argv";
|
||||
import { ffmpegArgs, spawnManaged, type ManagedProcess } from "../relay/ffmpeg";
|
||||
import type { DesiredChannel } from "../types";
|
||||
import { MediaSupervisor, type RelayJob } from "./index";
|
||||
import { supervisorSandboxEnabled } from "./sandbox";
|
||||
const pkg = (): {
|
||||
sandboxCapability(options: { rootfs: string; cgroupRoot: string }): { available: boolean; reason?: string };
|
||||
MediaCommandBuilder: new () => { supervisor(binary: "ffmpeg", args: readonly string[], scratch: string): { argv: readonly string[] } };
|
||||
} => require(resolve(__dirname, "../../../../packages/media-sandbox/dist"));
|
||||
const channel: DesiredChannel = {
|
||||
workspaceId: "sandbox-workspace", channelId: "sandbox-channel", workerId: "sandbox-worker", assignmentEpoch: 1,
|
||||
desiredVersion: 1, desiredStatus: "RUNNING", profile: "720p30", mode: "passthrough",
|
||||
source: { kind: "hls", url: "https://source.example/live.m3u8" },
|
||||
output: { endpointUrl: "rtmp://output.example/live", credentialHandleId: "opaque-handle" },
|
||||
};
|
||||
const store: ObservedStore = { async get() { return null; }, async put() {}, async heartbeat() {} };
|
||||
async function env<T>(values: Record<string, string | undefined>, action: () => Promise<T>): Promise<T> {
|
||||
const previous = Object.fromEntries(Object.keys(values).map(key => [key, process.env[key]]));
|
||||
for (const [key, value] of Object.entries(values)) { if (value === undefined) delete process.env[key]; else process.env[key] = value; }
|
||||
try { return await action(); }
|
||||
finally { for (const [key, value] of Object.entries(previous)) { if (value === undefined) delete process.env[key]; else process.env[key] = value; } }
|
||||
}
|
||||
function command(keys: ReturnType<typeof generateKeyPairSync>) {
|
||||
return signCommand({ workspaceId: channel.workspaceId, channelId: channel.channelId, workerId: channel.workerId,
|
||||
assignmentEpoch: 1, commandId: randomUUID(), desiredVersion: 1,
|
||||
issuedAt: new Date(Date.now() - 1000).toISOString(), expiresAt: new Date(Date.now() + 60_000).toISOString(), snapshot: channel }, "control", keys.privateKey);
|
||||
}
|
||||
test("sandbox flag defaults locally off and on in production; production off is rejected", async () => {
|
||||
await env({ NODE_ENV: "development", MEDIA_SANDBOX: undefined }, async () => assert.equal(supervisorSandboxEnabled(), false));
|
||||
await env({ NODE_ENV: "production", MEDIA_SANDBOX: undefined }, async () => assert.equal(supervisorSandboxEnabled(), true));
|
||||
await env({ NODE_ENV: "production", MEDIA_SANDBOX: "off" }, async () => assert.throws(supervisorSandboxEnabled, /requires/));
|
||||
});
|
||||
test("sandbox builder accepts existing relay/ingest/compositor/encoder/playlist/preview plans", () => {
|
||||
const builder = new (pkg().MediaCommandBuilder)(); const canvas = { width: 1280, height: 720, fps: 30 };
|
||||
const plans = [ffmpegArgs(channel), buildPlaylistRemuxArgs(0),
|
||||
buildIngestArgs({ canvas, input: { kind: "mpegtsPipe" }, audio: "source" }),
|
||||
buildIngestArgs({ canvas, input: { kind: "synthetic", pattern: "testsrc2", seed: 1, toneHz: 440 }, audio: "source" }),
|
||||
buildEncoderArgs({ canvas, videoBitrateKbps: 2800, audioBitrateKbps: 128, output: { kind: "null" }, programTap: { videoFd: 5, audioFd: 6, size: { width: 320, height: 180 } } }),
|
||||
buildSlateArgs({ canvas, seconds: 5, image: `/job/${"a".repeat(64)}.png`, offsetUs: 0 }),
|
||||
buildHlsTapArgs({ geometry: canvas, rendition: "video", outDir: "/job/taps/video" }),
|
||||
buildCompositorArgs({ sources: [{ video: { kind: "rawPipe", fd: 3 }, audio: { kind: "rawPipe", fd: 4 } }], assets: [`/job/${"a".repeat(64)}.png`],
|
||||
graph: { canvas, layout: "single", tiles: [{ source: 0, rect: { x: 0, y: 0, w: 1280, h: 720 }, fit: "cover" }],
|
||||
overlays: [{ kind: "chyron", asset: 0, y: 10, speedPxPerSec: 40 }], audio: { inputs: [{ source: 0, gainDb: 0 }], masterGainDb: 0 } }, output: { kind: "gate", videoFd: 1, audioFd: 3 } })];
|
||||
for (const args of plans) assert.ok(builder.supervisor("ffmpeg", args, "/job").argv.includes("-protocol_whitelist"));
|
||||
});
|
||||
test("supervisor fails closed if sandbox is enabled without configuration", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "media-sandbox-closed-")); const keys = generateKeyPairSync("ed25519");
|
||||
const lines: string[] = [];
|
||||
await env({ NODE_ENV: "development", MEDIA_SANDBOX: "on", MEDIA_SANDBOX_ROOTFS: undefined, MEDIA_SANDBOX_CGROUP_ROOT: undefined, MEDIA_SCRATCH_DIR: join(dir, "scratch") }, async () => {
|
||||
const worker = new MediaSupervisor(channel.workerId, [channel], store, new DurableFence(join(dir, "fence")), new Map([["control", keys.publicKey]]),
|
||||
(): RelayJob => ({ async start() { spawnManaged("ffmpeg", ffmpegArgs(channel), ["ignore", "ignore", "pipe"], { redact: v => v }, line => lines.push(line)); }, async stop() {} }),
|
||||
{ transition: event => lines.push(JSON.stringify(event)) }, { schedule() { return { cancel() {} }; } });
|
||||
try { await worker.start(); await worker.handle(command(keys)); assert.equal(worker.status()[0]?.state, "RESTARTING"); assert.ok(lines.join("\n").includes("SandboxUnavailable")); }
|
||||
finally { await worker.stop(); }
|
||||
});
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
for (const enabled of [false, true]) test(`secret-leak canaries through signed supervisor command (sandbox ${enabled ? "on" : "off"})`, { timeout: 20_000 }, async t => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "media-supervisor-canary-"));
|
||||
const rootfs = join(dir, "rootfs");
|
||||
for (const d of ["upload", "scratch", "tmp", "proc", "dev", "usr/bin"]) mkdirSync(join(rootfs, d), { recursive: true });
|
||||
writeFileSync(join(rootfs, "dev/null"), "");
|
||||
try {
|
||||
if (enabled) {
|
||||
const compiled = spawnSync("cc", ["-static", "-O0", resolve(__dirname, "../../../../packages/media-sandbox/native/test-child.c"), "-o", join(rootfs, "usr/bin/ffmpeg")]);
|
||||
const cap = compiled.status === 0 ? pkg().sandboxCapability({ rootfs, cgroupRoot: process.env.MEDIA_SANDBOX_CGROUP_ROOT ?? "/sys/fs/cgroup" }) : { available: false, reason: "static compiler unavailable" };
|
||||
if (!cap.available) { t.skip(`rootless sandbox canary skipped: ${cap.reason}`); return; }
|
||||
}
|
||||
const canaries = ["stream-key-canary-supervisor-531", "oauth-token-canary-supervisor-794", "srt-passphrase-canary-supervisor-652"];
|
||||
await env({ NODE_ENV: "development", MEDIA_SANDBOX: enabled ? "on" : "off", MEDIA_SANDBOX_ROOTFS: rootfs,
|
||||
MEDIA_SANDBOX_CGROUP_ROOT: process.env.MEDIA_SANDBOX_CGROUP_ROOT ?? "/sys/fs/cgroup", MEDIA_SCRATCH_DIR: join(dir, "scratch"),
|
||||
CANARY_STREAM_KEY: canaries[0], CANARY_OAUTH_TOKEN: canaries[1], CANARY_SRT_PASSPHRASE: canaries[2] }, async () => {
|
||||
const keys = generateKeyPairSync("ed25519"); const lines: string[] = []; let proc: ManagedProcess | undefined;
|
||||
const worker = new MediaSupervisor(channel.workerId, [channel], store, new DurableFence(join(dir, "fence")), new Map([["control", keys.publicKey]]),
|
||||
(_channel, onExit): RelayJob => ({
|
||||
async start() {
|
||||
proc = spawnManaged("ffmpeg", ["-hide_banner", "-nostdin", "-loglevel", "error", "-f", "mpegts", "-i", "pipe:0", "-f", "null", "pipe:1"],
|
||||
["pipe", "ignore", "pipe"], { redact: value => value }, line => lines.push(line));
|
||||
void proc.exited.then(({ code }) => onExit(`ffmpeg exited code=${code}`)); await proc.started;
|
||||
},
|
||||
async stop() { await proc?.terminate(100); },
|
||||
}), { transition: event => lines.push(JSON.stringify(event)) }, { schedule() { return { cancel() {} }; } });
|
||||
try {
|
||||
await worker.start(); await worker.handle(command(keys));
|
||||
assert.equal(worker.status()[0]?.state, "RUNNING"); assert.ok(proc?.child.pid);
|
||||
assert.equal(proc.child.spawnfile.endsWith("media-confine"), enabled);
|
||||
const cmdline = readFileSync(`/proc/${proc.child.pid}/cmdline`, "utf8");
|
||||
const environment = readFileSync(`/proc/${proc.child.pid}/environ`, "utf8");
|
||||
proc.child.stdin!.end("invalid-media-to-force-ffmpeg-error");
|
||||
const result = await proc.exited; assert.notEqual(result.code, 0);
|
||||
await worker.stop();
|
||||
assert.ok(lines.some(line => enabled ? line.includes("stderr suppressed") : /Invalid data|could not find codec|End of file/i.test(line)));
|
||||
for (const canary of canaries) for (const output of [cmdline, environment, ...lines]) assert.ok(!output.includes(canary));
|
||||
} finally { await worker.stop(); }
|
||||
});
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
99
apps/media-worker/src/supervisor/sandbox.ts
Normal file
99
apps/media-worker/src/supervisor/sandbox.ts
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
import { createHash } from "node:crypto";
|
||||
import type * as ChildProcessApi from "node:child_process";
|
||||
import { chmodSync, mkdirSync, constants, openSync, closeSync, fchmodSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { Transform } from "node:stream";
|
||||
import type { DesiredChannel } from "../types";
|
||||
import type { RelayJob } from "./index";
|
||||
|
||||
const childProcess: { spawn: typeof ChildProcessApi.spawn } = require("node:child_process");
|
||||
|
||||
// Resolve the CommonJS package from either src/supervisor or dist/supervisor.
|
||||
// A small structural boundary keeps worker typechecking independent of package build order.
|
||||
interface MediaCommand { binary: "ffmpeg" | "ffprobe"; argv: readonly string[] }
|
||||
interface SandboxModule {
|
||||
MediaCommandBuilder: new () => { supervisor(binary: MediaCommand["binary"], argv: readonly string[], scratch: string): MediaCommand };
|
||||
SandboxUnavailable: new () => Error;
|
||||
spawnSandbox(command: MediaCommand, options: { rootfs: string; cgroupRoot: string; scratchDir: string; limits: { wallTimeMs: number } }, spawnOptions: ChildProcessApi.SpawnOptions | undefined, spawner: typeof ChildProcessApi.spawn): { child: ChildProcessApi.ChildProcess };
|
||||
}
|
||||
const sandbox = (): SandboxModule => require(resolve(__dirname, "../../../../packages/media-sandbox/dist")) as SandboxModule;
|
||||
interface Context { scratch: string }
|
||||
const context = new AsyncLocalStorage<Context>();
|
||||
let installed = false;
|
||||
|
||||
/**
|
||||
* The only common spawn path lives in relay/, outside this change's boundary.
|
||||
* A scoped Node spawn adapter covers existing typed FFmpeg builders, including
|
||||
* asynchronous swaps/playlist producers, without altering their process API.
|
||||
* Non-media children and calls outside a supervisor job retain normal spawning.
|
||||
*/
|
||||
function installAdapter(): void {
|
||||
if (installed) return;
|
||||
const original = childProcess.spawn;
|
||||
childProcess.spawn = ((command: string, args?: readonly string[] | ChildProcessApi.SpawnOptions, options?: ChildProcessApi.SpawnOptions) => {
|
||||
const job = context.getStore();
|
||||
if (!job || !/(?:^|\/)(?:ffmpeg|ffprobe)$/.test(command)) {
|
||||
return typeof args === "object" && !Array.isArray(args)
|
||||
? original(command, args as ChildProcessApi.SpawnOptions)
|
||||
: original(command, (args as readonly string[] | undefined) ?? [], options ?? {});
|
||||
}
|
||||
if (!Array.isArray(args)) throw new Error("media argv required");
|
||||
const runtime = sandbox();
|
||||
const rootfs = process.env.MEDIA_SANDBOX_ROOTFS;
|
||||
const cgroupRoot = process.env.MEDIA_SANDBOX_CGROUP_ROOT;
|
||||
if (!rootfs || !cgroupRoot) throw new runtime.SandboxUnavailable();
|
||||
const media = new runtime.MediaCommandBuilder().supervisor(command.endsWith("ffprobe") ? "ffprobe" : "ffmpeg", args, job.scratch);
|
||||
// Make only generated output directories writable by the mapped UID.
|
||||
// Traverse through directory fds: a compromised sibling cannot race a
|
||||
// symlink replacement into chmod of a host file.
|
||||
for (const path of media.argv.filter(value => value.startsWith("file:/scratch/"))) {
|
||||
const parts = path.slice("file:/scratch/".length).split("/").slice(0, -1).filter(part => part && part !== ".");
|
||||
if (parts.length > 32 || parts.some(part => part === "..")) throw new runtime.SandboxUnavailable();
|
||||
let fd = openSync(job.scratch, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
||||
try {
|
||||
fchmodSync(fd, 0o777);
|
||||
for (const part of parts) {
|
||||
const next = openSync(`/proc/self/fd/${fd}/${part}`, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
||||
closeSync(fd); fd = next; fchmodSync(fd, 0o777);
|
||||
}
|
||||
} finally { closeSync(fd); }
|
||||
}
|
||||
const child = runtime.spawnSandbox(media, {
|
||||
rootfs, cgroupRoot, scratchDir: job.scratch,
|
||||
// Long-running channels still have a finite job lease; uploads use the package defaults.
|
||||
limits: { wallTimeMs: Number(process.env.MEDIA_SANDBOX_WALL_TIME_MS ?? 86_400_000) },
|
||||
}, options, original).child;
|
||||
// Replace the public stderr stream BEFORE spawnManaged attaches its logger.
|
||||
// Arbitrary media bytes cannot bypass a caller-supplied passthrough redactor.
|
||||
if (child.stderr) {
|
||||
const safe = new Transform({ transform(_chunk, _encoding, done) { done(null, Buffer.from("[media stderr suppressed]\n")); } });
|
||||
child.stderr.pipe(safe); child.stderr = safe; child.stdio[2] = safe;
|
||||
}
|
||||
return child;
|
||||
}) as typeof childProcess.spawn;
|
||||
installed = true;
|
||||
}
|
||||
function enabled(): boolean {
|
||||
const flag = process.env.MEDIA_SANDBOX;
|
||||
if (flag !== undefined && flag !== "on" && flag !== "off") throw new Error("MEDIA_SANDBOX must be on or off");
|
||||
if (process.env.NODE_ENV === "production" && flag === "off") throw new Error("production requires media sandbox");
|
||||
return flag === "on" || process.env.NODE_ENV === "production";
|
||||
}
|
||||
export function supervisorSandboxEnabled(): boolean { return enabled(); }
|
||||
|
||||
export function sandboxRelay(job: RelayJob, channel: DesiredChannel): RelayJob {
|
||||
if (!enabled()) return job;
|
||||
installAdapter();
|
||||
const id = createHash("sha256").update(`${channel.workspaceId}:${channel.channelId}`).digest("hex").slice(0, 32);
|
||||
const parent = resolve(process.env.MEDIA_SCRATCH_DIR ?? ".media-worker-state/scratch");
|
||||
const scratch = resolve(parent, id);
|
||||
mkdirSync(parent, { recursive: true }); chmodSync(parent, 0o700);
|
||||
mkdirSync(scratch, { recursive: true }); chmodSync(scratch, 0o777);
|
||||
const run = <T>(action: () => T): T => context.run({ scratch }, action);
|
||||
return {
|
||||
start: config => run(() => job.start(config)),
|
||||
stop: () => run(() => job.stop()),
|
||||
...(job.reconfigure ? { reconfigure: (config: DesiredChannel) => run(() => job.reconfigure!(config)) } : {}),
|
||||
};
|
||||
}
|
||||
135
apps/media-worker/src/supervisor/supervisor.test.ts
Normal file
135
apps/media-worker/src/supervisor/supervisor.test.ts
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { generateKeyPairSync, randomUUID } from "node:crypto";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { signCommand, type CommandBody } from "../bus/envelope";
|
||||
import { DurableFence } from "../bus/fence";
|
||||
import type { ObservedStore } from "../bus/redis";
|
||||
import { MediaSupervisor, type RelayJob, type RetryScheduler } from "./index";
|
||||
import type { DesiredChannel, ObservedChannel } from "../types";
|
||||
|
||||
const source: DesiredChannel = {
|
||||
workspaceId: "workspace-1", channelId: "channel-1", workerId: "worker-1",
|
||||
assignmentEpoch: 2, desiredVersion: 4, desiredStatus: "RUNNING",
|
||||
// Same HLS/RTMP endpoint shape supplied by connector-mock-livestream and the single-live simulator.
|
||||
source: { kind: "hls", url: "https://mock.local/hls/live-1/master.m3u8" },
|
||||
output: { endpointUrl: "rtmp://mock.local/live", credentialHandleId: "keyref-1" },
|
||||
profile: "720p30", mode: "passthrough",
|
||||
};
|
||||
|
||||
class MemoryObserved implements ObservedStore {
|
||||
rows = new Map<string, ObservedChannel>();
|
||||
heartbeats: Array<{ workerId: string; channels: ObservedChannel[] }> = [];
|
||||
async get(workspaceId: string, channelId: string): Promise<ObservedChannel | null> { return this.rows.get(`${workspaceId}:${channelId}`) ?? null; }
|
||||
async put(row: ObservedChannel): Promise<void> { this.rows.set(`${row.workspaceId}:${row.channelId}`, row); }
|
||||
async heartbeat(workerId: string, channels: ObservedChannel[]): Promise<void> {
|
||||
this.heartbeats.push({ workerId, channels });
|
||||
for (const row of channels) await this.put(row);
|
||||
}
|
||||
}
|
||||
|
||||
test("durable replay fence, stale version, and restart reconcile", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "media-fence-"));
|
||||
const store = new MemoryObserved();
|
||||
const keys = generateKeyPairSync("ed25519");
|
||||
const pinned = new Map([["control-v1", keys.publicKey]]);
|
||||
let starts = 0;
|
||||
const makeSupervisor = () => new MediaSupervisor("worker-1", [source], store,
|
||||
new DurableFence(join(dir, "fence.json")), pinned,
|
||||
(): RelayJob => ({ async start() { starts += 1; }, async stop() {} }),
|
||||
{ transition() {} });
|
||||
const makeBody = (commandId: string, version: number): CommandBody => ({
|
||||
workspaceId: source.workspaceId, channelId: source.channelId, workerId: source.workerId,
|
||||
assignmentEpoch: source.assignmentEpoch, commandId, desiredVersion: version,
|
||||
issuedAt: new Date(Date.now() - 1_000).toISOString(),
|
||||
expiresAt: new Date(Date.now() + 60_000).toISOString(),
|
||||
snapshot: { ...source, desiredVersion: version },
|
||||
});
|
||||
try {
|
||||
const first = signCommand(makeBody(randomUUID(), 4), "control-v1", keys.privateKey);
|
||||
const worker = makeSupervisor();
|
||||
await worker.start();
|
||||
assert.equal(worker.status()[0]?.realizedVersion, null);
|
||||
assert.equal(await worker.handle(first), "accepted");
|
||||
assert.equal(worker.status()[0]?.state, "RUNNING");
|
||||
assert.equal(worker.status()[0]?.realizedVersion, 4);
|
||||
assert.equal(starts, 1);
|
||||
await worker.stop();
|
||||
|
||||
const restarted = makeSupervisor();
|
||||
await restarted.start();
|
||||
assert.equal(restarted.status()[0]?.realizedVersion, null);
|
||||
assert.match(restarted.status()[0]?.reason ?? "", /restart reconcile/);
|
||||
assert.equal(await restarted.handle(first), "replay");
|
||||
assert.equal(starts, 1);
|
||||
const stale = signCommand(makeBody(randomUUID(), 3), "control-v1", keys.privateKey);
|
||||
assert.equal(await restarted.handle(stale), "stale");
|
||||
const freshSnapshot = signCommand(makeBody(randomUUID(), 4), "control-v1", keys.privateKey);
|
||||
assert.equal(await restarted.handle(freshSnapshot), "accepted");
|
||||
assert.equal(restarted.status()[0]?.realizedVersion, 4);
|
||||
assert.equal(starts, 2);
|
||||
await restarted.stop();
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("unsigned, expired, wrong target and tampered snapshots are rejected", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "media-command-"));
|
||||
const keys = generateKeyPairSync("ed25519");
|
||||
const worker = new MediaSupervisor("worker-1", [source], new MemoryObserved(),
|
||||
new DurableFence(join(dir, "fence.json")), new Map([["control-v1", keys.publicKey]]),
|
||||
(): RelayJob => ({ async start() {}, async stop() {} }), { transition() {} });
|
||||
const body: CommandBody = {
|
||||
workspaceId: source.workspaceId, channelId: source.channelId, workerId: source.workerId,
|
||||
assignmentEpoch: source.assignmentEpoch, commandId: randomUUID(), desiredVersion: 4,
|
||||
issuedAt: new Date(Date.now() - 2_000).toISOString(), expiresAt: new Date(Date.now() + 30_000).toISOString(), snapshot: source,
|
||||
};
|
||||
try {
|
||||
await worker.start();
|
||||
await assert.rejects(worker.handle({ kid: "control-v1", body, signature: "" }), /signature/);
|
||||
await assert.rejects(worker.handle(signCommand({ ...body, workerId: "other" }, "control-v1", keys.privateKey)), /fields/);
|
||||
const expired = { ...body, expiresAt: new Date(Date.now() - 1).toISOString() };
|
||||
await assert.rejects(worker.handle(signCommand(expired, "control-v1", keys.privateKey)), /expired/);
|
||||
const tampered = signCommand(body, "control-v1", keys.privateKey);
|
||||
tampered.body.snapshot = { ...source, desiredStatus: "STOPPED" };
|
||||
await assert.rejects(worker.handle(tampered), /signature/);
|
||||
await worker.stop();
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("relay failure follows DEGRADED, RESTARTING and bounded FAILED states", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "media-retry-"));
|
||||
const keys = generateKeyPairSync("ed25519");
|
||||
const queued: Array<{ action: () => Promise<void>; delay: number }> = [];
|
||||
const scheduler: RetryScheduler = {
|
||||
schedule(action, delay) { queued.push({ action, delay }); return { cancel() {} }; },
|
||||
};
|
||||
const transitions: string[] = [];
|
||||
const worker = new MediaSupervisor("worker-1", [source], new MemoryObserved(),
|
||||
new DurableFence(join(dir, "fence.json")), new Map([["control-v1", keys.publicKey]]),
|
||||
(): RelayJob => ({ async start() { throw new Error("ingest unavailable"); }, async stop() {} }),
|
||||
{ transition(event) { transitions.push(event.to); } }, scheduler);
|
||||
try {
|
||||
await worker.start();
|
||||
const body: CommandBody = {
|
||||
workspaceId: source.workspaceId, channelId: source.channelId, workerId: source.workerId,
|
||||
assignmentEpoch: source.assignmentEpoch, commandId: randomUUID(), desiredVersion: 4,
|
||||
issuedAt: new Date(Date.now() - 1_000).toISOString(), expiresAt: new Date(Date.now() + 60_000).toISOString(), snapshot: source,
|
||||
};
|
||||
assert.equal(await worker.handle(signCommand(body, "control-v1", keys.privateKey)), "accepted");
|
||||
assert.equal(worker.status()[0]?.state, "RESTARTING");
|
||||
const delays: number[] = [];
|
||||
while (queued.length) {
|
||||
const next = queued.shift();
|
||||
assert.ok(next);
|
||||
delays.push(next.delay);
|
||||
await next.action();
|
||||
}
|
||||
assert.deepEqual(delays, [1_000, 2_000, 4_000, 8_000, 16_000]);
|
||||
assert.equal(worker.status()[0]?.state, "FAILED");
|
||||
assert.equal(worker.status()[0]?.realizedVersion, null);
|
||||
assert.ok(transitions.includes("DEGRADED"));
|
||||
await worker.stop();
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue