Tactical Media Broadcast Hub — initial release
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
This commit is contained in:
commit
aa5e262f3a
880 changed files with 111042 additions and 0 deletions
14
apps/media-worker/src/bus/control-signer.ts
Normal file
14
apps/media-worker/src/bus/control-signer.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
import type { KeyObject } from "node:crypto";
|
||||
import { signCommand, type CommandBody, type SignedCommand } from "./envelope";
|
||||
|
||||
/** API-side stub: caller must commit desired state and re-authorize before signing. */
|
||||
export interface ControlCommandSigner {
|
||||
signCommittedSnapshot(body: CommandBody): Promise<SignedCommand>;
|
||||
}
|
||||
|
||||
export class Ed25519ControlSigner implements ControlCommandSigner {
|
||||
constructor(private readonly kid: string, private readonly privateKey: KeyObject) {}
|
||||
async signCommittedSnapshot(body: CommandBody): Promise<SignedCommand> {
|
||||
return signCommand(body, this.kid, this.privateKey);
|
||||
}
|
||||
}
|
||||
61
apps/media-worker/src/bus/envelope.ts
Normal file
61
apps/media-worker/src/bus/envelope.ts
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
import { sign, verify, type KeyObject } from "node:crypto";
|
||||
import { isDesiredChannel, snapshotHlsUrls, type DesiredChannel } from "../types";
|
||||
import { validateEndpointShape } from "../relay/endpoints";
|
||||
|
||||
export interface CommandBody {
|
||||
workspaceId: string;
|
||||
channelId: string;
|
||||
workerId: string;
|
||||
assignmentEpoch: number;
|
||||
commandId: string;
|
||||
desiredVersion: number;
|
||||
issuedAt: string;
|
||||
expiresAt: string;
|
||||
snapshot: DesiredChannel;
|
||||
}
|
||||
export interface SignedCommand { kid: string; body: CommandBody; signature: string }
|
||||
|
||||
/** Fixed field order is the wire-format contract for both signing and verifying. */
|
||||
export function commandBytes(body: CommandBody): Buffer {
|
||||
return Buffer.from(JSON.stringify({
|
||||
workspaceId: body.workspaceId, channelId: body.channelId, workerId: body.workerId,
|
||||
assignmentEpoch: body.assignmentEpoch, commandId: body.commandId,
|
||||
desiredVersion: body.desiredVersion, issuedAt: body.issuedAt,
|
||||
expiresAt: body.expiresAt, snapshot: body.snapshot,
|
||||
}));
|
||||
}
|
||||
|
||||
export function signCommand(body: CommandBody, kid: string, key: KeyObject): SignedCommand {
|
||||
if (!isDesiredChannel(body.snapshot)) throw new Error("invalid desired snapshot");
|
||||
validateSnapshotEndpoints(body.snapshot);
|
||||
return { kid, body, signature: sign(null, commandBytes(body), key).toString("base64") };
|
||||
}
|
||||
|
||||
function validateSnapshotEndpoints(snapshot: DesiredChannel): void {
|
||||
for (const url of snapshotHlsUrls(snapshot)) validateEndpointShape(url, "hls");
|
||||
if (snapshot.output) validateEndpointShape(snapshot.output.endpointUrl, "output");
|
||||
}
|
||||
|
||||
export function verifyCommand(value: unknown, keys: ReadonlyMap<string, KeyObject>, workerId: string, now = Date.now()): SignedCommand {
|
||||
if (typeof value !== "object" || value === null) throw new Error("invalid command");
|
||||
const command = value as Partial<SignedCommand>;
|
||||
const body = command.body;
|
||||
if (typeof command.kid !== "string" || typeof command.signature !== "string" || !body ||
|
||||
typeof body.workspaceId !== "string" || typeof body.channelId !== "string" ||
|
||||
body.workerId !== workerId || typeof body.commandId !== "string" || body.commandId.length === 0 ||
|
||||
!Number.isSafeInteger(body.assignmentEpoch) || (body.assignmentEpoch ?? -1) < 0 ||
|
||||
!Number.isSafeInteger(body.desiredVersion) || (body.desiredVersion ?? -1) < 0 ||
|
||||
typeof body.issuedAt !== "string" || typeof body.expiresAt !== "string" ||
|
||||
!isDesiredChannel(body.snapshot)) throw new Error("invalid command fields");
|
||||
const issued = Date.parse(body.issuedAt);
|
||||
const expires = Date.parse(body.expiresAt);
|
||||
if (!Number.isFinite(issued) || !Number.isFinite(expires) || issued > now + 30_000 || expires <= now || expires <= issued) throw new Error("expired command");
|
||||
const snapshot = body.snapshot;
|
||||
if (snapshot.workspaceId !== body.workspaceId || snapshot.channelId !== body.channelId ||
|
||||
snapshot.workerId !== body.workerId || snapshot.assignmentEpoch !== body.assignmentEpoch ||
|
||||
snapshot.desiredVersion !== body.desiredVersion) throw new Error("snapshot mismatch");
|
||||
validateSnapshotEndpoints(snapshot);
|
||||
const key = keys.get(command.kid);
|
||||
if (!key || !verify(null, commandBytes(body as CommandBody), key, Buffer.from(command.signature, "base64"))) throw new Error("invalid signature");
|
||||
return command as SignedCommand;
|
||||
}
|
||||
68
apps/media-worker/src/bus/fence.ts
Normal file
68
apps/media-worker/src/bus/fence.ts
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { CommandBody } from "./envelope";
|
||||
|
||||
interface FenceEntry { assignmentEpoch: number; acceptedVersion: number; seen: Record<string, number> }
|
||||
type FenceData = Record<string, FenceEntry>;
|
||||
|
||||
/** Local durable fence. Atomic rename plus fsync of file and directory precedes execute/ack. */
|
||||
export class DurableFence {
|
||||
private data: FenceData;
|
||||
constructor(private readonly path: string) {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
this.data = existsSync(path) ? this.read() : {};
|
||||
}
|
||||
|
||||
private read(): FenceData {
|
||||
const parsed: unknown = JSON.parse(readFileSync(this.path, "utf8"));
|
||||
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error("invalid fence file");
|
||||
for (const entry of Object.values(parsed)) {
|
||||
if (typeof entry !== "object" || entry === null) throw new Error("invalid fence entry");
|
||||
const record = entry as Partial<FenceEntry>;
|
||||
if (!Number.isSafeInteger(record.assignmentEpoch) || !Number.isSafeInteger(record.acceptedVersion) ||
|
||||
(record.assignmentEpoch ?? -1) < 0 || (record.acceptedVersion ?? -1) < 0 ||
|
||||
typeof record.seen !== "object" || record.seen === null || Array.isArray(record.seen) ||
|
||||
Object.values(record.seen).some((expiry) => !Number.isFinite(expiry))) throw new Error("invalid fence entry");
|
||||
}
|
||||
return parsed as FenceData;
|
||||
}
|
||||
|
||||
accepted(workspaceId: string, channelId: string): { assignmentEpoch: number; acceptedVersion: number } | null {
|
||||
const entry = this.data[`${workspaceId}:${channelId}`];
|
||||
return entry ? { assignmentEpoch: entry.assignmentEpoch, acceptedVersion: entry.acceptedVersion } : null;
|
||||
}
|
||||
|
||||
accept(body: CommandBody, now = Date.now()): "accepted" | "replay" | "stale" {
|
||||
const key = `${body.workspaceId}:${body.channelId}`;
|
||||
const previous = this.data[key];
|
||||
if (previous && body.assignmentEpoch < previous.assignmentEpoch) return "stale";
|
||||
if (previous && body.assignmentEpoch === previous.assignmentEpoch && body.desiredVersion < previous.acceptedVersion) return "stale";
|
||||
if (previous?.seen[body.commandId] !== undefined) return "replay";
|
||||
const seen = previous ? { ...previous.seen } : {};
|
||||
for (const [id, expiry] of Object.entries(seen)) if (expiry <= now) delete seen[id];
|
||||
seen[body.commandId] = Date.parse(body.expiresAt);
|
||||
const next: FenceData = { ...this.data, [key]: {
|
||||
assignmentEpoch: body.assignmentEpoch,
|
||||
acceptedVersion: previous?.assignmentEpoch === body.assignmentEpoch
|
||||
? Math.max(previous.acceptedVersion, body.desiredVersion) : body.desiredVersion,
|
||||
seen,
|
||||
} };
|
||||
this.persist(next);
|
||||
this.data = next;
|
||||
return "accepted";
|
||||
}
|
||||
|
||||
private persist(data: FenceData): void {
|
||||
const temp = `${this.path}.${randomUUID()}.tmp`;
|
||||
const fd = openSync(temp, "wx", 0o600);
|
||||
try {
|
||||
writeFileSync(fd, JSON.stringify(data));
|
||||
fsyncSync(fd);
|
||||
} catch (error) { unlinkSync(temp); throw error; }
|
||||
finally { closeSync(fd); }
|
||||
renameSync(temp, this.path);
|
||||
const dir = openSync(dirname(this.path), "r");
|
||||
try { fsyncSync(dir); } finally { closeSync(dir); }
|
||||
}
|
||||
}
|
||||
137
apps/media-worker/src/bus/redis.ts
Normal file
137
apps/media-worker/src/bus/redis.ts
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
import net, { type Socket } from "node:net";
|
||||
import tls from "node:tls";
|
||||
import type { ObservedChannel } from "../types";
|
||||
|
||||
type Reply = string | number | null | Reply[];
|
||||
function encode(parts: string[]): Buffer {
|
||||
return Buffer.from(`*${parts.length}\r\n${parts.map((part) => `\$${Buffer.byteLength(part)}\r\n${part}\r\n`).join("")}`);
|
||||
}
|
||||
function parse(buffer: Buffer, offset = 0): { reply: Reply; end: number } | null {
|
||||
if (offset >= buffer.length) return null;
|
||||
const endLine = buffer.indexOf("\r\n", offset);
|
||||
if (endLine < 0) return null;
|
||||
const length = Number(buffer.subarray(offset + 1, endLine).toString());
|
||||
const type = String.fromCharCode(buffer[offset] ?? 0);
|
||||
let pos = endLine + 2;
|
||||
if (type === "+" || type === ":" || type === "-") return { reply: type === ":" ? length : buffer.subarray(offset + 1, endLine).toString(), end: pos };
|
||||
if (type === "$") {
|
||||
if (length === -1) return { reply: null, end: pos };
|
||||
if (!Number.isInteger(length) || length < 0 || length > 2_000_000) throw new Error("invalid Redis bulk length");
|
||||
if (buffer.length < pos + length + 2) return null;
|
||||
const reply = buffer.subarray(pos, pos + length).toString();
|
||||
return { reply, end: pos + length + 2 };
|
||||
}
|
||||
if (type === "*") {
|
||||
if (length === -1) return { reply: null, end: pos };
|
||||
if (!Number.isInteger(length) || length < 0 || length > 100) throw new Error("invalid Redis array length");
|
||||
const replies: Reply[] = [];
|
||||
for (let i = 0; i < length; i += 1) {
|
||||
const item = parse(buffer, pos);
|
||||
if (!item) return null;
|
||||
replies.push(item.reply);
|
||||
pos = item.end;
|
||||
}
|
||||
return { reply: replies, end: pos };
|
||||
}
|
||||
throw new Error("invalid Redis reply");
|
||||
}
|
||||
|
||||
class RedisConnection {
|
||||
private socket: Socket;
|
||||
private buffer = Buffer.alloc(0);
|
||||
private pending: Array<{ resolve: (reply: Reply) => void; reject: (error: Error) => void }> = [];
|
||||
onPush: ((reply: Reply) => void) | null = null;
|
||||
private subscribing = false;
|
||||
|
||||
private constructor(socket: Socket) {
|
||||
this.socket = socket;
|
||||
socket.on("data", (chunk: Buffer) => {
|
||||
this.buffer = Buffer.concat([this.buffer, chunk]);
|
||||
if (this.buffer.length > 2_000_000) { socket.destroy(new Error("Redis reply too large")); return; }
|
||||
while (true) {
|
||||
const prefix = this.buffer[0];
|
||||
const parsed = parse(this.buffer);
|
||||
if (!parsed) break;
|
||||
this.buffer = this.buffer.subarray(parsed.end);
|
||||
if (prefix === 45) {
|
||||
if (this.subscribing) this.onPush?.(parsed.reply);
|
||||
else this.pending.shift()?.reject(new Error(`Redis: ${String(parsed.reply)}`));
|
||||
continue;
|
||||
}
|
||||
if (this.subscribing) this.onPush?.(parsed.reply);
|
||||
else this.pending.shift()?.resolve(parsed.reply);
|
||||
}
|
||||
});
|
||||
socket.on("error", (error) => { for (const item of this.pending.splice(0)) item.reject(error); });
|
||||
socket.on("close", () => { for (const item of this.pending.splice(0)) item.reject(new Error("Redis disconnected")); });
|
||||
}
|
||||
|
||||
static async connect(url: URL): Promise<RedisConnection> {
|
||||
const port = Number(url.port || (url.protocol === "rediss:" ? 6380 : 6379));
|
||||
const socket = url.protocol === "rediss:"
|
||||
? tls.connect({ host: url.hostname, port, servername: url.hostname, rejectUnauthorized: true })
|
||||
: net.connect({ host: url.hostname, port });
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
socket.once(url.protocol === "rediss:" ? "secureConnect" : "connect", () => resolve());
|
||||
socket.once("error", reject);
|
||||
});
|
||||
const connection = new RedisConnection(socket);
|
||||
if (url.password) await connection.send(["AUTH", decodeURIComponent(url.username || "default"), decodeURIComponent(url.password)]);
|
||||
return connection;
|
||||
}
|
||||
|
||||
send(parts: string[]): Promise<Reply> {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.pending.push({ resolve, reject });
|
||||
this.socket.write(encode(parts));
|
||||
});
|
||||
}
|
||||
subscribe(channel: string, onMessage: (message: string) => void): Promise<void> {
|
||||
this.subscribing = true;
|
||||
return new Promise((resolve, reject) => {
|
||||
this.onPush = (reply) => {
|
||||
if (typeof reply === "string") reject(new Error("Redis subscription rejected"));
|
||||
else if (Array.isArray(reply) && reply[0] === "subscribe" && reply[1] === channel) resolve();
|
||||
else if (Array.isArray(reply) && reply[0] === "message" && reply[1] === channel && typeof reply[2] === "string") onMessage(reply[2]);
|
||||
};
|
||||
this.socket.write(encode(["SUBSCRIBE", channel]));
|
||||
});
|
||||
}
|
||||
close(): void { this.socket.destroy(); }
|
||||
}
|
||||
|
||||
export interface ObservedStore {
|
||||
get(workspaceId: string, channelId: string): Promise<ObservedChannel | null>;
|
||||
put(observed: ObservedChannel): Promise<void>;
|
||||
heartbeat(workerId: string, channels: ObservedChannel[]): Promise<void>;
|
||||
}
|
||||
|
||||
export class RedisBus implements ObservedStore {
|
||||
private subscribers: RedisConnection[] = [];
|
||||
private constructor(private readonly url: URL, private readonly client: RedisConnection) {}
|
||||
static async connect(rawUrl: string): Promise<RedisBus> {
|
||||
const url = new URL(rawUrl);
|
||||
if (url.protocol !== "redis:" && url.protocol !== "rediss:") throw new Error("invalid Redis URL");
|
||||
return new RedisBus(url, await RedisConnection.connect(url));
|
||||
}
|
||||
async get(workspaceId: string, channelId: string): Promise<ObservedChannel | null> {
|
||||
const result = await this.client.send(["GET", `obs:${workspaceId}:${channelId}`]);
|
||||
return typeof result === "string" ? JSON.parse(result) as ObservedChannel : null;
|
||||
}
|
||||
async put(observed: ObservedChannel): Promise<void> {
|
||||
await this.client.send(["SET", `obs:${observed.workspaceId}:${observed.channelId}`, JSON.stringify(observed), "EX", "60"]);
|
||||
}
|
||||
async heartbeat(workerId: string, channels: ObservedChannel[]): Promise<void> {
|
||||
await this.client.send(["SET", `hb:${workerId}`, JSON.stringify({ workerId, sentAt: new Date().toISOString(), channels: channels.map((c) => ({ workspaceId: c.workspaceId, channelId: c.channelId, assignmentEpoch: c.assignmentEpoch, realizedVersion: c.realizedVersion })) }), "EX", "15"]);
|
||||
for (const channel of channels) await this.put(channel);
|
||||
}
|
||||
async subscribe(workerId: string, workspaces: string[], onCommand: (message: string) => void): Promise<void> {
|
||||
// A worker uses one subscription per assigned workspace; all channels are signature checked.
|
||||
for (const workspaceId of workspaces) {
|
||||
const subscriber = await RedisConnection.connect(this.url);
|
||||
await subscriber.subscribe(`cmd:${workspaceId}:${workerId}`, onCommand);
|
||||
this.subscribers.push(subscriber);
|
||||
}
|
||||
}
|
||||
close(): void { for (const subscriber of this.subscribers) subscriber.close(); this.client.close(); }
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue