Tactical Media Broadcast Hub — initial release

Self-hosted multi-platform broadcast newsroom for activist collectives:
publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style
column wall with hostile-content-safe rendering, live control room with Auto
Director, bridged chat governance with polls and quorum, distributed sandboxed
transcode workers, encrypted content-addressed archive, monthly analytics.

MIT licensed. See README.md and docs/.
This commit is contained in:
glass beadian 2026-10-10 11:56:42 +00:00
commit aa5e262f3a
880 changed files with 111042 additions and 0 deletions

View file

@ -0,0 +1,64 @@
# Monitor signal sources
`POST /live-monitor/webhooks/:endpointId` authenticates providers independently
of browser sessions. The module installs a raw parser before Nest's JSON parser;
the request cap is 64 KiB and compressed bodies are refused.
Configure `MONITOR_WEBHOOK_ENDPOINTS` as a server-controlled JSON array:
```json
[
{
"endpointId": "<32 server-generated random bytes, encoded as 64 lowercase hex characters>",
"workspaceId": "<workspace UUID>",
"connectionId": "<connection UUID>",
"credentialId": "<credential UUID>",
"subscriptionId": "<provider subscription id>"
}
]
```
The credential must belong to that workspace and connection, have kind
`monitor-webhook`, and contain a secret of at least 32 bytes sealed with
`vault.seal(workspaceId, connectionId, secret)`. Disconnection or credential
revocation disables the endpoint. Mappings contain references, never secrets.
Send these headers:
- `X-Monitor-Event-Id`: a unique provider message id (up to 256 characters).
- `X-Monitor-Timestamp`: an ISO UTC timestamp within five minutes of ingestion.
- `X-Monitor-Signature`: `sha256=` followed by lowercase hex HMAC-SHA256 over
`eventId + timestamp + rawBody`, using the vaulted secret.
The JSON body is `{ "subscriptionId": "...", "sourceId": "<UUID>",
"outcome": "ok|fail|error|unavailable" }`. A supplied workspace or connection
must also match the mapping. The source must belong to the mapped workspace
and connection. Successful delivery returns 200, replay 409, authentication
failure 401, unknown endpoint/source 404, invalid JSON 400, oversized body 413.
Replay ids are namespaced by endpoint in `monitor_seen_events`. Verification,
ownership checks, replay claim, reducer fold, state write and audit share a
tenant transaction. Journal pruning must retain at least the full ten-minute
timestamp acceptance span plus a clock margin; this module does not prune it.
`MEDIA_POLLER_SOURCES` entries support `playbackUrl`. Provider `hls` probes that
URL; provider `peertube` polls the video using the connection's `fields.instanceUrl`
and also probes `playbackUrl` when supplied. Canonical `video:<id>@<host>` refs
are supported. Existing mock configuration remains supported.
HLS probes fetch one master and one variant, validate all child URIs through
WP7, and emit `hls-probe` observations. Decoder reports retain `media-health`
precedence. VOD/endlist/empty/missing playlists fail; stale segments fail after
three target durations. Without program dates, successive playlist progression
provides recency and an initial playlist has the same bounded grace period.
Every outbound fetch uses the shared WP7 egress client and its database host
bucket (`monitor-probe` or `monitor-api`), including redirect hops. Requests
have a five-second total deadline, 256 KiB response cap, and 128-reference cap.
Per-source drivers back off with exponential jitter after failures, honoring
Retry-After seconds/dates and egress retry delays. Denied probe targets emit
`unavailable` with a policy reason so the existing reducer can fall back.
Production requires the built `packages/egress` artifact and its host-budget
migration/grants. Those remain owned by WP7; this module performs no DDL and
implements no additional limiter.

View file

@ -0,0 +1,84 @@
import { BadRequestException, Body, Controller, Get, Inject, Param, Post, Query, Req, UseGuards } from "@nestjs/common";
import type { Request } from "express";
import { parseId, parseLimit, requireObject, requireSession, requireString } from "../http/validation.js";
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
import { LiveMonitorService, type MonitorPage } from "./live-monitor.service.js";
import { LiveMonitorPoller } from "./poller/poller.js";
import type { MonitorObservation, MonitorRowState, ObservationOutcome, SignalTier } from "./reducer.js";
@Controller("live-monitor")
@UseGuards(RbacGuard)
export class LiveMonitorController {
constructor(
@Inject(LiveMonitorService) private readonly monitor: LiveMonitorService,
@Inject(LiveMonitorPoller) private readonly poller: LiveMonitorPoller,
) {}
@Post("poller/:action")
@RequirePermission("media.manage")
async controlPoller(@Param("action") action: string): Promise<{ running: boolean }> {
if (action === "start") return this.poller.start();
if (action === "stop") return this.poller.stop();
throw new BadRequestException("action must be start or stop");
}
@Post(":sourceId/observations")
@RequirePermission("media.manage")
async record(
@Req() request: Request & AuthenticatedRequest,
@Param("sourceId") sourceId: string,
@Body() body: unknown,
): Promise<MonitorRowState> {
const session = requireSession(request);
return this.monitor.recordObservation(session.workspaceId, parseId(sourceId, "sourceId"),
parseObservation(body), { userId: session.userId, role: session.role });
}
@Get()
@RequirePermission("media.view")
async list(
@Req() request: Request & AuthenticatedRequest,
@Query("cursor") cursor: string | undefined,
@Query("limit") limit: string | undefined,
): Promise<MonitorPage> {
return this.monitor.listStates(requireSession(request).workspaceId, cursor, parseLimit(limit, 50, 200));
}
@Get(":id")
@RequirePermission("media.view")
async get(
@Req() request: Request & AuthenticatedRequest,
@Param("id") id: string,
): Promise<MonitorRowState> {
return this.monitor.getState(requireSession(request).workspaceId, parseId(id));
}
}
function parseObservation(value: unknown): MonitorObservation {
const body = requireObject(value, "body");
const tier = body.tier;
if (tier !== "media-health" && tier !== "hls-probe" && tier !== "webhook" && tier !== "api-poll") {
throw new BadRequestException("tier is invalid");
}
const observedAt = requireString(body.observedAt, "observedAt");
if (!Number.isFinite(Date.parse(observedAt))) throw new BadRequestException("observedAt must be a timestamp");
const payload = requireObject(body.payload, "payload");
const outcome = payload.outcome;
if (outcome !== "ok" && outcome !== "fail" && outcome !== "error" && outcome !== "unavailable") {
throw new BadRequestException("outcome is invalid");
}
if (payload.mediaPresent !== undefined && typeof payload.mediaPresent !== "boolean") {
throw new BadRequestException("mediaPresent must be boolean");
}
return {
tier: tier as SignalTier,
observedAt,
receivedAt: new Date().toISOString(),
...(body.providerEventId === undefined ? {} : { providerEventId: requireString(body.providerEventId, "providerEventId") }),
payload: {
outcome: outcome as ObservationOutcome,
...(payload.reason === undefined ? {} : { reason: requireString(payload.reason, "reason") }),
...(payload.mediaPresent === undefined ? {} : { mediaPresent: payload.mediaPresent as boolean }),
},
};
}

View file

@ -0,0 +1,81 @@
import { randomUUID } from "node:crypto";
import { join } from "node:path";
import { Pool, type PoolClient } from "pg";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { runMigrations } from "../db/migrate.js";
import { LiveMonitorService, type LiveMonitorDependencies } from "./live-monitor.service.js";
const dbAvailable = process.env.DATABASE_URL !== undefined;
const ownerPool = new Pool({ connectionString: process.env.SUPER_DATABASE_URL ?? process.env.DATABASE_URL });
const appPool = new Pool({ connectionString: process.env.DATABASE_URL });
async function asTenant<T>(workspaceId: string, fn: (client: PoolClient) => Promise<T>): Promise<T> {
const client = await appPool.connect();
try {
await client.query("BEGIN");
await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]);
const result = await fn(client);
await client.query("COMMIT");
return result;
} catch (error) {
await client.query("ROLLBACK").catch(() => undefined);
throw error;
} finally {
client.release();
}
}
describe.skipIf(!dbAvailable)("live monitor CAS (live DB)", () => {
let workspaceId: string;
let sourceId: string;
let service: LiveMonitorService;
beforeAll(async () => {
await runMigrations(process.env.MIGRATIONS_DIR ?? join(process.cwd(), "migrations"));
const workspace = await ownerPool.query(
"INSERT INTO workspaces (name, slug) VALUES ('Monitor integration', $1) RETURNING id",
[`monitor-${randomUUID()}`],
);
workspaceId = String(workspace.rows[0]?.id);
sourceId = await asTenant(workspaceId, async (client) => {
const connection = await client.query(
"INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-livestream', 'monitor') RETURNING id",
[workspaceId],
);
const source = await client.query(
"INSERT INTO source_monitor_states (workspace_id, connection_id, remote_ref) VALUES ($1, $2, 'live:cas') RETURNING id",
[workspaceId, connection.rows[0]?.id],
);
return String(source.rows[0]?.id);
});
const dependencies: LiveMonitorDependencies = { tenantTransaction: asTenant };
service = new LiveMonitorService(dependencies);
});
afterAll(async () => {
if (workspaceId !== undefined) await ownerPool.query("DELETE FROM workspaces WHERE id = $1", [workspaceId]);
await appPool.end();
await ownerPool.end();
});
it("serializes two recorders and keeps one transition per successful CAS", async () => {
const observedAt = new Date().toISOString();
const [first, second] = await Promise.all([
service.recordObservation(workspaceId, sourceId, {
tier: "api-poll", observedAt, receivedAt: observedAt, payload: { outcome: "ok" },
}),
service.recordObservation(workspaceId, sourceId, {
tier: "hls-probe", observedAt, receivedAt: observedAt, payload: { outcome: "ok" },
}),
]);
expect([first.version, second.version].sort()).toEqual([1, 2]);
const current = await service.getState(workspaceId, sourceId);
expect(current.version).toBe(2);
expect(current.authoritativeTier).toBe("hls-probe");
const audit = await asTenant(workspaceId, (client) => client.query(
"SELECT count(*)::int AS count FROM audit_log WHERE action = 'monitor.state_changed' AND target_id = $1",
[sourceId],
));
expect(audit.rows[0]?.count).toBe(1);
});
});

View file

@ -0,0 +1,19 @@
import { Inject, Module } from "@nestjs/common";
import { HttpAdapterHost } from "@nestjs/core";
import type { ExpressAdapter } from "@nestjs/platform-express";
import { LiveMonitorController } from "./live-monitor.controller.js";
import { LiveMonitorService } from "./live-monitor.service.js";
import { LiveMonitorPoller } from "./poller/poller.js";
import { MonitorWebhookController } from "./webhook.controller.js";
import { MonitorWebhookService, WEBHOOK_MAX_BYTES } from "./webhook.service.js";
@Module({ controllers: [LiveMonitorController, MonitorWebhookController], providers: [LiveMonitorService, LiveMonitorPoller, MonitorWebhookService], exports: [LiveMonitorService] })
export class LiveMonitorModule {
constructor(@Inject(HttpAdapterHost) adapter: HttpAdapterHost) {
// Installed while Nest creates modules, before init registers its default JSON parser.
(adapter.httpAdapter as ExpressAdapter).useBodyParser("raw", false, {
type: (request: { url?: string }) => request.url?.startsWith("/live-monitor/webhooks/") === true,
limit: WEBHOOK_MAX_BYTES, inflate: false,
});
}
}

View file

@ -0,0 +1,181 @@
import { BadRequestException, ConflictException, Inject, Injectable, NotFoundException, Optional } from "@nestjs/common";
import type { PoolClient } from "pg";
import { withTenant } from "../db/client.js";
import { isUuid } from "../http/validation.js";
import { applyObservation, type MonitorObservation, type MonitorRowState, type MonitorState,
type SignalTier, type TierStatus, type MonitorPolicy, DEFAULT_MONITOR_POLICY } from "./reducer.js";
export interface MonitorActor { userId: string | null; role: string }
export interface MonitorPage { items: MonitorRowState[]; nextCursor?: string }
export interface LiveMonitorDependencies {
tenantTransaction: typeof withTenant;
policy?: Partial<MonitorPolicy>;
}
export const LIVE_MONITOR_DEPENDENCIES = Symbol("LIVE_MONITOR_DEPENDENCIES");
@Injectable()
export class LiveMonitorService {
private readonly dependencies: LiveMonitorDependencies;
private readonly policy: MonitorPolicy;
constructor(@Optional() @Inject(LIVE_MONITOR_DEPENDENCIES) dependencies?: Partial<LiveMonitorDependencies>) {
this.dependencies = { tenantTransaction: withTenant, ...dependencies };
this.policy = { ...DEFAULT_MONITOR_POLICY, ...dependencies?.policy };
}
async recordObservation(
workspaceId: string,
sourceId: string,
observation: MonitorObservation,
actor: MonitorActor = { userId: null, role: "monitor" },
): Promise<MonitorRowState> {
validateObservation(observation);
return this.dependencies.tenantTransaction(workspaceId, (client) =>
this.recordObservationInTransaction(client, workspaceId, sourceId, observation, actor));
}
/** Webhook verification, durable replay claim and reduction share one transaction. */
async recordObservationInTransaction(
client: PoolClient, workspaceId: string, sourceId: string, observation: MonitorObservation,
actor: MonitorActor = { userId: null, role: "monitor" }, rejectReplay = false,
): Promise<MonitorRowState> {
validateObservation(observation);
let seenInserted = false;
for (let attempt = 0; attempt < 5; attempt += 1) {
const locked = await client.query("SELECT * FROM source_monitor_states WHERE id = $1 FOR UPDATE", [sourceId]);
if (locked.rowCount !== 1) throw new NotFoundException(`live source ${sourceId} not found`);
const current = fromRow(locked.rows[0]);
if (observation.providerEventId !== undefined && !seenInserted) {
const seen = await client.query(
`INSERT INTO monitor_seen_events (workspace_id, provider_event_id, source_monitor_state_id)
VALUES ($1, $2, $3) ON CONFLICT (workspace_id, provider_event_id) DO NOTHING RETURNING provider_event_id`,
[workspaceId, observation.providerEventId, sourceId],
);
if (seen.rowCount === 0) {
if (rejectReplay) throw new ConflictException("webhook event already received");
return current;
}
seenInserted = true;
}
const reduction = applyObservation(current, observation, this.policy);
if (reduction.state === current) return current;
const next = reduction.state;
const changed = await client.query(
`UPDATE source_monitor_states SET state = $2, version = $3, authoritative_tier = $4,
last_media_health_at = $5, last_hls_probe_at = $6, last_webhook_at = $7,
last_api_poll_at = $8, tier_status = $9, ok_streak = $10, fail_streak = $11,
state_entered_at = $12, hold_until = $13, cooldown_until = $14,
previous_state = $15, last_transition_at = $16, last_transition_reason = $17,
updated_at = $18
WHERE id = $1 AND version = $19 RETURNING *`,
[sourceId, next.state, next.version, next.authoritativeTier, next.lastMediaHealthAt,
next.lastHlsProbeAt, next.lastWebhookAt, next.lastApiPollAt, next.tierStatus,
next.okStreak, next.failStreak, next.stateEnteredAt, next.holdUntil, next.cooldownUntil,
next.previousState, next.lastTransitionAt, next.lastTransitionReason, next.updatedAt, current.version],
);
if (changed.rowCount !== 1) continue;
if (reduction.transition !== undefined) {
await client.query(
"SELECT audit_append($1, $2, $3, 'monitor.state_changed', 'source_monitor_state', $4, $5)",
[workspaceId, actor.userId, actor.role, sourceId, {
from: reduction.transition.from, to: reduction.transition.to,
version: reduction.transition.version, reason: reduction.transition.reason,
tier: next.authoritativeTier,
}],
);
}
return fromRow(changed.rows[0]);
}
throw new Error(`live source ${sourceId} CAS conflict did not settle`);
}
async getState(workspaceId: string, sourceId: string): Promise<MonitorRowState> {
return this.dependencies.tenantTransaction(workspaceId, async (client) => {
const found = await client.query("SELECT * FROM source_monitor_states WHERE id = $1", [sourceId]);
if (found.rowCount !== 1) throw new NotFoundException(`live source ${sourceId} not found`);
return fromRow(found.rows[0]);
});
}
async getStates(workspaceId: string, sourceIds: string[]): Promise<MonitorRowState[]> {
if (sourceIds.length === 0) return [];
return this.dependencies.tenantTransaction(workspaceId, async (client) => {
const found = await client.query("SELECT * FROM source_monitor_states WHERE id = ANY($1::uuid[])", [sourceIds]);
return found.rows.map(fromRow);
});
}
async listStates(workspaceId: string, cursor?: string, limit = 50): Promise<MonitorPage> {
if (!Number.isInteger(limit) || limit < 1 || limit > 200) {
throw new BadRequestException("limit must be an integer between 1 and 200");
}
const decoded = cursor === undefined || cursor === "" ? undefined : decodeCursor(cursor);
return this.dependencies.tenantTransaction(workspaceId, async (client) => {
const result = await client.query(
`SELECT * FROM source_monitor_states
WHERE ($1::timestamptz IS NULL OR (created_at, id) < ($1::timestamptz, $2::uuid))
ORDER BY created_at DESC, id DESC LIMIT $3`,
[decoded?.createdAt ?? null, decoded?.id ?? null, limit + 1],
);
const rows = result.rows.map(fromRow);
const items = rows.slice(0, limit);
const last = items[items.length - 1];
return { items, ...(rows.length > limit && last !== undefined
? { nextCursor: Buffer.from(JSON.stringify({ createdAt: last.createdAt, id: last.id })).toString("base64url") }
: {}) };
});
}
}
function validateObservation(value: MonitorObservation): void {
if (!(["media-health", "hls-probe", "webhook", "api-poll"] as string[]).includes(value.tier)) {
throw new BadRequestException("invalid observation tier");
}
if (!Number.isFinite(Date.parse(value.observedAt)) || !Number.isFinite(Date.parse(value.receivedAt))) {
throw new BadRequestException("observation timestamps must be valid");
}
if (!(["ok", "fail", "error", "unavailable"] as string[]).includes(value.payload.outcome)) {
throw new BadRequestException("invalid observation outcome");
}
if (value.providerEventId !== undefined && value.providerEventId.trim() === "") {
throw new BadRequestException("providerEventId must be non-empty");
}
}
function fromRow(raw: unknown): MonitorRowState {
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) throw new Error("invalid monitor row");
const row = raw as Record<string, unknown>;
const version = Number(row.version);
if (!Number.isSafeInteger(version)) throw new Error("monitor version exceeds safe integer range");
return {
id: String(row.id), workspaceId: String(row.workspace_id), connectionId: String(row.connection_id),
remoteRef: String(row.remote_ref), state: row.state as MonitorState, version,
authoritativeTier: row.authoritative_tier as SignalTier | null,
lastMediaHealthAt: dateOrNull(row.last_media_health_at), lastHlsProbeAt: dateOrNull(row.last_hls_probe_at),
lastWebhookAt: dateOrNull(row.last_webhook_at), lastApiPollAt: dateOrNull(row.last_api_poll_at),
tierStatus: row.tier_status as Partial<Record<SignalTier, TierStatus>>,
okStreak: Number(row.ok_streak), failStreak: Number(row.fail_streak),
stateEnteredAt: dateOrNull(row.state_entered_at) ?? "",
holdUntil: dateOrNull(row.hold_until), cooldownUntil: dateOrNull(row.cooldown_until),
previousState: row.previous_state as MonitorState | null,
lastTransitionAt: dateOrNull(row.last_transition_at), lastTransitionReason: row.last_transition_reason as string | null,
createdAt: dateOrNull(row.created_at) ?? "", updatedAt: dateOrNull(row.updated_at) ?? "",
};
}
function dateOrNull(value: unknown): string | null {
return value === null || value === undefined ? null : new Date(String(value)).toISOString();
}
function decodeCursor(value: string): { createdAt: string; id: string } {
try {
const parsed: unknown = JSON.parse(Buffer.from(value, "base64url").toString("utf8"));
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error("invalid cursor");
const row = parsed as Record<string, unknown>;
if (typeof row.createdAt !== "string" || !Number.isFinite(Date.parse(row.createdAt))
|| typeof row.id !== "string" || !isUuid(row.id)) throw new Error("invalid cursor");
return { createdAt: row.createdAt, id: row.id };
} catch {
throw new BadRequestException("cursor is invalid");
}
}

View file

@ -0,0 +1,39 @@
import type { MonitorObservation, SignalTier } from "../reducer.js";
import type { PollDriver } from "./driver.js";
import { MonitorHttpError, retryAfterMs } from "./egress.js";
/** Scheduling state only: all host admission/debits remain in packages/egress. */
export class BackoffPollDriver implements PollDriver {
private failures = 0;
private retryAt = 0;
constructor(
private readonly driver: PollDriver, private readonly tier: SignalTier,
private readonly now: () => number = Date.now, private readonly random: () => number = Math.random,
) {}
get nextAttemptAt(): number { return this.retryAt; }
async poll(): Promise<MonitorObservation[]> {
if (this.now() < this.retryAt) return [];
try {
const observations = await this.driver.poll();
this.failures = 0;
this.retryAt = 0;
return observations;
} catch (error) {
const now = this.now();
const metadata = error as { code?: string; retryAfterMs?: number } | null;
const transient = error instanceof MonitorHttpError && (error.status === 429 || error.status >= 500)
|| metadata?.code === "BUDGET_EXHAUSTED" || metadata?.code === "BUDGET_UNAVAILABLE"
|| metadata?.code === "NETWORK_ERROR" || metadata?.code === "TIMEOUT" || metadata?.code === "ABORTED";
this.failures += 1;
const delay = Math.min(120_000, 1000 * 2 ** Math.min(7, this.failures - 1));
const serverDelay = error instanceof MonitorHttpError ? retryAfterMs(error.headers, now) : metadata?.retryAfterMs ?? 0;
this.retryAt = now + Math.max(serverDelay, delay * (0.8 + this.random() * 0.4));
const policyDenied = ["DENIED_PRIVATE_ADDRESS", "SCHEME_DENIED", "PORT_DENIED", "ORIGIN_DENIED", "INVALID_URL", "DNS_FAILED"].includes(metadata?.code ?? "");
const timestamp = new Date(now).toISOString();
return [{ tier: this.tier, observedAt: timestamp, receivedAt: timestamp,
payload: { outcome: transient || policyDenied ? "unavailable" : "error",
reason: policyDenied ? `egress-policy:${metadata?.code}` : transient ? "monitor-backoff" : "monitor-probe-error" } }];
}
}
}

View file

@ -0,0 +1,7 @@
import type { MonitorObservation } from "../reducer.js";
/** A provider probe emits observations ordered by its own source clock. */
export interface PollDriver {
poll(): Promise<MonitorObservation[]>;
}

View file

@ -0,0 +1,96 @@
import { existsSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { pathToFileURL } from "node:url";
import { pool } from "../../db/client.js";
import type { createEgressClient, EgressPolicy, rewriteSegmentUris, validateMasterPlaylist } from "../../../../../packages/egress/src/index.js";
type EgressPackage = typeof import("../../../../../packages/egress/src/index.js");
export interface MonitorEgress {
egressFetch: ReturnType<typeof createEgressClient>["egressFetch"];
policy: EgressPolicy;
validateMasterPlaylist: typeof validateMasterPlaylist;
rewriteSegmentUris: typeof rewriteSegmentUris;
}
export const MONITOR_TIMEOUT_MS = 5_000;
export const MONITOR_MAX_BYTES = 256 * 1024;
const nativeImport = new Function("specifier", "return import(specifier)") as (specifier: string) => Promise<EgressPackage>;
let shared: Promise<MonitorEgress> | undefined;
/** ESM package is loaded natively because the API emits CommonJS. No local limiter. */
export function monitorEgress(): Promise<MonitorEgress> {
shared ??= (async () => {
for (const start of [process.cwd(), __dirname]) {
for (let root = start; ; root = dirname(root)) {
const directory = resolve(root, "packages/egress");
const files = process.env.NODE_ENV === "production" ? ["dist/index.js"] : ["src/index.ts", "dist/index.js"];
for (const file of files) {
const path = resolve(directory, file);
if (!existsSync(path)) continue;
const pkg = await nativeImport(pathToFileURL(path).href);
const client = pkg.createEgressClient({ pool, timeoutMs: MONITOR_TIMEOUT_MS, maxResponseBytes: MONITOR_MAX_BYTES,
hostClasses: {
"monitor-probe": { capacity: 3, refillPerInterval: 10 },
"monitor-api": { capacity: 3, refillPerInterval: 10 },
} });
return { ...client, validateMasterPlaylist: pkg.validateMasterPlaylist, rewriteSegmentUris: pkg.rewriteSegmentUris };
}
if (dirname(root) === root) break;
}
}
throw new Error("Build packages/egress before starting the live monitor");
})();
return shared;
}
export class MonitorHttpError extends Error {
constructor(readonly status: number, readonly headers: Headers) { super(`monitor HTTP ${status}`); }
}
/** Retry-After remains a lower bound, including HTTP dates; never retry early. */
export function retryAfterMs(headers: Headers, now: number): number {
const raw = headers.get("retry-after");
if (raw === null) return 0;
if (/^\d+(?:\.\d+)?$/.test(raw.trim())) {
const delay = Number(raw) * 1000;
return Number.isFinite(delay) ? delay : 0;
}
const deadline = Date.parse(raw);
return Number.isFinite(deadline) ? Math.max(0, deadline - now) : 0;
}
/** Caps body reads even for injected clients, which need not buffer like WP7. */
export async function boundedText(response: Response, signal: AbortSignal): Promise<string> {
if (!response.body) return "";
const reader = response.body.getReader();
const chunks: Buffer[] = [];
let size = 0;
const cancel = () => { void reader.cancel().catch(() => undefined); };
signal.addEventListener("abort", cancel, { once: true });
try {
for (;;) {
if (signal.aborted) throw new Error("monitor deadline exceeded");
const part = await reader.read();
if (part.done) break;
size += part.value.byteLength;
if (size > MONITOR_MAX_BYTES) throw new Error("monitor response exceeds size cap");
chunks.push(Buffer.from(part.value));
}
if (signal.aborted) throw new Error("monitor deadline exceeded");
return Buffer.concat(chunks).toString("utf8");
} finally {
signal.removeEventListener("abort", cancel);
void reader.cancel().catch(() => undefined);
}
}
/** Covers DNS/URI validation as well as fetch and body consumption. */
export async function monitorDeadline<T>(run: (signal: AbortSignal) => Promise<T>): Promise<T> {
const controller = new AbortController();
let timer: ReturnType<typeof setTimeout> | undefined;
const timeout = new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => { controller.abort(); reject(new Error("monitor deadline exceeded")); }, MONITOR_TIMEOUT_MS);
});
try { return await Promise.race([run(controller.signal), timeout]); }
finally { clearTimeout(timer); controller.abort(); }
}

View file

@ -0,0 +1,8 @@
#EXTM3U
#EXT-X-TARGETDURATION:6
#EXT-X-MEDIA-SEQUENCE:20
#EXT-X-PROGRAM-DATE-TIME:2026-01-01T00:00:00Z
#EXTINF:6,
segment20.ts
#EXTINF:6,
segment21.ts

View file

@ -0,0 +1,4 @@
#EXTM3U
#EXT-X-TARGETDURATION:garbage
#EXTINF:6,
segment20.ts

View file

@ -0,0 +1,5 @@
#EXTM3U
#EXT-X-STREAM-INF:BANDWIDTH=800000
live.m3u8
#EXT-X-STREAM-INF:BANDWIDTH=2000000
unused.m3u8

View file

@ -0,0 +1,6 @@
#EXTM3U
#EXT-X-PLAYLIST-TYPE:VOD
#EXT-X-TARGETDURATION:6
#EXTINF:6,
segment20.ts
#EXT-X-ENDLIST

View file

@ -0,0 +1,127 @@
import type { MonitorObservation } from "../reducer.js";
import type { PollDriver } from "./driver.js";
import { boundedText, monitorDeadline, MonitorHttpError, type MonitorEgress } from "./egress.js";
export interface HlsMediaPlaylist {
targetDuration: number;
sequence: number;
segments: Array<{ uri: string; duration: number; endAt?: number }>;
ended: boolean;
vod: boolean;
}
export function parseMediaPlaylist(text: string): HlsMediaPlaylist {
const lines = text.replace(/^\uFEFF/, "").split(/\r?\n/).map((line) => line.trim());
if (lines[0] !== "#EXTM3U" || text.includes("\0")) throw new Error("malformed HLS header");
let targetDuration = 0;
let sequence = 0;
let duration: number | undefined;
let programTime: number | undefined;
let ended = false;
let vod = false;
const segments: HlsMediaPlaylist["segments"] = [];
for (const line of lines.slice(1)) {
if (line.startsWith("#EXT-X-TARGETDURATION:")) {
const raw = line.slice(22);
if (targetDuration || !/^\d+$/.test(raw)) throw new Error("invalid HLS target duration");
targetDuration = Number(raw);
} else if (line.startsWith("#EXT-X-MEDIA-SEQUENCE:")) {
const raw = line.slice(22);
if (!/^\d+$/.test(raw) || !Number.isSafeInteger(Number(raw))) throw new Error("invalid HLS sequence");
sequence = Number(raw);
} else if (line.startsWith("#EXT-X-PLAYLIST-TYPE:")) {
const type = line.slice(21);
if (type !== "EVENT" && type !== "VOD") throw new Error("invalid HLS playlist type");
vod = type === "VOD";
} else if (line.startsWith("#EXT-X-PROGRAM-DATE-TIME:")) {
programTime = Date.parse(line.slice(25));
if (!Number.isFinite(programTime)) throw new Error("invalid HLS program time");
} else if (line.startsWith("#EXTINF:")) {
if (duration !== undefined) throw new Error("HLS segment URI missing");
const raw = line.slice(8).split(",")[0] ?? "";
if (!/^\d+(?:\.\d+)?$/.test(raw)) throw new Error("invalid HLS duration");
duration = Number(raw);
if (!Number.isFinite(duration) || duration <= 0) throw new Error("invalid HLS duration");
} else if (line === "#EXT-X-ENDLIST") ended = true;
else if (line && !line.startsWith("#")) {
if (duration === undefined) throw new Error("HLS segment duration missing");
if (programTime !== undefined) programTime += duration * 1000;
segments.push({ uri: line, duration, ...(programTime === undefined ? {} : { endAt: programTime }) });
duration = undefined;
}
}
if (duration !== undefined || !Number.isSafeInteger(targetDuration) || targetDuration <= 0 || targetDuration > 3600
|| segments.some((segment) => Math.round(segment.duration) > targetDuration) || segments.length > 128) {
throw new Error("invalid HLS media playlist");
}
return { targetDuration, sequence, segments, ended, vod };
}
/** Playlist ground truth uses hls-probe, below decoder media-health and above provider signals. */
export class HlsProbeDriver implements PollDriver {
private previous: { fingerprint: string; advancedAt: number; playlistUrl: string } | undefined;
constructor(private readonly playbackUrl: string, private readonly egress: MonitorEgress,
private readonly now: () => number = Date.now) {}
async poll(): Promise<MonitorObservation[]> {
return monitorDeadline(async (signal) => {
let url = this.playbackUrl;
let response = await this.fetch(url, signal);
if (response.status === 404 || response.status === 410) return [this.observation(false, "hls-missing")];
let text = await boundedText(response, signal);
url = response.url || url;
// Bound URI/DNS work before the WP7 helper validates every reference (including keys/maps).
this.capReferences(text);
if (/^#EXT-X-STREAM-INF:/m.test(text)) {
await this.egress.validateMasterPlaylist(text, url, this.egress.policy);
const lines = text.split(/\r?\n/).map((line) => line.trim());
const variantTag = lines.findIndex((line) => line.startsWith("#EXT-X-STREAM-INF:"));
const variant = lines.slice(variantTag + 1).find((line) => line && !line.startsWith("#"));
if (!variant) throw new Error("HLS master has no variant");
if (signal.aborted) throw new Error("monitor deadline exceeded");
url = new URL(variant, url).href;
response = await this.fetch(url, signal);
if (response.status === 404 || response.status === 410) return [this.observation(false, "hls-variant-missing")];
text = await boundedText(response, signal);
url = response.url || url;
this.capReferences(text);
}
const validated = await this.egress.rewriteSegmentUris(text, url, this.egress.policy);
if (signal.aborted) throw new Error("monitor deadline exceeded");
const playlist = parseMediaPlaylist(validated);
const now = this.now();
const latest = playlist.segments.at(-1);
if (playlist.ended || playlist.vod || !latest) return [this.observation(false, "hls-ended-or-empty")];
const fingerprint = JSON.stringify([playlist.sequence, playlist.segments.length, latest.uri]);
const old = this.previous;
if (!old || old.playlistUrl !== url || old.fingerprint !== fingerprint) {
this.previous = { fingerprint, advancedAt: now, playlistUrl: url };
}
// Three target durations allow one missed reload; unchanged windows eventually fail even without PDT.
const graceMs = playlist.targetDuration * 3 * 1000;
const fresh = latest.endAt === undefined
? now - this.previous!.advancedAt <= graceMs
: latest.endAt >= now - graceMs && latest.endAt <= now + graceMs;
return [this.observation(fresh, fresh ? "hls-recent-segments" : "hls-stale-segments")];
});
}
private async fetch(url: string, signal: AbortSignal): Promise<Response> {
if (signal.aborted) throw new Error("monitor deadline exceeded");
const response = await this.egress.egressFetch(url, { signal, headers: { accept: "application/vnd.apple.mpegurl" } },
{ hostClass: "monitor-probe" });
if (!response.ok && response.status !== 404 && response.status !== 410) throw new MonitorHttpError(response.status, response.headers);
return response;
}
private observation(live: boolean, reason: string): MonitorObservation {
const timestamp = new Date(this.now()).toISOString();
return { tier: "hls-probe", observedAt: timestamp, receivedAt: timestamp,
payload: { outcome: live ? "ok" : "fail", mediaPresent: live, reason } };
}
private capReferences(text: string): void {
if (text.split(/\r?\n/).filter((line) => line.trim() && !line.trim().startsWith("#")).length
+ (text.match(/(?:SERVER-)?URI=/g)?.length ?? 0) > 128) throw new Error("too many HLS references");
}
}

View file

@ -0,0 +1,136 @@
import { existsSync } from "node:fs";
import { resolve } from "node:path";
import { pathToFileURL } from "node:url";
import type { MonitorObservation } from "../reducer.js";
import type { PollDriver } from "./driver.js";
interface ScenarioStep {
source: "A" | "B" | "C";
action: "go-live" | "fail" | "recover" | "go-offline";
holdSeconds: number;
}
interface Scenario { steps: ScenarioStep[] }
type MockLiveState = "created" | "waiting" | "live" | "ended";
interface MockStream { remoteId: string; state: MockLiveState }
interface MockConnection {
id: string;
workspaceId: string;
provider: string;
channelId: string;
credentialId: { kind: "credential-ref"; id: string };
fields: Record<string, string>;
createdAt: Date;
}
interface MockConnector {
manifest(): { id: string };
createLiveStream?(connection: MockConnection, request: {
channelId: string; title: string; permanent: boolean; saveReplay: boolean;
}): Promise<MockStream>;
getLiveStreams?(connection: MockConnection, request: { limit: number }): Promise<{ items: MockStream[] }>;
}
interface MockModule {
createMockLivestreamState(): unknown;
createMockLivestreamConnector(state: unknown): MockConnector;
transitionMockLive(state: unknown, remoteId: string, to: MockLiveState): void;
}
interface ScenarioModule { SCENARIOS: Record<string, Scenario> }
export interface MockDriverPackages { scenarios: ScenarioModule; mock: MockModule }
export interface MockDriverOptions {
scenario: string;
source: "A" | "B" | "C";
sourceRef: string;
workspaceId: string;
connectionId: string;
tickMs?: number;
epoch?: string;
ingestNow?: () => string;
}
/** Native import keeps the ESM simulator and mock connector usable from the CJS API build. */
const nativeImport = new Function("specifier", "return import(specifier)") as (specifier: string) => Promise<unknown>;
async function loadPackage<T>(packageName: string, file: string): Promise<T> {
const roots = [process.cwd(), resolve(process.cwd(), "../..")];
for (const root of roots) {
const built = resolve(root, "packages", packageName, "dist", file);
const source = resolve(root, "packages", packageName, "src", file.replace(/\.js$/, ".ts"));
const candidates = process.env.NODE_ENV === "production" ? [built, source] : [source, built];
for (const candidate of candidates) {
if (existsSync(candidate)) return await nativeImport(pathToFileURL(candidate).href) as T;
}
}
throw new Error(`missing ${packageName}/${file}; build the package before starting the poller`);
}
export class MockPollDriver implements PollDriver {
private elapsedMs = 0;
private nextStep = 0;
private nextStepAt = 0;
private tick = 0;
private action: ScenarioStep["action"] | null = null;
private constructor(
private readonly options: MockDriverOptions,
private readonly scenario: Scenario,
private readonly connector: MockConnector,
private readonly state: unknown,
private readonly transition: MockModule["transitionMockLive"],
private readonly connection: MockConnection,
private readonly stream: MockStream,
) {}
static async create(options: MockDriverOptions, packages?: MockDriverPackages): Promise<MockPollDriver> {
const normalized = { ...options, epoch: options.epoch ?? new Date().toISOString() };
if (!Number.isFinite(Date.parse(normalized.epoch))) throw new Error("mock epoch must be a timestamp");
const [{ SCENARIOS }, mock] = packages === undefined ? await Promise.all([
loadPackage<ScenarioModule>("testing", "scenarios.js"),
loadPackage<MockModule>("connector-mock-livestream", "index.js"),
]) : [packages.scenarios, packages.mock];
const scenario = SCENARIOS[normalized.scenario];
if (scenario === undefined) throw new Error(`unknown mock scenario ${normalized.scenario}`);
const state = mock.createMockLivestreamState();
const connector = mock.createMockLivestreamConnector(state);
const connection: MockConnection = {
id: normalized.connectionId, workspaceId: normalized.workspaceId, provider: connector.manifest().id,
channelId: normalized.sourceRef, credentialId: { kind: "credential-ref", id: "mock" },
fields: {}, createdAt: new Date(normalized.epoch),
};
if (connector.createLiveStream === undefined) throw new Error("mock connector has no live stream factory");
const stream = await connector.createLiveStream(connection, {
channelId: normalized.sourceRef, title: normalized.sourceRef, permanent: false, saveReplay: false,
});
return new MockPollDriver(normalized, scenario, connector, state, mock.transitionMockLive, connection, stream);
}
async poll(): Promise<MonitorObservation[]> {
const tickMs = this.options.tickMs ?? 10_000;
if (!Number.isFinite(tickMs) || tickMs <= 0) throw new Error("mock tickMs must be positive");
while (this.nextStep < this.scenario.steps.length && this.elapsedMs >= this.nextStepAt) {
const step = this.scenario.steps[this.nextStep];
if (step === undefined) break;
if (step.source === this.options.source) {
this.action = step.action;
const state: MockLiveState = step.action === "go-live" || step.action === "recover"
? "live" : step.action === "fail" ? "waiting" : "ended";
this.transition(this.state, this.stream.remoteId, state);
}
this.nextStepAt += step.holdSeconds * 1000;
this.nextStep += 1;
}
if (this.connector.getLiveStreams === undefined) throw new Error("mock connector has no live stream query");
const page = await this.connector.getLiveStreams(this.connection, { limit: 100 });
const live = page.items.find((item) => item.remoteId === this.stream.remoteId);
if (live === undefined) throw new Error(`mock live ${this.stream.remoteId} disappeared`);
const observedAt = new Date(Date.parse(this.options.epoch ?? "") + this.elapsedMs).toISOString();
const outcome = this.action === "fail" ? "error" : live.state === "live" ? "ok" : "fail";
const observation: MonitorObservation = {
tier: "api-poll", observedAt, receivedAt: this.options.ingestNow?.() ?? new Date().toISOString(),
providerEventId: `mock:${this.options.workspaceId}:${this.options.connectionId}:${this.options.sourceRef}:${this.options.epoch}:${this.tick}`,
payload: { outcome, reason: this.action ?? "not-live", mediaPresent: live.state === "live" },
};
this.tick += 1;
this.elapsedMs += tickMs;
return [observation];
}
}

View file

@ -0,0 +1,44 @@
import type { MonitorObservation } from "../reducer.js";
import type { PollDriver } from "./driver.js";
import { boundedText, monitorDeadline, MonitorHttpError, type MonitorEgress } from "./egress.js";
/** Injectable provider surface; production uses the WP7-backed client below. */
export interface PeerTubeProbeClient {
probe(videoId: string): Promise<MonitorObservation[]>;
}
export class EgressPeerTubeProbeClient implements PeerTubeProbeClient {
constructor(private readonly instanceUrl: string, private readonly egress: MonitorEgress,
private readonly now: () => number = Date.now) {}
async probe(videoId: string): Promise<MonitorObservation[]> {
return monitorDeadline(async (signal) => {
const url = new URL(`/api/v1/videos/${encodeURIComponent(videoId)}`, this.instanceUrl);
const response = await this.egress.egressFetch(url, { signal, headers: { accept: "application/json" } },
{ hostClass: "monitor-api", pinnedOrigin: true });
const timestamp = new Date(this.now()).toISOString();
if (response.status === 404 || response.status === 410) return [{ tier: "api-poll", observedAt: timestamp,
receivedAt: timestamp, payload: { outcome: "fail", reason: "api-video-missing" } }];
if (!response.ok) throw new MonitorHttpError(response.status, response.headers);
const parsed: unknown = JSON.parse(await boundedText(response, signal));
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("invalid PeerTube response");
const video = parsed as { isLive?: unknown; state?: unknown };
const state = typeof video.state === "object" && video.state !== null
? (video.state as { id?: unknown }).id : video.state;
if (typeof video.isLive !== "boolean" || !Number.isInteger(state)) throw new Error("invalid PeerTube live state");
// A running live remains PUBLISHED (1); WAITING_FOR_LIVE (4)/LIVE_ENDED (5) are offline.
const live = video.isLive && state === 1;
return [{ tier: "api-poll", observedAt: timestamp, receivedAt: timestamp,
payload: { outcome: live ? "ok" : "fail", reason: live ? "api-published-live" : "api-not-live" } }];
});
}
}
export class PeerTubePollDriver implements PollDriver {
constructor(private readonly videoId: string, private readonly egress?: PeerTubeProbeClient) {}
async poll(): Promise<MonitorObservation[]> {
if (this.egress === undefined) throw new Error("NOT_IMPLEMENTED: PeerTube poll requires WP7 egress client");
return await this.egress.probe(this.videoId);
}
}

View file

@ -0,0 +1,166 @@
import type { PoolClient } from "pg";
import { describe, expect, it, vi } from "vitest";
import { LiveMonitorService, type LiveMonitorDependencies } from "../live-monitor.service.js";
import type { MonitorObservation, MonitorRowState } from "../reducer.js";
import { MockPollDriver, type MockDriverPackages } from "./mock-driver.js";
import { PeerTubePollDriver } from "./peertube-driver.js";
import { jitteredInterval, LiveMonitorPoller, parsePollSources, type PollerDependencies } from "./poller.js";
const epoch = "2026-01-01T00:00:00.000Z";
async function mockPackages(): Promise<MockDriverPackages> {
const [mock, scenarios] = await Promise.all([
vi.importActual<MockDriverPackages["mock"]>("../../../../../packages/connector-mock-livestream/src/index.js"),
vi.importActual<MockDriverPackages["scenarios"]>("../../../../../packages/testing/src/scenarios.js"),
]);
return { mock, scenarios };
}
function fixture(): { service: LiveMonitorService; get: () => Promise<MonitorRowState>; audits: () => number } {
const row: Record<string, unknown> = {
id: "source", workspace_id: "workspace", connection_id: "connection", remote_ref: "live:1",
state: "OFFLINE", version: 0, authoritative_tier: null,
last_media_health_at: null, last_hls_probe_at: null, last_webhook_at: null, last_api_poll_at: null,
tier_status: {}, ok_streak: 0, fail_streak: 0, state_entered_at: epoch,
hold_until: null, cooldown_until: null, previous_state: null, last_transition_at: null,
last_transition_reason: null, created_at: epoch, updated_at: epoch,
};
const seen = new Set<string>();
let auditCount = 0;
const client = {
async query(sql: string, params: unknown[] = []) {
if (sql.startsWith("SELECT * FROM source_monitor_states")) return { rowCount: 1, rows: [{ ...row }] };
if (sql.startsWith("INSERT INTO monitor_seen_events")) {
const id = String(params[1]);
if (seen.has(id)) return { rowCount: 0, rows: [] };
seen.add(id);
return { rowCount: 1, rows: [{ provider_event_id: id }] };
}
if (sql.startsWith("UPDATE source_monitor_states")) {
if (Number(params[18]) !== Number(row.version)) return { rowCount: 0, rows: [] };
const columns = ["state", "version", "authoritative_tier", "last_media_health_at", "last_hls_probe_at",
"last_webhook_at", "last_api_poll_at", "tier_status", "ok_streak", "fail_streak", "state_entered_at",
"hold_until", "cooldown_until", "previous_state", "last_transition_at", "last_transition_reason", "updated_at"];
for (const [index, column] of columns.entries()) row[column] = params[index + 1];
return { rowCount: 1, rows: [{ ...row }] };
}
if (sql.startsWith("SELECT audit_append")) {
auditCount += 1;
return { rowCount: 1, rows: [{}] };
}
throw new Error(`unexpected query: ${sql}`);
},
} as unknown as PoolClient;
const dependencies: LiveMonitorDependencies = { tenantTransaction: async (_workspace, fn) => fn(client) };
const service = new LiveMonitorService(dependencies);
return { service, get: () => service.getState("workspace", "source"), audits: () => auditCount };
}
describe("live monitor poller", () => {
it("drives dod-flap through the service and reducer without rapid terminal cuts", async () => {
const { service, get, audits } = fixture();
const observations: MonitorObservation[] = [];
const driver = await MockPollDriver.create({
scenario: "dod-flap", source: "A", sourceRef: "live:1", workspaceId: "workspace",
connectionId: "connection", epoch,
ingestNow: () => new Date(Date.parse(epoch) + observations.length * 10_000).toISOString(),
}, await mockPackages());
const states: string[] = [];
for (let tick = 0; tick < 10; tick += 1) {
const [observation] = await driver.poll();
if (observation === undefined) throw new Error("mock poll emitted no observation");
observations.push(observation);
const state = await service.recordObservation("workspace", "source", observation);
expect(state.version).toBe(tick + 1);
states.push(state.state);
}
expect(states).toEqual(["PROBING", "LIVE", "LIVE", "LIVE", "ERROR", "ERROR", "ERROR", "PROBING", "LIVE", "LIVE"]);
expect(observations.map((item) => item.observedAt)).toEqual(
Array.from({ length: 10 }, (_, tick) => new Date(Date.parse(epoch) + tick * 10_000).toISOString()),
);
expect(observations.every((item) => item.tier === "api-poll")).toBe(true);
expect((await get()).version).toBe(10);
expect(audits()).toBe(5);
const replay = observations[8];
if (replay === undefined) throw new Error("missing replay event");
expect((await service.recordObservation("workspace", "source", replay)).version).toBe(10);
expect(audits()).toBe(5);
});
it("plays the single-live scenario and keeps a replayed event from confirming LIVE", async () => {
const { service } = fixture();
const driver = await MockPollDriver.create({
scenario: "single-live", source: "A", sourceRef: "live:1", workspaceId: "workspace",
connectionId: "connection", epoch,
}, await mockPackages());
const [first] = await driver.poll();
if (first === undefined) throw new Error("missing first observation");
expect((await service.recordObservation("workspace", "source", first)).state).toBe("PROBING");
expect((await service.recordObservation("workspace", "source", first)).version).toBe(1);
const [second] = await driver.poll();
if (second === undefined) throw new Error("missing second observation");
const live = await service.recordObservation("workspace", "source", second);
expect(live.state).toBe("LIVE");
expect(live.version).toBe(2);
});
it("applies the same provider event only once across two poll ticks", async () => {
const { service, get } = fixture();
const observation: MonitorObservation = {
tier: "api-poll", observedAt: epoch, receivedAt: epoch, providerEventId: "same-event",
payload: { outcome: "ok" },
};
const poller = new LiveMonitorPoller(service, {
enabled: false,
sources: [{ provider: "mock-livestream", sourceRef: "live:1", workspaceId: "workspace", connectionId: "connection" }],
schedule: () => 1 as unknown as ReturnType<typeof setTimeout>,
cancel: () => undefined,
resolveSourceId: async () => "source",
makeDriver: async () => ({ poll: async () => [observation] }),
});
poller.start();
await poller.pollOnce();
await poller.pollOnce();
expect((await get()).version).toBe(1);
expect((await get()).state).toBe("PROBING");
await poller.stop();
});
it("uses a 10 second interval with bounded ±20% jitter and schedules the next tick", async () => {
expect([0, 0.5, 1].map((value) => jitteredInterval(10_000, value))).toEqual([8_000, 10_000, 12_000]);
const scheduled: Array<{ callback: () => void; delay: number }> = [];
const recorded: MonitorObservation[] = [];
const observation: MonitorObservation = {
tier: "api-poll", observedAt: epoch, receivedAt: epoch, providerEventId: "event-1",
payload: { outcome: "ok" },
};
const monitor = { recordObservation: async (_workspace: string, _source: string, value: MonitorObservation) => {
recorded.push(value);
} } as unknown as LiveMonitorService;
const overrides: Partial<PollerDependencies> = {
enabled: false, intervalMs: 10_000, random: () => 0,
sources: [{ provider: "mock-livestream", sourceRef: "live:1", workspaceId: "workspace", connectionId: "connection" }],
schedule: (callback, delay) => {
scheduled.push({ callback, delay });
return 1 as unknown as ReturnType<typeof setTimeout>;
},
cancel: () => undefined,
resolveSourceId: async () => "source",
makeDriver: async () => ({ poll: async () => [observation] }),
};
const poller = new LiveMonitorPoller(monitor, overrides);
poller.start();
expect(scheduled[0]?.delay).toBe(0);
scheduled[0]?.callback();
await new Promise((resolve) => setImmediate(resolve));
expect(recorded).toEqual([observation]);
expect(scheduled[1]?.delay).toBe(8_000);
await poller.stop();
});
it("parses source config and leaves the real probe stubbed", async () => {
expect(parsePollSources(JSON.stringify([{ provider: "mock-livestream", sourceRef: "live:1",
workspaceId: "workspace", connectionId: "connection" }]))).toHaveLength(1);
await expect(new PeerTubePollDriver("video-id").poll()).rejects.toThrow("NOT_IMPLEMENTED");
});
});

View file

@ -0,0 +1,215 @@
import { Inject, Injectable, Logger, NotFoundException, OnApplicationBootstrap, OnApplicationShutdown, Optional } from "@nestjs/common";
import { withTenant } from "../../db/client.js";
import { LiveMonitorService } from "../live-monitor.service.js";
import type { PollDriver } from "./driver.js";
import { MockPollDriver } from "./mock-driver.js";
// eslint-disable-next-line no-restricted-syntax
import { EgressPeerTubeProbeClient, PeerTubePollDriver } from "./peertube-driver.js";
import { monitorEgress } from "./egress.js";
import { HlsProbeDriver } from "./hls-driver.js";
import { BackoffPollDriver } from "./backoff-driver.js";
export interface PollSource {
provider: string;
sourceRef: string;
workspaceId: string;
connectionId: string;
scenario?: string;
scenarioSource?: "A" | "B" | "C";
playbackUrl?: string;
}
export interface PollerDependencies {
sources: PollSource[];
intervalMs: number;
enabled: boolean;
random: () => number;
schedule: (callback: () => void, delayMs: number) => ReturnType<typeof setTimeout>;
cancel: (timer: ReturnType<typeof setTimeout>) => void;
resolveSourceId: (source: PollSource) => Promise<string>;
makeDriver: (source: PollSource, intervalMs: number) => Promise<PollDriver>;
}
export const MEDIA_POLLER_DEPENDENCIES = Symbol("MEDIA_POLLER_DEPENDENCIES");
export function jitteredInterval(intervalMs: number, random: number): number {
if (!Number.isFinite(intervalMs) || intervalMs <= 0) throw new Error("poll interval must be positive");
if (!Number.isFinite(random) || random < 0 || random > 1) throw new Error("random must be between 0 and 1");
return Math.round(intervalMs * (0.8 + random * 0.4));
}
function requiredString(value: unknown, name: string): string {
if (typeof value !== "string" || value.trim() === "") throw new Error(`${name} must be a non-empty string`);
return value;
}
export function parsePollSources(raw: string | undefined): PollSource[] {
if (raw === undefined || raw.trim() === "") return [];
const parsed: unknown = JSON.parse(raw);
if (!Array.isArray(parsed)) throw new Error("MEDIA_POLLER_SOURCES must be a JSON array");
return parsed.map((entry: unknown, index): PollSource => {
if (typeof entry !== "object" || entry === null || Array.isArray(entry)) {
throw new Error(`MEDIA_POLLER_SOURCES[${index}] must be an object`);
}
const row = entry as Record<string, unknown>;
const source: PollSource = {
provider: requiredString(row.provider, "provider"),
sourceRef: requiredString(row.sourceRef, "sourceRef"),
workspaceId: requiredString(row.workspaceId, "workspaceId"),
connectionId: requiredString(row.connectionId, "connectionId"),
};
if (row.scenario !== undefined) source.scenario = requiredString(row.scenario, "scenario");
if (row.playbackUrl !== undefined) source.playbackUrl = requiredString(row.playbackUrl, "playbackUrl");
if (row.scenarioSource !== undefined) {
if (row.scenarioSource !== "A" && row.scenarioSource !== "B" && row.scenarioSource !== "C") {
throw new Error("scenarioSource must be A, B, or C");
}
source.scenarioSource = row.scenarioSource;
}
return source;
});
}
async function resolveSourceId(source: PollSource): Promise<string> {
return withTenant(source.workspaceId, async (client) => {
const result = await client.query(
"SELECT id FROM source_monitor_states WHERE connection_id = $1 AND remote_ref = $2",
[source.connectionId, source.sourceRef],
);
if (result.rowCount !== 1) throw new NotFoundException(`monitored source ${source.sourceRef} not found`);
return String(result.rows[0]?.id);
});
}
async function makeDriver(source: PollSource, intervalMs: number): Promise<PollDriver> {
// Provider routing lives here until the connector registry exposes monitor probes.
if (source.provider === "mock-livestream") {
return MockPollDriver.create({
scenario: source.scenario ?? "single-live", source: source.scenarioSource ?? "A",
sourceRef: source.sourceRef, workspaceId: source.workspaceId, connectionId: source.connectionId,
tickMs: intervalMs,
});
}
// eslint-disable-next-line no-restricted-syntax
if (source.provider === "peertube" || source.provider === "hls") {
const egress = await monitorEgress();
const drivers: PollDriver[] = [];
// eslint-disable-next-line no-restricted-syntax
if (source.provider === "peertube") {
const fields = await withTenant(source.workspaceId, async (client) => {
const result = await client.query<{ fields: Record<string, unknown> }>(
"SELECT fields FROM connections WHERE id = $1 AND workspace_id = $2 AND disconnected_at IS NULL",
[source.connectionId, source.workspaceId]);
if (result.rowCount !== 1) throw new NotFoundException("monitor connection not found");
return result.rows[0]!.fields;
});
const instanceUrl = requiredString(fields.instanceUrl, "connection instanceUrl");
const videoId = source.sourceRef.startsWith("video:")
? source.sourceRef.slice(6).split("@")[0]! : source.sourceRef;
drivers.push(new BackoffPollDriver(new PeerTubePollDriver(videoId,
new EgressPeerTubeProbeClient(instanceUrl, egress)), "api-poll"));
}
if (source.playbackUrl) drivers.push(new BackoffPollDriver(new HlsProbeDriver(source.playbackUrl, egress), "hls-probe"));
if (!drivers.length) throw new Error("HLS monitor source requires playbackUrl");
return { poll: async () => {
const observations = [];
for (const driver of drivers) observations.push(...await driver.poll());
return observations;
} };
}
throw new Error(`no live poll driver for provider ${source.provider}`);
}
function defaults(): PollerDependencies {
const interval = Number(process.env.MEDIA_POLLER_INTERVAL_MS ?? 10_000);
if (!Number.isFinite(interval) || interval <= 0) throw new Error("MEDIA_POLLER_INTERVAL_MS must be positive");
const enabled = process.env.MEDIA_POLLER_ENABLED === undefined
? process.env.NODE_ENV !== "production" : process.env.MEDIA_POLLER_ENABLED === "true";
return {
sources: parsePollSources(process.env.MEDIA_POLLER_SOURCES), intervalMs: interval, enabled,
random: Math.random, schedule: setTimeout, cancel: clearTimeout, resolveSourceId, makeDriver,
};
}
@Injectable()
export class LiveMonitorPoller implements OnApplicationBootstrap, OnApplicationShutdown {
private readonly logger = new Logger(LiveMonitorPoller.name);
private readonly config: PollerDependencies;
private readonly drivers = new Map<string, PollDriver>();
private readonly sourceIds = new Map<string, string>();
private timer: ReturnType<typeof setTimeout> | undefined;
private active: Promise<void> | undefined;
private running = false;
constructor(
@Inject(LiveMonitorService) private readonly monitor: LiveMonitorService,
@Optional() @Inject(MEDIA_POLLER_DEPENDENCIES) dependencies?: Partial<PollerDependencies>,
) {
this.config = { ...defaults(), ...dependencies };
}
onApplicationBootstrap(): void {
if (this.config.enabled) this.start();
}
async onApplicationShutdown(): Promise<void> {
await this.stop();
}
get isRunning(): boolean { return this.running; }
start(): { running: boolean } {
if (this.running) return { running: true };
this.running = true;
this.schedule(0);
return { running: true };
}
async stop(): Promise<{ running: boolean }> {
this.running = false;
if (this.timer !== undefined) this.config.cancel(this.timer);
this.timer = undefined;
await this.active;
return { running: false };
}
private schedule(delayMs: number): void {
if (!this.running) return;
this.timer = this.config.schedule(() => {
this.timer = undefined;
this.active = this.pollOnce().catch((error: unknown) => {
this.logger.error(`poll tick failed: ${String(error)}`);
}).finally(() => {
this.active = undefined;
this.schedule(jitteredInterval(this.config.intervalMs, this.config.random()));
});
}, delayMs);
}
/** One serialized tick; each source fails independently. */
async pollOnce(): Promise<void> {
for (const source of this.config.sources) {
if (!this.running) return;
const key = `${source.workspaceId}:${source.connectionId}:${source.sourceRef}`;
try {
let driver = this.drivers.get(key);
if (driver === undefined) {
driver = await this.config.makeDriver(source, this.config.intervalMs);
this.drivers.set(key, driver);
}
let sourceId = this.sourceIds.get(key);
if (sourceId === undefined) {
sourceId = await this.config.resolveSourceId(source);
this.sourceIds.set(key, sourceId);
}
const observations = await driver.poll();
for (const observation of observations) {
if (!this.running) return;
await this.monitor.recordObservation(source.workspaceId, sourceId, observation);
}
} catch (error) {
this.logger.error(`poll failed for ${key}: ${String(error)}`);
}
}
}
}

View file

@ -0,0 +1,172 @@
import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { EgressError, EgressPolicy, rewriteSegmentUris, validateMasterPlaylist } from "../../../../../packages/egress/src/index.js";
import { BackoffPollDriver } from "./backoff-driver.js";
import { MONITOR_MAX_BYTES, MONITOR_TIMEOUT_MS, type MonitorEgress } from "./egress.js";
import { HlsProbeDriver, parseMediaPlaylist } from "./hls-driver.js";
import { EgressPeerTubeProbeClient, PeerTubePollDriver } from "./peertube-driver.js";
const epoch = Date.parse("2026-01-01T00:00:12.000Z");
const origin = "https://stream.example";
const masterUrl = `${origin}/hls/master.m3u8`;
const liveUrl = `${origin}/hls/live.m3u8`;
const fixture = (name: string) => readFileSync(resolve(__dirname, "fixtures", name), "utf8");
const live = fixture("live.m3u8");
/** A fake transport tracks each egress fetch/debit; real WP7 URI policy/helpers still run. */
function fakeEgress(routes: Record<string, string | Response | (() => Response | Promise<Response>)>) {
const draws: Array<{ host: string; hostClass: string; url: string }> = [];
const policy = new EgressPolicy({ resolver: async () => [{ address: "8.8.8.8", family: 4 }] });
const egress: MonitorEgress = {
policy, validateMasterPlaylist: vi.fn(validateMasterPlaylist), rewriteSegmentUris: vi.fn(rewriteSegmentUris),
egressFetch: vi.fn(async (raw, _init, context) => {
const endpoint = await policy.validate(raw);
draws.push({ host: endpoint.hostKey, hostClass: context.hostClass, url: endpoint.url.href });
const route = routes[endpoint.url.href];
if (route === undefined) throw new Error("unexpected outbound URL");
const response = typeof route === "function" ? await route() : typeof route === "string" ? new Response(route) : route.clone();
if (!response.url) Object.defineProperty(response, "url", { value: endpoint.url.href });
return response;
}),
};
return { egress, draws };
}
afterEach(() => { vi.useRealTimers(); });
describe("HLS media ground truth", () => {
it("fetches a master and exactly one variant through monitor-probe egress, validates all URIs", async () => {
const f = fakeEgress({ [masterUrl]: fixture("master.m3u8"), [liveUrl]: live });
const observation = (await new HlsProbeDriver(masterUrl, f.egress, () => epoch).poll())[0];
expect(observation).toMatchObject({ tier: "hls-probe", payload: { outcome: "ok", mediaPresent: true } });
expect(f.draws).toEqual([
{ url: masterUrl, host: "stream.example:443", hostClass: "monitor-probe" },
{ url: liveUrl, host: "stream.example:443", hostClass: "monitor-probe" },
]);
expect(f.egress.validateMasterPlaylist).toHaveBeenCalledOnce();
expect(f.egress.rewriteSegmentUris).toHaveBeenCalledOnce();
expect(f.egress.rewriteSegmentUris).toHaveBeenCalledWith(live, liveUrl, f.egress.policy);
});
it.each(["offline.m3u8", "empty", "404"])("reports %s as offline", async (name) => {
const body = name === "empty" ? "#EXTM3U\n#EXT-X-TARGETDURATION:6\n"
: name === "404" ? new Response(null, { status: 404 }) : fixture(name);
const f = fakeEgress({ [liveUrl]: body });
expect((await new HlsProbeDriver(liveUrl, f.egress, () => epoch).poll())[0]?.payload)
.toMatchObject({ outcome: "fail", mediaPresent: false });
});
it("reports stale program dates and unreasonable future dates as offline", async () => {
const f = fakeEgress({ [liveUrl]: live });
expect((await new HlsProbeDriver(liveUrl, f.egress, () => epoch + 19_000).poll())[0]?.payload.outcome).toBe("fail");
expect((await new HlsProbeDriver(liveUrl, f.egress, () => epoch - 19_000).poll())[0]?.payload.outcome).toBe("fail");
});
it("uses media-sequence/last-segment progression and three target durations when PDT is absent", async () => {
let now = epoch; let body = live.replace(/#EXT-X-PROGRAM-DATE-TIME:.*\n/, "");
const f = fakeEgress({ [liveUrl]: () => new Response(body) });
const driver = new HlsProbeDriver(liveUrl, f.egress, () => now);
expect((await driver.poll())[0]?.payload.outcome).toBe("ok");
now += 18_001;
expect((await driver.poll())[0]?.payload.outcome).toBe("fail");
body = body.replace("SEQUENCE:20", "SEQUENCE:21").replace("segment21.ts", "segment22.ts");
expect((await driver.poll())[0]?.payload.outcome).toBe("ok");
});
it.each(["malformed.m3u8", "header", "missing-duration", "missing-uri"])("rejects malformed playlist %s", async (name) => {
const body = name === "header" ? "not a playlist" : name === "missing-duration" ? "#EXTM3U\n#EXT-X-TARGETDURATION:6\nseg.ts\n"
: name === "missing-uri" ? "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXTINF:6,\n" : fixture(name);
const f = fakeEgress({ [liveUrl]: body });
await expect(new HlsProbeDriver(liveUrl, f.egress, () => epoch).poll()).rejects.toThrow();
});
it("accepts EVENT playlists and rejects invalid target-duration math", () => {
expect(parseMediaPlaylist(live.replace("#EXT-X-TARGETDURATION", "#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-TARGETDURATION")).vod).toBe(false);
expect(() => parseMediaPlaylist(live.replace("EXTINF:6", "EXTINF:7"))).toThrow();
});
it.each(["variant", "unused-variant", "segment", "key"])("denies private %s references before any child request", async (kind) => {
const master = kind === "variant" ? "#EXTM3U\n#EXT-X-STREAM-INF:BANDWIDTH=800000\nhttp://169.254.169.254/private.m3u8\n"
: fixture("master.m3u8").replace("unused.m3u8", "http://127.0.0.1/unused.m3u8");
const media = kind === "segment" ? live.replace("segment20.ts", "http://10.0.0.1/seg.ts")
: live.replace("#EXTINF:6,", '#EXT-X-KEY:METHOD=AES-128,URI="http://169.254.169.254/key"\n#EXTINF:6,');
const isMaster = kind === "variant" || kind === "unused-variant";
const f = fakeEgress({ [isMaster ? masterUrl : liveUrl]: isMaster ? master : media });
await expect(new HlsProbeDriver(isMaster ? masterUrl : liveUrl, f.egress, () => epoch).poll())
.rejects.toMatchObject({ code: "DENIED_PRIVATE_ADDRESS" });
expect(f.draws).toHaveLength(1);
});
it("resolves variant and segment URIs against the final redirected playlist URL", async () => {
const response = new Response(fixture("master.m3u8"));
Object.defineProperty(response, "url", { value: `${origin}/redirected/master.m3u8` });
const f = fakeEgress({ [masterUrl]: () => response, [`${origin}/redirected/live.m3u8`]: live });
await new HlsProbeDriver(masterUrl, f.egress, () => epoch).poll();
expect(f.draws[1]?.url).toBe(`${origin}/redirected/live.m3u8`);
});
it("enforces body size, URI count and whole-probe deadlines", async () => {
const oversized = fakeEgress({ [liveUrl]: "x".repeat(MONITOR_MAX_BYTES + 1) });
await expect(new HlsProbeDriver(liveUrl, oversized.egress).poll()).rejects.toThrow("size cap");
const references = fakeEgress({ [liveUrl]: `#EXTM3U\n${"#EXTINF:6,\nsegment.ts\n".repeat(129)}` });
await expect(new HlsProbeDriver(liveUrl, references.egress).poll()).rejects.toThrow("too many HLS references");
expect(references.egress.rewriteSegmentUris).not.toHaveBeenCalled();
vi.useFakeTimers();
const hanging = fakeEgress({ [liveUrl]: () => new Promise<Response>(() => undefined) });
const pending = new HlsProbeDriver(liveUrl, hanging.egress).poll();
const rejected = expect(pending).rejects.toThrow("deadline");
await vi.advanceTimersByTimeAsync(MONITOR_TIMEOUT_MS);
await rejected;
expect(vi.getTimerCount()).toBe(0);
});
});
describe("monitor API budgets and adaptive source backoff", () => {
it("counts API + master + variant draws on the same host, with the required host classes", async () => {
const f = fakeEgress({ [`${origin}/api/v1/videos/video-1`]: JSON.stringify({ isLive: true, state: { id: 1 } }),
[masterUrl]: fixture("master.m3u8"), [liveUrl]: live });
const api = new PeerTubePollDriver("video-1", new EgressPeerTubeProbeClient(origin, f.egress, () => epoch));
expect((await api.poll())[0]?.payload.outcome).toBe("ok");
await new HlsProbeDriver(masterUrl, f.egress, () => epoch).poll();
expect(f.draws.map((draw) => draw.hostClass)).toEqual(["monitor-api", "monitor-probe", "monitor-probe"]);
expect(new Set(f.draws.map((draw) => draw.host))).toEqual(new Set(["stream.example:443"]));
expect(f.egress.egressFetch).toHaveBeenNthCalledWith(1, expect.any(URL), expect.anything(), { hostClass: "monitor-api", pinnedOrigin: true });
});
it.each([4, 5])("does not consider PeerTube state %s live even when isLive=true", async (state) => {
const f = fakeEgress({ [`${origin}/api/v1/videos/video-1`]: JSON.stringify({ isLive: true, state: { id: state } }) });
expect((await new EgressPeerTubeProbeClient(origin, f.egress).probe("video-1"))[0]?.payload.outcome).toBe("fail");
});
it.each([429, 503])("backs off source after HTTP %s, honors Retry-After and resets on recovery", async (status) => {
let now = epoch; let attempt = 0;
const f = fakeEgress({ [liveUrl]: () => ++attempt === 1 ? new Response(null, { status, headers: { "retry-after": "30" } }) : new Response(live) });
const driver = new BackoffPollDriver(new HlsProbeDriver(liveUrl, f.egress, () => now), "hls-probe", () => now, () => 0.5);
expect((await driver.poll())[0]?.payload.outcome).toBe("unavailable");
expect(driver.nextAttemptAt).toBe(epoch + 30_000);
now += 29_999; expect(await driver.poll()).toEqual([]); expect(f.draws).toHaveLength(1);
now += 1; await driver.poll(); expect(f.draws).toHaveLength(2); expect(driver.nextAttemptAt).toBe(0);
});
it("honors an HTTP-date Retry-After for API polls and grows source-local delays on repeated 5xx", async () => {
let now = epoch; let attempt = 0;
const f = fakeEgress({ [`${origin}/api/v1/videos/v`]: () => new Response(null, { status: 503,
headers: ++attempt === 1 ? { "retry-after": new Date(epoch + 60_000).toUTCString() } : {} }) });
const driver = new BackoffPollDriver(new PeerTubePollDriver("v", new EgressPeerTubeProbeClient(origin, f.egress)), "api-poll", () => now, () => 0.5);
await driver.poll(); expect(driver.nextAttemptAt).toBe(epoch + 60_000);
now += 60_000; await driver.poll(); expect(driver.nextAttemptAt).toBe(now + 2000);
now += 2000; await driver.poll(); expect(driver.nextAttemptAt).toBe(now + 4000);
});
it("respects WP7 budget exhaustion and allows a different source to proceed", async () => {
let now = epoch;
const egress = fakeEgress({ [liveUrl]: live });
const blocked = { ...egress.egress, egressFetch: vi.fn(async () => { throw new EgressError("BUDGET_EXHAUSTED", "budget", 10_000); }) };
const one = new BackoffPollDriver(new HlsProbeDriver(liveUrl, blocked, () => now), "hls-probe", () => now, () => 0.5);
const two = new BackoffPollDriver(new HlsProbeDriver(liveUrl, egress.egress, () => now), "hls-probe", () => now, () => 0.5);
await one.poll(); now += 1000; expect(await one.poll()).toEqual([]);
expect((await two.poll())[0]?.payload.outcome).toBe("ok");
expect(blocked.egressFetch).toHaveBeenCalledOnce(); expect(egress.draws).toHaveLength(1);
});
});

View file

@ -0,0 +1,97 @@
import { describe, expect, it } from "vitest";
import { applyObservation, type MonitorObservation, type MonitorRowState, type SignalTier } from "./reducer.js";
const start = Date.parse("2026-01-01T00:00:00.000Z");
const at = (seconds: number): string => new Date(start + seconds * 1000).toISOString();
function initial(): MonitorRowState {
return {
id: "source", workspaceId: "workspace", connectionId: "connection", remoteRef: "live:1",
state: "OFFLINE", version: 0, authoritativeTier: null,
lastMediaHealthAt: null, lastHlsProbeAt: null, lastWebhookAt: null, lastApiPollAt: null,
tierStatus: {}, okStreak: 0, failStreak: 0, stateEnteredAt: at(0),
holdUntil: null, cooldownUntil: null, previousState: null,
lastTransitionAt: null, lastTransitionReason: null, createdAt: at(0), updatedAt: at(0),
};
}
function obs(tier: SignalTier, outcome: MonitorObservation["payload"]["outcome"], seconds: number,
extra: Partial<MonitorObservation> = {}): MonitorObservation {
return { tier, observedAt: at(seconds), receivedAt: at(seconds), payload: { outcome }, ...extra };
}
function fold(state: MonitorRowState, ...observations: MonitorObservation[]): MonitorRowState {
return observations.reduce((current, observation) => applyObservation(current, observation).state, state);
}
describe("live source reducer", () => {
it("confirms LIVE after two ok signals from a poll-only source", () => {
const first = applyObservation(initial(), obs("api-poll", "ok", 1));
expect(first.state.state).toBe("PROBING");
const second = applyObservation(first.state, obs("api-poll", "ok", 2));
expect(second.state.state).toBe("LIVE");
expect(second.state.authoritativeTier).toBe("api-poll");
expect(second.transition?.version).toBe(2);
});
it("confirms OFFLINE after three failures and waits out hysteresis", () => {
const live = fold(initial(), obs("api-poll", "ok", 1), obs("api-poll", "ok", 2));
const one = applyObservation(live, obs("api-poll", "fail", 3));
expect(one.state.state).toBe("LIVE");
const two = applyObservation(one.state, obs("api-poll", "fail", 4));
expect(two.state.state).toBe("UNSTABLE");
const three = applyObservation(two.state, obs("api-poll", "fail", 5));
expect(three.state.state).toBe("UNSTABLE");
expect(applyObservation(three.state, obs("api-poll", "fail", 12)).state.state).toBe("OFFLINE");
});
it("holds a flapping source without rapid LIVE/OFFLINE cuts", () => {
let state = fold(initial(), obs("api-poll", "ok", 1), obs("api-poll", "ok", 2));
const terminal: string[] = [state.state];
for (let second = 3; second <= 9; second += 1) {
const outcome = second % 2 === 0 ? "ok" : "fail";
const result = applyObservation(state, obs("api-poll", outcome, second));
state = result.state;
if (result.transition?.to === "LIVE" || result.transition?.to === "OFFLINE") terminal.push(result.transition.to);
}
expect(terminal).toEqual(["LIVE"]);
});
it("discards stale observations within a tier and repeated provider IDs", () => {
const first = fold(initial(), obs("webhook", "ok", 3, { providerEventId: "event-1" }));
expect(applyObservation(first, obs("webhook", "fail", 2)).state).toBe(first);
const second = fold(first, obs("webhook", "ok", 4, { providerEventId: "event-2" }));
expect(applyObservation(second, obs("webhook", "ok", 5, { providerEventId: "event-1" })).state).toBe(second);
expect(first.version).toBe(1);
});
it("keeps a newer HLS probe authoritative against a late API poll and clock-skewed webhook", () => {
const futureWebhook = obs("webhook", "fail", 1000, { receivedAt: at(1) });
const state = fold(initial(), futureWebhook,
obs("hls-probe", "ok", 2), obs("hls-probe", "ok", 3),
obs("api-poll", "fail", 1, { receivedAt: at(6) }));
expect(state.authoritativeTier).toBe("hls-probe");
expect(state.state).toBe("UNSTABLE");
expect(state.tierStatus["hls-probe"]?.okStreak).toBe(2);
const confirmed = applyObservation(state, obs("hls-probe", "ok", 13)).state;
expect(confirmed.state).toBe("LIVE");
});
it("raises and lowers authority when HLS becomes available then denied", () => {
const poll = fold(initial(), obs("api-poll", "ok", 1), obs("api-poll", "ok", 2));
const raised = applyObservation(poll, obs("hls-probe", "ok", 3)).state;
expect(raised.authoritativeTier).toBe("hls-probe");
const lowered = applyObservation(raised, obs("hls-probe", "unavailable", 4)).state;
expect(lowered.authoritativeTier).toBe("api-poll");
expect(lowered.state).toBe("LIVE");
});
it("uses ten seconds without media as a failure confirmation", () => {
const live = fold(initial(),
{ ...obs("media-health", "ok", 1), payload: { outcome: "ok", mediaPresent: true } },
{ ...obs("media-health", "ok", 2), payload: { outcome: "ok", mediaPresent: true } });
const failed = applyObservation(live, obs("media-health", "fail", 13));
expect(failed.state.state).toBe("OFFLINE");
expect(failed.transition?.reason).toBe("media-health:no-media-timeout");
});
});

View file

@ -0,0 +1,182 @@
/** Pure per-source fold. Provider time is used only to order one tier's events. */
export type MonitorState = "OFFLINE" | "PROBING" | "LIVE" | "UNSTABLE" | "ERROR";
export type SignalTier = "media-health" | "hls-probe" | "webhook" | "api-poll";
export type ObservationOutcome = "ok" | "fail" | "error" | "unavailable";
export interface MonitorObservation {
tier: SignalTier;
observedAt: string;
/** Ingest clock, never a provider clock. */
receivedAt: string;
providerEventId?: string;
payload: { outcome: ObservationOutcome; reason?: string; mediaPresent?: boolean };
}
export interface TierStatus {
available: boolean;
lastOutcome?: ObservationOutcome;
lastReceivedAt?: string;
lastProviderEventId?: string;
/** Recent replay guard; the database journal provides durable full-history dedupe. */
seenProviderEventIds?: string[];
okStreak: number;
failStreak: number;
lastMediaAt?: string;
}
export interface MonitorRowState {
id: string;
workspaceId: string;
connectionId: string;
remoteRef: string;
state: MonitorState;
version: number;
authoritativeTier: SignalTier | null;
lastMediaHealthAt: string | null;
lastHlsProbeAt: string | null;
lastWebhookAt: string | null;
lastApiPollAt: string | null;
tierStatus: Partial<Record<SignalTier, TierStatus>>;
okStreak: number;
failStreak: number;
stateEnteredAt: string;
holdUntil: string | null;
cooldownUntil: string | null;
previousState: MonitorState | null;
lastTransitionAt: string | null;
lastTransitionReason: string | null;
createdAt: string;
updatedAt: string;
}
export interface MonitorPolicy {
liveConfirmations: number;
offlineConfirmations: number;
noMediaMs: number;
minHoldMs: number;
cooldownMs: number;
}
export const DEFAULT_MONITOR_POLICY: MonitorPolicy = {
liveConfirmations: 2,
offlineConfirmations: 3,
noMediaMs: 10_000,
minHoldMs: 2_000,
cooldownMs: 10_000,
};
export interface MonitorTransition {
from: MonitorState;
to: MonitorState;
version: number;
reason: string;
at: string;
}
export interface Reduction {
state: MonitorRowState;
transition?: MonitorTransition;
}
const PRECEDENCE: readonly SignalTier[] = ["media-health", "hls-probe", "webhook", "api-poll"];
const OBSERVED_FIELD: Record<SignalTier, keyof Pick<MonitorRowState,
"lastMediaHealthAt" | "lastHlsProbeAt" | "lastWebhookAt" | "lastApiPollAt">> = {
"media-health": "lastMediaHealthAt",
"hls-probe": "lastHlsProbeAt",
webhook: "lastWebhookAt",
"api-poll": "lastApiPollAt",
};
export function applyObservation(
state: MonitorRowState,
observation: MonitorObservation,
policy: MonitorPolicy = DEFAULT_MONITOR_POLICY,
): Reduction {
const now = Date.parse(observation.receivedAt);
const observed = Date.parse(observation.observedAt);
if (!Number.isFinite(now) || !Number.isFinite(observed)) throw new Error("observation timestamps must be valid");
const field = OBSERVED_FIELD[observation.tier];
const last = state[field];
if (last !== null && observed <= Date.parse(last)) return { state };
const oldTier = state.tierStatus[observation.tier];
if (observation.providerEventId !== undefined && (oldTier?.lastProviderEventId === observation.providerEventId
|| oldTier?.seenProviderEventIds?.includes(observation.providerEventId) === true)) {
return { state };
}
const outcome = observation.payload.outcome;
const available = outcome !== "unavailable";
const okStreak = outcome === "ok" ? (oldTier?.okStreak ?? 0) + 1 : 0;
const failStreak = outcome === "fail" ? (oldTier?.failStreak ?? 0) + 1 : 0;
const tierStatus: MonitorRowState["tierStatus"] = {
...state.tierStatus,
[observation.tier]: {
available,
lastOutcome: outcome,
lastReceivedAt: observation.receivedAt,
...(observation.providerEventId === undefined
? oldTier?.lastProviderEventId === undefined ? {} : { lastProviderEventId: oldTier.lastProviderEventId }
: { lastProviderEventId: observation.providerEventId }),
...(observation.providerEventId === undefined && oldTier?.seenProviderEventIds === undefined
? {} : { seenProviderEventIds: observation.providerEventId === undefined
? oldTier?.seenProviderEventIds
: [...(oldTier?.seenProviderEventIds ?? []), observation.providerEventId].slice(-64) }),
okStreak,
failStreak,
...(observation.payload.mediaPresent === true
? { lastMediaAt: observation.receivedAt }
: oldTier?.lastMediaAt === undefined ? {} : { lastMediaAt: oldTier.lastMediaAt }),
},
};
const authoritativeTier = PRECEDENCE.find((tier) => tierStatus[tier]?.available === true) ?? null;
const authority = authoritativeTier === null ? undefined : tierStatus[authoritativeTier];
let next: MonitorRowState = {
...state,
[field]: observation.observedAt,
tierStatus,
authoritativeTier,
version: state.version + 1,
okStreak: authority?.okStreak ?? 0,
failStreak: authority?.failStreak ?? 0,
updatedAt: observation.receivedAt,
};
let target: MonitorState | undefined;
let reason = `${observation.tier}:${outcome}`;
if (outcome === "error" && observation.tier === authoritativeTier) {
target = "ERROR";
} else if (authoritativeTier !== null) {
const lowerSignal = observation.tier !== authoritativeTier;
if (lowerSignal && outcome !== "unavailable") {
if (outcome === "fail" || outcome === "error") target = state.state === "LIVE" ? "UNSTABLE" : "PROBING";
} else if (authority?.lastOutcome === "ok") {
if (authority.okStreak >= policy.liveConfirmations) target = "LIVE";
else if (state.state !== "LIVE") target = "PROBING";
} else if (authority?.lastOutcome === "fail") {
const noMedia = authoritativeTier === "media-health" && authority.lastMediaAt !== undefined
&& now - Date.parse(authority.lastMediaAt) >= policy.noMediaMs;
if (authority.failStreak >= policy.offlineConfirmations || noMedia) target = "OFFLINE";
else if (state.state === "LIVE") target = "UNSTABLE";
else if (state.state !== "OFFLINE" && state.state !== "UNSTABLE") target = "PROBING";
if (noMedia) reason = "media-health:no-media-timeout";
}
}
if (target === undefined || target === state.state) return { state: next };
const terminal = target === "LIVE" || target === "OFFLINE";
const held = target !== "ERROR" && state.state !== "PROBING" && state.state !== "UNSTABLE"
&& state.holdUntil !== null && now < Date.parse(state.holdUntil);
const cooling = terminal && state.cooldownUntil !== null && now < Date.parse(state.cooldownUntil);
if (held || cooling) return { state: next };
next = {
...next,
state: target,
previousState: state.state,
stateEnteredAt: observation.receivedAt,
holdUntil: terminal ? new Date(now + policy.minHoldMs).toISOString() : state.holdUntil,
cooldownUntil: terminal ? new Date(now + policy.cooldownMs).toISOString() : state.cooldownUntil,
lastTransitionAt: observation.receivedAt,
lastTransitionReason: reason,
};
return { state: next, transition: { from: state.state, to: target, version: next.version, reason, at: observation.receivedAt } };
}

View file

@ -0,0 +1,19 @@
import { Controller, HttpCode, Inject, Param, Post, Req } from "@nestjs/common";
import type { Request } from "express";
import { MonitorWebhookService } from "./webhook.service.js";
/** Provider authentication is performed over raw bytes; browser session guards do not apply. */
@Controller("live-monitor/webhooks")
export class MonitorWebhookController {
constructor(@Inject(MonitorWebhookService) private readonly webhook: MonitorWebhookService) {}
@Post(":endpointId")
@HttpCode(200)
async ingest(@Param("endpointId") endpointId: string, @Req() request: Request): Promise<{ accepted: true }> {
await this.webhook.ingest(endpointId, request.body as Buffer, {
signature: request.get("x-monitor-signature"), timestamp: request.get("x-monitor-timestamp"),
eventId: request.get("x-monitor-event-id"),
});
return { accepted: true };
}
}

View file

@ -0,0 +1,91 @@
import { createHmac, randomBytes, randomUUID } from "node:crypto";
import { join } from "node:path";
import { Pool, type PoolClient } from "pg";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { runMigrations } from "../db/migrate.js";
import { vault } from "../vault/vault.js";
import { LiveMonitorService } from "./live-monitor.service.js";
import { MonitorWebhookService, type MonitorWebhookEndpoint } from "./webhook.service.js";
const available = process.env.DATABASE_URL !== undefined && process.env.SUPER_DATABASE_URL !== undefined;
const owner = new Pool({ connectionString: process.env.SUPER_DATABASE_URL });
const app = new Pool({ connectionString: process.env.DATABASE_URL });
async function asTenant<T>(workspaceId: string, fn: (client: PoolClient) => Promise<T>): Promise<T> {
const client = await app.connect();
try {
await client.query("BEGIN");
await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]);
const result = await fn(client);
await client.query("COMMIT");
return result;
} catch (error) { await client.query("ROLLBACK"); throw error; }
finally { client.release(); }
}
describe.skipIf(!available)("monitor webhook durable journal and RLS (PostgreSQL)", () => {
const secret = randomBytes(32);
const workspaces: string[] = [];
let endpoint: MonitorWebhookEndpoint;
let sourceId: string;
let foreignSourceId: string;
let monitor: LiveMonitorService;
const service = () => new MonitorWebhookService(monitor, { endpoints: [endpoint], tenantTransaction: asTenant });
beforeAll(async () => {
await runMigrations(process.env.MIGRATIONS_DIR ?? join(process.cwd(), "migrations"));
for (let index = 0; index < 2; index++) {
const workspace = await owner.query<{ id: string }>(
"INSERT INTO workspaces (name, slug) VALUES ('Webhook integration', $1) RETURNING id", [`monitor-webhook-${randomUUID()}`]);
const workspaceId = workspace.rows[0]!.id;
workspaces.push(workspaceId);
await asTenant(workspaceId, async (client) => {
const connection = await client.query<{ id: string }>(
"INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-livestream', 'webhook') RETURNING id", [workspaceId]);
const connectionId = connection.rows[0]!.id;
const source = await client.query<{ id: string }>(
"INSERT INTO source_monitor_states (workspace_id, connection_id, remote_ref) VALUES ($1, $2, 'live:webhook') RETURNING id", [workspaceId, connectionId]);
if (index === 0) {
sourceId = source.rows[0]!.id;
const sealed = await vault.seal(workspaceId, connectionId, secret);
const credential = await client.query<{ id: string }>(
`INSERT INTO credentials (workspace_id, connection_id, kind, ciphertext, key_version)
VALUES ($1, $2, 'monitor-webhook', $3, $4) RETURNING id`, [workspaceId, connectionId, sealed.ciphertext, sealed.keyVersion]);
endpoint = { endpointId: randomBytes(32).toString("hex"), workspaceId, connectionId,
credentialId: credential.rows[0]!.id, subscriptionId: "db-subscription" };
} else foreignSourceId = source.rows[0]!.id;
});
}
monitor = new LiveMonitorService({ tenantTransaction: asTenant });
});
afterAll(async () => {
for (const workspace of workspaces) await owner.query("DELETE FROM workspaces WHERE id = $1", [workspace]);
await app.end(); await owner.end(); secret.fill(0);
});
function request(source = sourceId, eventId = randomUUID()) {
const raw = Buffer.from(JSON.stringify({ sourceId: source, subscriptionId: endpoint.subscriptionId, outcome: "ok" }));
const timestamp = new Date().toISOString();
const signature = `sha256=${createHmac("sha256", secret).update(eventId).update(timestamp).update(raw).digest("hex")}`;
return { raw, headers: { eventId, timestamp, signature } };
}
it("accepts one concurrent delivery, rejects its replay after service restart, persists exactly one journal row", async () => {
const { raw, headers } = request();
const results = await Promise.allSettled([service().ingest(endpoint.endpointId, raw, headers), service().ingest(endpoint.endpointId, raw, headers)]);
expect(results.filter((result) => result.status === "fulfilled")).toHaveLength(1);
expect(results.find((result) => result.status === "rejected")).toMatchObject({ reason: { status: 409 } });
await expect(service().ingest(endpoint.endpointId, raw, headers)).rejects.toMatchObject({ status: 409 });
expect((await monitor.getState(endpoint.workspaceId, sourceId)).version).toBe(1);
const journal = await asTenant(endpoint.workspaceId, (client) => client.query<{ count: number }>(
"SELECT count(*)::int AS count FROM monitor_seen_events WHERE source_monitor_state_id = $1", [sourceId]));
expect(journal.rows[0]!.count).toBe(1);
});
it("rejects an authenticated event targeting another workspace without touching either source", async () => {
const { raw, headers } = request(foreignSourceId);
await expect(service().ingest(endpoint.endpointId, raw, headers)).rejects.toMatchObject({ status: 404 });
expect((await monitor.getState(workspaces[1]!, foreignSourceId)).version).toBe(0);
expect((await monitor.getState(endpoint.workspaceId, sourceId)).version).toBe(1);
});
});

View file

@ -0,0 +1,136 @@
import { createHash, createHmac, timingSafeEqual } from "node:crypto";
import { BadRequestException, Inject, Injectable, NotFoundException, Optional, PayloadTooLargeException, UnauthorizedException } from "@nestjs/common";
import { withTenant } from "../db/client.js";
import { isUuid, requireObject, requireString } from "../http/validation.js";
import { vault } from "../vault/vault.js";
import { LiveMonitorService } from "./live-monitor.service.js";
import type { MonitorRowState, ObservationOutcome } from "./reducer.js";
export const WEBHOOK_MAX_BYTES = 64 * 1024;
export const WEBHOOK_WINDOW_MS = 5 * 60 * 1000;
export interface MonitorWebhookEndpoint {
endpointId: string;
workspaceId: string;
connectionId: string;
credentialId: string;
subscriptionId: string;
}
export interface MonitorWebhookHeaders { signature?: string; timestamp?: string; eventId?: string }
export interface MonitorWebhookDependencies {
endpoints: readonly MonitorWebhookEndpoint[];
tenantTransaction: typeof withTenant;
openSecret: typeof vault.open;
now: () => number;
}
export const MONITOR_WEBHOOK_DEPENDENCIES = Symbol("MONITOR_WEBHOOK_DEPENDENCIES");
/** Server-controlled mapping, never a workspace/connection supplied by the sender. */
export function parseWebhookEndpoints(raw: string | undefined): MonitorWebhookEndpoint[] {
if (!raw) return [];
const value: unknown = JSON.parse(raw);
if (!Array.isArray(value)) throw new Error("MONITOR_WEBHOOK_ENDPOINTS must be an array");
const endpoints = value.map((entry) => {
const row = requireObject(entry, "endpoint");
const endpoint: MonitorWebhookEndpoint = {
endpointId: requireString(row.endpointId, "endpointId"),
workspaceId: requireString(row.workspaceId, "workspaceId"),
connectionId: requireString(row.connectionId, "connectionId"),
credentialId: requireString(row.credentialId, "credentialId"),
subscriptionId: requireString(row.subscriptionId, "subscriptionId"),
};
if (!/^[a-f0-9]{64}$/.test(endpoint.endpointId)
|| ![endpoint.workspaceId, endpoint.connectionId, endpoint.credentialId].every(isUuid)) {
throw new Error("invalid monitor webhook endpoint mapping (endpointId must be 32 random bytes in hex)");
}
return endpoint;
});
if (new Set(endpoints.map((entry) => entry.endpointId)).size !== endpoints.length) {
throw new Error("duplicate monitor webhook endpoint");
}
return endpoints;
}
/** Signature covers event id + ISO timestamp + exact bytes, including whitespace. */
export function verifyMonitorSignature(raw: Buffer, headers: MonitorWebhookHeaders, secret: Buffer, now: number): void {
const { signature, timestamp, eventId } = headers;
if (!signature || !/^sha256=[a-f0-9]{64}$/.test(signature) || !timestamp
|| !/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d{1,3})?Z$/.test(timestamp)
|| !eventId || eventId.length > 256 || /[\s\p{Cc}]/u.test(eventId)
|| !Number.isFinite(Date.parse(timestamp)) || Math.abs(now - Date.parse(timestamp)) > WEBHOOK_WINDOW_MS) {
throw new UnauthorizedException("invalid webhook authentication");
}
const expected = createHmac("sha256", secret).update(eventId).update(timestamp).update(raw).digest();
if (!timingSafeEqual(expected, Buffer.from(signature.slice(7), "hex"))) {
throw new UnauthorizedException("invalid webhook authentication");
}
}
@Injectable()
export class MonitorWebhookService {
private readonly config: MonitorWebhookDependencies;
constructor(
@Inject(LiveMonitorService) private readonly monitor: LiveMonitorService,
@Optional() @Inject(MONITOR_WEBHOOK_DEPENDENCIES) dependencies?: Partial<MonitorWebhookDependencies>,
) {
this.config = {
endpoints: parseWebhookEndpoints(process.env.MONITOR_WEBHOOK_ENDPOINTS),
tenantTransaction: withTenant, openSecret: vault.open.bind(vault), now: Date.now,
...dependencies,
};
}
async ingest(endpointId: string, raw: Buffer, headers: MonitorWebhookHeaders): Promise<MonitorRowState> {
const endpoint = this.config.endpoints.find((entry) => entry.endpointId === endpointId);
if (!endpoint) throw new NotFoundException("webhook endpoint not found");
if (!Buffer.isBuffer(raw)) throw new BadRequestException("raw webhook body required");
if (raw.length > WEBHOOK_MAX_BYTES) throw new PayloadTooLargeException();
return this.config.tenantTransaction(endpoint.workspaceId, async (client) => {
// A removed connection/revoked credential disables even a configured endpoint.
const credential = await client.query<{ ciphertext: Buffer; key_version: number }>(
`SELECT cr.ciphertext, cr.key_version FROM credentials cr
JOIN connections c ON c.id = cr.connection_id AND c.workspace_id = cr.workspace_id
WHERE cr.id = $1 AND cr.connection_id = $2 AND cr.workspace_id = $3
AND cr.kind = 'monitor-webhook' AND cr.revoked_at IS NULL AND c.disconnected_at IS NULL
FOR SHARE OF cr, c`, [endpoint.credentialId, endpoint.connectionId, endpoint.workspaceId],
);
const sealed = credential.rows[0];
if (!sealed) throw new UnauthorizedException("invalid webhook authentication");
let secret: Buffer;
try {
secret = await this.config.openSecret(endpoint.workspaceId, endpoint.connectionId,
{ ciphertext: sealed.ciphertext, keyVersion: sealed.key_version });
} catch { throw new UnauthorizedException("invalid webhook authentication"); }
const now = this.config.now();
try {
if (secret.length < 32) throw new UnauthorizedException("invalid webhook authentication");
verifyMonitorSignature(raw, headers, secret, now);
} finally { secret.fill(0); }
let parsed: unknown;
try { parsed = JSON.parse(raw.toString("utf8")); }
catch { throw new BadRequestException("invalid webhook JSON"); }
const body = requireObject(parsed, "event");
if (body.subscriptionId !== endpoint.subscriptionId
|| body.workspaceId !== undefined && body.workspaceId !== endpoint.workspaceId
|| body.connectionId !== undefined && body.connectionId !== endpoint.connectionId) {
throw new UnauthorizedException("webhook binding mismatch");
}
const sourceId = requireString(body.sourceId, "sourceId");
if (!isUuid(sourceId)) throw new BadRequestException("sourceId must be a UUID");
const outcome = body.outcome;
if (outcome !== "ok" && outcome !== "fail" && outcome !== "error" && outcome !== "unavailable") {
throw new BadRequestException("invalid webhook outcome");
}
const source = await client.query(
`SELECT id FROM source_monitor_states WHERE id = $1 AND workspace_id = $2 AND connection_id = $3 FOR UPDATE`,
[sourceId, endpoint.workspaceId, endpoint.connectionId],
);
if (source.rowCount !== 1) throw new NotFoundException("webhook source not found");
// Journal retention exceeds the acceptance window. The tuple scopes ids per endpoint without ambiguity.
const providerEventId = `webhook:${createHash("sha256").update(JSON.stringify([endpointId, headers.eventId])).digest("hex")}`;
return this.monitor.recordObservationInTransaction(client, endpoint.workspaceId, sourceId, {
tier: "webhook", observedAt: headers.timestamp!, receivedAt: new Date(now).toISOString(), providerEventId,
payload: { outcome: outcome as ObservationOutcome, reason: "signed-webhook" },
}, { userId: null, role: "monitor-webhook" }, true);
});
}
}

View file

@ -0,0 +1,199 @@
import "reflect-metadata";
import { createHmac } from "node:crypto";
import { BadRequestException, ConflictException, NotFoundException, UnauthorizedException, type INestApplication } from "@nestjs/common";
import { NestFactory } from "@nestjs/core";
import type { PoolClient } from "pg";
import { afterEach, describe, expect, it } from "vitest";
import { injectHttp } from "../connections/http-test-transport.js";
import { vault } from "../vault/vault.js";
import { LiveMonitorModule } from "./live-monitor.module.js";
import { LIVE_MONITOR_DEPENDENCIES, LiveMonitorService } from "./live-monitor.service.js";
import { MEDIA_POLLER_DEPENDENCIES } from "./poller/poller.js";
import { MONITOR_WEBHOOK_DEPENDENCIES, MonitorWebhookService, parseWebhookEndpoints, verifyMonitorSignature,
WEBHOOK_MAX_BYTES, WEBHOOK_WINDOW_MS, type MonitorWebhookDependencies, type MonitorWebhookHeaders } from "./webhook.service.js";
const workspaceId = "11111111-1111-4111-8111-111111111111";
const connectionId = "22222222-2222-4222-8222-222222222222";
const credentialId = "33333333-3333-4333-8333-333333333333";
const sourceId = "44444444-4444-4444-8444-444444444444";
const endpointId = "ab".repeat(32);
const endpoint = { endpointId, workspaceId, connectionId, credentialId, subscriptionId: "subscription-1" };
const secret = Buffer.alloc(32, 42);
const epoch = Date.parse("2026-01-01T00:00:00.000Z");
const event = { sourceId, subscriptionId: endpoint.subscriptionId, outcome: "ok" };
function signed(raw: Buffer, eventId = "event-1", time = epoch): MonitorWebhookHeaders {
const timestamp = new Date(time).toISOString();
return { eventId, timestamp, signature: `sha256=${createHmac("sha256", secret).update(eventId).update(timestamp).update(raw).digest("hex")}` };
}
async function fixture() {
const row: Record<string, unknown> = {
id: sourceId, workspace_id: workspaceId, connection_id: connectionId, remote_ref: "live:1",
state: "OFFLINE", version: 0, authoritative_tier: null, last_media_health_at: null, last_hls_probe_at: null,
last_webhook_at: null, last_api_poll_at: null, tier_status: {}, ok_streak: 0, fail_streak: 0,
state_entered_at: new Date(epoch).toISOString(), hold_until: null, cooldown_until: null, previous_state: null,
last_transition_at: null, last_transition_reason: null, created_at: new Date(epoch).toISOString(), updated_at: new Date(epoch).toISOString(),
};
const sealed = await vault.seal(workspaceId, connectionId, secret);
const journal = new Set<string>();
let now = epoch;
let failUpdate = false;
let revoked = false;
let audits = 0;
const client = { async query(sql: string, params: unknown[] = []) {
if (sql.startsWith("SELECT cr.ciphertext")) {
expect(params).toEqual([credentialId, connectionId, workspaceId]);
return { rowCount: revoked ? 0 : 1, rows: revoked ? [] : [{ ciphertext: sealed.ciphertext, key_version: sealed.keyVersion }] };
}
if (sql.startsWith("SELECT id FROM source_monitor_states")) {
const owns = params[0] === row.id && params[1] === row.workspace_id && params[2] === row.connection_id;
return { rowCount: owns ? 1 : 0, rows: owns ? [{ id: sourceId }] : [] };
}
if (sql.startsWith("SELECT * FROM source_monitor_states")) return { rowCount: 1, rows: [{ ...row }] };
if (sql.startsWith("INSERT INTO monitor_seen_events")) {
const key = JSON.stringify([params[0], params[1]]);
if (journal.has(key)) return { rowCount: 0, rows: [] };
journal.add(key);
return { rowCount: 1, rows: [{}] };
}
if (sql.startsWith("UPDATE source_monitor_states")) {
if (failUpdate) throw new Error("database update failed");
const columns = ["state", "version", "authoritative_tier", "last_media_health_at", "last_hls_probe_at",
"last_webhook_at", "last_api_poll_at", "tier_status", "ok_streak", "fail_streak", "state_entered_at", "hold_until",
"cooldown_until", "previous_state", "last_transition_at", "last_transition_reason", "updated_at"];
columns.forEach((column, index) => { row[column] = params[index + 1]; });
return { rowCount: 1, rows: [{ ...row }] };
}
if (sql.startsWith("SELECT audit_append")) { audits++; return { rowCount: 1, rows: [{}] }; }
throw new Error(`unexpected query ${sql}`);
} } as unknown as PoolClient;
let pending = Promise.resolve();
const tenantTransaction: MonitorWebhookDependencies["tenantTransaction"] = async (workspace, fn) => {
expect(workspace).toBe(workspaceId);
const previous = pending;
let release!: () => void;
pending = new Promise<void>((resolve) => { release = resolve; });
await previous;
const snapshot = { row: structuredClone(row), journal: [...journal], audits };
try { return await fn(client); }
catch (error) {
Object.assign(row, snapshot.row); journal.clear(); snapshot.journal.forEach((key) => journal.add(key));
audits = snapshot.audits;
throw error;
} finally { release(); }
};
const dependencies: MonitorWebhookDependencies = {
endpoints: [endpoint, { ...endpoint, endpointId: "cd".repeat(32) }],
tenantTransaction, now: () => now, openSecret: vault.open.bind(vault),
};
const monitor = new LiveMonitorService({ tenantTransaction });
const service = () => new MonitorWebhookService(monitor, dependencies);
return { service, dependencies, monitor, row, journal, audits: () => audits,
clock: (time: number) => { now = time; }, fail: (value: boolean) => { failUpdate = value; }, revoke: () => { revoked = true; } };
}
describe("monitor webhook authentication and ingest", () => {
it("verifies exact raw bytes, event id and timestamp in constant-time HMAC", () => {
const raw = Buffer.from(JSON.stringify(event));
expect(() => verifyMonitorSignature(raw, signed(raw), secret, epoch)).not.toThrow();
expect(() => verifyMonitorSignature(raw, signed(raw), Buffer.alloc(32, 43), epoch)).toThrow(UnauthorizedException);
expect(() => verifyMonitorSignature(Buffer.concat([raw, Buffer.from(" ")]), signed(raw), secret, epoch)).toThrow(UnauthorizedException);
expect(() => verifyMonitorSignature(raw, { ...signed(raw), eventId: "event-2" }, secret, epoch)).toThrow(UnauthorizedException);
expect(() => verifyMonitorSignature(raw, { ...signed(raw), timestamp: new Date(epoch + 1000).toISOString() }, secret, epoch)).toThrow(UnauthorizedException);
expect(() => verifyMonitorSignature(raw, { ...signed(raw), signature: "sha256=bad" }, secret, epoch)).toThrow(UnauthorizedException);
});
it.each([-1, 1])("rejects signed timestamps outside the window in direction %s", (direction) => {
const raw = Buffer.from(JSON.stringify(event));
expect(() => verifyMonitorSignature(raw, signed(raw, "event-1", epoch + direction * (WEBHOOK_WINDOW_MS + 1)), secret, epoch)).toThrow(UnauthorizedException);
expect(() => verifyMonitorSignature(raw, signed(raw, "event-1", epoch + direction * WEBHOOK_WINDOW_MS), secret, epoch)).not.toThrow();
});
it("folds authenticated events through the existing service and reducer; replay survives service recreation", async () => {
const f = await fixture(); const raw = Buffer.from(JSON.stringify(event));
expect((await f.service().ingest(endpointId, raw, signed(raw))).state).toBe("PROBING");
await expect(f.service().ingest(endpointId, raw, signed(raw))).rejects.toBeInstanceOf(ConflictException);
expect(f.row.version).toBe(1); expect(f.audits()).toBe(1); expect(f.journal.size).toBe(1);
});
it("retains dedupe across the full early-to-late timestamp acceptance span", async () => {
const f = await fixture(); const raw = Buffer.from(JSON.stringify(event));
f.clock(epoch - WEBHOOK_WINDOW_MS);
await f.service().ingest(endpointId, raw, signed(raw));
f.clock(epoch + 4 * 60 * 1000);
await expect(f.service().ingest(endpointId, raw, signed(raw))).rejects.toBeInstanceOf(ConflictException);
expect(f.row.version).toBe(1);
});
it("scopes event ids per endpoint, including under concurrent delivery", async () => {
const f = await fixture(); const raw = Buffer.from(JSON.stringify(event));
const results = await Promise.allSettled([f.service().ingest(endpointId, raw, signed(raw)), f.service().ingest(endpointId, raw, signed(raw))]);
expect(results.filter((result) => result.status === "fulfilled")).toHaveLength(1);
await f.service().ingest("cd".repeat(32), raw, signed(raw));
expect(f.journal.size).toBe(2); expect(f.row.version).toBe(1); // same-tier stale timestamp is still folded only once
});
it.each(["workspace_id", "connection_id"])("rejects a source owned by a different %s before journaling", async (column) => {
const f = await fixture(); f.row[column] = "55555555-5555-4555-8555-555555555555";
const raw = Buffer.from(JSON.stringify(event));
await expect(f.service().ingest(endpointId, raw, signed(raw))).rejects.toBeInstanceOf(NotFoundException);
expect(f.journal.size).toBe(0); expect(f.row.version).toBe(0);
});
it.each([{ subscriptionId: "other" }, { workspaceId: "other" }, { connectionId: "other" }])("rejects provider binding mismatch %j", async (binding) => {
const f = await fixture(); const raw = Buffer.from(JSON.stringify({ ...event, ...binding }));
await expect(f.service().ingest(endpointId, raw, signed(raw))).rejects.toBeInstanceOf(UnauthorizedException);
expect(f.journal.size).toBe(0); expect(f.row.version).toBe(0);
});
it("rolls back the replay claim when state persistence fails", async () => {
const f = await fixture(); const raw = Buffer.from(JSON.stringify(event)); f.fail(true);
await expect(f.service().ingest(endpointId, raw, signed(raw))).rejects.toThrow("database update failed");
expect(f.journal.size).toBe(0); f.fail(false);
expect((await f.service().ingest(endpointId, raw, signed(raw))).version).toBe(1);
});
it("rejects unknown endpoints, revoked credentials, malformed JSON and bad authentication without writes", async () => {
const f = await fixture(); const raw = Buffer.from(JSON.stringify(event));
await expect(f.service().ingest("unknown", raw, signed(raw))).rejects.toBeInstanceOf(NotFoundException);
await expect(f.service().ingest(endpointId, raw, {})).rejects.toBeInstanceOf(UnauthorizedException);
const malformed = Buffer.from("{");
await expect(f.service().ingest(endpointId, malformed, signed(malformed))).rejects.toBeInstanceOf(BadRequestException);
f.revoke();
await expect(f.service().ingest(endpointId, raw, signed(raw))).rejects.toBeInstanceOf(UnauthorizedException);
expect(f.journal.size).toBe(0); expect(f.row.version).toBe(0);
});
it("requires valid and unique server-controlled endpoint mappings", () => {
expect(parseWebhookEndpoints(JSON.stringify([endpoint]))).toEqual([endpoint]);
expect(() => parseWebhookEndpoints(JSON.stringify([endpoint, endpoint]))).toThrow("duplicate");
expect(() => parseWebhookEndpoints(JSON.stringify([{ ...endpoint, endpointId: "guessable" }]))).toThrow("invalid");
});
});
describe("monitor webhook raw HTTP route", () => {
let app: INestApplication | undefined;
afterEach(async () => { await app?.close(); app = undefined; });
it("captures whitespace before Nest JSON parsing, needs no browser session, rejects replay/tampering/oversize", async () => {
const f = await fixture();
app = await NestFactory.create({ module: LiveMonitorModule, providers: [
{ provide: LIVE_MONITOR_DEPENDENCIES, useValue: { tenantTransaction: f.dependencies.tenantTransaction } },
{ provide: MONITOR_WEBHOOK_DEPENDENCIES, useValue: f.dependencies },
{ provide: MEDIA_POLLER_DEPENDENCIES, useValue: { enabled: false } },
] }, { logger: false, abortOnError: false });
await app.init();
const body = ` ${JSON.stringify(event, null, 2)}\n`;
const auth = signed(Buffer.from(body));
const headers = { "content-type": "application/json", "x-monitor-signature": auth.signature!,
"x-monitor-timestamp": auth.timestamp!, "x-monitor-event-id": auth.eventId! };
const post = (value: string) => injectHttp(app!.getHttpAdapter().getInstance(), `http://api.test/live-monitor/webhooks/${endpointId}`,
{ method: "POST", body: value, headers });
expect((await post(body)).status).toBe(200);
expect((await post(body)).status).toBe(409);
expect((await post(`${body} `)).status).toBe(401);
expect((await post(" ".repeat(WEBHOOK_MAX_BYTES + 1))).status).toBe(413);
expect(f.row.version).toBe(1);
});
});