Tactical Media Broadcast Hub — initial release
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
This commit is contained in:
commit
aa5e262f3a
880 changed files with 111042 additions and 0 deletions
64
apps/api/src/live-monitor/README.md
Normal file
64
apps/api/src/live-monitor/README.md
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
# Monitor signal sources
|
||||
|
||||
`POST /live-monitor/webhooks/:endpointId` authenticates providers independently
|
||||
of browser sessions. The module installs a raw parser before Nest's JSON parser;
|
||||
the request cap is 64 KiB and compressed bodies are refused.
|
||||
|
||||
Configure `MONITOR_WEBHOOK_ENDPOINTS` as a server-controlled JSON array:
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"endpointId": "<32 server-generated random bytes, encoded as 64 lowercase hex characters>",
|
||||
"workspaceId": "<workspace UUID>",
|
||||
"connectionId": "<connection UUID>",
|
||||
"credentialId": "<credential UUID>",
|
||||
"subscriptionId": "<provider subscription id>"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
The credential must belong to that workspace and connection, have kind
|
||||
`monitor-webhook`, and contain a secret of at least 32 bytes sealed with
|
||||
`vault.seal(workspaceId, connectionId, secret)`. Disconnection or credential
|
||||
revocation disables the endpoint. Mappings contain references, never secrets.
|
||||
|
||||
Send these headers:
|
||||
|
||||
- `X-Monitor-Event-Id`: a unique provider message id (up to 256 characters).
|
||||
- `X-Monitor-Timestamp`: an ISO UTC timestamp within five minutes of ingestion.
|
||||
- `X-Monitor-Signature`: `sha256=` followed by lowercase hex HMAC-SHA256 over
|
||||
`eventId + timestamp + rawBody`, using the vaulted secret.
|
||||
|
||||
The JSON body is `{ "subscriptionId": "...", "sourceId": "<UUID>",
|
||||
"outcome": "ok|fail|error|unavailable" }`. A supplied workspace or connection
|
||||
must also match the mapping. The source must belong to the mapped workspace
|
||||
and connection. Successful delivery returns 200, replay 409, authentication
|
||||
failure 401, unknown endpoint/source 404, invalid JSON 400, oversized body 413.
|
||||
|
||||
Replay ids are namespaced by endpoint in `monitor_seen_events`. Verification,
|
||||
ownership checks, replay claim, reducer fold, state write and audit share a
|
||||
tenant transaction. Journal pruning must retain at least the full ten-minute
|
||||
timestamp acceptance span plus a clock margin; this module does not prune it.
|
||||
|
||||
`MEDIA_POLLER_SOURCES` entries support `playbackUrl`. Provider `hls` probes that
|
||||
URL; provider `peertube` polls the video using the connection's `fields.instanceUrl`
|
||||
and also probes `playbackUrl` when supplied. Canonical `video:<id>@<host>` refs
|
||||
are supported. Existing mock configuration remains supported.
|
||||
|
||||
HLS probes fetch one master and one variant, validate all child URIs through
|
||||
WP7, and emit `hls-probe` observations. Decoder reports retain `media-health`
|
||||
precedence. VOD/endlist/empty/missing playlists fail; stale segments fail after
|
||||
three target durations. Without program dates, successive playlist progression
|
||||
provides recency and an initial playlist has the same bounded grace period.
|
||||
|
||||
Every outbound fetch uses the shared WP7 egress client and its database host
|
||||
bucket (`monitor-probe` or `monitor-api`), including redirect hops. Requests
|
||||
have a five-second total deadline, 256 KiB response cap, and 128-reference cap.
|
||||
Per-source drivers back off with exponential jitter after failures, honoring
|
||||
Retry-After seconds/dates and egress retry delays. Denied probe targets emit
|
||||
`unavailable` with a policy reason so the existing reducer can fall back.
|
||||
|
||||
Production requires the built `packages/egress` artifact and its host-budget
|
||||
migration/grants. Those remain owned by WP7; this module performs no DDL and
|
||||
implements no additional limiter.
|
||||
84
apps/api/src/live-monitor/live-monitor.controller.ts
Normal file
84
apps/api/src/live-monitor/live-monitor.controller.ts
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
import { BadRequestException, Body, Controller, Get, Inject, Param, Post, Query, Req, UseGuards } from "@nestjs/common";
|
||||
import type { Request } from "express";
|
||||
import { parseId, parseLimit, requireObject, requireSession, requireString } from "../http/validation.js";
|
||||
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import { LiveMonitorService, type MonitorPage } from "./live-monitor.service.js";
|
||||
import { LiveMonitorPoller } from "./poller/poller.js";
|
||||
import type { MonitorObservation, MonitorRowState, ObservationOutcome, SignalTier } from "./reducer.js";
|
||||
|
||||
@Controller("live-monitor")
|
||||
@UseGuards(RbacGuard)
|
||||
export class LiveMonitorController {
|
||||
constructor(
|
||||
@Inject(LiveMonitorService) private readonly monitor: LiveMonitorService,
|
||||
@Inject(LiveMonitorPoller) private readonly poller: LiveMonitorPoller,
|
||||
) {}
|
||||
|
||||
@Post("poller/:action")
|
||||
@RequirePermission("media.manage")
|
||||
async controlPoller(@Param("action") action: string): Promise<{ running: boolean }> {
|
||||
if (action === "start") return this.poller.start();
|
||||
if (action === "stop") return this.poller.stop();
|
||||
throw new BadRequestException("action must be start or stop");
|
||||
}
|
||||
|
||||
@Post(":sourceId/observations")
|
||||
@RequirePermission("media.manage")
|
||||
async record(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Param("sourceId") sourceId: string,
|
||||
@Body() body: unknown,
|
||||
): Promise<MonitorRowState> {
|
||||
const session = requireSession(request);
|
||||
return this.monitor.recordObservation(session.workspaceId, parseId(sourceId, "sourceId"),
|
||||
parseObservation(body), { userId: session.userId, role: session.role });
|
||||
}
|
||||
|
||||
@Get()
|
||||
@RequirePermission("media.view")
|
||||
async list(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Query("cursor") cursor: string | undefined,
|
||||
@Query("limit") limit: string | undefined,
|
||||
): Promise<MonitorPage> {
|
||||
return this.monitor.listStates(requireSession(request).workspaceId, cursor, parseLimit(limit, 50, 200));
|
||||
}
|
||||
|
||||
@Get(":id")
|
||||
@RequirePermission("media.view")
|
||||
async get(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Param("id") id: string,
|
||||
): Promise<MonitorRowState> {
|
||||
return this.monitor.getState(requireSession(request).workspaceId, parseId(id));
|
||||
}
|
||||
}
|
||||
|
||||
function parseObservation(value: unknown): MonitorObservation {
|
||||
const body = requireObject(value, "body");
|
||||
const tier = body.tier;
|
||||
if (tier !== "media-health" && tier !== "hls-probe" && tier !== "webhook" && tier !== "api-poll") {
|
||||
throw new BadRequestException("tier is invalid");
|
||||
}
|
||||
const observedAt = requireString(body.observedAt, "observedAt");
|
||||
if (!Number.isFinite(Date.parse(observedAt))) throw new BadRequestException("observedAt must be a timestamp");
|
||||
const payload = requireObject(body.payload, "payload");
|
||||
const outcome = payload.outcome;
|
||||
if (outcome !== "ok" && outcome !== "fail" && outcome !== "error" && outcome !== "unavailable") {
|
||||
throw new BadRequestException("outcome is invalid");
|
||||
}
|
||||
if (payload.mediaPresent !== undefined && typeof payload.mediaPresent !== "boolean") {
|
||||
throw new BadRequestException("mediaPresent must be boolean");
|
||||
}
|
||||
return {
|
||||
tier: tier as SignalTier,
|
||||
observedAt,
|
||||
receivedAt: new Date().toISOString(),
|
||||
...(body.providerEventId === undefined ? {} : { providerEventId: requireString(body.providerEventId, "providerEventId") }),
|
||||
payload: {
|
||||
outcome: outcome as ObservationOutcome,
|
||||
...(payload.reason === undefined ? {} : { reason: requireString(payload.reason, "reason") }),
|
||||
...(payload.mediaPresent === undefined ? {} : { mediaPresent: payload.mediaPresent as boolean }),
|
||||
},
|
||||
};
|
||||
}
|
||||
81
apps/api/src/live-monitor/live-monitor.integration.test.ts
Normal file
81
apps/api/src/live-monitor/live-monitor.integration.test.ts
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { join } from "node:path";
|
||||
import { Pool, type PoolClient } from "pg";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { runMigrations } from "../db/migrate.js";
|
||||
import { LiveMonitorService, type LiveMonitorDependencies } from "./live-monitor.service.js";
|
||||
|
||||
const dbAvailable = process.env.DATABASE_URL !== undefined;
|
||||
const ownerPool = new Pool({ connectionString: process.env.SUPER_DATABASE_URL ?? process.env.DATABASE_URL });
|
||||
const appPool = new Pool({ connectionString: process.env.DATABASE_URL });
|
||||
|
||||
async function asTenant<T>(workspaceId: string, fn: (client: PoolClient) => Promise<T>): Promise<T> {
|
||||
const client = await appPool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]);
|
||||
const result = await fn(client);
|
||||
await client.query("COMMIT");
|
||||
return result;
|
||||
} catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
describe.skipIf(!dbAvailable)("live monitor CAS (live DB)", () => {
|
||||
let workspaceId: string;
|
||||
let sourceId: string;
|
||||
let service: LiveMonitorService;
|
||||
|
||||
beforeAll(async () => {
|
||||
await runMigrations(process.env.MIGRATIONS_DIR ?? join(process.cwd(), "migrations"));
|
||||
const workspace = await ownerPool.query(
|
||||
"INSERT INTO workspaces (name, slug) VALUES ('Monitor integration', $1) RETURNING id",
|
||||
[`monitor-${randomUUID()}`],
|
||||
);
|
||||
workspaceId = String(workspace.rows[0]?.id);
|
||||
sourceId = await asTenant(workspaceId, async (client) => {
|
||||
const connection = await client.query(
|
||||
"INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-livestream', 'monitor') RETURNING id",
|
||||
[workspaceId],
|
||||
);
|
||||
const source = await client.query(
|
||||
"INSERT INTO source_monitor_states (workspace_id, connection_id, remote_ref) VALUES ($1, $2, 'live:cas') RETURNING id",
|
||||
[workspaceId, connection.rows[0]?.id],
|
||||
);
|
||||
return String(source.rows[0]?.id);
|
||||
});
|
||||
const dependencies: LiveMonitorDependencies = { tenantTransaction: asTenant };
|
||||
service = new LiveMonitorService(dependencies);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (workspaceId !== undefined) await ownerPool.query("DELETE FROM workspaces WHERE id = $1", [workspaceId]);
|
||||
await appPool.end();
|
||||
await ownerPool.end();
|
||||
});
|
||||
|
||||
it("serializes two recorders and keeps one transition per successful CAS", async () => {
|
||||
const observedAt = new Date().toISOString();
|
||||
const [first, second] = await Promise.all([
|
||||
service.recordObservation(workspaceId, sourceId, {
|
||||
tier: "api-poll", observedAt, receivedAt: observedAt, payload: { outcome: "ok" },
|
||||
}),
|
||||
service.recordObservation(workspaceId, sourceId, {
|
||||
tier: "hls-probe", observedAt, receivedAt: observedAt, payload: { outcome: "ok" },
|
||||
}),
|
||||
]);
|
||||
expect([first.version, second.version].sort()).toEqual([1, 2]);
|
||||
const current = await service.getState(workspaceId, sourceId);
|
||||
expect(current.version).toBe(2);
|
||||
expect(current.authoritativeTier).toBe("hls-probe");
|
||||
const audit = await asTenant(workspaceId, (client) => client.query(
|
||||
"SELECT count(*)::int AS count FROM audit_log WHERE action = 'monitor.state_changed' AND target_id = $1",
|
||||
[sourceId],
|
||||
));
|
||||
expect(audit.rows[0]?.count).toBe(1);
|
||||
});
|
||||
});
|
||||
19
apps/api/src/live-monitor/live-monitor.module.ts
Normal file
19
apps/api/src/live-monitor/live-monitor.module.ts
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import { Inject, Module } from "@nestjs/common";
|
||||
import { HttpAdapterHost } from "@nestjs/core";
|
||||
import type { ExpressAdapter } from "@nestjs/platform-express";
|
||||
import { LiveMonitorController } from "./live-monitor.controller.js";
|
||||
import { LiveMonitorService } from "./live-monitor.service.js";
|
||||
import { LiveMonitorPoller } from "./poller/poller.js";
|
||||
import { MonitorWebhookController } from "./webhook.controller.js";
|
||||
import { MonitorWebhookService, WEBHOOK_MAX_BYTES } from "./webhook.service.js";
|
||||
|
||||
@Module({ controllers: [LiveMonitorController, MonitorWebhookController], providers: [LiveMonitorService, LiveMonitorPoller, MonitorWebhookService], exports: [LiveMonitorService] })
|
||||
export class LiveMonitorModule {
|
||||
constructor(@Inject(HttpAdapterHost) adapter: HttpAdapterHost) {
|
||||
// Installed while Nest creates modules, before init registers its default JSON parser.
|
||||
(adapter.httpAdapter as ExpressAdapter).useBodyParser("raw", false, {
|
||||
type: (request: { url?: string }) => request.url?.startsWith("/live-monitor/webhooks/") === true,
|
||||
limit: WEBHOOK_MAX_BYTES, inflate: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
181
apps/api/src/live-monitor/live-monitor.service.ts
Normal file
181
apps/api/src/live-monitor/live-monitor.service.ts
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
import { BadRequestException, ConflictException, Inject, Injectable, NotFoundException, Optional } from "@nestjs/common";
|
||||
import type { PoolClient } from "pg";
|
||||
import { withTenant } from "../db/client.js";
|
||||
import { isUuid } from "../http/validation.js";
|
||||
import { applyObservation, type MonitorObservation, type MonitorRowState, type MonitorState,
|
||||
type SignalTier, type TierStatus, type MonitorPolicy, DEFAULT_MONITOR_POLICY } from "./reducer.js";
|
||||
|
||||
export interface MonitorActor { userId: string | null; role: string }
|
||||
export interface MonitorPage { items: MonitorRowState[]; nextCursor?: string }
|
||||
export interface LiveMonitorDependencies {
|
||||
tenantTransaction: typeof withTenant;
|
||||
policy?: Partial<MonitorPolicy>;
|
||||
}
|
||||
export const LIVE_MONITOR_DEPENDENCIES = Symbol("LIVE_MONITOR_DEPENDENCIES");
|
||||
|
||||
@Injectable()
|
||||
export class LiveMonitorService {
|
||||
private readonly dependencies: LiveMonitorDependencies;
|
||||
private readonly policy: MonitorPolicy;
|
||||
|
||||
constructor(@Optional() @Inject(LIVE_MONITOR_DEPENDENCIES) dependencies?: Partial<LiveMonitorDependencies>) {
|
||||
this.dependencies = { tenantTransaction: withTenant, ...dependencies };
|
||||
this.policy = { ...DEFAULT_MONITOR_POLICY, ...dependencies?.policy };
|
||||
}
|
||||
|
||||
async recordObservation(
|
||||
workspaceId: string,
|
||||
sourceId: string,
|
||||
observation: MonitorObservation,
|
||||
actor: MonitorActor = { userId: null, role: "monitor" },
|
||||
): Promise<MonitorRowState> {
|
||||
validateObservation(observation);
|
||||
return this.dependencies.tenantTransaction(workspaceId, (client) =>
|
||||
this.recordObservationInTransaction(client, workspaceId, sourceId, observation, actor));
|
||||
}
|
||||
|
||||
/** Webhook verification, durable replay claim and reduction share one transaction. */
|
||||
async recordObservationInTransaction(
|
||||
client: PoolClient, workspaceId: string, sourceId: string, observation: MonitorObservation,
|
||||
actor: MonitorActor = { userId: null, role: "monitor" }, rejectReplay = false,
|
||||
): Promise<MonitorRowState> {
|
||||
validateObservation(observation);
|
||||
let seenInserted = false;
|
||||
for (let attempt = 0; attempt < 5; attempt += 1) {
|
||||
const locked = await client.query("SELECT * FROM source_monitor_states WHERE id = $1 FOR UPDATE", [sourceId]);
|
||||
if (locked.rowCount !== 1) throw new NotFoundException(`live source ${sourceId} not found`);
|
||||
const current = fromRow(locked.rows[0]);
|
||||
if (observation.providerEventId !== undefined && !seenInserted) {
|
||||
const seen = await client.query(
|
||||
`INSERT INTO monitor_seen_events (workspace_id, provider_event_id, source_monitor_state_id)
|
||||
VALUES ($1, $2, $3) ON CONFLICT (workspace_id, provider_event_id) DO NOTHING RETURNING provider_event_id`,
|
||||
[workspaceId, observation.providerEventId, sourceId],
|
||||
);
|
||||
if (seen.rowCount === 0) {
|
||||
if (rejectReplay) throw new ConflictException("webhook event already received");
|
||||
return current;
|
||||
}
|
||||
seenInserted = true;
|
||||
}
|
||||
const reduction = applyObservation(current, observation, this.policy);
|
||||
if (reduction.state === current) return current;
|
||||
const next = reduction.state;
|
||||
const changed = await client.query(
|
||||
`UPDATE source_monitor_states SET state = $2, version = $3, authoritative_tier = $4,
|
||||
last_media_health_at = $5, last_hls_probe_at = $6, last_webhook_at = $7,
|
||||
last_api_poll_at = $8, tier_status = $9, ok_streak = $10, fail_streak = $11,
|
||||
state_entered_at = $12, hold_until = $13, cooldown_until = $14,
|
||||
previous_state = $15, last_transition_at = $16, last_transition_reason = $17,
|
||||
updated_at = $18
|
||||
WHERE id = $1 AND version = $19 RETURNING *`,
|
||||
[sourceId, next.state, next.version, next.authoritativeTier, next.lastMediaHealthAt,
|
||||
next.lastHlsProbeAt, next.lastWebhookAt, next.lastApiPollAt, next.tierStatus,
|
||||
next.okStreak, next.failStreak, next.stateEnteredAt, next.holdUntil, next.cooldownUntil,
|
||||
next.previousState, next.lastTransitionAt, next.lastTransitionReason, next.updatedAt, current.version],
|
||||
);
|
||||
if (changed.rowCount !== 1) continue;
|
||||
if (reduction.transition !== undefined) {
|
||||
await client.query(
|
||||
"SELECT audit_append($1, $2, $3, 'monitor.state_changed', 'source_monitor_state', $4, $5)",
|
||||
[workspaceId, actor.userId, actor.role, sourceId, {
|
||||
from: reduction.transition.from, to: reduction.transition.to,
|
||||
version: reduction.transition.version, reason: reduction.transition.reason,
|
||||
tier: next.authoritativeTier,
|
||||
}],
|
||||
);
|
||||
}
|
||||
return fromRow(changed.rows[0]);
|
||||
}
|
||||
throw new Error(`live source ${sourceId} CAS conflict did not settle`);
|
||||
}
|
||||
|
||||
async getState(workspaceId: string, sourceId: string): Promise<MonitorRowState> {
|
||||
return this.dependencies.tenantTransaction(workspaceId, async (client) => {
|
||||
const found = await client.query("SELECT * FROM source_monitor_states WHERE id = $1", [sourceId]);
|
||||
if (found.rowCount !== 1) throw new NotFoundException(`live source ${sourceId} not found`);
|
||||
return fromRow(found.rows[0]);
|
||||
});
|
||||
}
|
||||
|
||||
async getStates(workspaceId: string, sourceIds: string[]): Promise<MonitorRowState[]> {
|
||||
if (sourceIds.length === 0) return [];
|
||||
return this.dependencies.tenantTransaction(workspaceId, async (client) => {
|
||||
const found = await client.query("SELECT * FROM source_monitor_states WHERE id = ANY($1::uuid[])", [sourceIds]);
|
||||
return found.rows.map(fromRow);
|
||||
});
|
||||
}
|
||||
|
||||
async listStates(workspaceId: string, cursor?: string, limit = 50): Promise<MonitorPage> {
|
||||
if (!Number.isInteger(limit) || limit < 1 || limit > 200) {
|
||||
throw new BadRequestException("limit must be an integer between 1 and 200");
|
||||
}
|
||||
const decoded = cursor === undefined || cursor === "" ? undefined : decodeCursor(cursor);
|
||||
return this.dependencies.tenantTransaction(workspaceId, async (client) => {
|
||||
const result = await client.query(
|
||||
`SELECT * FROM source_monitor_states
|
||||
WHERE ($1::timestamptz IS NULL OR (created_at, id) < ($1::timestamptz, $2::uuid))
|
||||
ORDER BY created_at DESC, id DESC LIMIT $3`,
|
||||
[decoded?.createdAt ?? null, decoded?.id ?? null, limit + 1],
|
||||
);
|
||||
const rows = result.rows.map(fromRow);
|
||||
const items = rows.slice(0, limit);
|
||||
const last = items[items.length - 1];
|
||||
return { items, ...(rows.length > limit && last !== undefined
|
||||
? { nextCursor: Buffer.from(JSON.stringify({ createdAt: last.createdAt, id: last.id })).toString("base64url") }
|
||||
: {}) };
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function validateObservation(value: MonitorObservation): void {
|
||||
if (!(["media-health", "hls-probe", "webhook", "api-poll"] as string[]).includes(value.tier)) {
|
||||
throw new BadRequestException("invalid observation tier");
|
||||
}
|
||||
if (!Number.isFinite(Date.parse(value.observedAt)) || !Number.isFinite(Date.parse(value.receivedAt))) {
|
||||
throw new BadRequestException("observation timestamps must be valid");
|
||||
}
|
||||
if (!(["ok", "fail", "error", "unavailable"] as string[]).includes(value.payload.outcome)) {
|
||||
throw new BadRequestException("invalid observation outcome");
|
||||
}
|
||||
if (value.providerEventId !== undefined && value.providerEventId.trim() === "") {
|
||||
throw new BadRequestException("providerEventId must be non-empty");
|
||||
}
|
||||
}
|
||||
|
||||
function fromRow(raw: unknown): MonitorRowState {
|
||||
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) throw new Error("invalid monitor row");
|
||||
const row = raw as Record<string, unknown>;
|
||||
const version = Number(row.version);
|
||||
if (!Number.isSafeInteger(version)) throw new Error("monitor version exceeds safe integer range");
|
||||
return {
|
||||
id: String(row.id), workspaceId: String(row.workspace_id), connectionId: String(row.connection_id),
|
||||
remoteRef: String(row.remote_ref), state: row.state as MonitorState, version,
|
||||
authoritativeTier: row.authoritative_tier as SignalTier | null,
|
||||
lastMediaHealthAt: dateOrNull(row.last_media_health_at), lastHlsProbeAt: dateOrNull(row.last_hls_probe_at),
|
||||
lastWebhookAt: dateOrNull(row.last_webhook_at), lastApiPollAt: dateOrNull(row.last_api_poll_at),
|
||||
tierStatus: row.tier_status as Partial<Record<SignalTier, TierStatus>>,
|
||||
okStreak: Number(row.ok_streak), failStreak: Number(row.fail_streak),
|
||||
stateEnteredAt: dateOrNull(row.state_entered_at) ?? "",
|
||||
holdUntil: dateOrNull(row.hold_until), cooldownUntil: dateOrNull(row.cooldown_until),
|
||||
previousState: row.previous_state as MonitorState | null,
|
||||
lastTransitionAt: dateOrNull(row.last_transition_at), lastTransitionReason: row.last_transition_reason as string | null,
|
||||
createdAt: dateOrNull(row.created_at) ?? "", updatedAt: dateOrNull(row.updated_at) ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
function dateOrNull(value: unknown): string | null {
|
||||
return value === null || value === undefined ? null : new Date(String(value)).toISOString();
|
||||
}
|
||||
|
||||
function decodeCursor(value: string): { createdAt: string; id: string } {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(Buffer.from(value, "base64url").toString("utf8"));
|
||||
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error("invalid cursor");
|
||||
const row = parsed as Record<string, unknown>;
|
||||
if (typeof row.createdAt !== "string" || !Number.isFinite(Date.parse(row.createdAt))
|
||||
|| typeof row.id !== "string" || !isUuid(row.id)) throw new Error("invalid cursor");
|
||||
return { createdAt: row.createdAt, id: row.id };
|
||||
} catch {
|
||||
throw new BadRequestException("cursor is invalid");
|
||||
}
|
||||
}
|
||||
39
apps/api/src/live-monitor/poller/backoff-driver.ts
Normal file
39
apps/api/src/live-monitor/poller/backoff-driver.ts
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
import type { MonitorObservation, SignalTier } from "../reducer.js";
|
||||
import type { PollDriver } from "./driver.js";
|
||||
import { MonitorHttpError, retryAfterMs } from "./egress.js";
|
||||
|
||||
/** Scheduling state only: all host admission/debits remain in packages/egress. */
|
||||
export class BackoffPollDriver implements PollDriver {
|
||||
private failures = 0;
|
||||
private retryAt = 0;
|
||||
constructor(
|
||||
private readonly driver: PollDriver, private readonly tier: SignalTier,
|
||||
private readonly now: () => number = Date.now, private readonly random: () => number = Math.random,
|
||||
) {}
|
||||
get nextAttemptAt(): number { return this.retryAt; }
|
||||
|
||||
async poll(): Promise<MonitorObservation[]> {
|
||||
if (this.now() < this.retryAt) return [];
|
||||
try {
|
||||
const observations = await this.driver.poll();
|
||||
this.failures = 0;
|
||||
this.retryAt = 0;
|
||||
return observations;
|
||||
} catch (error) {
|
||||
const now = this.now();
|
||||
const metadata = error as { code?: string; retryAfterMs?: number } | null;
|
||||
const transient = error instanceof MonitorHttpError && (error.status === 429 || error.status >= 500)
|
||||
|| metadata?.code === "BUDGET_EXHAUSTED" || metadata?.code === "BUDGET_UNAVAILABLE"
|
||||
|| metadata?.code === "NETWORK_ERROR" || metadata?.code === "TIMEOUT" || metadata?.code === "ABORTED";
|
||||
this.failures += 1;
|
||||
const delay = Math.min(120_000, 1000 * 2 ** Math.min(7, this.failures - 1));
|
||||
const serverDelay = error instanceof MonitorHttpError ? retryAfterMs(error.headers, now) : metadata?.retryAfterMs ?? 0;
|
||||
this.retryAt = now + Math.max(serverDelay, delay * (0.8 + this.random() * 0.4));
|
||||
const policyDenied = ["DENIED_PRIVATE_ADDRESS", "SCHEME_DENIED", "PORT_DENIED", "ORIGIN_DENIED", "INVALID_URL", "DNS_FAILED"].includes(metadata?.code ?? "");
|
||||
const timestamp = new Date(now).toISOString();
|
||||
return [{ tier: this.tier, observedAt: timestamp, receivedAt: timestamp,
|
||||
payload: { outcome: transient || policyDenied ? "unavailable" : "error",
|
||||
reason: policyDenied ? `egress-policy:${metadata?.code}` : transient ? "monitor-backoff" : "monitor-probe-error" } }];
|
||||
}
|
||||
}
|
||||
}
|
||||
7
apps/api/src/live-monitor/poller/driver.ts
Normal file
7
apps/api/src/live-monitor/poller/driver.ts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
import type { MonitorObservation } from "../reducer.js";
|
||||
|
||||
/** A provider probe emits observations ordered by its own source clock. */
|
||||
export interface PollDriver {
|
||||
poll(): Promise<MonitorObservation[]>;
|
||||
}
|
||||
|
||||
96
apps/api/src/live-monitor/poller/egress.ts
Normal file
96
apps/api/src/live-monitor/poller/egress.ts
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
import { existsSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { pool } from "../../db/client.js";
|
||||
import type { createEgressClient, EgressPolicy, rewriteSegmentUris, validateMasterPlaylist } from "../../../../../packages/egress/src/index.js";
|
||||
|
||||
type EgressPackage = typeof import("../../../../../packages/egress/src/index.js");
|
||||
export interface MonitorEgress {
|
||||
egressFetch: ReturnType<typeof createEgressClient>["egressFetch"];
|
||||
policy: EgressPolicy;
|
||||
validateMasterPlaylist: typeof validateMasterPlaylist;
|
||||
rewriteSegmentUris: typeof rewriteSegmentUris;
|
||||
}
|
||||
export const MONITOR_TIMEOUT_MS = 5_000;
|
||||
export const MONITOR_MAX_BYTES = 256 * 1024;
|
||||
|
||||
const nativeImport = new Function("specifier", "return import(specifier)") as (specifier: string) => Promise<EgressPackage>;
|
||||
let shared: Promise<MonitorEgress> | undefined;
|
||||
|
||||
/** ESM package is loaded natively because the API emits CommonJS. No local limiter. */
|
||||
export function monitorEgress(): Promise<MonitorEgress> {
|
||||
shared ??= (async () => {
|
||||
for (const start of [process.cwd(), __dirname]) {
|
||||
for (let root = start; ; root = dirname(root)) {
|
||||
const directory = resolve(root, "packages/egress");
|
||||
const files = process.env.NODE_ENV === "production" ? ["dist/index.js"] : ["src/index.ts", "dist/index.js"];
|
||||
for (const file of files) {
|
||||
const path = resolve(directory, file);
|
||||
if (!existsSync(path)) continue;
|
||||
const pkg = await nativeImport(pathToFileURL(path).href);
|
||||
const client = pkg.createEgressClient({ pool, timeoutMs: MONITOR_TIMEOUT_MS, maxResponseBytes: MONITOR_MAX_BYTES,
|
||||
hostClasses: {
|
||||
"monitor-probe": { capacity: 3, refillPerInterval: 10 },
|
||||
"monitor-api": { capacity: 3, refillPerInterval: 10 },
|
||||
} });
|
||||
return { ...client, validateMasterPlaylist: pkg.validateMasterPlaylist, rewriteSegmentUris: pkg.rewriteSegmentUris };
|
||||
}
|
||||
if (dirname(root) === root) break;
|
||||
}
|
||||
}
|
||||
throw new Error("Build packages/egress before starting the live monitor");
|
||||
})();
|
||||
return shared;
|
||||
}
|
||||
|
||||
export class MonitorHttpError extends Error {
|
||||
constructor(readonly status: number, readonly headers: Headers) { super(`monitor HTTP ${status}`); }
|
||||
}
|
||||
|
||||
/** Retry-After remains a lower bound, including HTTP dates; never retry early. */
|
||||
export function retryAfterMs(headers: Headers, now: number): number {
|
||||
const raw = headers.get("retry-after");
|
||||
if (raw === null) return 0;
|
||||
if (/^\d+(?:\.\d+)?$/.test(raw.trim())) {
|
||||
const delay = Number(raw) * 1000;
|
||||
return Number.isFinite(delay) ? delay : 0;
|
||||
}
|
||||
const deadline = Date.parse(raw);
|
||||
return Number.isFinite(deadline) ? Math.max(0, deadline - now) : 0;
|
||||
}
|
||||
|
||||
/** Caps body reads even for injected clients, which need not buffer like WP7. */
|
||||
export async function boundedText(response: Response, signal: AbortSignal): Promise<string> {
|
||||
if (!response.body) return "";
|
||||
const reader = response.body.getReader();
|
||||
const chunks: Buffer[] = [];
|
||||
let size = 0;
|
||||
const cancel = () => { void reader.cancel().catch(() => undefined); };
|
||||
signal.addEventListener("abort", cancel, { once: true });
|
||||
try {
|
||||
for (;;) {
|
||||
if (signal.aborted) throw new Error("monitor deadline exceeded");
|
||||
const part = await reader.read();
|
||||
if (part.done) break;
|
||||
size += part.value.byteLength;
|
||||
if (size > MONITOR_MAX_BYTES) throw new Error("monitor response exceeds size cap");
|
||||
chunks.push(Buffer.from(part.value));
|
||||
}
|
||||
if (signal.aborted) throw new Error("monitor deadline exceeded");
|
||||
return Buffer.concat(chunks).toString("utf8");
|
||||
} finally {
|
||||
signal.removeEventListener("abort", cancel);
|
||||
void reader.cancel().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
/** Covers DNS/URI validation as well as fetch and body consumption. */
|
||||
export async function monitorDeadline<T>(run: (signal: AbortSignal) => Promise<T>): Promise<T> {
|
||||
const controller = new AbortController();
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const timeout = new Promise<never>((_resolve, reject) => {
|
||||
timer = setTimeout(() => { controller.abort(); reject(new Error("monitor deadline exceeded")); }, MONITOR_TIMEOUT_MS);
|
||||
});
|
||||
try { return await Promise.race([run(controller.signal), timeout]); }
|
||||
finally { clearTimeout(timer); controller.abort(); }
|
||||
}
|
||||
8
apps/api/src/live-monitor/poller/fixtures/live.m3u8
Normal file
8
apps/api/src/live-monitor/poller/fixtures/live.m3u8
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
#EXTM3U
|
||||
#EXT-X-TARGETDURATION:6
|
||||
#EXT-X-MEDIA-SEQUENCE:20
|
||||
#EXT-X-PROGRAM-DATE-TIME:2026-01-01T00:00:00Z
|
||||
#EXTINF:6,
|
||||
segment20.ts
|
||||
#EXTINF:6,
|
||||
segment21.ts
|
||||
4
apps/api/src/live-monitor/poller/fixtures/malformed.m3u8
Normal file
4
apps/api/src/live-monitor/poller/fixtures/malformed.m3u8
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
#EXTM3U
|
||||
#EXT-X-TARGETDURATION:garbage
|
||||
#EXTINF:6,
|
||||
segment20.ts
|
||||
5
apps/api/src/live-monitor/poller/fixtures/master.m3u8
Normal file
5
apps/api/src/live-monitor/poller/fixtures/master.m3u8
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
#EXTM3U
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=800000
|
||||
live.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=2000000
|
||||
unused.m3u8
|
||||
6
apps/api/src/live-monitor/poller/fixtures/offline.m3u8
Normal file
6
apps/api/src/live-monitor/poller/fixtures/offline.m3u8
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
#EXTM3U
|
||||
#EXT-X-PLAYLIST-TYPE:VOD
|
||||
#EXT-X-TARGETDURATION:6
|
||||
#EXTINF:6,
|
||||
segment20.ts
|
||||
#EXT-X-ENDLIST
|
||||
127
apps/api/src/live-monitor/poller/hls-driver.ts
Normal file
127
apps/api/src/live-monitor/poller/hls-driver.ts
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
import type { MonitorObservation } from "../reducer.js";
|
||||
import type { PollDriver } from "./driver.js";
|
||||
import { boundedText, monitorDeadline, MonitorHttpError, type MonitorEgress } from "./egress.js";
|
||||
|
||||
export interface HlsMediaPlaylist {
|
||||
targetDuration: number;
|
||||
sequence: number;
|
||||
segments: Array<{ uri: string; duration: number; endAt?: number }>;
|
||||
ended: boolean;
|
||||
vod: boolean;
|
||||
}
|
||||
|
||||
export function parseMediaPlaylist(text: string): HlsMediaPlaylist {
|
||||
const lines = text.replace(/^\uFEFF/, "").split(/\r?\n/).map((line) => line.trim());
|
||||
if (lines[0] !== "#EXTM3U" || text.includes("\0")) throw new Error("malformed HLS header");
|
||||
let targetDuration = 0;
|
||||
let sequence = 0;
|
||||
let duration: number | undefined;
|
||||
let programTime: number | undefined;
|
||||
let ended = false;
|
||||
let vod = false;
|
||||
const segments: HlsMediaPlaylist["segments"] = [];
|
||||
for (const line of lines.slice(1)) {
|
||||
if (line.startsWith("#EXT-X-TARGETDURATION:")) {
|
||||
const raw = line.slice(22);
|
||||
if (targetDuration || !/^\d+$/.test(raw)) throw new Error("invalid HLS target duration");
|
||||
targetDuration = Number(raw);
|
||||
} else if (line.startsWith("#EXT-X-MEDIA-SEQUENCE:")) {
|
||||
const raw = line.slice(22);
|
||||
if (!/^\d+$/.test(raw) || !Number.isSafeInteger(Number(raw))) throw new Error("invalid HLS sequence");
|
||||
sequence = Number(raw);
|
||||
} else if (line.startsWith("#EXT-X-PLAYLIST-TYPE:")) {
|
||||
const type = line.slice(21);
|
||||
if (type !== "EVENT" && type !== "VOD") throw new Error("invalid HLS playlist type");
|
||||
vod = type === "VOD";
|
||||
} else if (line.startsWith("#EXT-X-PROGRAM-DATE-TIME:")) {
|
||||
programTime = Date.parse(line.slice(25));
|
||||
if (!Number.isFinite(programTime)) throw new Error("invalid HLS program time");
|
||||
} else if (line.startsWith("#EXTINF:")) {
|
||||
if (duration !== undefined) throw new Error("HLS segment URI missing");
|
||||
const raw = line.slice(8).split(",")[0] ?? "";
|
||||
if (!/^\d+(?:\.\d+)?$/.test(raw)) throw new Error("invalid HLS duration");
|
||||
duration = Number(raw);
|
||||
if (!Number.isFinite(duration) || duration <= 0) throw new Error("invalid HLS duration");
|
||||
} else if (line === "#EXT-X-ENDLIST") ended = true;
|
||||
else if (line && !line.startsWith("#")) {
|
||||
if (duration === undefined) throw new Error("HLS segment duration missing");
|
||||
if (programTime !== undefined) programTime += duration * 1000;
|
||||
segments.push({ uri: line, duration, ...(programTime === undefined ? {} : { endAt: programTime }) });
|
||||
duration = undefined;
|
||||
}
|
||||
}
|
||||
if (duration !== undefined || !Number.isSafeInteger(targetDuration) || targetDuration <= 0 || targetDuration > 3600
|
||||
|| segments.some((segment) => Math.round(segment.duration) > targetDuration) || segments.length > 128) {
|
||||
throw new Error("invalid HLS media playlist");
|
||||
}
|
||||
return { targetDuration, sequence, segments, ended, vod };
|
||||
}
|
||||
|
||||
/** Playlist ground truth uses hls-probe, below decoder media-health and above provider signals. */
|
||||
export class HlsProbeDriver implements PollDriver {
|
||||
private previous: { fingerprint: string; advancedAt: number; playlistUrl: string } | undefined;
|
||||
constructor(private readonly playbackUrl: string, private readonly egress: MonitorEgress,
|
||||
private readonly now: () => number = Date.now) {}
|
||||
|
||||
async poll(): Promise<MonitorObservation[]> {
|
||||
return monitorDeadline(async (signal) => {
|
||||
let url = this.playbackUrl;
|
||||
let response = await this.fetch(url, signal);
|
||||
if (response.status === 404 || response.status === 410) return [this.observation(false, "hls-missing")];
|
||||
let text = await boundedText(response, signal);
|
||||
url = response.url || url;
|
||||
// Bound URI/DNS work before the WP7 helper validates every reference (including keys/maps).
|
||||
this.capReferences(text);
|
||||
if (/^#EXT-X-STREAM-INF:/m.test(text)) {
|
||||
await this.egress.validateMasterPlaylist(text, url, this.egress.policy);
|
||||
const lines = text.split(/\r?\n/).map((line) => line.trim());
|
||||
const variantTag = lines.findIndex((line) => line.startsWith("#EXT-X-STREAM-INF:"));
|
||||
const variant = lines.slice(variantTag + 1).find((line) => line && !line.startsWith("#"));
|
||||
if (!variant) throw new Error("HLS master has no variant");
|
||||
if (signal.aborted) throw new Error("monitor deadline exceeded");
|
||||
url = new URL(variant, url).href;
|
||||
response = await this.fetch(url, signal);
|
||||
if (response.status === 404 || response.status === 410) return [this.observation(false, "hls-variant-missing")];
|
||||
text = await boundedText(response, signal);
|
||||
url = response.url || url;
|
||||
this.capReferences(text);
|
||||
}
|
||||
const validated = await this.egress.rewriteSegmentUris(text, url, this.egress.policy);
|
||||
if (signal.aborted) throw new Error("monitor deadline exceeded");
|
||||
const playlist = parseMediaPlaylist(validated);
|
||||
const now = this.now();
|
||||
const latest = playlist.segments.at(-1);
|
||||
if (playlist.ended || playlist.vod || !latest) return [this.observation(false, "hls-ended-or-empty")];
|
||||
const fingerprint = JSON.stringify([playlist.sequence, playlist.segments.length, latest.uri]);
|
||||
const old = this.previous;
|
||||
if (!old || old.playlistUrl !== url || old.fingerprint !== fingerprint) {
|
||||
this.previous = { fingerprint, advancedAt: now, playlistUrl: url };
|
||||
}
|
||||
// Three target durations allow one missed reload; unchanged windows eventually fail even without PDT.
|
||||
const graceMs = playlist.targetDuration * 3 * 1000;
|
||||
const fresh = latest.endAt === undefined
|
||||
? now - this.previous!.advancedAt <= graceMs
|
||||
: latest.endAt >= now - graceMs && latest.endAt <= now + graceMs;
|
||||
return [this.observation(fresh, fresh ? "hls-recent-segments" : "hls-stale-segments")];
|
||||
});
|
||||
}
|
||||
|
||||
private async fetch(url: string, signal: AbortSignal): Promise<Response> {
|
||||
if (signal.aborted) throw new Error("monitor deadline exceeded");
|
||||
const response = await this.egress.egressFetch(url, { signal, headers: { accept: "application/vnd.apple.mpegurl" } },
|
||||
{ hostClass: "monitor-probe" });
|
||||
if (!response.ok && response.status !== 404 && response.status !== 410) throw new MonitorHttpError(response.status, response.headers);
|
||||
return response;
|
||||
}
|
||||
|
||||
private observation(live: boolean, reason: string): MonitorObservation {
|
||||
const timestamp = new Date(this.now()).toISOString();
|
||||
return { tier: "hls-probe", observedAt: timestamp, receivedAt: timestamp,
|
||||
payload: { outcome: live ? "ok" : "fail", mediaPresent: live, reason } };
|
||||
}
|
||||
|
||||
private capReferences(text: string): void {
|
||||
if (text.split(/\r?\n/).filter((line) => line.trim() && !line.trim().startsWith("#")).length
|
||||
+ (text.match(/(?:SERVER-)?URI=/g)?.length ?? 0) > 128) throw new Error("too many HLS references");
|
||||
}
|
||||
}
|
||||
136
apps/api/src/live-monitor/poller/mock-driver.ts
Normal file
136
apps/api/src/live-monitor/poller/mock-driver.ts
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
import { existsSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import type { MonitorObservation } from "../reducer.js";
|
||||
import type { PollDriver } from "./driver.js";
|
||||
|
||||
interface ScenarioStep {
|
||||
source: "A" | "B" | "C";
|
||||
action: "go-live" | "fail" | "recover" | "go-offline";
|
||||
holdSeconds: number;
|
||||
}
|
||||
interface Scenario { steps: ScenarioStep[] }
|
||||
type MockLiveState = "created" | "waiting" | "live" | "ended";
|
||||
interface MockStream { remoteId: string; state: MockLiveState }
|
||||
interface MockConnection {
|
||||
id: string;
|
||||
workspaceId: string;
|
||||
provider: string;
|
||||
channelId: string;
|
||||
credentialId: { kind: "credential-ref"; id: string };
|
||||
fields: Record<string, string>;
|
||||
createdAt: Date;
|
||||
}
|
||||
interface MockConnector {
|
||||
manifest(): { id: string };
|
||||
createLiveStream?(connection: MockConnection, request: {
|
||||
channelId: string; title: string; permanent: boolean; saveReplay: boolean;
|
||||
}): Promise<MockStream>;
|
||||
getLiveStreams?(connection: MockConnection, request: { limit: number }): Promise<{ items: MockStream[] }>;
|
||||
}
|
||||
interface MockModule {
|
||||
createMockLivestreamState(): unknown;
|
||||
createMockLivestreamConnector(state: unknown): MockConnector;
|
||||
transitionMockLive(state: unknown, remoteId: string, to: MockLiveState): void;
|
||||
}
|
||||
interface ScenarioModule { SCENARIOS: Record<string, Scenario> }
|
||||
export interface MockDriverPackages { scenarios: ScenarioModule; mock: MockModule }
|
||||
|
||||
export interface MockDriverOptions {
|
||||
scenario: string;
|
||||
source: "A" | "B" | "C";
|
||||
sourceRef: string;
|
||||
workspaceId: string;
|
||||
connectionId: string;
|
||||
tickMs?: number;
|
||||
epoch?: string;
|
||||
ingestNow?: () => string;
|
||||
}
|
||||
|
||||
/** Native import keeps the ESM simulator and mock connector usable from the CJS API build. */
|
||||
const nativeImport = new Function("specifier", "return import(specifier)") as (specifier: string) => Promise<unknown>;
|
||||
|
||||
async function loadPackage<T>(packageName: string, file: string): Promise<T> {
|
||||
const roots = [process.cwd(), resolve(process.cwd(), "../..")];
|
||||
for (const root of roots) {
|
||||
const built = resolve(root, "packages", packageName, "dist", file);
|
||||
const source = resolve(root, "packages", packageName, "src", file.replace(/\.js$/, ".ts"));
|
||||
const candidates = process.env.NODE_ENV === "production" ? [built, source] : [source, built];
|
||||
for (const candidate of candidates) {
|
||||
if (existsSync(candidate)) return await nativeImport(pathToFileURL(candidate).href) as T;
|
||||
}
|
||||
}
|
||||
throw new Error(`missing ${packageName}/${file}; build the package before starting the poller`);
|
||||
}
|
||||
|
||||
export class MockPollDriver implements PollDriver {
|
||||
private elapsedMs = 0;
|
||||
private nextStep = 0;
|
||||
private nextStepAt = 0;
|
||||
private tick = 0;
|
||||
private action: ScenarioStep["action"] | null = null;
|
||||
|
||||
private constructor(
|
||||
private readonly options: MockDriverOptions,
|
||||
private readonly scenario: Scenario,
|
||||
private readonly connector: MockConnector,
|
||||
private readonly state: unknown,
|
||||
private readonly transition: MockModule["transitionMockLive"],
|
||||
private readonly connection: MockConnection,
|
||||
private readonly stream: MockStream,
|
||||
) {}
|
||||
|
||||
static async create(options: MockDriverOptions, packages?: MockDriverPackages): Promise<MockPollDriver> {
|
||||
const normalized = { ...options, epoch: options.epoch ?? new Date().toISOString() };
|
||||
if (!Number.isFinite(Date.parse(normalized.epoch))) throw new Error("mock epoch must be a timestamp");
|
||||
const [{ SCENARIOS }, mock] = packages === undefined ? await Promise.all([
|
||||
loadPackage<ScenarioModule>("testing", "scenarios.js"),
|
||||
loadPackage<MockModule>("connector-mock-livestream", "index.js"),
|
||||
]) : [packages.scenarios, packages.mock];
|
||||
const scenario = SCENARIOS[normalized.scenario];
|
||||
if (scenario === undefined) throw new Error(`unknown mock scenario ${normalized.scenario}`);
|
||||
const state = mock.createMockLivestreamState();
|
||||
const connector = mock.createMockLivestreamConnector(state);
|
||||
const connection: MockConnection = {
|
||||
id: normalized.connectionId, workspaceId: normalized.workspaceId, provider: connector.manifest().id,
|
||||
channelId: normalized.sourceRef, credentialId: { kind: "credential-ref", id: "mock" },
|
||||
fields: {}, createdAt: new Date(normalized.epoch),
|
||||
};
|
||||
if (connector.createLiveStream === undefined) throw new Error("mock connector has no live stream factory");
|
||||
const stream = await connector.createLiveStream(connection, {
|
||||
channelId: normalized.sourceRef, title: normalized.sourceRef, permanent: false, saveReplay: false,
|
||||
});
|
||||
return new MockPollDriver(normalized, scenario, connector, state, mock.transitionMockLive, connection, stream);
|
||||
}
|
||||
|
||||
async poll(): Promise<MonitorObservation[]> {
|
||||
const tickMs = this.options.tickMs ?? 10_000;
|
||||
if (!Number.isFinite(tickMs) || tickMs <= 0) throw new Error("mock tickMs must be positive");
|
||||
while (this.nextStep < this.scenario.steps.length && this.elapsedMs >= this.nextStepAt) {
|
||||
const step = this.scenario.steps[this.nextStep];
|
||||
if (step === undefined) break;
|
||||
if (step.source === this.options.source) {
|
||||
this.action = step.action;
|
||||
const state: MockLiveState = step.action === "go-live" || step.action === "recover"
|
||||
? "live" : step.action === "fail" ? "waiting" : "ended";
|
||||
this.transition(this.state, this.stream.remoteId, state);
|
||||
}
|
||||
this.nextStepAt += step.holdSeconds * 1000;
|
||||
this.nextStep += 1;
|
||||
}
|
||||
if (this.connector.getLiveStreams === undefined) throw new Error("mock connector has no live stream query");
|
||||
const page = await this.connector.getLiveStreams(this.connection, { limit: 100 });
|
||||
const live = page.items.find((item) => item.remoteId === this.stream.remoteId);
|
||||
if (live === undefined) throw new Error(`mock live ${this.stream.remoteId} disappeared`);
|
||||
const observedAt = new Date(Date.parse(this.options.epoch ?? "") + this.elapsedMs).toISOString();
|
||||
const outcome = this.action === "fail" ? "error" : live.state === "live" ? "ok" : "fail";
|
||||
const observation: MonitorObservation = {
|
||||
tier: "api-poll", observedAt, receivedAt: this.options.ingestNow?.() ?? new Date().toISOString(),
|
||||
providerEventId: `mock:${this.options.workspaceId}:${this.options.connectionId}:${this.options.sourceRef}:${this.options.epoch}:${this.tick}`,
|
||||
payload: { outcome, reason: this.action ?? "not-live", mediaPresent: live.state === "live" },
|
||||
};
|
||||
this.tick += 1;
|
||||
this.elapsedMs += tickMs;
|
||||
return [observation];
|
||||
}
|
||||
}
|
||||
44
apps/api/src/live-monitor/poller/peertube-driver.ts
Normal file
44
apps/api/src/live-monitor/poller/peertube-driver.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
import type { MonitorObservation } from "../reducer.js";
|
||||
import type { PollDriver } from "./driver.js";
|
||||
import { boundedText, monitorDeadline, MonitorHttpError, type MonitorEgress } from "./egress.js";
|
||||
|
||||
/** Injectable provider surface; production uses the WP7-backed client below. */
|
||||
export interface PeerTubeProbeClient {
|
||||
probe(videoId: string): Promise<MonitorObservation[]>;
|
||||
}
|
||||
|
||||
export class EgressPeerTubeProbeClient implements PeerTubeProbeClient {
|
||||
constructor(private readonly instanceUrl: string, private readonly egress: MonitorEgress,
|
||||
private readonly now: () => number = Date.now) {}
|
||||
|
||||
async probe(videoId: string): Promise<MonitorObservation[]> {
|
||||
return monitorDeadline(async (signal) => {
|
||||
const url = new URL(`/api/v1/videos/${encodeURIComponent(videoId)}`, this.instanceUrl);
|
||||
const response = await this.egress.egressFetch(url, { signal, headers: { accept: "application/json" } },
|
||||
{ hostClass: "monitor-api", pinnedOrigin: true });
|
||||
const timestamp = new Date(this.now()).toISOString();
|
||||
if (response.status === 404 || response.status === 410) return [{ tier: "api-poll", observedAt: timestamp,
|
||||
receivedAt: timestamp, payload: { outcome: "fail", reason: "api-video-missing" } }];
|
||||
if (!response.ok) throw new MonitorHttpError(response.status, response.headers);
|
||||
const parsed: unknown = JSON.parse(await boundedText(response, signal));
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("invalid PeerTube response");
|
||||
const video = parsed as { isLive?: unknown; state?: unknown };
|
||||
const state = typeof video.state === "object" && video.state !== null
|
||||
? (video.state as { id?: unknown }).id : video.state;
|
||||
if (typeof video.isLive !== "boolean" || !Number.isInteger(state)) throw new Error("invalid PeerTube live state");
|
||||
// A running live remains PUBLISHED (1); WAITING_FOR_LIVE (4)/LIVE_ENDED (5) are offline.
|
||||
const live = video.isLive && state === 1;
|
||||
return [{ tier: "api-poll", observedAt: timestamp, receivedAt: timestamp,
|
||||
payload: { outcome: live ? "ok" : "fail", reason: live ? "api-published-live" : "api-not-live" } }];
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export class PeerTubePollDriver implements PollDriver {
|
||||
constructor(private readonly videoId: string, private readonly egress?: PeerTubeProbeClient) {}
|
||||
|
||||
async poll(): Promise<MonitorObservation[]> {
|
||||
if (this.egress === undefined) throw new Error("NOT_IMPLEMENTED: PeerTube poll requires WP7 egress client");
|
||||
return await this.egress.probe(this.videoId);
|
||||
}
|
||||
}
|
||||
166
apps/api/src/live-monitor/poller/poller.test.ts
Normal file
166
apps/api/src/live-monitor/poller/poller.test.ts
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
import type { PoolClient } from "pg";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { LiveMonitorService, type LiveMonitorDependencies } from "../live-monitor.service.js";
|
||||
import type { MonitorObservation, MonitorRowState } from "../reducer.js";
|
||||
import { MockPollDriver, type MockDriverPackages } from "./mock-driver.js";
|
||||
import { PeerTubePollDriver } from "./peertube-driver.js";
|
||||
import { jitteredInterval, LiveMonitorPoller, parsePollSources, type PollerDependencies } from "./poller.js";
|
||||
|
||||
const epoch = "2026-01-01T00:00:00.000Z";
|
||||
|
||||
async function mockPackages(): Promise<MockDriverPackages> {
|
||||
const [mock, scenarios] = await Promise.all([
|
||||
vi.importActual<MockDriverPackages["mock"]>("../../../../../packages/connector-mock-livestream/src/index.js"),
|
||||
vi.importActual<MockDriverPackages["scenarios"]>("../../../../../packages/testing/src/scenarios.js"),
|
||||
]);
|
||||
return { mock, scenarios };
|
||||
}
|
||||
|
||||
function fixture(): { service: LiveMonitorService; get: () => Promise<MonitorRowState>; audits: () => number } {
|
||||
const row: Record<string, unknown> = {
|
||||
id: "source", workspace_id: "workspace", connection_id: "connection", remote_ref: "live:1",
|
||||
state: "OFFLINE", version: 0, authoritative_tier: null,
|
||||
last_media_health_at: null, last_hls_probe_at: null, last_webhook_at: null, last_api_poll_at: null,
|
||||
tier_status: {}, ok_streak: 0, fail_streak: 0, state_entered_at: epoch,
|
||||
hold_until: null, cooldown_until: null, previous_state: null, last_transition_at: null,
|
||||
last_transition_reason: null, created_at: epoch, updated_at: epoch,
|
||||
};
|
||||
const seen = new Set<string>();
|
||||
let auditCount = 0;
|
||||
const client = {
|
||||
async query(sql: string, params: unknown[] = []) {
|
||||
if (sql.startsWith("SELECT * FROM source_monitor_states")) return { rowCount: 1, rows: [{ ...row }] };
|
||||
if (sql.startsWith("INSERT INTO monitor_seen_events")) {
|
||||
const id = String(params[1]);
|
||||
if (seen.has(id)) return { rowCount: 0, rows: [] };
|
||||
seen.add(id);
|
||||
return { rowCount: 1, rows: [{ provider_event_id: id }] };
|
||||
}
|
||||
if (sql.startsWith("UPDATE source_monitor_states")) {
|
||||
if (Number(params[18]) !== Number(row.version)) return { rowCount: 0, rows: [] };
|
||||
const columns = ["state", "version", "authoritative_tier", "last_media_health_at", "last_hls_probe_at",
|
||||
"last_webhook_at", "last_api_poll_at", "tier_status", "ok_streak", "fail_streak", "state_entered_at",
|
||||
"hold_until", "cooldown_until", "previous_state", "last_transition_at", "last_transition_reason", "updated_at"];
|
||||
for (const [index, column] of columns.entries()) row[column] = params[index + 1];
|
||||
return { rowCount: 1, rows: [{ ...row }] };
|
||||
}
|
||||
if (sql.startsWith("SELECT audit_append")) {
|
||||
auditCount += 1;
|
||||
return { rowCount: 1, rows: [{}] };
|
||||
}
|
||||
throw new Error(`unexpected query: ${sql}`);
|
||||
},
|
||||
} as unknown as PoolClient;
|
||||
const dependencies: LiveMonitorDependencies = { tenantTransaction: async (_workspace, fn) => fn(client) };
|
||||
const service = new LiveMonitorService(dependencies);
|
||||
return { service, get: () => service.getState("workspace", "source"), audits: () => auditCount };
|
||||
}
|
||||
|
||||
describe("live monitor poller", () => {
|
||||
it("drives dod-flap through the service and reducer without rapid terminal cuts", async () => {
|
||||
const { service, get, audits } = fixture();
|
||||
const observations: MonitorObservation[] = [];
|
||||
const driver = await MockPollDriver.create({
|
||||
scenario: "dod-flap", source: "A", sourceRef: "live:1", workspaceId: "workspace",
|
||||
connectionId: "connection", epoch,
|
||||
ingestNow: () => new Date(Date.parse(epoch) + observations.length * 10_000).toISOString(),
|
||||
}, await mockPackages());
|
||||
const states: string[] = [];
|
||||
for (let tick = 0; tick < 10; tick += 1) {
|
||||
const [observation] = await driver.poll();
|
||||
if (observation === undefined) throw new Error("mock poll emitted no observation");
|
||||
observations.push(observation);
|
||||
const state = await service.recordObservation("workspace", "source", observation);
|
||||
expect(state.version).toBe(tick + 1);
|
||||
states.push(state.state);
|
||||
}
|
||||
expect(states).toEqual(["PROBING", "LIVE", "LIVE", "LIVE", "ERROR", "ERROR", "ERROR", "PROBING", "LIVE", "LIVE"]);
|
||||
expect(observations.map((item) => item.observedAt)).toEqual(
|
||||
Array.from({ length: 10 }, (_, tick) => new Date(Date.parse(epoch) + tick * 10_000).toISOString()),
|
||||
);
|
||||
expect(observations.every((item) => item.tier === "api-poll")).toBe(true);
|
||||
expect((await get()).version).toBe(10);
|
||||
expect(audits()).toBe(5);
|
||||
const replay = observations[8];
|
||||
if (replay === undefined) throw new Error("missing replay event");
|
||||
expect((await service.recordObservation("workspace", "source", replay)).version).toBe(10);
|
||||
expect(audits()).toBe(5);
|
||||
});
|
||||
|
||||
it("plays the single-live scenario and keeps a replayed event from confirming LIVE", async () => {
|
||||
const { service } = fixture();
|
||||
const driver = await MockPollDriver.create({
|
||||
scenario: "single-live", source: "A", sourceRef: "live:1", workspaceId: "workspace",
|
||||
connectionId: "connection", epoch,
|
||||
}, await mockPackages());
|
||||
const [first] = await driver.poll();
|
||||
if (first === undefined) throw new Error("missing first observation");
|
||||
expect((await service.recordObservation("workspace", "source", first)).state).toBe("PROBING");
|
||||
expect((await service.recordObservation("workspace", "source", first)).version).toBe(1);
|
||||
const [second] = await driver.poll();
|
||||
if (second === undefined) throw new Error("missing second observation");
|
||||
const live = await service.recordObservation("workspace", "source", second);
|
||||
expect(live.state).toBe("LIVE");
|
||||
expect(live.version).toBe(2);
|
||||
});
|
||||
|
||||
it("applies the same provider event only once across two poll ticks", async () => {
|
||||
const { service, get } = fixture();
|
||||
const observation: MonitorObservation = {
|
||||
tier: "api-poll", observedAt: epoch, receivedAt: epoch, providerEventId: "same-event",
|
||||
payload: { outcome: "ok" },
|
||||
};
|
||||
const poller = new LiveMonitorPoller(service, {
|
||||
enabled: false,
|
||||
sources: [{ provider: "mock-livestream", sourceRef: "live:1", workspaceId: "workspace", connectionId: "connection" }],
|
||||
schedule: () => 1 as unknown as ReturnType<typeof setTimeout>,
|
||||
cancel: () => undefined,
|
||||
resolveSourceId: async () => "source",
|
||||
makeDriver: async () => ({ poll: async () => [observation] }),
|
||||
});
|
||||
poller.start();
|
||||
await poller.pollOnce();
|
||||
await poller.pollOnce();
|
||||
expect((await get()).version).toBe(1);
|
||||
expect((await get()).state).toBe("PROBING");
|
||||
await poller.stop();
|
||||
});
|
||||
|
||||
it("uses a 10 second interval with bounded ±20% jitter and schedules the next tick", async () => {
|
||||
expect([0, 0.5, 1].map((value) => jitteredInterval(10_000, value))).toEqual([8_000, 10_000, 12_000]);
|
||||
const scheduled: Array<{ callback: () => void; delay: number }> = [];
|
||||
const recorded: MonitorObservation[] = [];
|
||||
const observation: MonitorObservation = {
|
||||
tier: "api-poll", observedAt: epoch, receivedAt: epoch, providerEventId: "event-1",
|
||||
payload: { outcome: "ok" },
|
||||
};
|
||||
const monitor = { recordObservation: async (_workspace: string, _source: string, value: MonitorObservation) => {
|
||||
recorded.push(value);
|
||||
} } as unknown as LiveMonitorService;
|
||||
const overrides: Partial<PollerDependencies> = {
|
||||
enabled: false, intervalMs: 10_000, random: () => 0,
|
||||
sources: [{ provider: "mock-livestream", sourceRef: "live:1", workspaceId: "workspace", connectionId: "connection" }],
|
||||
schedule: (callback, delay) => {
|
||||
scheduled.push({ callback, delay });
|
||||
return 1 as unknown as ReturnType<typeof setTimeout>;
|
||||
},
|
||||
cancel: () => undefined,
|
||||
resolveSourceId: async () => "source",
|
||||
makeDriver: async () => ({ poll: async () => [observation] }),
|
||||
};
|
||||
const poller = new LiveMonitorPoller(monitor, overrides);
|
||||
poller.start();
|
||||
expect(scheduled[0]?.delay).toBe(0);
|
||||
scheduled[0]?.callback();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(recorded).toEqual([observation]);
|
||||
expect(scheduled[1]?.delay).toBe(8_000);
|
||||
await poller.stop();
|
||||
});
|
||||
|
||||
it("parses source config and leaves the real probe stubbed", async () => {
|
||||
expect(parsePollSources(JSON.stringify([{ provider: "mock-livestream", sourceRef: "live:1",
|
||||
workspaceId: "workspace", connectionId: "connection" }]))).toHaveLength(1);
|
||||
await expect(new PeerTubePollDriver("video-id").poll()).rejects.toThrow("NOT_IMPLEMENTED");
|
||||
});
|
||||
});
|
||||
215
apps/api/src/live-monitor/poller/poller.ts
Normal file
215
apps/api/src/live-monitor/poller/poller.ts
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
import { Inject, Injectable, Logger, NotFoundException, OnApplicationBootstrap, OnApplicationShutdown, Optional } from "@nestjs/common";
|
||||
import { withTenant } from "../../db/client.js";
|
||||
import { LiveMonitorService } from "../live-monitor.service.js";
|
||||
import type { PollDriver } from "./driver.js";
|
||||
import { MockPollDriver } from "./mock-driver.js";
|
||||
// eslint-disable-next-line no-restricted-syntax
|
||||
import { EgressPeerTubeProbeClient, PeerTubePollDriver } from "./peertube-driver.js";
|
||||
import { monitorEgress } from "./egress.js";
|
||||
import { HlsProbeDriver } from "./hls-driver.js";
|
||||
import { BackoffPollDriver } from "./backoff-driver.js";
|
||||
|
||||
export interface PollSource {
|
||||
provider: string;
|
||||
sourceRef: string;
|
||||
workspaceId: string;
|
||||
connectionId: string;
|
||||
scenario?: string;
|
||||
scenarioSource?: "A" | "B" | "C";
|
||||
playbackUrl?: string;
|
||||
}
|
||||
|
||||
export interface PollerDependencies {
|
||||
sources: PollSource[];
|
||||
intervalMs: number;
|
||||
enabled: boolean;
|
||||
random: () => number;
|
||||
schedule: (callback: () => void, delayMs: number) => ReturnType<typeof setTimeout>;
|
||||
cancel: (timer: ReturnType<typeof setTimeout>) => void;
|
||||
resolveSourceId: (source: PollSource) => Promise<string>;
|
||||
makeDriver: (source: PollSource, intervalMs: number) => Promise<PollDriver>;
|
||||
}
|
||||
|
||||
export const MEDIA_POLLER_DEPENDENCIES = Symbol("MEDIA_POLLER_DEPENDENCIES");
|
||||
|
||||
export function jitteredInterval(intervalMs: number, random: number): number {
|
||||
if (!Number.isFinite(intervalMs) || intervalMs <= 0) throw new Error("poll interval must be positive");
|
||||
if (!Number.isFinite(random) || random < 0 || random > 1) throw new Error("random must be between 0 and 1");
|
||||
return Math.round(intervalMs * (0.8 + random * 0.4));
|
||||
}
|
||||
|
||||
function requiredString(value: unknown, name: string): string {
|
||||
if (typeof value !== "string" || value.trim() === "") throw new Error(`${name} must be a non-empty string`);
|
||||
return value;
|
||||
}
|
||||
|
||||
export function parsePollSources(raw: string | undefined): PollSource[] {
|
||||
if (raw === undefined || raw.trim() === "") return [];
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (!Array.isArray(parsed)) throw new Error("MEDIA_POLLER_SOURCES must be a JSON array");
|
||||
return parsed.map((entry: unknown, index): PollSource => {
|
||||
if (typeof entry !== "object" || entry === null || Array.isArray(entry)) {
|
||||
throw new Error(`MEDIA_POLLER_SOURCES[${index}] must be an object`);
|
||||
}
|
||||
const row = entry as Record<string, unknown>;
|
||||
const source: PollSource = {
|
||||
provider: requiredString(row.provider, "provider"),
|
||||
sourceRef: requiredString(row.sourceRef, "sourceRef"),
|
||||
workspaceId: requiredString(row.workspaceId, "workspaceId"),
|
||||
connectionId: requiredString(row.connectionId, "connectionId"),
|
||||
};
|
||||
if (row.scenario !== undefined) source.scenario = requiredString(row.scenario, "scenario");
|
||||
if (row.playbackUrl !== undefined) source.playbackUrl = requiredString(row.playbackUrl, "playbackUrl");
|
||||
if (row.scenarioSource !== undefined) {
|
||||
if (row.scenarioSource !== "A" && row.scenarioSource !== "B" && row.scenarioSource !== "C") {
|
||||
throw new Error("scenarioSource must be A, B, or C");
|
||||
}
|
||||
source.scenarioSource = row.scenarioSource;
|
||||
}
|
||||
return source;
|
||||
});
|
||||
}
|
||||
|
||||
async function resolveSourceId(source: PollSource): Promise<string> {
|
||||
return withTenant(source.workspaceId, async (client) => {
|
||||
const result = await client.query(
|
||||
"SELECT id FROM source_monitor_states WHERE connection_id = $1 AND remote_ref = $2",
|
||||
[source.connectionId, source.sourceRef],
|
||||
);
|
||||
if (result.rowCount !== 1) throw new NotFoundException(`monitored source ${source.sourceRef} not found`);
|
||||
return String(result.rows[0]?.id);
|
||||
});
|
||||
}
|
||||
|
||||
async function makeDriver(source: PollSource, intervalMs: number): Promise<PollDriver> {
|
||||
// Provider routing lives here until the connector registry exposes monitor probes.
|
||||
if (source.provider === "mock-livestream") {
|
||||
return MockPollDriver.create({
|
||||
scenario: source.scenario ?? "single-live", source: source.scenarioSource ?? "A",
|
||||
sourceRef: source.sourceRef, workspaceId: source.workspaceId, connectionId: source.connectionId,
|
||||
tickMs: intervalMs,
|
||||
});
|
||||
}
|
||||
// eslint-disable-next-line no-restricted-syntax
|
||||
if (source.provider === "peertube" || source.provider === "hls") {
|
||||
const egress = await monitorEgress();
|
||||
const drivers: PollDriver[] = [];
|
||||
// eslint-disable-next-line no-restricted-syntax
|
||||
if (source.provider === "peertube") {
|
||||
const fields = await withTenant(source.workspaceId, async (client) => {
|
||||
const result = await client.query<{ fields: Record<string, unknown> }>(
|
||||
"SELECT fields FROM connections WHERE id = $1 AND workspace_id = $2 AND disconnected_at IS NULL",
|
||||
[source.connectionId, source.workspaceId]);
|
||||
if (result.rowCount !== 1) throw new NotFoundException("monitor connection not found");
|
||||
return result.rows[0]!.fields;
|
||||
});
|
||||
const instanceUrl = requiredString(fields.instanceUrl, "connection instanceUrl");
|
||||
const videoId = source.sourceRef.startsWith("video:")
|
||||
? source.sourceRef.slice(6).split("@")[0]! : source.sourceRef;
|
||||
drivers.push(new BackoffPollDriver(new PeerTubePollDriver(videoId,
|
||||
new EgressPeerTubeProbeClient(instanceUrl, egress)), "api-poll"));
|
||||
}
|
||||
if (source.playbackUrl) drivers.push(new BackoffPollDriver(new HlsProbeDriver(source.playbackUrl, egress), "hls-probe"));
|
||||
if (!drivers.length) throw new Error("HLS monitor source requires playbackUrl");
|
||||
return { poll: async () => {
|
||||
const observations = [];
|
||||
for (const driver of drivers) observations.push(...await driver.poll());
|
||||
return observations;
|
||||
} };
|
||||
}
|
||||
throw new Error(`no live poll driver for provider ${source.provider}`);
|
||||
}
|
||||
|
||||
function defaults(): PollerDependencies {
|
||||
const interval = Number(process.env.MEDIA_POLLER_INTERVAL_MS ?? 10_000);
|
||||
if (!Number.isFinite(interval) || interval <= 0) throw new Error("MEDIA_POLLER_INTERVAL_MS must be positive");
|
||||
const enabled = process.env.MEDIA_POLLER_ENABLED === undefined
|
||||
? process.env.NODE_ENV !== "production" : process.env.MEDIA_POLLER_ENABLED === "true";
|
||||
return {
|
||||
sources: parsePollSources(process.env.MEDIA_POLLER_SOURCES), intervalMs: interval, enabled,
|
||||
random: Math.random, schedule: setTimeout, cancel: clearTimeout, resolveSourceId, makeDriver,
|
||||
};
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class LiveMonitorPoller implements OnApplicationBootstrap, OnApplicationShutdown {
|
||||
private readonly logger = new Logger(LiveMonitorPoller.name);
|
||||
private readonly config: PollerDependencies;
|
||||
private readonly drivers = new Map<string, PollDriver>();
|
||||
private readonly sourceIds = new Map<string, string>();
|
||||
private timer: ReturnType<typeof setTimeout> | undefined;
|
||||
private active: Promise<void> | undefined;
|
||||
private running = false;
|
||||
|
||||
constructor(
|
||||
@Inject(LiveMonitorService) private readonly monitor: LiveMonitorService,
|
||||
@Optional() @Inject(MEDIA_POLLER_DEPENDENCIES) dependencies?: Partial<PollerDependencies>,
|
||||
) {
|
||||
this.config = { ...defaults(), ...dependencies };
|
||||
}
|
||||
|
||||
onApplicationBootstrap(): void {
|
||||
if (this.config.enabled) this.start();
|
||||
}
|
||||
|
||||
async onApplicationShutdown(): Promise<void> {
|
||||
await this.stop();
|
||||
}
|
||||
|
||||
get isRunning(): boolean { return this.running; }
|
||||
|
||||
start(): { running: boolean } {
|
||||
if (this.running) return { running: true };
|
||||
this.running = true;
|
||||
this.schedule(0);
|
||||
return { running: true };
|
||||
}
|
||||
|
||||
async stop(): Promise<{ running: boolean }> {
|
||||
this.running = false;
|
||||
if (this.timer !== undefined) this.config.cancel(this.timer);
|
||||
this.timer = undefined;
|
||||
await this.active;
|
||||
return { running: false };
|
||||
}
|
||||
|
||||
private schedule(delayMs: number): void {
|
||||
if (!this.running) return;
|
||||
this.timer = this.config.schedule(() => {
|
||||
this.timer = undefined;
|
||||
this.active = this.pollOnce().catch((error: unknown) => {
|
||||
this.logger.error(`poll tick failed: ${String(error)}`);
|
||||
}).finally(() => {
|
||||
this.active = undefined;
|
||||
this.schedule(jitteredInterval(this.config.intervalMs, this.config.random()));
|
||||
});
|
||||
}, delayMs);
|
||||
}
|
||||
|
||||
/** One serialized tick; each source fails independently. */
|
||||
async pollOnce(): Promise<void> {
|
||||
for (const source of this.config.sources) {
|
||||
if (!this.running) return;
|
||||
const key = `${source.workspaceId}:${source.connectionId}:${source.sourceRef}`;
|
||||
try {
|
||||
let driver = this.drivers.get(key);
|
||||
if (driver === undefined) {
|
||||
driver = await this.config.makeDriver(source, this.config.intervalMs);
|
||||
this.drivers.set(key, driver);
|
||||
}
|
||||
let sourceId = this.sourceIds.get(key);
|
||||
if (sourceId === undefined) {
|
||||
sourceId = await this.config.resolveSourceId(source);
|
||||
this.sourceIds.set(key, sourceId);
|
||||
}
|
||||
const observations = await driver.poll();
|
||||
for (const observation of observations) {
|
||||
if (!this.running) return;
|
||||
await this.monitor.recordObservation(source.workspaceId, sourceId, observation);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(`poll failed for ${key}: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
172
apps/api/src/live-monitor/poller/signals.test.ts
Normal file
172
apps/api/src/live-monitor/poller/signals.test.ts
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
import { readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { EgressError, EgressPolicy, rewriteSegmentUris, validateMasterPlaylist } from "../../../../../packages/egress/src/index.js";
|
||||
import { BackoffPollDriver } from "./backoff-driver.js";
|
||||
import { MONITOR_MAX_BYTES, MONITOR_TIMEOUT_MS, type MonitorEgress } from "./egress.js";
|
||||
import { HlsProbeDriver, parseMediaPlaylist } from "./hls-driver.js";
|
||||
import { EgressPeerTubeProbeClient, PeerTubePollDriver } from "./peertube-driver.js";
|
||||
|
||||
const epoch = Date.parse("2026-01-01T00:00:12.000Z");
|
||||
const origin = "https://stream.example";
|
||||
const masterUrl = `${origin}/hls/master.m3u8`;
|
||||
const liveUrl = `${origin}/hls/live.m3u8`;
|
||||
const fixture = (name: string) => readFileSync(resolve(__dirname, "fixtures", name), "utf8");
|
||||
const live = fixture("live.m3u8");
|
||||
|
||||
/** A fake transport tracks each egress fetch/debit; real WP7 URI policy/helpers still run. */
|
||||
function fakeEgress(routes: Record<string, string | Response | (() => Response | Promise<Response>)>) {
|
||||
const draws: Array<{ host: string; hostClass: string; url: string }> = [];
|
||||
const policy = new EgressPolicy({ resolver: async () => [{ address: "8.8.8.8", family: 4 }] });
|
||||
const egress: MonitorEgress = {
|
||||
policy, validateMasterPlaylist: vi.fn(validateMasterPlaylist), rewriteSegmentUris: vi.fn(rewriteSegmentUris),
|
||||
egressFetch: vi.fn(async (raw, _init, context) => {
|
||||
const endpoint = await policy.validate(raw);
|
||||
draws.push({ host: endpoint.hostKey, hostClass: context.hostClass, url: endpoint.url.href });
|
||||
const route = routes[endpoint.url.href];
|
||||
if (route === undefined) throw new Error("unexpected outbound URL");
|
||||
const response = typeof route === "function" ? await route() : typeof route === "string" ? new Response(route) : route.clone();
|
||||
if (!response.url) Object.defineProperty(response, "url", { value: endpoint.url.href });
|
||||
return response;
|
||||
}),
|
||||
};
|
||||
return { egress, draws };
|
||||
}
|
||||
|
||||
afterEach(() => { vi.useRealTimers(); });
|
||||
|
||||
describe("HLS media ground truth", () => {
|
||||
it("fetches a master and exactly one variant through monitor-probe egress, validates all URIs", async () => {
|
||||
const f = fakeEgress({ [masterUrl]: fixture("master.m3u8"), [liveUrl]: live });
|
||||
const observation = (await new HlsProbeDriver(masterUrl, f.egress, () => epoch).poll())[0];
|
||||
expect(observation).toMatchObject({ tier: "hls-probe", payload: { outcome: "ok", mediaPresent: true } });
|
||||
expect(f.draws).toEqual([
|
||||
{ url: masterUrl, host: "stream.example:443", hostClass: "monitor-probe" },
|
||||
{ url: liveUrl, host: "stream.example:443", hostClass: "monitor-probe" },
|
||||
]);
|
||||
expect(f.egress.validateMasterPlaylist).toHaveBeenCalledOnce();
|
||||
expect(f.egress.rewriteSegmentUris).toHaveBeenCalledOnce();
|
||||
expect(f.egress.rewriteSegmentUris).toHaveBeenCalledWith(live, liveUrl, f.egress.policy);
|
||||
});
|
||||
|
||||
it.each(["offline.m3u8", "empty", "404"])("reports %s as offline", async (name) => {
|
||||
const body = name === "empty" ? "#EXTM3U\n#EXT-X-TARGETDURATION:6\n"
|
||||
: name === "404" ? new Response(null, { status: 404 }) : fixture(name);
|
||||
const f = fakeEgress({ [liveUrl]: body });
|
||||
expect((await new HlsProbeDriver(liveUrl, f.egress, () => epoch).poll())[0]?.payload)
|
||||
.toMatchObject({ outcome: "fail", mediaPresent: false });
|
||||
});
|
||||
|
||||
it("reports stale program dates and unreasonable future dates as offline", async () => {
|
||||
const f = fakeEgress({ [liveUrl]: live });
|
||||
expect((await new HlsProbeDriver(liveUrl, f.egress, () => epoch + 19_000).poll())[0]?.payload.outcome).toBe("fail");
|
||||
expect((await new HlsProbeDriver(liveUrl, f.egress, () => epoch - 19_000).poll())[0]?.payload.outcome).toBe("fail");
|
||||
});
|
||||
|
||||
it("uses media-sequence/last-segment progression and three target durations when PDT is absent", async () => {
|
||||
let now = epoch; let body = live.replace(/#EXT-X-PROGRAM-DATE-TIME:.*\n/, "");
|
||||
const f = fakeEgress({ [liveUrl]: () => new Response(body) });
|
||||
const driver = new HlsProbeDriver(liveUrl, f.egress, () => now);
|
||||
expect((await driver.poll())[0]?.payload.outcome).toBe("ok");
|
||||
now += 18_001;
|
||||
expect((await driver.poll())[0]?.payload.outcome).toBe("fail");
|
||||
body = body.replace("SEQUENCE:20", "SEQUENCE:21").replace("segment21.ts", "segment22.ts");
|
||||
expect((await driver.poll())[0]?.payload.outcome).toBe("ok");
|
||||
});
|
||||
|
||||
it.each(["malformed.m3u8", "header", "missing-duration", "missing-uri"])("rejects malformed playlist %s", async (name) => {
|
||||
const body = name === "header" ? "not a playlist" : name === "missing-duration" ? "#EXTM3U\n#EXT-X-TARGETDURATION:6\nseg.ts\n"
|
||||
: name === "missing-uri" ? "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXTINF:6,\n" : fixture(name);
|
||||
const f = fakeEgress({ [liveUrl]: body });
|
||||
await expect(new HlsProbeDriver(liveUrl, f.egress, () => epoch).poll()).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("accepts EVENT playlists and rejects invalid target-duration math", () => {
|
||||
expect(parseMediaPlaylist(live.replace("#EXT-X-TARGETDURATION", "#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-TARGETDURATION")).vod).toBe(false);
|
||||
expect(() => parseMediaPlaylist(live.replace("EXTINF:6", "EXTINF:7"))).toThrow();
|
||||
});
|
||||
|
||||
it.each(["variant", "unused-variant", "segment", "key"])("denies private %s references before any child request", async (kind) => {
|
||||
const master = kind === "variant" ? "#EXTM3U\n#EXT-X-STREAM-INF:BANDWIDTH=800000\nhttp://169.254.169.254/private.m3u8\n"
|
||||
: fixture("master.m3u8").replace("unused.m3u8", "http://127.0.0.1/unused.m3u8");
|
||||
const media = kind === "segment" ? live.replace("segment20.ts", "http://10.0.0.1/seg.ts")
|
||||
: live.replace("#EXTINF:6,", '#EXT-X-KEY:METHOD=AES-128,URI="http://169.254.169.254/key"\n#EXTINF:6,');
|
||||
const isMaster = kind === "variant" || kind === "unused-variant";
|
||||
const f = fakeEgress({ [isMaster ? masterUrl : liveUrl]: isMaster ? master : media });
|
||||
await expect(new HlsProbeDriver(isMaster ? masterUrl : liveUrl, f.egress, () => epoch).poll())
|
||||
.rejects.toMatchObject({ code: "DENIED_PRIVATE_ADDRESS" });
|
||||
expect(f.draws).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("resolves variant and segment URIs against the final redirected playlist URL", async () => {
|
||||
const response = new Response(fixture("master.m3u8"));
|
||||
Object.defineProperty(response, "url", { value: `${origin}/redirected/master.m3u8` });
|
||||
const f = fakeEgress({ [masterUrl]: () => response, [`${origin}/redirected/live.m3u8`]: live });
|
||||
await new HlsProbeDriver(masterUrl, f.egress, () => epoch).poll();
|
||||
expect(f.draws[1]?.url).toBe(`${origin}/redirected/live.m3u8`);
|
||||
});
|
||||
|
||||
it("enforces body size, URI count and whole-probe deadlines", async () => {
|
||||
const oversized = fakeEgress({ [liveUrl]: "x".repeat(MONITOR_MAX_BYTES + 1) });
|
||||
await expect(new HlsProbeDriver(liveUrl, oversized.egress).poll()).rejects.toThrow("size cap");
|
||||
const references = fakeEgress({ [liveUrl]: `#EXTM3U\n${"#EXTINF:6,\nsegment.ts\n".repeat(129)}` });
|
||||
await expect(new HlsProbeDriver(liveUrl, references.egress).poll()).rejects.toThrow("too many HLS references");
|
||||
expect(references.egress.rewriteSegmentUris).not.toHaveBeenCalled();
|
||||
vi.useFakeTimers();
|
||||
const hanging = fakeEgress({ [liveUrl]: () => new Promise<Response>(() => undefined) });
|
||||
const pending = new HlsProbeDriver(liveUrl, hanging.egress).poll();
|
||||
const rejected = expect(pending).rejects.toThrow("deadline");
|
||||
await vi.advanceTimersByTimeAsync(MONITOR_TIMEOUT_MS);
|
||||
await rejected;
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("monitor API budgets and adaptive source backoff", () => {
|
||||
it("counts API + master + variant draws on the same host, with the required host classes", async () => {
|
||||
const f = fakeEgress({ [`${origin}/api/v1/videos/video-1`]: JSON.stringify({ isLive: true, state: { id: 1 } }),
|
||||
[masterUrl]: fixture("master.m3u8"), [liveUrl]: live });
|
||||
const api = new PeerTubePollDriver("video-1", new EgressPeerTubeProbeClient(origin, f.egress, () => epoch));
|
||||
expect((await api.poll())[0]?.payload.outcome).toBe("ok");
|
||||
await new HlsProbeDriver(masterUrl, f.egress, () => epoch).poll();
|
||||
expect(f.draws.map((draw) => draw.hostClass)).toEqual(["monitor-api", "monitor-probe", "monitor-probe"]);
|
||||
expect(new Set(f.draws.map((draw) => draw.host))).toEqual(new Set(["stream.example:443"]));
|
||||
expect(f.egress.egressFetch).toHaveBeenNthCalledWith(1, expect.any(URL), expect.anything(), { hostClass: "monitor-api", pinnedOrigin: true });
|
||||
});
|
||||
|
||||
it.each([4, 5])("does not consider PeerTube state %s live even when isLive=true", async (state) => {
|
||||
const f = fakeEgress({ [`${origin}/api/v1/videos/video-1`]: JSON.stringify({ isLive: true, state: { id: state } }) });
|
||||
expect((await new EgressPeerTubeProbeClient(origin, f.egress).probe("video-1"))[0]?.payload.outcome).toBe("fail");
|
||||
});
|
||||
|
||||
it.each([429, 503])("backs off source after HTTP %s, honors Retry-After and resets on recovery", async (status) => {
|
||||
let now = epoch; let attempt = 0;
|
||||
const f = fakeEgress({ [liveUrl]: () => ++attempt === 1 ? new Response(null, { status, headers: { "retry-after": "30" } }) : new Response(live) });
|
||||
const driver = new BackoffPollDriver(new HlsProbeDriver(liveUrl, f.egress, () => now), "hls-probe", () => now, () => 0.5);
|
||||
expect((await driver.poll())[0]?.payload.outcome).toBe("unavailable");
|
||||
expect(driver.nextAttemptAt).toBe(epoch + 30_000);
|
||||
now += 29_999; expect(await driver.poll()).toEqual([]); expect(f.draws).toHaveLength(1);
|
||||
now += 1; await driver.poll(); expect(f.draws).toHaveLength(2); expect(driver.nextAttemptAt).toBe(0);
|
||||
});
|
||||
|
||||
it("honors an HTTP-date Retry-After for API polls and grows source-local delays on repeated 5xx", async () => {
|
||||
let now = epoch; let attempt = 0;
|
||||
const f = fakeEgress({ [`${origin}/api/v1/videos/v`]: () => new Response(null, { status: 503,
|
||||
headers: ++attempt === 1 ? { "retry-after": new Date(epoch + 60_000).toUTCString() } : {} }) });
|
||||
const driver = new BackoffPollDriver(new PeerTubePollDriver("v", new EgressPeerTubeProbeClient(origin, f.egress)), "api-poll", () => now, () => 0.5);
|
||||
await driver.poll(); expect(driver.nextAttemptAt).toBe(epoch + 60_000);
|
||||
now += 60_000; await driver.poll(); expect(driver.nextAttemptAt).toBe(now + 2000);
|
||||
now += 2000; await driver.poll(); expect(driver.nextAttemptAt).toBe(now + 4000);
|
||||
});
|
||||
|
||||
it("respects WP7 budget exhaustion and allows a different source to proceed", async () => {
|
||||
let now = epoch;
|
||||
const egress = fakeEgress({ [liveUrl]: live });
|
||||
const blocked = { ...egress.egress, egressFetch: vi.fn(async () => { throw new EgressError("BUDGET_EXHAUSTED", "budget", 10_000); }) };
|
||||
const one = new BackoffPollDriver(new HlsProbeDriver(liveUrl, blocked, () => now), "hls-probe", () => now, () => 0.5);
|
||||
const two = new BackoffPollDriver(new HlsProbeDriver(liveUrl, egress.egress, () => now), "hls-probe", () => now, () => 0.5);
|
||||
await one.poll(); now += 1000; expect(await one.poll()).toEqual([]);
|
||||
expect((await two.poll())[0]?.payload.outcome).toBe("ok");
|
||||
expect(blocked.egressFetch).toHaveBeenCalledOnce(); expect(egress.draws).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
97
apps/api/src/live-monitor/reducer.test.ts
Normal file
97
apps/api/src/live-monitor/reducer.test.ts
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { applyObservation, type MonitorObservation, type MonitorRowState, type SignalTier } from "./reducer.js";
|
||||
|
||||
const start = Date.parse("2026-01-01T00:00:00.000Z");
|
||||
const at = (seconds: number): string => new Date(start + seconds * 1000).toISOString();
|
||||
|
||||
function initial(): MonitorRowState {
|
||||
return {
|
||||
id: "source", workspaceId: "workspace", connectionId: "connection", remoteRef: "live:1",
|
||||
state: "OFFLINE", version: 0, authoritativeTier: null,
|
||||
lastMediaHealthAt: null, lastHlsProbeAt: null, lastWebhookAt: null, lastApiPollAt: null,
|
||||
tierStatus: {}, okStreak: 0, failStreak: 0, stateEnteredAt: at(0),
|
||||
holdUntil: null, cooldownUntil: null, previousState: null,
|
||||
lastTransitionAt: null, lastTransitionReason: null, createdAt: at(0), updatedAt: at(0),
|
||||
};
|
||||
}
|
||||
|
||||
function obs(tier: SignalTier, outcome: MonitorObservation["payload"]["outcome"], seconds: number,
|
||||
extra: Partial<MonitorObservation> = {}): MonitorObservation {
|
||||
return { tier, observedAt: at(seconds), receivedAt: at(seconds), payload: { outcome }, ...extra };
|
||||
}
|
||||
|
||||
function fold(state: MonitorRowState, ...observations: MonitorObservation[]): MonitorRowState {
|
||||
return observations.reduce((current, observation) => applyObservation(current, observation).state, state);
|
||||
}
|
||||
|
||||
describe("live source reducer", () => {
|
||||
it("confirms LIVE after two ok signals from a poll-only source", () => {
|
||||
const first = applyObservation(initial(), obs("api-poll", "ok", 1));
|
||||
expect(first.state.state).toBe("PROBING");
|
||||
const second = applyObservation(first.state, obs("api-poll", "ok", 2));
|
||||
expect(second.state.state).toBe("LIVE");
|
||||
expect(second.state.authoritativeTier).toBe("api-poll");
|
||||
expect(second.transition?.version).toBe(2);
|
||||
});
|
||||
|
||||
it("confirms OFFLINE after three failures and waits out hysteresis", () => {
|
||||
const live = fold(initial(), obs("api-poll", "ok", 1), obs("api-poll", "ok", 2));
|
||||
const one = applyObservation(live, obs("api-poll", "fail", 3));
|
||||
expect(one.state.state).toBe("LIVE");
|
||||
const two = applyObservation(one.state, obs("api-poll", "fail", 4));
|
||||
expect(two.state.state).toBe("UNSTABLE");
|
||||
const three = applyObservation(two.state, obs("api-poll", "fail", 5));
|
||||
expect(three.state.state).toBe("UNSTABLE");
|
||||
expect(applyObservation(three.state, obs("api-poll", "fail", 12)).state.state).toBe("OFFLINE");
|
||||
});
|
||||
|
||||
it("holds a flapping source without rapid LIVE/OFFLINE cuts", () => {
|
||||
let state = fold(initial(), obs("api-poll", "ok", 1), obs("api-poll", "ok", 2));
|
||||
const terminal: string[] = [state.state];
|
||||
for (let second = 3; second <= 9; second += 1) {
|
||||
const outcome = second % 2 === 0 ? "ok" : "fail";
|
||||
const result = applyObservation(state, obs("api-poll", outcome, second));
|
||||
state = result.state;
|
||||
if (result.transition?.to === "LIVE" || result.transition?.to === "OFFLINE") terminal.push(result.transition.to);
|
||||
}
|
||||
expect(terminal).toEqual(["LIVE"]);
|
||||
});
|
||||
|
||||
it("discards stale observations within a tier and repeated provider IDs", () => {
|
||||
const first = fold(initial(), obs("webhook", "ok", 3, { providerEventId: "event-1" }));
|
||||
expect(applyObservation(first, obs("webhook", "fail", 2)).state).toBe(first);
|
||||
const second = fold(first, obs("webhook", "ok", 4, { providerEventId: "event-2" }));
|
||||
expect(applyObservation(second, obs("webhook", "ok", 5, { providerEventId: "event-1" })).state).toBe(second);
|
||||
expect(first.version).toBe(1);
|
||||
});
|
||||
|
||||
it("keeps a newer HLS probe authoritative against a late API poll and clock-skewed webhook", () => {
|
||||
const futureWebhook = obs("webhook", "fail", 1000, { receivedAt: at(1) });
|
||||
const state = fold(initial(), futureWebhook,
|
||||
obs("hls-probe", "ok", 2), obs("hls-probe", "ok", 3),
|
||||
obs("api-poll", "fail", 1, { receivedAt: at(6) }));
|
||||
expect(state.authoritativeTier).toBe("hls-probe");
|
||||
expect(state.state).toBe("UNSTABLE");
|
||||
expect(state.tierStatus["hls-probe"]?.okStreak).toBe(2);
|
||||
const confirmed = applyObservation(state, obs("hls-probe", "ok", 13)).state;
|
||||
expect(confirmed.state).toBe("LIVE");
|
||||
});
|
||||
|
||||
it("raises and lowers authority when HLS becomes available then denied", () => {
|
||||
const poll = fold(initial(), obs("api-poll", "ok", 1), obs("api-poll", "ok", 2));
|
||||
const raised = applyObservation(poll, obs("hls-probe", "ok", 3)).state;
|
||||
expect(raised.authoritativeTier).toBe("hls-probe");
|
||||
const lowered = applyObservation(raised, obs("hls-probe", "unavailable", 4)).state;
|
||||
expect(lowered.authoritativeTier).toBe("api-poll");
|
||||
expect(lowered.state).toBe("LIVE");
|
||||
});
|
||||
|
||||
it("uses ten seconds without media as a failure confirmation", () => {
|
||||
const live = fold(initial(),
|
||||
{ ...obs("media-health", "ok", 1), payload: { outcome: "ok", mediaPresent: true } },
|
||||
{ ...obs("media-health", "ok", 2), payload: { outcome: "ok", mediaPresent: true } });
|
||||
const failed = applyObservation(live, obs("media-health", "fail", 13));
|
||||
expect(failed.state.state).toBe("OFFLINE");
|
||||
expect(failed.transition?.reason).toBe("media-health:no-media-timeout");
|
||||
});
|
||||
});
|
||||
182
apps/api/src/live-monitor/reducer.ts
Normal file
182
apps/api/src/live-monitor/reducer.ts
Normal file
|
|
@ -0,0 +1,182 @@
|
|||
/** Pure per-source fold. Provider time is used only to order one tier's events. */
|
||||
export type MonitorState = "OFFLINE" | "PROBING" | "LIVE" | "UNSTABLE" | "ERROR";
|
||||
export type SignalTier = "media-health" | "hls-probe" | "webhook" | "api-poll";
|
||||
export type ObservationOutcome = "ok" | "fail" | "error" | "unavailable";
|
||||
|
||||
export interface MonitorObservation {
|
||||
tier: SignalTier;
|
||||
observedAt: string;
|
||||
/** Ingest clock, never a provider clock. */
|
||||
receivedAt: string;
|
||||
providerEventId?: string;
|
||||
payload: { outcome: ObservationOutcome; reason?: string; mediaPresent?: boolean };
|
||||
}
|
||||
|
||||
export interface TierStatus {
|
||||
available: boolean;
|
||||
lastOutcome?: ObservationOutcome;
|
||||
lastReceivedAt?: string;
|
||||
lastProviderEventId?: string;
|
||||
/** Recent replay guard; the database journal provides durable full-history dedupe. */
|
||||
seenProviderEventIds?: string[];
|
||||
okStreak: number;
|
||||
failStreak: number;
|
||||
lastMediaAt?: string;
|
||||
}
|
||||
|
||||
export interface MonitorRowState {
|
||||
id: string;
|
||||
workspaceId: string;
|
||||
connectionId: string;
|
||||
remoteRef: string;
|
||||
state: MonitorState;
|
||||
version: number;
|
||||
authoritativeTier: SignalTier | null;
|
||||
lastMediaHealthAt: string | null;
|
||||
lastHlsProbeAt: string | null;
|
||||
lastWebhookAt: string | null;
|
||||
lastApiPollAt: string | null;
|
||||
tierStatus: Partial<Record<SignalTier, TierStatus>>;
|
||||
okStreak: number;
|
||||
failStreak: number;
|
||||
stateEnteredAt: string;
|
||||
holdUntil: string | null;
|
||||
cooldownUntil: string | null;
|
||||
previousState: MonitorState | null;
|
||||
lastTransitionAt: string | null;
|
||||
lastTransitionReason: string | null;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface MonitorPolicy {
|
||||
liveConfirmations: number;
|
||||
offlineConfirmations: number;
|
||||
noMediaMs: number;
|
||||
minHoldMs: number;
|
||||
cooldownMs: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_MONITOR_POLICY: MonitorPolicy = {
|
||||
liveConfirmations: 2,
|
||||
offlineConfirmations: 3,
|
||||
noMediaMs: 10_000,
|
||||
minHoldMs: 2_000,
|
||||
cooldownMs: 10_000,
|
||||
};
|
||||
|
||||
export interface MonitorTransition {
|
||||
from: MonitorState;
|
||||
to: MonitorState;
|
||||
version: number;
|
||||
reason: string;
|
||||
at: string;
|
||||
}
|
||||
|
||||
export interface Reduction {
|
||||
state: MonitorRowState;
|
||||
transition?: MonitorTransition;
|
||||
}
|
||||
|
||||
const PRECEDENCE: readonly SignalTier[] = ["media-health", "hls-probe", "webhook", "api-poll"];
|
||||
const OBSERVED_FIELD: Record<SignalTier, keyof Pick<MonitorRowState,
|
||||
"lastMediaHealthAt" | "lastHlsProbeAt" | "lastWebhookAt" | "lastApiPollAt">> = {
|
||||
"media-health": "lastMediaHealthAt",
|
||||
"hls-probe": "lastHlsProbeAt",
|
||||
webhook: "lastWebhookAt",
|
||||
"api-poll": "lastApiPollAt",
|
||||
};
|
||||
|
||||
export function applyObservation(
|
||||
state: MonitorRowState,
|
||||
observation: MonitorObservation,
|
||||
policy: MonitorPolicy = DEFAULT_MONITOR_POLICY,
|
||||
): Reduction {
|
||||
const now = Date.parse(observation.receivedAt);
|
||||
const observed = Date.parse(observation.observedAt);
|
||||
if (!Number.isFinite(now) || !Number.isFinite(observed)) throw new Error("observation timestamps must be valid");
|
||||
const field = OBSERVED_FIELD[observation.tier];
|
||||
const last = state[field];
|
||||
if (last !== null && observed <= Date.parse(last)) return { state };
|
||||
const oldTier = state.tierStatus[observation.tier];
|
||||
if (observation.providerEventId !== undefined && (oldTier?.lastProviderEventId === observation.providerEventId
|
||||
|| oldTier?.seenProviderEventIds?.includes(observation.providerEventId) === true)) {
|
||||
return { state };
|
||||
}
|
||||
|
||||
const outcome = observation.payload.outcome;
|
||||
const available = outcome !== "unavailable";
|
||||
const okStreak = outcome === "ok" ? (oldTier?.okStreak ?? 0) + 1 : 0;
|
||||
const failStreak = outcome === "fail" ? (oldTier?.failStreak ?? 0) + 1 : 0;
|
||||
const tierStatus: MonitorRowState["tierStatus"] = {
|
||||
...state.tierStatus,
|
||||
[observation.tier]: {
|
||||
available,
|
||||
lastOutcome: outcome,
|
||||
lastReceivedAt: observation.receivedAt,
|
||||
...(observation.providerEventId === undefined
|
||||
? oldTier?.lastProviderEventId === undefined ? {} : { lastProviderEventId: oldTier.lastProviderEventId }
|
||||
: { lastProviderEventId: observation.providerEventId }),
|
||||
...(observation.providerEventId === undefined && oldTier?.seenProviderEventIds === undefined
|
||||
? {} : { seenProviderEventIds: observation.providerEventId === undefined
|
||||
? oldTier?.seenProviderEventIds
|
||||
: [...(oldTier?.seenProviderEventIds ?? []), observation.providerEventId].slice(-64) }),
|
||||
okStreak,
|
||||
failStreak,
|
||||
...(observation.payload.mediaPresent === true
|
||||
? { lastMediaAt: observation.receivedAt }
|
||||
: oldTier?.lastMediaAt === undefined ? {} : { lastMediaAt: oldTier.lastMediaAt }),
|
||||
},
|
||||
};
|
||||
const authoritativeTier = PRECEDENCE.find((tier) => tierStatus[tier]?.available === true) ?? null;
|
||||
const authority = authoritativeTier === null ? undefined : tierStatus[authoritativeTier];
|
||||
let next: MonitorRowState = {
|
||||
...state,
|
||||
[field]: observation.observedAt,
|
||||
tierStatus,
|
||||
authoritativeTier,
|
||||
version: state.version + 1,
|
||||
okStreak: authority?.okStreak ?? 0,
|
||||
failStreak: authority?.failStreak ?? 0,
|
||||
updatedAt: observation.receivedAt,
|
||||
};
|
||||
|
||||
let target: MonitorState | undefined;
|
||||
let reason = `${observation.tier}:${outcome}`;
|
||||
if (outcome === "error" && observation.tier === authoritativeTier) {
|
||||
target = "ERROR";
|
||||
} else if (authoritativeTier !== null) {
|
||||
const lowerSignal = observation.tier !== authoritativeTier;
|
||||
if (lowerSignal && outcome !== "unavailable") {
|
||||
if (outcome === "fail" || outcome === "error") target = state.state === "LIVE" ? "UNSTABLE" : "PROBING";
|
||||
} else if (authority?.lastOutcome === "ok") {
|
||||
if (authority.okStreak >= policy.liveConfirmations) target = "LIVE";
|
||||
else if (state.state !== "LIVE") target = "PROBING";
|
||||
} else if (authority?.lastOutcome === "fail") {
|
||||
const noMedia = authoritativeTier === "media-health" && authority.lastMediaAt !== undefined
|
||||
&& now - Date.parse(authority.lastMediaAt) >= policy.noMediaMs;
|
||||
if (authority.failStreak >= policy.offlineConfirmations || noMedia) target = "OFFLINE";
|
||||
else if (state.state === "LIVE") target = "UNSTABLE";
|
||||
else if (state.state !== "OFFLINE" && state.state !== "UNSTABLE") target = "PROBING";
|
||||
if (noMedia) reason = "media-health:no-media-timeout";
|
||||
}
|
||||
}
|
||||
|
||||
if (target === undefined || target === state.state) return { state: next };
|
||||
const terminal = target === "LIVE" || target === "OFFLINE";
|
||||
const held = target !== "ERROR" && state.state !== "PROBING" && state.state !== "UNSTABLE"
|
||||
&& state.holdUntil !== null && now < Date.parse(state.holdUntil);
|
||||
const cooling = terminal && state.cooldownUntil !== null && now < Date.parse(state.cooldownUntil);
|
||||
if (held || cooling) return { state: next };
|
||||
next = {
|
||||
...next,
|
||||
state: target,
|
||||
previousState: state.state,
|
||||
stateEnteredAt: observation.receivedAt,
|
||||
holdUntil: terminal ? new Date(now + policy.minHoldMs).toISOString() : state.holdUntil,
|
||||
cooldownUntil: terminal ? new Date(now + policy.cooldownMs).toISOString() : state.cooldownUntil,
|
||||
lastTransitionAt: observation.receivedAt,
|
||||
lastTransitionReason: reason,
|
||||
};
|
||||
return { state: next, transition: { from: state.state, to: target, version: next.version, reason, at: observation.receivedAt } };
|
||||
}
|
||||
19
apps/api/src/live-monitor/webhook.controller.ts
Normal file
19
apps/api/src/live-monitor/webhook.controller.ts
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import { Controller, HttpCode, Inject, Param, Post, Req } from "@nestjs/common";
|
||||
import type { Request } from "express";
|
||||
import { MonitorWebhookService } from "./webhook.service.js";
|
||||
|
||||
/** Provider authentication is performed over raw bytes; browser session guards do not apply. */
|
||||
@Controller("live-monitor/webhooks")
|
||||
export class MonitorWebhookController {
|
||||
constructor(@Inject(MonitorWebhookService) private readonly webhook: MonitorWebhookService) {}
|
||||
|
||||
@Post(":endpointId")
|
||||
@HttpCode(200)
|
||||
async ingest(@Param("endpointId") endpointId: string, @Req() request: Request): Promise<{ accepted: true }> {
|
||||
await this.webhook.ingest(endpointId, request.body as Buffer, {
|
||||
signature: request.get("x-monitor-signature"), timestamp: request.get("x-monitor-timestamp"),
|
||||
eventId: request.get("x-monitor-event-id"),
|
||||
});
|
||||
return { accepted: true };
|
||||
}
|
||||
}
|
||||
91
apps/api/src/live-monitor/webhook.integration.test.ts
Normal file
91
apps/api/src/live-monitor/webhook.integration.test.ts
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
import { createHmac, randomBytes, randomUUID } from "node:crypto";
|
||||
import { join } from "node:path";
|
||||
import { Pool, type PoolClient } from "pg";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { runMigrations } from "../db/migrate.js";
|
||||
import { vault } from "../vault/vault.js";
|
||||
import { LiveMonitorService } from "./live-monitor.service.js";
|
||||
import { MonitorWebhookService, type MonitorWebhookEndpoint } from "./webhook.service.js";
|
||||
|
||||
const available = process.env.DATABASE_URL !== undefined && process.env.SUPER_DATABASE_URL !== undefined;
|
||||
const owner = new Pool({ connectionString: process.env.SUPER_DATABASE_URL });
|
||||
const app = new Pool({ connectionString: process.env.DATABASE_URL });
|
||||
async function asTenant<T>(workspaceId: string, fn: (client: PoolClient) => Promise<T>): Promise<T> {
|
||||
const client = await app.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]);
|
||||
const result = await fn(client);
|
||||
await client.query("COMMIT");
|
||||
return result;
|
||||
} catch (error) { await client.query("ROLLBACK"); throw error; }
|
||||
finally { client.release(); }
|
||||
}
|
||||
|
||||
describe.skipIf(!available)("monitor webhook durable journal and RLS (PostgreSQL)", () => {
|
||||
const secret = randomBytes(32);
|
||||
const workspaces: string[] = [];
|
||||
let endpoint: MonitorWebhookEndpoint;
|
||||
let sourceId: string;
|
||||
let foreignSourceId: string;
|
||||
let monitor: LiveMonitorService;
|
||||
const service = () => new MonitorWebhookService(monitor, { endpoints: [endpoint], tenantTransaction: asTenant });
|
||||
|
||||
beforeAll(async () => {
|
||||
await runMigrations(process.env.MIGRATIONS_DIR ?? join(process.cwd(), "migrations"));
|
||||
for (let index = 0; index < 2; index++) {
|
||||
const workspace = await owner.query<{ id: string }>(
|
||||
"INSERT INTO workspaces (name, slug) VALUES ('Webhook integration', $1) RETURNING id", [`monitor-webhook-${randomUUID()}`]);
|
||||
const workspaceId = workspace.rows[0]!.id;
|
||||
workspaces.push(workspaceId);
|
||||
await asTenant(workspaceId, async (client) => {
|
||||
const connection = await client.query<{ id: string }>(
|
||||
"INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-livestream', 'webhook') RETURNING id", [workspaceId]);
|
||||
const connectionId = connection.rows[0]!.id;
|
||||
const source = await client.query<{ id: string }>(
|
||||
"INSERT INTO source_monitor_states (workspace_id, connection_id, remote_ref) VALUES ($1, $2, 'live:webhook') RETURNING id", [workspaceId, connectionId]);
|
||||
if (index === 0) {
|
||||
sourceId = source.rows[0]!.id;
|
||||
const sealed = await vault.seal(workspaceId, connectionId, secret);
|
||||
const credential = await client.query<{ id: string }>(
|
||||
`INSERT INTO credentials (workspace_id, connection_id, kind, ciphertext, key_version)
|
||||
VALUES ($1, $2, 'monitor-webhook', $3, $4) RETURNING id`, [workspaceId, connectionId, sealed.ciphertext, sealed.keyVersion]);
|
||||
endpoint = { endpointId: randomBytes(32).toString("hex"), workspaceId, connectionId,
|
||||
credentialId: credential.rows[0]!.id, subscriptionId: "db-subscription" };
|
||||
} else foreignSourceId = source.rows[0]!.id;
|
||||
});
|
||||
}
|
||||
monitor = new LiveMonitorService({ tenantTransaction: asTenant });
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
for (const workspace of workspaces) await owner.query("DELETE FROM workspaces WHERE id = $1", [workspace]);
|
||||
await app.end(); await owner.end(); secret.fill(0);
|
||||
});
|
||||
|
||||
function request(source = sourceId, eventId = randomUUID()) {
|
||||
const raw = Buffer.from(JSON.stringify({ sourceId: source, subscriptionId: endpoint.subscriptionId, outcome: "ok" }));
|
||||
const timestamp = new Date().toISOString();
|
||||
const signature = `sha256=${createHmac("sha256", secret).update(eventId).update(timestamp).update(raw).digest("hex")}`;
|
||||
return { raw, headers: { eventId, timestamp, signature } };
|
||||
}
|
||||
|
||||
it("accepts one concurrent delivery, rejects its replay after service restart, persists exactly one journal row", async () => {
|
||||
const { raw, headers } = request();
|
||||
const results = await Promise.allSettled([service().ingest(endpoint.endpointId, raw, headers), service().ingest(endpoint.endpointId, raw, headers)]);
|
||||
expect(results.filter((result) => result.status === "fulfilled")).toHaveLength(1);
|
||||
expect(results.find((result) => result.status === "rejected")).toMatchObject({ reason: { status: 409 } });
|
||||
await expect(service().ingest(endpoint.endpointId, raw, headers)).rejects.toMatchObject({ status: 409 });
|
||||
expect((await monitor.getState(endpoint.workspaceId, sourceId)).version).toBe(1);
|
||||
const journal = await asTenant(endpoint.workspaceId, (client) => client.query<{ count: number }>(
|
||||
"SELECT count(*)::int AS count FROM monitor_seen_events WHERE source_monitor_state_id = $1", [sourceId]));
|
||||
expect(journal.rows[0]!.count).toBe(1);
|
||||
});
|
||||
|
||||
it("rejects an authenticated event targeting another workspace without touching either source", async () => {
|
||||
const { raw, headers } = request(foreignSourceId);
|
||||
await expect(service().ingest(endpoint.endpointId, raw, headers)).rejects.toMatchObject({ status: 404 });
|
||||
expect((await monitor.getState(workspaces[1]!, foreignSourceId)).version).toBe(0);
|
||||
expect((await monitor.getState(endpoint.workspaceId, sourceId)).version).toBe(1);
|
||||
});
|
||||
});
|
||||
136
apps/api/src/live-monitor/webhook.service.ts
Normal file
136
apps/api/src/live-monitor/webhook.service.ts
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
import { createHash, createHmac, timingSafeEqual } from "node:crypto";
|
||||
import { BadRequestException, Inject, Injectable, NotFoundException, Optional, PayloadTooLargeException, UnauthorizedException } from "@nestjs/common";
|
||||
import { withTenant } from "../db/client.js";
|
||||
import { isUuid, requireObject, requireString } from "../http/validation.js";
|
||||
import { vault } from "../vault/vault.js";
|
||||
import { LiveMonitorService } from "./live-monitor.service.js";
|
||||
import type { MonitorRowState, ObservationOutcome } from "./reducer.js";
|
||||
|
||||
export const WEBHOOK_MAX_BYTES = 64 * 1024;
|
||||
export const WEBHOOK_WINDOW_MS = 5 * 60 * 1000;
|
||||
export interface MonitorWebhookEndpoint {
|
||||
endpointId: string;
|
||||
workspaceId: string;
|
||||
connectionId: string;
|
||||
credentialId: string;
|
||||
subscriptionId: string;
|
||||
}
|
||||
export interface MonitorWebhookHeaders { signature?: string; timestamp?: string; eventId?: string }
|
||||
export interface MonitorWebhookDependencies {
|
||||
endpoints: readonly MonitorWebhookEndpoint[];
|
||||
tenantTransaction: typeof withTenant;
|
||||
openSecret: typeof vault.open;
|
||||
now: () => number;
|
||||
}
|
||||
export const MONITOR_WEBHOOK_DEPENDENCIES = Symbol("MONITOR_WEBHOOK_DEPENDENCIES");
|
||||
|
||||
/** Server-controlled mapping, never a workspace/connection supplied by the sender. */
|
||||
export function parseWebhookEndpoints(raw: string | undefined): MonitorWebhookEndpoint[] {
|
||||
if (!raw) return [];
|
||||
const value: unknown = JSON.parse(raw);
|
||||
if (!Array.isArray(value)) throw new Error("MONITOR_WEBHOOK_ENDPOINTS must be an array");
|
||||
const endpoints = value.map((entry) => {
|
||||
const row = requireObject(entry, "endpoint");
|
||||
const endpoint: MonitorWebhookEndpoint = {
|
||||
endpointId: requireString(row.endpointId, "endpointId"),
|
||||
workspaceId: requireString(row.workspaceId, "workspaceId"),
|
||||
connectionId: requireString(row.connectionId, "connectionId"),
|
||||
credentialId: requireString(row.credentialId, "credentialId"),
|
||||
subscriptionId: requireString(row.subscriptionId, "subscriptionId"),
|
||||
};
|
||||
if (!/^[a-f0-9]{64}$/.test(endpoint.endpointId)
|
||||
|| ![endpoint.workspaceId, endpoint.connectionId, endpoint.credentialId].every(isUuid)) {
|
||||
throw new Error("invalid monitor webhook endpoint mapping (endpointId must be 32 random bytes in hex)");
|
||||
}
|
||||
return endpoint;
|
||||
});
|
||||
if (new Set(endpoints.map((entry) => entry.endpointId)).size !== endpoints.length) {
|
||||
throw new Error("duplicate monitor webhook endpoint");
|
||||
}
|
||||
return endpoints;
|
||||
}
|
||||
|
||||
/** Signature covers event id + ISO timestamp + exact bytes, including whitespace. */
|
||||
export function verifyMonitorSignature(raw: Buffer, headers: MonitorWebhookHeaders, secret: Buffer, now: number): void {
|
||||
const { signature, timestamp, eventId } = headers;
|
||||
if (!signature || !/^sha256=[a-f0-9]{64}$/.test(signature) || !timestamp
|
||||
|| !/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d{1,3})?Z$/.test(timestamp)
|
||||
|| !eventId || eventId.length > 256 || /[\s\p{Cc}]/u.test(eventId)
|
||||
|| !Number.isFinite(Date.parse(timestamp)) || Math.abs(now - Date.parse(timestamp)) > WEBHOOK_WINDOW_MS) {
|
||||
throw new UnauthorizedException("invalid webhook authentication");
|
||||
}
|
||||
const expected = createHmac("sha256", secret).update(eventId).update(timestamp).update(raw).digest();
|
||||
if (!timingSafeEqual(expected, Buffer.from(signature.slice(7), "hex"))) {
|
||||
throw new UnauthorizedException("invalid webhook authentication");
|
||||
}
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class MonitorWebhookService {
|
||||
private readonly config: MonitorWebhookDependencies;
|
||||
constructor(
|
||||
@Inject(LiveMonitorService) private readonly monitor: LiveMonitorService,
|
||||
@Optional() @Inject(MONITOR_WEBHOOK_DEPENDENCIES) dependencies?: Partial<MonitorWebhookDependencies>,
|
||||
) {
|
||||
this.config = {
|
||||
endpoints: parseWebhookEndpoints(process.env.MONITOR_WEBHOOK_ENDPOINTS),
|
||||
tenantTransaction: withTenant, openSecret: vault.open.bind(vault), now: Date.now,
|
||||
...dependencies,
|
||||
};
|
||||
}
|
||||
|
||||
async ingest(endpointId: string, raw: Buffer, headers: MonitorWebhookHeaders): Promise<MonitorRowState> {
|
||||
const endpoint = this.config.endpoints.find((entry) => entry.endpointId === endpointId);
|
||||
if (!endpoint) throw new NotFoundException("webhook endpoint not found");
|
||||
if (!Buffer.isBuffer(raw)) throw new BadRequestException("raw webhook body required");
|
||||
if (raw.length > WEBHOOK_MAX_BYTES) throw new PayloadTooLargeException();
|
||||
return this.config.tenantTransaction(endpoint.workspaceId, async (client) => {
|
||||
// A removed connection/revoked credential disables even a configured endpoint.
|
||||
const credential = await client.query<{ ciphertext: Buffer; key_version: number }>(
|
||||
`SELECT cr.ciphertext, cr.key_version FROM credentials cr
|
||||
JOIN connections c ON c.id = cr.connection_id AND c.workspace_id = cr.workspace_id
|
||||
WHERE cr.id = $1 AND cr.connection_id = $2 AND cr.workspace_id = $3
|
||||
AND cr.kind = 'monitor-webhook' AND cr.revoked_at IS NULL AND c.disconnected_at IS NULL
|
||||
FOR SHARE OF cr, c`, [endpoint.credentialId, endpoint.connectionId, endpoint.workspaceId],
|
||||
);
|
||||
const sealed = credential.rows[0];
|
||||
if (!sealed) throw new UnauthorizedException("invalid webhook authentication");
|
||||
let secret: Buffer;
|
||||
try {
|
||||
secret = await this.config.openSecret(endpoint.workspaceId, endpoint.connectionId,
|
||||
{ ciphertext: sealed.ciphertext, keyVersion: sealed.key_version });
|
||||
} catch { throw new UnauthorizedException("invalid webhook authentication"); }
|
||||
const now = this.config.now();
|
||||
try {
|
||||
if (secret.length < 32) throw new UnauthorizedException("invalid webhook authentication");
|
||||
verifyMonitorSignature(raw, headers, secret, now);
|
||||
} finally { secret.fill(0); }
|
||||
let parsed: unknown;
|
||||
try { parsed = JSON.parse(raw.toString("utf8")); }
|
||||
catch { throw new BadRequestException("invalid webhook JSON"); }
|
||||
const body = requireObject(parsed, "event");
|
||||
if (body.subscriptionId !== endpoint.subscriptionId
|
||||
|| body.workspaceId !== undefined && body.workspaceId !== endpoint.workspaceId
|
||||
|| body.connectionId !== undefined && body.connectionId !== endpoint.connectionId) {
|
||||
throw new UnauthorizedException("webhook binding mismatch");
|
||||
}
|
||||
const sourceId = requireString(body.sourceId, "sourceId");
|
||||
if (!isUuid(sourceId)) throw new BadRequestException("sourceId must be a UUID");
|
||||
const outcome = body.outcome;
|
||||
if (outcome !== "ok" && outcome !== "fail" && outcome !== "error" && outcome !== "unavailable") {
|
||||
throw new BadRequestException("invalid webhook outcome");
|
||||
}
|
||||
const source = await client.query(
|
||||
`SELECT id FROM source_monitor_states WHERE id = $1 AND workspace_id = $2 AND connection_id = $3 FOR UPDATE`,
|
||||
[sourceId, endpoint.workspaceId, endpoint.connectionId],
|
||||
);
|
||||
if (source.rowCount !== 1) throw new NotFoundException("webhook source not found");
|
||||
// Journal retention exceeds the acceptance window. The tuple scopes ids per endpoint without ambiguity.
|
||||
const providerEventId = `webhook:${createHash("sha256").update(JSON.stringify([endpointId, headers.eventId])).digest("hex")}`;
|
||||
return this.monitor.recordObservationInTransaction(client, endpoint.workspaceId, sourceId, {
|
||||
tier: "webhook", observedAt: headers.timestamp!, receivedAt: new Date(now).toISOString(), providerEventId,
|
||||
payload: { outcome: outcome as ObservationOutcome, reason: "signed-webhook" },
|
||||
}, { userId: null, role: "monitor-webhook" }, true);
|
||||
});
|
||||
}
|
||||
}
|
||||
199
apps/api/src/live-monitor/webhook.test.ts
Normal file
199
apps/api/src/live-monitor/webhook.test.ts
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
import "reflect-metadata";
|
||||
import { createHmac } from "node:crypto";
|
||||
import { BadRequestException, ConflictException, NotFoundException, UnauthorizedException, type INestApplication } from "@nestjs/common";
|
||||
import { NestFactory } from "@nestjs/core";
|
||||
import type { PoolClient } from "pg";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { injectHttp } from "../connections/http-test-transport.js";
|
||||
import { vault } from "../vault/vault.js";
|
||||
import { LiveMonitorModule } from "./live-monitor.module.js";
|
||||
import { LIVE_MONITOR_DEPENDENCIES, LiveMonitorService } from "./live-monitor.service.js";
|
||||
import { MEDIA_POLLER_DEPENDENCIES } from "./poller/poller.js";
|
||||
import { MONITOR_WEBHOOK_DEPENDENCIES, MonitorWebhookService, parseWebhookEndpoints, verifyMonitorSignature,
|
||||
WEBHOOK_MAX_BYTES, WEBHOOK_WINDOW_MS, type MonitorWebhookDependencies, type MonitorWebhookHeaders } from "./webhook.service.js";
|
||||
|
||||
const workspaceId = "11111111-1111-4111-8111-111111111111";
|
||||
const connectionId = "22222222-2222-4222-8222-222222222222";
|
||||
const credentialId = "33333333-3333-4333-8333-333333333333";
|
||||
const sourceId = "44444444-4444-4444-8444-444444444444";
|
||||
const endpointId = "ab".repeat(32);
|
||||
const endpoint = { endpointId, workspaceId, connectionId, credentialId, subscriptionId: "subscription-1" };
|
||||
const secret = Buffer.alloc(32, 42);
|
||||
const epoch = Date.parse("2026-01-01T00:00:00.000Z");
|
||||
const event = { sourceId, subscriptionId: endpoint.subscriptionId, outcome: "ok" };
|
||||
|
||||
function signed(raw: Buffer, eventId = "event-1", time = epoch): MonitorWebhookHeaders {
|
||||
const timestamp = new Date(time).toISOString();
|
||||
return { eventId, timestamp, signature: `sha256=${createHmac("sha256", secret).update(eventId).update(timestamp).update(raw).digest("hex")}` };
|
||||
}
|
||||
|
||||
async function fixture() {
|
||||
const row: Record<string, unknown> = {
|
||||
id: sourceId, workspace_id: workspaceId, connection_id: connectionId, remote_ref: "live:1",
|
||||
state: "OFFLINE", version: 0, authoritative_tier: null, last_media_health_at: null, last_hls_probe_at: null,
|
||||
last_webhook_at: null, last_api_poll_at: null, tier_status: {}, ok_streak: 0, fail_streak: 0,
|
||||
state_entered_at: new Date(epoch).toISOString(), hold_until: null, cooldown_until: null, previous_state: null,
|
||||
last_transition_at: null, last_transition_reason: null, created_at: new Date(epoch).toISOString(), updated_at: new Date(epoch).toISOString(),
|
||||
};
|
||||
const sealed = await vault.seal(workspaceId, connectionId, secret);
|
||||
const journal = new Set<string>();
|
||||
let now = epoch;
|
||||
let failUpdate = false;
|
||||
let revoked = false;
|
||||
let audits = 0;
|
||||
const client = { async query(sql: string, params: unknown[] = []) {
|
||||
if (sql.startsWith("SELECT cr.ciphertext")) {
|
||||
expect(params).toEqual([credentialId, connectionId, workspaceId]);
|
||||
return { rowCount: revoked ? 0 : 1, rows: revoked ? [] : [{ ciphertext: sealed.ciphertext, key_version: sealed.keyVersion }] };
|
||||
}
|
||||
if (sql.startsWith("SELECT id FROM source_monitor_states")) {
|
||||
const owns = params[0] === row.id && params[1] === row.workspace_id && params[2] === row.connection_id;
|
||||
return { rowCount: owns ? 1 : 0, rows: owns ? [{ id: sourceId }] : [] };
|
||||
}
|
||||
if (sql.startsWith("SELECT * FROM source_monitor_states")) return { rowCount: 1, rows: [{ ...row }] };
|
||||
if (sql.startsWith("INSERT INTO monitor_seen_events")) {
|
||||
const key = JSON.stringify([params[0], params[1]]);
|
||||
if (journal.has(key)) return { rowCount: 0, rows: [] };
|
||||
journal.add(key);
|
||||
return { rowCount: 1, rows: [{}] };
|
||||
}
|
||||
if (sql.startsWith("UPDATE source_monitor_states")) {
|
||||
if (failUpdate) throw new Error("database update failed");
|
||||
const columns = ["state", "version", "authoritative_tier", "last_media_health_at", "last_hls_probe_at",
|
||||
"last_webhook_at", "last_api_poll_at", "tier_status", "ok_streak", "fail_streak", "state_entered_at", "hold_until",
|
||||
"cooldown_until", "previous_state", "last_transition_at", "last_transition_reason", "updated_at"];
|
||||
columns.forEach((column, index) => { row[column] = params[index + 1]; });
|
||||
return { rowCount: 1, rows: [{ ...row }] };
|
||||
}
|
||||
if (sql.startsWith("SELECT audit_append")) { audits++; return { rowCount: 1, rows: [{}] }; }
|
||||
throw new Error(`unexpected query ${sql}`);
|
||||
} } as unknown as PoolClient;
|
||||
let pending = Promise.resolve();
|
||||
const tenantTransaction: MonitorWebhookDependencies["tenantTransaction"] = async (workspace, fn) => {
|
||||
expect(workspace).toBe(workspaceId);
|
||||
const previous = pending;
|
||||
let release!: () => void;
|
||||
pending = new Promise<void>((resolve) => { release = resolve; });
|
||||
await previous;
|
||||
const snapshot = { row: structuredClone(row), journal: [...journal], audits };
|
||||
try { return await fn(client); }
|
||||
catch (error) {
|
||||
Object.assign(row, snapshot.row); journal.clear(); snapshot.journal.forEach((key) => journal.add(key));
|
||||
audits = snapshot.audits;
|
||||
throw error;
|
||||
} finally { release(); }
|
||||
};
|
||||
const dependencies: MonitorWebhookDependencies = {
|
||||
endpoints: [endpoint, { ...endpoint, endpointId: "cd".repeat(32) }],
|
||||
tenantTransaction, now: () => now, openSecret: vault.open.bind(vault),
|
||||
};
|
||||
const monitor = new LiveMonitorService({ tenantTransaction });
|
||||
const service = () => new MonitorWebhookService(monitor, dependencies);
|
||||
return { service, dependencies, monitor, row, journal, audits: () => audits,
|
||||
clock: (time: number) => { now = time; }, fail: (value: boolean) => { failUpdate = value; }, revoke: () => { revoked = true; } };
|
||||
}
|
||||
|
||||
describe("monitor webhook authentication and ingest", () => {
|
||||
it("verifies exact raw bytes, event id and timestamp in constant-time HMAC", () => {
|
||||
const raw = Buffer.from(JSON.stringify(event));
|
||||
expect(() => verifyMonitorSignature(raw, signed(raw), secret, epoch)).not.toThrow();
|
||||
expect(() => verifyMonitorSignature(raw, signed(raw), Buffer.alloc(32, 43), epoch)).toThrow(UnauthorizedException);
|
||||
expect(() => verifyMonitorSignature(Buffer.concat([raw, Buffer.from(" ")]), signed(raw), secret, epoch)).toThrow(UnauthorizedException);
|
||||
expect(() => verifyMonitorSignature(raw, { ...signed(raw), eventId: "event-2" }, secret, epoch)).toThrow(UnauthorizedException);
|
||||
expect(() => verifyMonitorSignature(raw, { ...signed(raw), timestamp: new Date(epoch + 1000).toISOString() }, secret, epoch)).toThrow(UnauthorizedException);
|
||||
expect(() => verifyMonitorSignature(raw, { ...signed(raw), signature: "sha256=bad" }, secret, epoch)).toThrow(UnauthorizedException);
|
||||
});
|
||||
|
||||
it.each([-1, 1])("rejects signed timestamps outside the window in direction %s", (direction) => {
|
||||
const raw = Buffer.from(JSON.stringify(event));
|
||||
expect(() => verifyMonitorSignature(raw, signed(raw, "event-1", epoch + direction * (WEBHOOK_WINDOW_MS + 1)), secret, epoch)).toThrow(UnauthorizedException);
|
||||
expect(() => verifyMonitorSignature(raw, signed(raw, "event-1", epoch + direction * WEBHOOK_WINDOW_MS), secret, epoch)).not.toThrow();
|
||||
});
|
||||
|
||||
it("folds authenticated events through the existing service and reducer; replay survives service recreation", async () => {
|
||||
const f = await fixture(); const raw = Buffer.from(JSON.stringify(event));
|
||||
expect((await f.service().ingest(endpointId, raw, signed(raw))).state).toBe("PROBING");
|
||||
await expect(f.service().ingest(endpointId, raw, signed(raw))).rejects.toBeInstanceOf(ConflictException);
|
||||
expect(f.row.version).toBe(1); expect(f.audits()).toBe(1); expect(f.journal.size).toBe(1);
|
||||
});
|
||||
|
||||
it("retains dedupe across the full early-to-late timestamp acceptance span", async () => {
|
||||
const f = await fixture(); const raw = Buffer.from(JSON.stringify(event));
|
||||
f.clock(epoch - WEBHOOK_WINDOW_MS);
|
||||
await f.service().ingest(endpointId, raw, signed(raw));
|
||||
f.clock(epoch + 4 * 60 * 1000);
|
||||
await expect(f.service().ingest(endpointId, raw, signed(raw))).rejects.toBeInstanceOf(ConflictException);
|
||||
expect(f.row.version).toBe(1);
|
||||
});
|
||||
|
||||
it("scopes event ids per endpoint, including under concurrent delivery", async () => {
|
||||
const f = await fixture(); const raw = Buffer.from(JSON.stringify(event));
|
||||
const results = await Promise.allSettled([f.service().ingest(endpointId, raw, signed(raw)), f.service().ingest(endpointId, raw, signed(raw))]);
|
||||
expect(results.filter((result) => result.status === "fulfilled")).toHaveLength(1);
|
||||
await f.service().ingest("cd".repeat(32), raw, signed(raw));
|
||||
expect(f.journal.size).toBe(2); expect(f.row.version).toBe(1); // same-tier stale timestamp is still folded only once
|
||||
});
|
||||
|
||||
it.each(["workspace_id", "connection_id"])("rejects a source owned by a different %s before journaling", async (column) => {
|
||||
const f = await fixture(); f.row[column] = "55555555-5555-4555-8555-555555555555";
|
||||
const raw = Buffer.from(JSON.stringify(event));
|
||||
await expect(f.service().ingest(endpointId, raw, signed(raw))).rejects.toBeInstanceOf(NotFoundException);
|
||||
expect(f.journal.size).toBe(0); expect(f.row.version).toBe(0);
|
||||
});
|
||||
|
||||
it.each([{ subscriptionId: "other" }, { workspaceId: "other" }, { connectionId: "other" }])("rejects provider binding mismatch %j", async (binding) => {
|
||||
const f = await fixture(); const raw = Buffer.from(JSON.stringify({ ...event, ...binding }));
|
||||
await expect(f.service().ingest(endpointId, raw, signed(raw))).rejects.toBeInstanceOf(UnauthorizedException);
|
||||
expect(f.journal.size).toBe(0); expect(f.row.version).toBe(0);
|
||||
});
|
||||
|
||||
it("rolls back the replay claim when state persistence fails", async () => {
|
||||
const f = await fixture(); const raw = Buffer.from(JSON.stringify(event)); f.fail(true);
|
||||
await expect(f.service().ingest(endpointId, raw, signed(raw))).rejects.toThrow("database update failed");
|
||||
expect(f.journal.size).toBe(0); f.fail(false);
|
||||
expect((await f.service().ingest(endpointId, raw, signed(raw))).version).toBe(1);
|
||||
});
|
||||
|
||||
it("rejects unknown endpoints, revoked credentials, malformed JSON and bad authentication without writes", async () => {
|
||||
const f = await fixture(); const raw = Buffer.from(JSON.stringify(event));
|
||||
await expect(f.service().ingest("unknown", raw, signed(raw))).rejects.toBeInstanceOf(NotFoundException);
|
||||
await expect(f.service().ingest(endpointId, raw, {})).rejects.toBeInstanceOf(UnauthorizedException);
|
||||
const malformed = Buffer.from("{");
|
||||
await expect(f.service().ingest(endpointId, malformed, signed(malformed))).rejects.toBeInstanceOf(BadRequestException);
|
||||
f.revoke();
|
||||
await expect(f.service().ingest(endpointId, raw, signed(raw))).rejects.toBeInstanceOf(UnauthorizedException);
|
||||
expect(f.journal.size).toBe(0); expect(f.row.version).toBe(0);
|
||||
});
|
||||
|
||||
it("requires valid and unique server-controlled endpoint mappings", () => {
|
||||
expect(parseWebhookEndpoints(JSON.stringify([endpoint]))).toEqual([endpoint]);
|
||||
expect(() => parseWebhookEndpoints(JSON.stringify([endpoint, endpoint]))).toThrow("duplicate");
|
||||
expect(() => parseWebhookEndpoints(JSON.stringify([{ ...endpoint, endpointId: "guessable" }]))).toThrow("invalid");
|
||||
});
|
||||
});
|
||||
|
||||
describe("monitor webhook raw HTTP route", () => {
|
||||
let app: INestApplication | undefined;
|
||||
afterEach(async () => { await app?.close(); app = undefined; });
|
||||
|
||||
it("captures whitespace before Nest JSON parsing, needs no browser session, rejects replay/tampering/oversize", async () => {
|
||||
const f = await fixture();
|
||||
app = await NestFactory.create({ module: LiveMonitorModule, providers: [
|
||||
{ provide: LIVE_MONITOR_DEPENDENCIES, useValue: { tenantTransaction: f.dependencies.tenantTransaction } },
|
||||
{ provide: MONITOR_WEBHOOK_DEPENDENCIES, useValue: f.dependencies },
|
||||
{ provide: MEDIA_POLLER_DEPENDENCIES, useValue: { enabled: false } },
|
||||
] }, { logger: false, abortOnError: false });
|
||||
await app.init();
|
||||
const body = ` ${JSON.stringify(event, null, 2)}\n`;
|
||||
const auth = signed(Buffer.from(body));
|
||||
const headers = { "content-type": "application/json", "x-monitor-signature": auth.signature!,
|
||||
"x-monitor-timestamp": auth.timestamp!, "x-monitor-event-id": auth.eventId! };
|
||||
const post = (value: string) => injectHttp(app!.getHttpAdapter().getInstance(), `http://api.test/live-monitor/webhooks/${endpointId}`,
|
||||
{ method: "POST", body: value, headers });
|
||||
expect((await post(body)).status).toBe(200);
|
||||
expect((await post(body)).status).toBe(409);
|
||||
expect((await post(`${body} `)).status).toBe(401);
|
||||
expect((await post(" ".repeat(WEBHOOK_MAX_BYTES + 1))).status).toBe(413);
|
||||
expect(f.row.version).toBe(1);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue