Tactical Media Broadcast Hub — initial release
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
This commit is contained in:
commit
aa5e262f3a
880 changed files with 111042 additions and 0 deletions
75
apps/scheduler/src/director/adapters.ts
Normal file
75
apps/scheduler/src/director/adapters.ts
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
/**
|
||||
* Port adapters for the scheduler side of the director wiring (WP10).
|
||||
*
|
||||
* The engine's ports (director.activity.ts) speak engine types; the API
|
||||
* answers in transport shapes (ports.ts). These adapters bridge them:
|
||||
* sources are converted with the same semantics engine.sourceInput uses, and
|
||||
* the desired-state calls are forwarded to the API's internal endpoints.
|
||||
*/
|
||||
import type { DirectorSourceInput } from "./engine.js";
|
||||
import type { ChannelRef, ChannelSnapshot, DesiredFence, DesiredStatePort, DirectorStatePort, ProgramSwitchCommand } from "./director.activity.js";
|
||||
import type { DirectorApi, DirectorApplyRequest, DirectorChannelSource, DirectorStateRead } from "./ports.js";
|
||||
|
||||
/** Engine source input from the transport shape: the shapes agree field-for-field. */
|
||||
export function engineSource(source: DirectorChannelSource): DirectorSourceInput {
|
||||
return { ...source };
|
||||
}
|
||||
|
||||
/** Transport snapshot -> engine snapshot. */
|
||||
export function engineSnapshot(read: DirectorStateRead & { kind: "current" }): ChannelSnapshot {
|
||||
return {
|
||||
kind: "current",
|
||||
desiredVersion: read.desiredVersion,
|
||||
authzEpoch: read.authzEpoch,
|
||||
sources: read.sources.map(engineSource),
|
||||
rules: read.rules,
|
||||
policy: read.policy,
|
||||
fallbackSceneId: read.fallbackSceneId,
|
||||
override: read.override,
|
||||
principalPermissions: read.principalPermissions,
|
||||
};
|
||||
}
|
||||
|
||||
/** DirectorStatePort backed by the API's internal state endpoint. */
|
||||
export class HttpDirectorStatePort implements DirectorStatePort {
|
||||
constructor(private readonly api: DirectorApi) {}
|
||||
|
||||
async read(ref: ChannelRef): Promise<ChannelSnapshot> {
|
||||
const read = await this.api.channelState(ref);
|
||||
return read.kind === "superseded" ? { kind: "superseded", code: "SUPERSEDED" } : engineSnapshot(read);
|
||||
}
|
||||
}
|
||||
|
||||
/** DesiredStatePort backed by the API's apply/audit endpoints. */
|
||||
export class HttpDesiredStatePort implements DesiredStatePort {
|
||||
constructor(private readonly api: DirectorApi) {}
|
||||
|
||||
async fence(ref: ChannelRef): Promise<DesiredFence | undefined> {
|
||||
return this.api.fence(ref);
|
||||
}
|
||||
|
||||
async findCommand(_ref: ChannelRef, _commandId: string): Promise<{ desiredVersion: number } | undefined> {
|
||||
// Idempotency is the API's job: the apply endpoint records the commandId in
|
||||
// the production_command audit chain and replays the stored receipt. A
|
||||
// pre-check here would double the round trips for no extra safety.
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async writeProgram(command: ProgramSwitchCommand): Promise<{ kind: "applied"; desiredVersion: number } | { kind: "conflict" }> {
|
||||
const body: DirectorApplyRequest = {
|
||||
commandId: command.commandId,
|
||||
expectedDesiredVersion: command.expectedDesiredVersion,
|
||||
authzEpoch: command.authzEpoch,
|
||||
program: command.program,
|
||||
layout: command.layout,
|
||||
reasons: command.reasons,
|
||||
...(command.releaseOverride === undefined ? {} : { releaseOverride: command.releaseOverride }),
|
||||
};
|
||||
const result = await this.api.applyProgramSwitch(command.ref, body);
|
||||
return result.kind === "conflict" ? { kind: "conflict" } : { kind: "applied", desiredVersion: result.desiredVersion };
|
||||
}
|
||||
|
||||
async audit(ref: ChannelRef, event: { commandId: string; outcome: "SUPERSEDED" | "AUTHZ_REVOKED"; reason: string }): Promise<void> {
|
||||
await this.api.audit(ref, event);
|
||||
}
|
||||
}
|
||||
144
apps/scheduler/src/director/config.ts
Normal file
144
apps/scheduler/src/director/config.ts
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
import type { AutoDirectorRule, DirectorAction, DirectorCondition, SceneLayout, SwitchPolicy } from "@tactical-media-broadcast-hub/core";
|
||||
|
||||
/**
|
||||
* Director channel configuration (WP10 wiring slice).
|
||||
*
|
||||
* `DIRECTOR_CHANNELS` is a server-controlled JSON array — the same pattern as
|
||||
* MEDIA_POLLER_SOURCES / MEDIA_CONTROL_ASSIGNMENTS. Rule CRUD is a later
|
||||
* work package; this slice makes the configured set live, so the array is the
|
||||
* authority for which channels get an evaluator, which rules it evaluates and
|
||||
* which generation fences its writes.
|
||||
*
|
||||
* The API side parses the same document (apps/api/src/director/config.ts) to
|
||||
* answer snapshot/fence reads, so the two parsers must stay shape-compatible:
|
||||
* both are tested against the same fixtures.
|
||||
*/
|
||||
export interface DirectorChannelConfig {
|
||||
readonly workspaceId: string;
|
||||
readonly channelId: string;
|
||||
/** Assignment generation; a newer value supersedes running evaluators. */
|
||||
readonly generation: string;
|
||||
readonly rules: readonly AutoDirectorRule[];
|
||||
readonly policy: SwitchPolicy | undefined;
|
||||
readonly fallbackSceneId: string | null;
|
||||
}
|
||||
|
||||
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
||||
const GENERATION = /^[\x21-\x7e]{1,64}$/;
|
||||
const POLICY_NAMES: readonly SwitchPolicy["name"][] = [
|
||||
"highest_priority", "oldest_live", "newest_live", "round_robin", "operator_required",
|
||||
"picture_in_picture", "two_up", "split_screen", "grid",
|
||||
];
|
||||
const LAYOUTS: readonly SceneLayout[] = ["single", "pip", "two_up", "split_horizontal", "split_vertical", "grid", "custom"];
|
||||
const CONDITION_FIELDS: readonly DirectorCondition["field"][] = ["live_source_count", "max_priority", "source_state"];
|
||||
const CONDITION_OPS: readonly DirectorCondition["op"][] = [">=", "<=", "==", ">"];
|
||||
const SELECTIONS: readonly DirectorAction["selection"][] = ["top_priority", "all_live", "oldest_live", "explicit"];
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function uuid(value: unknown, field: string): string {
|
||||
if (typeof value !== "string" || !UUID.test(value)) throw new Error(`${field} must be a UUID`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function nonNegativeInt(value: unknown, field: string): number {
|
||||
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) throw new Error(`${field} must be a non-negative integer`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseCondition(value: unknown, field: string): DirectorCondition {
|
||||
if (!isRecord(value)) throw new Error(`${field} must be an object`);
|
||||
const conditionField = value.field;
|
||||
const op = value.op;
|
||||
if (typeof conditionField !== "string" || !CONDITION_FIELDS.includes(conditionField as DirectorCondition["field"])) {
|
||||
throw new Error(`${field}.field must be one of ${CONDITION_FIELDS.join(", ")}`);
|
||||
}
|
||||
if (typeof op !== "string" || !CONDITION_OPS.includes(op as DirectorCondition["op"])) {
|
||||
throw new Error(`${field}.op must be one of ${CONDITION_OPS.join(", ")}`);
|
||||
}
|
||||
const ruleValue = value.value;
|
||||
if (typeof ruleValue !== "number" && typeof ruleValue !== "string") throw new Error(`${field}.value must be a number or string`);
|
||||
return { field: conditionField as DirectorCondition["field"], op: op as DirectorCondition["op"], value: ruleValue };
|
||||
}
|
||||
|
||||
function parseAction(value: unknown, field: string): DirectorAction {
|
||||
if (!isRecord(value)) throw new Error(`${field} must be an object`);
|
||||
const selection = value.selection;
|
||||
if (typeof selection !== "string" || !SELECTIONS.includes(selection as DirectorAction["selection"])) {
|
||||
throw new Error(`${field}.selection must be one of ${SELECTIONS.join(", ")}`);
|
||||
}
|
||||
const layout = value.layout;
|
||||
if (typeof layout !== "string" || !LAYOUTS.includes(layout as SceneLayout)) {
|
||||
throw new Error(`${field}.layout must be one of ${LAYOUTS.join(", ")}`);
|
||||
}
|
||||
let sourceIds: string[] | undefined;
|
||||
if (value.sourceIds !== undefined) {
|
||||
if (!Array.isArray(value.sourceIds)) throw new Error(`${field}.sourceIds must be an array`);
|
||||
sourceIds = value.sourceIds.map((id, i) => uuid(id, `${field}.sourceIds[${i}]`));
|
||||
if (new Set(sourceIds).size !== sourceIds.length) throw new Error(`${field}.sourceIds must be unique`);
|
||||
}
|
||||
if (selection === "explicit" && (sourceIds === undefined || sourceIds.length === 0)) {
|
||||
throw new Error(`${field}.selection explicit requires sourceIds`);
|
||||
}
|
||||
return { layout: layout as SceneLayout, selection: selection as DirectorAction["selection"], ...(sourceIds === undefined ? {} : { sourceIds }) };
|
||||
}
|
||||
|
||||
function parseRule(value: unknown, channelId: string, index: number): AutoDirectorRule {
|
||||
const field = `DIRECTOR_CHANNELS[${index}].rules[${index}]`;
|
||||
if (!isRecord(value)) throw new Error(`${field} must be an object`);
|
||||
if (value.virtualChannelId !== undefined && value.virtualChannelId !== channelId) {
|
||||
throw new Error(`${field}.virtualChannelId must match the channel it is configured on`);
|
||||
}
|
||||
return {
|
||||
id: uuid(value.id, `${field}.id`),
|
||||
virtualChannelId: channelId,
|
||||
priority: nonNegativeInt(value.priority ?? 0, `${field}.priority`),
|
||||
condition: parseCondition(value.condition, `${field}.condition`),
|
||||
action: parseAction(value.action, `${field}.action`),
|
||||
};
|
||||
}
|
||||
|
||||
function parsePolicy(value: unknown, field: string): SwitchPolicy {
|
||||
if (!isRecord(value)) throw new Error(`${field} must be an object`);
|
||||
const name = value.name;
|
||||
if (typeof name !== "string" || !POLICY_NAMES.includes(name as SwitchPolicy["name"])) {
|
||||
throw new Error(`${field}.name must be one of ${POLICY_NAMES.join(", ")}`);
|
||||
}
|
||||
return {
|
||||
name: name as SwitchPolicy["name"],
|
||||
liveConfirmations: nonNegativeInt(value.liveConfirmations ?? 2, `${field}.liveConfirmations`),
|
||||
offlineConfirmations: nonNegativeInt(value.offlineConfirmations ?? 3, `${field}.offlineConfirmations`),
|
||||
noMediaSeconds: nonNegativeInt(value.noMediaSeconds ?? 10, `${field}.noMediaSeconds`),
|
||||
minHoldMs: nonNegativeInt(value.minHoldMs ?? 2_000, `${field}.minHoldMs`),
|
||||
switchCooldownMs: nonNegativeInt(value.switchCooldownMs ?? 10_000, `${field}.switchCooldownMs`),
|
||||
};
|
||||
}
|
||||
|
||||
/** Parse and validate DIRECTOR_CHANNELS; throws with a field path on the first problem. */
|
||||
export function parseDirectorChannels(raw: string | undefined): DirectorChannelConfig[] {
|
||||
if (raw === undefined || raw.trim() === "") return [];
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (!Array.isArray(parsed)) throw new Error("DIRECTOR_CHANNELS must be a JSON array");
|
||||
return parsed.map((entry, index) => {
|
||||
const field = `DIRECTOR_CHANNELS[${index}]`;
|
||||
if (!isRecord(entry)) throw new Error(`${field} must be an object`);
|
||||
const channelId = uuid(entry.channelId, `${field}.channelId`);
|
||||
const generation = entry.generation;
|
||||
if (typeof generation !== "string" || !GENERATION.test(generation)) {
|
||||
throw new Error(`${field}.generation must be 1-64 printable characters`);
|
||||
}
|
||||
const rules = Array.isArray(entry.rules) ? entry.rules.map((rule, i) => parseRule(rule, channelId, i)) : [];
|
||||
const fallbackSceneId = entry.fallbackSceneId === undefined || entry.fallbackSceneId === null
|
||||
? null : uuid(entry.fallbackSceneId, `${field}.fallbackSceneId`);
|
||||
return {
|
||||
workspaceId: uuid(entry.workspaceId, `${field}.workspaceId`),
|
||||
channelId,
|
||||
generation,
|
||||
rules,
|
||||
policy: entry.policy === undefined ? undefined : parsePolicy(entry.policy, `${field}.policy`),
|
||||
fallbackSceneId,
|
||||
};
|
||||
});
|
||||
}
|
||||
75
apps/scheduler/src/director/director.activity.test.ts
Normal file
75
apps/scheduler/src/director/director.activity.test.ts
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
import { Client, WorkflowExecutionAlreadyStartedError } from "@temporalio/client";
|
||||
import { DirectorActivity, unwiredDesiredPort, type DesiredFence, type DesiredStatePort, type ProgramSwitchCommand } from "./director.activity.js";
|
||||
import { ensureDirector, evaluatorWorkflowId } from "./starter.js";
|
||||
|
||||
const ref = { workspaceId: "ws", channelId: "vc", generation: "g2" };
|
||||
const command: ProgramSwitchCommand = { ref, commandId: "vc:g2:1", expectedDesiredVersion: 7, authzEpoch: 3, program: { kind: "sources", sourceIds: ["a"] }, layout: "single", reasons: [] };
|
||||
function port(fence: DesiredFence | undefined, over: Partial<DesiredStatePort> = {}): DesiredStatePort & { writeProgram: ReturnType<typeof vi.fn>; audit: ReturnType<typeof vi.fn> } {
|
||||
return { fence: async () => fence, findCommand: async () => undefined, writeProgram: vi.fn(async () => ({ kind: "applied" as const, desiredVersion: 8 })), audit: vi.fn(async () => undefined), ...over } as never;
|
||||
}
|
||||
const current: DesiredFence = { generation: "g2", desiredVersion: 7, authzEpoch: 3, directorAuthorized: true };
|
||||
const activity = (desired: DesiredStatePort): DirectorActivity => new DirectorActivity(undefined, desired);
|
||||
|
||||
describe("applyProgramSwitch generation fences", () => {
|
||||
it("applies under a current fence", async () => {
|
||||
const desired = port(current);
|
||||
await expect(activity(desired).applyProgramSwitch(command)).resolves.toEqual({ kind: "applied", desiredVersion: 8 });
|
||||
expect(desired.writeProgram).toHaveBeenCalledWith(command);
|
||||
});
|
||||
it("a retried command with a recorded commandId is a no-op duplicate", async () => {
|
||||
const desired = port(current, { findCommand: async () => ({ desiredVersion: 8 }) });
|
||||
await expect(activity(desired).applyProgramSwitch(command)).resolves.toEqual({ kind: "duplicate", desiredVersion: 8 });
|
||||
expect(desired.writeProgram).not.toHaveBeenCalled();
|
||||
});
|
||||
it("a superseded channel generation is audited SUPERSEDED and never writes", async () => {
|
||||
for (const fence of [{ ...current, generation: "g3" }, undefined]) {
|
||||
const desired = port(fence);
|
||||
await expect(activity(desired).applyProgramSwitch(command)).resolves.toEqual({ kind: "generation_superseded", code: "SUPERSEDED" });
|
||||
expect(desired.writeProgram).not.toHaveBeenCalled();
|
||||
expect(desired.audit).toHaveBeenCalledWith(ref, expect.objectContaining({ outcome: "SUPERSEDED" }));
|
||||
}
|
||||
});
|
||||
it("a stale authzEpoch whose principal lost director.* is audited AUTHZ_REVOKED", async () => {
|
||||
const desired = port({ ...current, authzEpoch: 4, directorAuthorized: false });
|
||||
await expect(activity(desired).applyProgramSwitch(command)).resolves.toEqual({ kind: "authz_revoked", code: "AUTHZ_REVOKED" });
|
||||
expect(desired.writeProgram).not.toHaveBeenCalled();
|
||||
expect(desired.audit).toHaveBeenCalledWith(ref, expect.objectContaining({ outcome: "AUTHZ_REVOKED" }));
|
||||
});
|
||||
it("an unrelated epoch bump with the principal still authorized proceeds", async () => {
|
||||
await expect(activity(port({ ...current, authzEpoch: 4 })).applyProgramSwitch(command)).resolves.toMatchObject({ kind: "applied" });
|
||||
});
|
||||
it("a superseded desiredVersion or lost CAS is SUPERSEDED", async () => {
|
||||
const stale = port({ ...current, desiredVersion: 9 });
|
||||
await expect(activity(stale).applyProgramSwitch(command)).resolves.toEqual({ kind: "superseded", code: "SUPERSEDED" });
|
||||
expect(stale.writeProgram).not.toHaveBeenCalled();
|
||||
const raced = port(current, { writeProgram: vi.fn(async () => ({ kind: "conflict" as const })) });
|
||||
await expect(activity(raced).applyProgramSwitch(command)).resolves.toEqual({ kind: "superseded", code: "SUPERSEDED" });
|
||||
});
|
||||
it("is inert until the go-live slice wires the port", async () => {
|
||||
await expect(activity(unwiredDesiredPort).applyProgramSwitch(command)).rejects.toMatchObject({ type: "DIRECTOR_NOT_WIRED", nonRetryable: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe("ensureDirector", () => {
|
||||
const already = (): WorkflowExecutionAlreadyStartedError => new WorkflowExecutionAlreadyStartedError("existing", evaluatorWorkflowId(ref), "autoDirectorEvaluator");
|
||||
it("starts one evaluator per channel", async () => {
|
||||
const start = vi.fn(async () => ({}));
|
||||
await expect(ensureDirector({ workflow: { start } } as unknown as Client, ref, "q")).resolves.toBe("started");
|
||||
expect(start).toHaveBeenCalledWith("autoDirectorEvaluator", expect.objectContaining({ workflowId: "auto-director-ws-vc", args: [{ ref }] }));
|
||||
});
|
||||
it("nudges a running evaluator instead of starting a duplicate", async () => {
|
||||
const start = vi.fn().mockRejectedValue(already());
|
||||
const signal = vi.fn(async () => undefined);
|
||||
const client = { workflow: { start, getHandle: () => ({ describe: async () => ({ status: { name: "RUNNING" } }), signal }) } } as unknown as Client;
|
||||
await expect(ensureDirector(client, ref, "q")).resolves.toBe("running");
|
||||
expect(start).toHaveBeenCalledTimes(1);
|
||||
expect(signal).toHaveBeenCalledWith("sourceChanged", { sourceId: "*", version: 0 });
|
||||
});
|
||||
it("restarts once when the evaluator closed between start and describe", async () => {
|
||||
const start = vi.fn().mockRejectedValueOnce(already()).mockResolvedValue({});
|
||||
const client = { workflow: { start, getHandle: () => ({ describe: async () => ({ status: { name: "COMPLETED" } }), signal: vi.fn() }) } } as unknown as Client;
|
||||
await expect(ensureDirector(client, ref, "q")).resolves.toBe("restarted");
|
||||
expect(start).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
103
apps/scheduler/src/director/director.activity.ts
Normal file
103
apps/scheduler/src/director/director.activity.ts
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
import { ApplicationFailure } from "@temporalio/common";
|
||||
import type { AutoDirectorRule, DirectorOverrideReleaseReason, SceneLayout, SwitchPolicy } from "@tactical-media-broadcast-hub/core";
|
||||
import type { DirectorOverride, DirectorProgram, DirectorSourceInput } from "./engine.js";
|
||||
|
||||
/** Workflow identity. `generation` is the channel assignment generation; a newer one supersedes this evaluator. */
|
||||
export interface ChannelRef { workspaceId: string; channelId: string; generation: string }
|
||||
|
||||
/** Fresh monitor + desired-state snapshot; signals carry only hints, every evaluation re-reads this. */
|
||||
export type ChannelSnapshot =
|
||||
| {
|
||||
kind: "current";
|
||||
desiredVersion: number;
|
||||
authzEpoch: number;
|
||||
sources: DirectorSourceInput[];
|
||||
rules: AutoDirectorRule[];
|
||||
policy: SwitchPolicy;
|
||||
fallbackSceneId: string | null;
|
||||
override: DirectorOverride | null;
|
||||
principalPermissions: Record<string, string[]>;
|
||||
}
|
||||
| { kind: "superseded"; code: "SUPERSEDED" };
|
||||
|
||||
export interface ProgramSwitchCommand {
|
||||
ref: ChannelRef;
|
||||
/** Deterministic per workflow decision: retries and replays reuse it (idempotency key). */
|
||||
commandId: string;
|
||||
expectedDesiredVersion: number;
|
||||
authzEpoch: number;
|
||||
program: DirectorProgram;
|
||||
layout: SceneLayout;
|
||||
reasons: string[];
|
||||
releaseOverride?: DirectorOverrideReleaseReason;
|
||||
}
|
||||
|
||||
export type SwitchOutcome =
|
||||
| { kind: "applied" | "duplicate"; desiredVersion: number }
|
||||
| { kind: "superseded"; code: "SUPERSEDED" }
|
||||
| { kind: "generation_superseded"; code: "SUPERSEDED" }
|
||||
| { kind: "authz_revoked"; code: "AUTHZ_REVOKED" };
|
||||
|
||||
export interface DesiredFence {
|
||||
generation: string;
|
||||
desiredVersion: number;
|
||||
authzEpoch: number;
|
||||
/** The evaluator's recorded principal still holds `director.*` under the current epoch. */
|
||||
directorAuthorized: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ports owned by the media-control go-live slice. `writeProgram` must be one
|
||||
* transaction: CAS on `expectedDesiredVersion`, desired row + `desiredVersion`
|
||||
* bump + outbox row keyed by `commandId`.
|
||||
*/
|
||||
export interface DirectorStatePort { read(ref: ChannelRef): Promise<ChannelSnapshot> }
|
||||
export interface DesiredStatePort {
|
||||
fence(ref: ChannelRef): Promise<DesiredFence | undefined>;
|
||||
findCommand(ref: ChannelRef, commandId: string): Promise<{ desiredVersion: number } | undefined>;
|
||||
writeProgram(command: ProgramSwitchCommand): Promise<{ kind: "applied"; desiredVersion: number } | { kind: "conflict" }>;
|
||||
audit(ref: ChannelRef, event: { commandId: string; outcome: "SUPERSEDED" | "AUTHZ_REVOKED"; reason: string }): Promise<void>;
|
||||
}
|
||||
|
||||
function notWired(): never {
|
||||
throw ApplicationFailure.nonRetryable("director desired-state port is not wired", "DIRECTOR_NOT_WIRED");
|
||||
}
|
||||
/** Placeholder until the go-live slice provides the media-control desired-state adapter. */
|
||||
export const unwiredStatePort: DirectorStatePort = { read: async () => notWired() };
|
||||
export const unwiredDesiredPort: DesiredStatePort = {
|
||||
fence: async () => notWired(), findCommand: async () => notWired(), writeProgram: async () => notWired(), audit: async () => notWired(),
|
||||
};
|
||||
|
||||
export class DirectorActivity {
|
||||
constructor(private readonly state: DirectorStatePort = unwiredStatePort, private readonly desired: DesiredStatePort = unwiredDesiredPort) {}
|
||||
|
||||
async readChannelState(ref: ChannelRef): Promise<ChannelSnapshot> {
|
||||
return this.state.read(ref);
|
||||
}
|
||||
|
||||
/** Generation fences are re-checked here, at emission time, never trusted from the workflow. */
|
||||
async applyProgramSwitch(command: ProgramSwitchCommand): Promise<SwitchOutcome> {
|
||||
const { ref, commandId } = command;
|
||||
const prior = await this.desired.findCommand(ref, commandId);
|
||||
if (prior) return { kind: "duplicate", desiredVersion: prior.desiredVersion };
|
||||
const fence = await this.desired.fence(ref);
|
||||
if (!fence || fence.generation !== ref.generation) {
|
||||
await this.desired.audit(ref, { commandId, outcome: "SUPERSEDED", reason: "channel generation superseded" });
|
||||
return { kind: "generation_superseded", code: "SUPERSEDED" };
|
||||
}
|
||||
if (fence.authzEpoch !== command.authzEpoch && !fence.directorAuthorized) {
|
||||
await this.desired.audit(ref, { commandId, outcome: "AUTHZ_REVOKED", reason: "director principal lost director.*" });
|
||||
return { kind: "authz_revoked", code: "AUTHZ_REVOKED" };
|
||||
}
|
||||
if (fence.desiredVersion !== command.expectedDesiredVersion) {
|
||||
await this.desired.audit(ref, { commandId, outcome: "SUPERSEDED", reason: "desiredVersion advanced since snapshot" });
|
||||
return { kind: "superseded", code: "SUPERSEDED" };
|
||||
}
|
||||
const written = await this.desired.writeProgram(command);
|
||||
if (written.kind === "conflict") {
|
||||
await this.desired.audit(ref, { commandId, outcome: "SUPERSEDED", reason: "desiredVersion CAS lost" });
|
||||
return { kind: "superseded", code: "SUPERSEDED" };
|
||||
}
|
||||
return written;
|
||||
}
|
||||
}
|
||||
54
apps/scheduler/src/director/discovery.ts
Normal file
54
apps/scheduler/src/director/discovery.ts
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
/**
|
||||
* Director evaluator discovery (WP10 wiring slice).
|
||||
*
|
||||
* Mirrors sweepDue: starts are repair hints, and ensureDirector handles
|
||||
* AlreadyStarted itself (nudge a running evaluator, restart a closed one).
|
||||
* A discovery pass that fails must never take the scheduler down, so every
|
||||
* channel error is logged and skipped.
|
||||
*/
|
||||
import type { Client } from "@temporalio/client";
|
||||
import { ensureDirector } from "./starter.js";
|
||||
import { parseDirectorChannels, type DirectorChannelConfig } from "./config.js";
|
||||
|
||||
export interface DirectorSweepOutcome {
|
||||
started: number;
|
||||
running: number;
|
||||
restarted: number;
|
||||
}
|
||||
|
||||
/** Configured channels whose generation (and therefore evaluator identity) is still current. */
|
||||
export function currentChannels(channels: readonly DirectorChannelConfig[]): DirectorChannelConfig[] {
|
||||
const latest = new Map<string, DirectorChannelConfig>();
|
||||
for (const channel of channels) {
|
||||
const key = `${channel.workspaceId}:${channel.channelId}`;
|
||||
const existing = latest.get(key);
|
||||
// Generations are opaque tokens; the highest wins only when comparable, so
|
||||
// keep the last-declared entry for equal tokens (config is authoritative).
|
||||
if (existing === undefined || existing.generation !== channel.generation) latest.set(key, channel);
|
||||
}
|
||||
return [...latest.values()];
|
||||
}
|
||||
|
||||
/** Start (or nudge) one evaluator per configured channel. */
|
||||
export async function sweepDirectors(
|
||||
client: Client,
|
||||
options: { taskQueue: string; channels: readonly DirectorChannelConfig[] },
|
||||
): Promise<DirectorSweepOutcome> {
|
||||
const outcome: DirectorSweepOutcome = { started: 0, running: 0, restarted: 0 };
|
||||
for (const channel of currentChannels(options.channels)) {
|
||||
const ref = { workspaceId: channel.workspaceId, channelId: channel.channelId, generation: channel.generation };
|
||||
try {
|
||||
const result = await ensureDirector(client, ref, options.taskQueue);
|
||||
outcome[result] += 1;
|
||||
} catch {
|
||||
// Temporal unavailable or the start refused: the next sweep retries.
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return outcome;
|
||||
}
|
||||
|
||||
/** Parse DIRECTOR_CHANNELS, failing startup loudly (a malformed doc must not silently disable the director). */
|
||||
export function directorChannelsFromEnv(env: NodeJS.ProcessEnv = process.env): DirectorChannelConfig[] {
|
||||
return parseDirectorChannels(env.DIRECTOR_CHANNELS);
|
||||
}
|
||||
279
apps/scheduler/src/director/engine.test.ts
Normal file
279
apps/scheduler/src/director/engine.test.ts
Normal file
|
|
@ -0,0 +1,279 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import type { AutoDirectorRule, DirectorCondition, DirectorAction, SwitchPolicy } from "@tactical-media-broadcast-hub/core";
|
||||
import { decide, type DecideInput, type Decision, type DirectorMemory, type DirectorOverride, type DirectorSourceInput, type LastSwitch } from "./engine.js";
|
||||
|
||||
const policy = (over: Partial<SwitchPolicy> = {}): SwitchPolicy => ({ name: "highest_priority", liveConfirmations: 2, offlineConfirmations: 3, noMediaSeconds: 10, minHoldMs: 2_000, switchCooldownMs: 10_000, ...over });
|
||||
const src = (id: string, priority: number, state: DirectorSourceInput["state"] = "LIVE", extra: Partial<DirectorSourceInput> = {}): DirectorSourceInput => ({ id, priority, enabled: true, autoSelect: true, state, ...extra });
|
||||
const confirmed = (...ids: string[]): DirectorMemory => ({ sources: Object.fromEntries(ids.map((id) => [id, { liveStreak: 2, offlineStreak: 0, confirmed: true, confirmedAtMs: 0 }])) });
|
||||
const rule = (id: string, priority: number, condition: DirectorCondition, action: DirectorAction): AutoDirectorRule => ({ id, virtualChannelId: "vc", priority, condition, action });
|
||||
const on = (ids: string[], atMs = 0, layout: LastSwitch["layout"] = "single", heldSinceMs = atMs): LastSwitch => ({ atMs, program: { kind: "sources", sourceIds: ids }, layout, heldSinceMs });
|
||||
const program = (decision: Decision): string[] => decision.program.kind === "sources" ? decision.program.sourceIds : [`FALLBACK:${decision.program.sceneId ?? ""}`];
|
||||
const base = (over: Partial<DecideInput>): DecideInput => ({ sources: [], rules: [], policy: policy(), now: 100_000, ...over });
|
||||
|
||||
/** Feed successive snapshots through decide, carrying memory and lastSwitch like the workflow does. */
|
||||
function run(steps: Array<{ now: number; sources: DirectorSourceInput[] }>, over: Partial<DecideInput> = {}): Decision[] {
|
||||
let memory: DirectorMemory | undefined = over.memory;
|
||||
let lastSwitch: LastSwitch | undefined = over.lastSwitch;
|
||||
return steps.map(({ now, sources }) => {
|
||||
const decision = decide({ ...base(over), now, sources, ...(memory ? { memory } : {}), ...(lastSwitch ? { lastSwitch } : {}) });
|
||||
memory = decision.memory; lastSwitch = decision.lastSwitch;
|
||||
return decision;
|
||||
});
|
||||
}
|
||||
|
||||
describe("rule evaluation", () => {
|
||||
const sources = [src("a", 10), src("b", 50), src("c", 30)];
|
||||
it("evaluates by priority desc and the first matching rule wins regardless of list order", () => {
|
||||
const rules = [
|
||||
rule("low", 1, { field: "live_source_count", op: ">=", value: 1 }, { layout: "single", selection: "top_priority" }),
|
||||
rule("high", 9, { field: "live_source_count", op: ">=", value: 3 }, { layout: "grid", selection: "all_live" }),
|
||||
rule("mid", 5, { field: "live_source_count", op: ">=", value: 2 }, { layout: "two_up", selection: "top_priority" }),
|
||||
];
|
||||
const decision = decide(base({ sources, rules, memory: confirmed("a", "b", "c") }));
|
||||
expect(decision.ruleId).toBe("high");
|
||||
expect(decision.layout).toBe("grid");
|
||||
expect(program(decision)).toEqual(["b", "c", "a"]);
|
||||
});
|
||||
it("breaks priority ties by rule id and skips non-matching rules", () => {
|
||||
const rules = [
|
||||
rule("z", 5, { field: "live_source_count", op: ">=", value: 1 }, { layout: "single", selection: "top_priority" }),
|
||||
rule("y", 5, { field: "live_source_count", op: ">=", value: 1 }, { layout: "pip", selection: "top_priority" }),
|
||||
rule("never", 99, { field: "live_source_count", op: ">", value: 3 }, { layout: "grid", selection: "all_live" }),
|
||||
];
|
||||
expect(decide(base({ sources, rules, memory: confirmed("a", "b", "c") })).ruleId).toBe("y");
|
||||
});
|
||||
it("supports every condition op on live_source_count and max_priority", () => {
|
||||
const pick = (condition: DirectorCondition): string | undefined => decide(base({ sources, memory: confirmed("a", "b", "c"),
|
||||
rules: [rule("r", 1, condition, { layout: "single", selection: "oldest_live" })] })).ruleId;
|
||||
expect(pick({ field: "live_source_count", op: "==", value: 3 })).toBe("r");
|
||||
expect(pick({ field: "live_source_count", op: "<=", value: 2 })).toBeUndefined();
|
||||
expect(pick({ field: "max_priority", op: ">", value: 49 })).toBe("r");
|
||||
expect(pick({ field: "max_priority", op: ">", value: 50 })).toBeUndefined();
|
||||
expect(pick({ field: "max_priority", op: ">=", value: "50" })).toBe("r");
|
||||
expect(pick({ field: "max_priority", op: "<=", value: 10 })).toBeUndefined();
|
||||
});
|
||||
it("matches source_state on the debounced state, globally or per source", () => {
|
||||
const action: DirectorAction = { layout: "single", selection: "explicit", sourceIds: ["c"] };
|
||||
const memory = confirmed("a", "c");
|
||||
const pick = (value: string, op: DirectorCondition["op"] = "=="): string | undefined =>
|
||||
decide(base({ sources, memory, rules: [rule("r", 1, { field: "source_state", op, value }, action)] })).ruleId;
|
||||
expect(pick("c:LIVE")).toBe("r");
|
||||
// b reports LIVE but is unconfirmed: debounced state is PROBING.
|
||||
expect(pick("b:LIVE")).toBeUndefined();
|
||||
expect(pick("b:PROBING")).toBe("r");
|
||||
expect(pick("LIVE")).toBe("r");
|
||||
expect(pick("LIVE", ">=")).toBeUndefined();
|
||||
});
|
||||
it("ignores rules for other channels when channelId is given", () => {
|
||||
const foreign = { ...rule("x", 9, { field: "live_source_count", op: ">=", value: 1 }, { layout: "pip", selection: "top_priority" }), virtualChannelId: "other" };
|
||||
expect(decide(base({ channelId: "vc", sources, rules: [foreign], memory: confirmed("a", "b") })).ruleId).toBeUndefined();
|
||||
});
|
||||
it("explicit sourceIds pick the listed sources, even lower priority or non-auto ones", () => {
|
||||
const decision = decide(base({ sources: [src("a", 10, "LIVE", { autoSelect: false }), src("b", 50)], memory: confirmed("a", "b"),
|
||||
rules: [rule("r", 1, { field: "live_source_count", op: ">=", value: 1 }, { layout: "single", selection: "explicit", sourceIds: ["a"] })] }));
|
||||
expect(program(decision)).toEqual(["a"]);
|
||||
});
|
||||
it("falls through an explicit rule whose sources are all dead", () => {
|
||||
const decision = decide(base({ sources: [src("a", 10, "OFFLINE"), src("b", 50)], memory: confirmed("b"),
|
||||
rules: [rule("r", 9, { field: "live_source_count", op: ">=", value: 1 }, { layout: "single", selection: "explicit", sourceIds: ["a"] })] }));
|
||||
expect(decision.ruleId).toBeUndefined();
|
||||
expect(program(decision)).toEqual(["b"]);
|
||||
expect(decision.reasons.join("\n")).toContain("falling through");
|
||||
});
|
||||
it("oldest_live rule selection orders by liveSince", () => {
|
||||
const decision = decide(base({ sources: [src("a", 99, "LIVE", { liveSinceMs: 5 }), src("b", 1, "LIVE", { liveSinceMs: 1 })], memory: confirmed("a", "b"),
|
||||
rules: [rule("r", 1, { field: "live_source_count", op: ">=", value: 1 }, { layout: "single", selection: "oldest_live" })] }));
|
||||
expect(program(decision)).toEqual(["b"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("selection policies", () => {
|
||||
const sources = [src("a", 10, "LIVE", { liveSinceMs: 300 }), src("b", 50, "LIVE", { liveSinceMs: 100 }), src("c", 30, "LIVE", { liveSinceMs: 200 }), src("d", 20, "LIVE", { liveSinceMs: 400 }), src("e", 5, "LIVE", { liveSinceMs: 500 })];
|
||||
const memory = confirmed("a", "b", "c", "d", "e");
|
||||
const pick = (name: SwitchPolicy["name"], over: Partial<DecideInput> = {}): Decision => decide(base({ sources, memory, policy: policy({ name }), ...over }));
|
||||
it("highest_priority", () => { expect(program(pick("highest_priority"))).toEqual(["b"]); });
|
||||
it("oldest_live", () => { expect(program(pick("oldest_live"))).toEqual(["b"]); expect(program(pick("oldest_live", { sources: sources.filter((s) => s.id !== "b") }))).toEqual(["c"]); });
|
||||
it("newest_live", () => { expect(program(pick("newest_live"))).toEqual(["e"]); });
|
||||
it("round_robin holds for minHold then rotates in a stable order", () => {
|
||||
const rr = policy({ name: "round_robin", minHoldMs: 30_000, switchCooldownMs: 0 });
|
||||
const decisions = run([0, 10_000, 30_000, 60_000, 90_000, 120_000, 150_000].map((now) => ({ now, sources })), { policy: rr, memory });
|
||||
expect(decisions.map((d) => program(d)[0])).toEqual(["a", "a", "b", "c", "d", "e", "a"]);
|
||||
});
|
||||
it("operator_required never auto-selects; keeps a healthy program; dead program -> fallback", () => {
|
||||
expect(program(pick("operator_required", { fallbackSceneId: "slate" }))).toEqual(["FALLBACK:slate"]);
|
||||
expect(pick("operator_required").reasons).toContain("operator_required: awaiting operator TAKE");
|
||||
expect(program(pick("operator_required", { lastSwitch: on(["e"]) }))).toEqual(["e"]);
|
||||
const dead = pick("operator_required", { lastSwitch: on(["e"]), sources: sources.map((s) => s.id === "e" ? { ...s, state: "ERROR" as const } : s) });
|
||||
expect(program(dead)).toEqual(["FALLBACK:"]);
|
||||
expect(dead.forced).toBe(true);
|
||||
});
|
||||
it("picture_in_picture / two_up / split_screen take the top two", () => {
|
||||
expect([pick("picture_in_picture").layout, program(pick("picture_in_picture"))]).toEqual(["pip", ["b", "c"]]);
|
||||
expect([pick("two_up").layout, program(pick("two_up"))]).toEqual(["two_up", ["b", "c"]]);
|
||||
expect([pick("split_screen").layout, program(pick("split_screen"))]).toEqual(["split_horizontal", ["b", "c"]]);
|
||||
});
|
||||
it("grid caps at four slots", () => {
|
||||
expect([pick("grid").layout, program(pick("grid"))]).toEqual(["grid", ["b", "c", "d", "a"]]);
|
||||
});
|
||||
it("composites degrade to single with one live source", () => {
|
||||
for (const name of ["picture_in_picture", "two_up", "split_screen", "grid"] as const) {
|
||||
const decision = decide(base({ sources: [src("a", 1)], memory: confirmed("a"), policy: policy({ name }) }));
|
||||
expect([decision.layout, program(decision)]).toEqual(["single", ["a"]]);
|
||||
}
|
||||
});
|
||||
it("skips disabled and non-autoSelect sources", () => {
|
||||
const decision = decide(base({ sources: [src("a", 99, "LIVE", { enabled: false }), src("b", 50, "LIVE", { autoSelect: false }), src("c", 1)], memory: confirmed("a", "b", "c") }));
|
||||
expect(program(decision)).toEqual(["c"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("anti-flapping", () => {
|
||||
it("needs liveConfirmations samples before a source is eligible", () => {
|
||||
const decisions = run([{ now: 1_000, sources: [src("a", 1, "LIVE", { version: 1 })] }, { now: 2_000, sources: [src("a", 1, "LIVE", { version: 2 })] }]);
|
||||
expect(program(decisions[0]!)).toEqual(["FALLBACK:"]);
|
||||
expect(program(decisions[1]!)).toEqual(["a"]);
|
||||
});
|
||||
it("does not count a re-read of the same monitor version as a confirmation", () => {
|
||||
const decisions = run([1_000, 2_000, 3_000].map((now) => ({ now, sources: [src("a", 1, "LIVE", { version: 7 })] })));
|
||||
expect(decisions.map((d) => program(d)[0])).toEqual(["FALLBACK:", "FALLBACK:", "FALLBACK:"]);
|
||||
});
|
||||
it("keeps program through offlineConfirmations - 1 failures (UNSTABLE counts) then fails over", () => {
|
||||
const a = (state: DirectorSourceInput["state"], version: number): DirectorSourceInput => src("a", 10, state, { version });
|
||||
const decisions = run([
|
||||
{ now: 1_000, sources: [a("UNSTABLE", 1), src("b", 1)] },
|
||||
{ now: 2_000, sources: [a("OFFLINE", 2), src("b", 1)] },
|
||||
{ now: 3_000, sources: [a("OFFLINE", 3), src("b", 1)] },
|
||||
], { memory: confirmed("a", "b"), lastSwitch: on(["a"], 0) });
|
||||
expect(decisions.map((d) => program(d)[0])).toEqual(["a", "a", "b"]);
|
||||
expect(decisions[2]!.forced).toBe(true);
|
||||
expect(decisions[2]!.reasons.join("\n")).toContain("failover: a offline_confirmed");
|
||||
});
|
||||
it("a LIVE sample resets the offline streak", () => {
|
||||
const decisions = run([["OFFLINE", 1], ["OFFLINE", 2], ["LIVE", 3], ["OFFLINE", 4], ["OFFLINE", 5]].map(([state, version], i) => ({ now: 1_000 * (i + 1), sources: [src("a", 10, state as "LIVE", { version: version as number }), src("b", 1)] })),
|
||||
{ memory: confirmed("a", "b"), lastSwitch: on(["a"], 0) });
|
||||
expect(decisions.every((d) => program(d)[0] === "a")).toBe(true);
|
||||
});
|
||||
it("fails over after noMediaSeconds without media and wakes exactly at the timeout", () => {
|
||||
const sources = (now: number): DirectorSourceInput[] => [src("a", 10, "LIVE", { lastMediaAtMs: 50_000 }), src("b", 1, "LIVE", { lastMediaAtMs: now })];
|
||||
const before = decide(base({ now: 59_999, sources: sources(59_999), memory: confirmed("a", "b"), lastSwitch: on(["a"], 0) }));
|
||||
expect(program(before)).toEqual(["a"]);
|
||||
expect(before.nextWakeMs).toBe(60_000);
|
||||
const after = decide(base({ now: 60_000, sources: sources(60_000), memory: before.memory, lastSwitch: before.lastSwitch }));
|
||||
expect(program(after)).toEqual(["b"]);
|
||||
expect(after.memory.sources.a).toMatchObject({ confirmed: false, degraded: "no_media" });
|
||||
});
|
||||
it("media-health failure bypasses confirmations", () => {
|
||||
const decision = decide(base({ sources: [src("a", 10, "LIVE", { mediaFailed: true }), src("b", 1)], memory: confirmed("a", "b"), lastSwitch: on(["a"], 99_000) }));
|
||||
expect(program(decision)).toEqual(["b"]);
|
||||
expect(decision.forced).toBe(true);
|
||||
});
|
||||
it("switch cooldown blocks a non-forced preemption until it lapses", () => {
|
||||
const sources = [src("a", 10), src("hi", 99)];
|
||||
const opts = { memory: confirmed("a", "hi"), lastSwitch: on(["a"], 95_000, "single", 0) };
|
||||
const held = decide(base({ now: 100_000, sources, ...opts }));
|
||||
expect(program(held)).toEqual(["a"]);
|
||||
expect(held.switched).toBe(false);
|
||||
expect(held.nextWakeMs).toBe(105_000);
|
||||
expect(program(decide(base({ now: 105_000, sources, ...opts })))).toEqual(["hi"]);
|
||||
});
|
||||
it("minHold keeps the main source but cooldown alone does not apply when zero", () => {
|
||||
const sources = [src("a", 10), src("hi", 99)];
|
||||
const opts = { memory: confirmed("a", "hi"), policy: policy({ switchCooldownMs: 0, minHoldMs: 15_000 }), lastSwitch: on(["a"], 90_000) };
|
||||
const held = decide(base({ now: 100_000, sources, ...opts }));
|
||||
expect(program(held)).toEqual(["a"]);
|
||||
expect(held.reasons.join("\n")).toContain("min hold");
|
||||
const moved = decide(base({ now: 105_000, sources, ...opts }));
|
||||
expect(program(moved)).toEqual(["hi"]);
|
||||
expect(moved.lastSwitch).toEqual({ atMs: 105_000, program: { kind: "sources", sourceIds: ["hi"] }, layout: "single", heldSinceMs: 105_000 });
|
||||
});
|
||||
it("high-priority interrupt happens after hold and cooldown", () => {
|
||||
const decisions = run([0, 2_000, 4_000].map((now) => ({ now, sources: [src("a", 10), src("hi", 99, now === 0 ? "OFFLINE" : "LIVE", { version: now })] })),
|
||||
{ memory: confirmed("a"), policy: policy({ minHoldMs: 1_000, switchCooldownMs: 3_000 }) });
|
||||
expect(decisions.map((d) => program(d)[0])).toEqual(["a", "a", "hi"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("failover ladder and fallback", () => {
|
||||
it("degrades down the priority ladder then to the fallback scene, then recovers", () => {
|
||||
const step = (now: number, a: DirectorSourceInput["state"], b: DirectorSourceInput["state"], c: DirectorSourceInput["state"]): { now: number; sources: DirectorSourceInput[] } =>
|
||||
({ now, sources: [src("a", 30, a), src("b", 20, b), src("c", 10, c)] });
|
||||
const decisions = run([
|
||||
step(1_000, "LIVE", "LIVE", "LIVE"),
|
||||
step(1_100, "ERROR", "LIVE", "LIVE"),
|
||||
step(1_200, "ERROR", "ERROR", "LIVE"),
|
||||
step(1_300, "ERROR", "ERROR", "ERROR"),
|
||||
step(1_400, "LIVE", "ERROR", "ERROR"),
|
||||
step(1_500, "LIVE", "ERROR", "ERROR"),
|
||||
], { memory: confirmed("a", "b", "c"), fallbackSceneId: "slate" });
|
||||
expect(decisions.map((d) => program(d)[0])).toEqual(["a", "b", "c", "FALLBACK:slate", "FALLBACK:slate", "a"]);
|
||||
expect(decisions.slice(1, 4).every((d) => d.forced && d.switched)).toBe(true);
|
||||
expect(decisions[3]!.mode).toBe("FALLBACK");
|
||||
expect(decisions[5]!.reasons).toContain("leaving fallback scene");
|
||||
});
|
||||
it("a composite with one dead slot re-forms immediately", () => {
|
||||
const decision = decide(base({ policy: policy({ name: "two_up" }), sources: [src("a", 30), src("b", 20, "ERROR"), src("c", 10)], memory: confirmed("a", "b", "c"), lastSwitch: on(["a", "b"], 99_000, "two_up") }));
|
||||
expect([decision.layout, program(decision), decision.forced]).toEqual(["two_up", ["a", "c"], true]);
|
||||
});
|
||||
it("all sources OFFLINE from a cold start -> fallback with null scene", () => {
|
||||
const decision = decide(base({ sources: [src("a", 1, "OFFLINE"), src("b", 2, "OFFLINE")] }));
|
||||
expect(decision.program).toEqual({ kind: "fallback", sceneId: null });
|
||||
expect(decision.reasons).toContain("all sources dead: fallback scene");
|
||||
});
|
||||
it("no sources at all -> fallback", () => {
|
||||
expect(decide(base({ fallbackSceneId: "loop" })).program).toEqual({ kind: "fallback", sceneId: "loop" });
|
||||
});
|
||||
it("single source stays on program and is not re-switched", () => {
|
||||
const first = decide(base({ sources: [src("only", 1)], memory: confirmed("only") }));
|
||||
expect([program(first), first.switched]).toEqual([["only"], true]);
|
||||
const second = decide(base({ now: 200_000, sources: [src("only", 1)], memory: first.memory, lastSwitch: first.lastSwitch }));
|
||||
expect([program(second), second.switched]).toEqual([["only"], false]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("operator override", () => {
|
||||
const take = (over: Partial<DirectorOverride> = {}): DirectorOverride => ({ kind: "take", release: "until_released", program: { kind: "sources", sourceIds: ["low"] }, layout: "single", principalId: "op", takenAtMs: 90_000, ...over });
|
||||
const sources = [src("low", 1), src("hi", 99)];
|
||||
const permitted = { op: ["director.switch"] };
|
||||
const rules = [rule("r", 9, { field: "live_source_count", op: ">=", value: 1 }, { layout: "single", selection: "top_priority" })];
|
||||
it("operator TAKE wins over AUTO rules and is not undone by automation", () => {
|
||||
const decision = decide(base({ sources, rules, override: take(), principalPermissions: permitted, memory: confirmed("low", "hi"), lastSwitch: on(["low"], 90_000) }));
|
||||
expect([program(decision), decision.mode, decision.switched, decision.override]).toEqual([["low"], "MANUAL", false, take()]);
|
||||
});
|
||||
it("program lock holds even when its source is dead", () => {
|
||||
const decision = decide(base({ sources: [src("low", 1, "ERROR"), src("hi", 99)], override: take({ kind: "lock" }), principalPermissions: permitted, memory: confirmed("low", "hi"), lastSwitch: on(["low"], 0) }));
|
||||
expect([program(decision), decision.mode]).toEqual([["low"], "LOCKED"]);
|
||||
expect(decision.reasons).toContain("program lock holds an unhealthy source");
|
||||
});
|
||||
it("revoking director.* releases the override back to AUTO immediately (revocation matrix)", () => {
|
||||
for (const [kind, permissions] of [["take", {}], ["lock", { op: ["posts.read"] }], ["take", { other: ["director.switch"] }]] as const) {
|
||||
const decision = decide(base({ sources, override: take({ kind }), principalPermissions: permissions, memory: confirmed("low", "hi"), lastSwitch: on(["low"], 99_999) }));
|
||||
expect([decision.overrideReleased, decision.override, decision.mode, program(decision), decision.forced]).toEqual(["AUTHZ_REVOKED", null, "AUTO", ["hi"], true]);
|
||||
}
|
||||
});
|
||||
it("timed override expires; until_source_offline and until_next_event release on their triggers", () => {
|
||||
const opts = { sources, principalPermissions: permitted, lastSwitch: on(["low"], 99_000) };
|
||||
const expired = decide(base({ ...opts, override: take({ release: "timed", expiresAtMs: 100_000 }), memory: confirmed("low", "hi") }));
|
||||
expect([expired.overrideReleased, program(expired)]).toEqual(["OVERRIDE_EXPIRED", ["hi"]]);
|
||||
const pending = decide(base({ ...opts, now: 99_000, override: take({ release: "timed", expiresAtMs: 100_000 }), memory: confirmed("low", "hi") }));
|
||||
expect([pending.override?.release, pending.nextWakeMs]).toEqual(["timed", 100_000]);
|
||||
|
||||
const offline = decide(base({ ...opts, sources: [src("low", 1, "ERROR"), src("hi", 99)], override: take({ release: "until_source_offline" }), memory: confirmed("low", "hi") }));
|
||||
expect([offline.overrideReleased, program(offline)]).toEqual(["SOURCE_OFFLINE", ["hi"]]);
|
||||
|
||||
const armed = decide(base({ ...opts, override: take({ release: "until_next_event" }), memory: confirmed("low", "hi") }));
|
||||
expect(armed.override?.eligibleAtTake).toEqual(["hi", "low"]);
|
||||
const same = decide(base({ ...opts, override: armed.override, memory: armed.memory }));
|
||||
expect(same.override).not.toBeNull();
|
||||
const event = decide(base({ ...opts, sources: [...sources, src("new", 50)], override: armed.override, memory: confirmed("low", "hi", "new") }));
|
||||
expect([event.overrideReleased, program(event)]).toEqual(["NEXT_EVENT", ["hi"]]);
|
||||
});
|
||||
it("a lock ignores source-offline release modes", () => {
|
||||
const decision = decide(base({ sources: [src("low", 1, "ERROR")], override: take({ kind: "lock", release: "until_source_offline" }), principalPermissions: permitted, memory: confirmed("low") }));
|
||||
expect([decision.override?.kind, program(decision)]).toEqual(["lock", ["low"]]);
|
||||
});
|
||||
it("override applies with switch and keeps counting confirmations underneath", () => {
|
||||
const decision = decide(base({ sources: [src("low", 1), src("hi", 99, "LIVE", { version: 1 })], override: take(), principalPermissions: permitted, memory: confirmed("low"), lastSwitch: on(["hi"], 0) }));
|
||||
expect([decision.switched, program(decision)]).toEqual([true, ["low"]]);
|
||||
expect(decision.memory.sources.hi).toMatchObject({ liveStreak: 1, confirmed: false });
|
||||
});
|
||||
});
|
||||
344
apps/scheduler/src/director/engine.ts
Normal file
344
apps/scheduler/src/director/engine.ts
Normal file
|
|
@ -0,0 +1,344 @@
|
|||
/**
|
||||
* WP10 Auto Director decision engine. Pure and deterministic: no I/O, no clock,
|
||||
* no Temporal. Every timestamp is epoch milliseconds so inputs and outputs stay
|
||||
* JSON-serializable across activity boundaries and continue-as-new.
|
||||
* Imports from core are type-only so the workflow bundle stays sandbox-safe.
|
||||
*/
|
||||
import type {
|
||||
AutoDirectorRule, DirectorCondition, DirectorMode, DirectorOverrideKind, DirectorOverrideRelease,
|
||||
DirectorOverrideReleaseReason, LiveSourceState, LiveSourceStateType, SceneLayout, SwitchPolicy, VirtualChannelSource,
|
||||
} from "@tactical-media-broadcast-hub/core";
|
||||
|
||||
export interface DirectorSourceInput {
|
||||
id: string;
|
||||
priority: number;
|
||||
enabled: boolean;
|
||||
autoSelect: boolean;
|
||||
state: LiveSourceStateType;
|
||||
/** Monitor row version; a re-read of the same version is not a new confirmation sample. */
|
||||
version?: number;
|
||||
liveSinceMs?: number;
|
||||
/** Last media-health "media present" instant; drives the no-media timeout. */
|
||||
lastMediaAtMs?: number;
|
||||
/** Media-health tier reports failure: bypasses confirmations (proposal §5.2 ladder step 2). */
|
||||
mediaFailed?: boolean;
|
||||
}
|
||||
|
||||
export interface SourceTrack {
|
||||
liveStreak: number;
|
||||
offlineStreak: number;
|
||||
confirmed: boolean;
|
||||
confirmedAtMs?: number;
|
||||
lastVersion?: number;
|
||||
/** Why the source last lost (or never had) eligibility. */
|
||||
degraded?: string;
|
||||
}
|
||||
|
||||
export interface DirectorMemory { sources: Record<string, SourceTrack> }
|
||||
|
||||
export type DirectorProgram =
|
||||
| { kind: "sources"; sourceIds: string[] }
|
||||
| { kind: "fallback"; sceneId: string | null };
|
||||
|
||||
export interface LastSwitch {
|
||||
atMs: number;
|
||||
program: DirectorProgram;
|
||||
layout: SceneLayout;
|
||||
/** When the program's main (first-slot) source went on air; minHold is measured from here. */
|
||||
heldSinceMs: number;
|
||||
}
|
||||
|
||||
export interface DirectorOverride {
|
||||
kind: DirectorOverrideKind;
|
||||
release: DirectorOverrideRelease;
|
||||
program: DirectorProgram;
|
||||
layout: SceneLayout;
|
||||
principalId: string;
|
||||
takenAtMs: number;
|
||||
expiresAtMs?: number;
|
||||
/** Eligible set at TAKE time, filled by the engine for `until_next_event`. */
|
||||
eligibleAtTake?: string[];
|
||||
}
|
||||
|
||||
export interface DecideInput {
|
||||
sources: readonly DirectorSourceInput[];
|
||||
rules: readonly AutoDirectorRule[];
|
||||
policy: SwitchPolicy;
|
||||
now: number;
|
||||
lastSwitch?: LastSwitch;
|
||||
memory?: DirectorMemory;
|
||||
channelId?: string;
|
||||
fallbackSceneId?: string | null;
|
||||
override?: DirectorOverride | null;
|
||||
/** Current permissions per principal (re-read every evaluation, never the TAKE-time snapshot). */
|
||||
principalPermissions?: Readonly<Record<string, readonly string[]>>;
|
||||
}
|
||||
|
||||
export interface Decision {
|
||||
program: DirectorProgram;
|
||||
layout: SceneLayout;
|
||||
reasons: string[];
|
||||
mode: DirectorMode;
|
||||
/** Program or layout differs from `lastSwitch`; the wrapper must emit a desired-state write. */
|
||||
switched: boolean;
|
||||
/** Switch bypassed minHold/cooldown (failover, fallback exit, override end). */
|
||||
forced: boolean;
|
||||
ruleId?: string;
|
||||
memory: DirectorMemory;
|
||||
override: DirectorOverride | null;
|
||||
overrideReleased?: DirectorOverrideReleaseReason;
|
||||
lastSwitch: LastSwitch;
|
||||
/** Earliest instant a purely time-based change could alter the decision. */
|
||||
nextWakeMs?: number;
|
||||
}
|
||||
|
||||
interface Candidate { id: string; priority: number; liveSinceMs: number }
|
||||
interface Selection { program: DirectorProgram; layout: SceneLayout; ruleId?: string }
|
||||
|
||||
const CAPACITY: Record<SceneLayout, number> = { single: 1, pip: 2, two_up: 2, split_horizontal: 2, split_vertical: 2, grid: 4, custom: Number.POSITIVE_INFINITY };
|
||||
const COMPOSITE_POLICY: Partial<Record<SwitchPolicy["name"], SceneLayout>> = { picture_in_picture: "pip", two_up: "two_up", split_screen: "split_horizontal", grid: "grid" };
|
||||
|
||||
/** Adapt the persisted monitor/domain types into engine input. */
|
||||
export function sourceInput(source: Pick<VirtualChannelSource, "id" | "priority" | "enabled" | "autoSelect">, state: LiveSourceState, extra: { version?: number; mediaFailed?: boolean } = {}): DirectorSourceInput {
|
||||
return {
|
||||
id: source.id, priority: source.priority, enabled: source.enabled, autoSelect: source.autoSelect, state: state.state,
|
||||
...(extra.version === undefined ? {} : { version: extra.version }),
|
||||
...(state.liveSince ? { liveSinceMs: state.liveSince.getTime() } : {}),
|
||||
...(state.lastHealthyAt ? { lastMediaAtMs: state.lastHealthyAt.getTime() } : {}),
|
||||
...(extra.mediaFailed === undefined ? {} : { mediaFailed: extra.mediaFailed }),
|
||||
};
|
||||
}
|
||||
|
||||
/** Fold one snapshot into per-source confirmation tracking. */
|
||||
function track(previous: SourceTrack | undefined, source: DirectorSourceInput, policy: SwitchPolicy, now: number): SourceTrack {
|
||||
let next: SourceTrack = previous ? { ...previous } : { liveStreak: 0, offlineStreak: 0, confirmed: false };
|
||||
const fresh = source.version === undefined || previous?.lastVersion !== source.version;
|
||||
if (fresh) {
|
||||
if (source.version !== undefined) next.lastVersion = source.version;
|
||||
switch (source.state) {
|
||||
case "LIVE":
|
||||
next.liveStreak += 1; next.offlineStreak = 0;
|
||||
if (!next.confirmed && next.liveStreak >= policy.liveConfirmations) next = { ...next, confirmed: true, confirmedAtMs: now, degraded: undefined };
|
||||
break;
|
||||
case "OFFLINE": case "ENDING": case "UNSTABLE":
|
||||
next.offlineStreak += 1; next.liveStreak = 0;
|
||||
if (next.confirmed && next.offlineStreak >= policy.offlineConfirmations) next = { ...next, confirmed: false, degraded: "offline_confirmed" };
|
||||
break;
|
||||
case "ERROR":
|
||||
next = { ...next, liveStreak: 0, offlineStreak: 0, confirmed: false, degraded: next.confirmed ? "error" : next.degraded };
|
||||
break;
|
||||
default:
|
||||
next.liveStreak = 0;
|
||||
}
|
||||
}
|
||||
// Hard media failures override confirmations and force re-confirmation.
|
||||
if (next.confirmed && source.mediaFailed === true) next = { ...next, confirmed: false, liveStreak: 0, degraded: "media_failed" };
|
||||
if (next.confirmed && source.lastMediaAtMs !== undefined && now - source.lastMediaAtMs >= policy.noMediaSeconds * 1000) {
|
||||
next = { ...next, confirmed: false, liveStreak: 0, degraded: "no_media" };
|
||||
}
|
||||
if (!source.enabled && next.confirmed) next = { ...next, confirmed: false, degraded: "disabled" };
|
||||
if (next.degraded === undefined) delete next.degraded;
|
||||
return next;
|
||||
}
|
||||
|
||||
function byPriority(a: Candidate, b: Candidate): number {
|
||||
return b.priority - a.priority || a.liveSinceMs - b.liveSinceMs || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
|
||||
}
|
||||
function byAge(a: Candidate, b: Candidate): number {
|
||||
return a.liveSinceMs - b.liveSinceMs || byPriority(a, b);
|
||||
}
|
||||
function compare(actual: number, op: DirectorCondition["op"], expected: number): boolean {
|
||||
switch (op) {
|
||||
case ">=": return actual >= expected;
|
||||
case "<=": return actual <= expected;
|
||||
case "==": return actual === expected;
|
||||
case ">": return actual > expected;
|
||||
}
|
||||
}
|
||||
function programKey(program: DirectorProgram, layout: SceneLayout): string {
|
||||
return program.kind === "fallback" ? `fallback:${program.sceneId ?? ""}` : `${layout}:${program.sourceIds.join(",")}`;
|
||||
}
|
||||
function mainSource(program: DirectorProgram): string | undefined {
|
||||
return program.kind === "sources" ? program.sourceIds[0] : undefined;
|
||||
}
|
||||
/** Composite layouts collapse to `single` when only one source can fill them. */
|
||||
function fill(ids: string[], layout: SceneLayout, reasons: string[]): { program: DirectorProgram; layout: SceneLayout } | undefined {
|
||||
const picked = ids.slice(0, CAPACITY[layout]);
|
||||
if (picked.length === 0) return undefined;
|
||||
if (layout !== "single" && layout !== "custom" && picked.length < 2) {
|
||||
reasons.push(`layout ${layout} degraded to single: one eligible source`);
|
||||
return { program: { kind: "sources", sourceIds: picked }, layout: "single" };
|
||||
}
|
||||
return { program: { kind: "sources", sourceIds: picked }, layout };
|
||||
}
|
||||
|
||||
export function decide(input: DecideInput): Decision {
|
||||
const { policy, now } = input;
|
||||
const reasons: string[] = [];
|
||||
const memory: DirectorMemory = { sources: {} };
|
||||
for (const source of input.sources) memory.sources[source.id] = track(input.memory?.sources[source.id], source, policy, now);
|
||||
|
||||
const live: Candidate[] = input.sources
|
||||
.filter((source) => source.enabled && memory.sources[source.id]?.confirmed === true)
|
||||
.map((source) => ({ id: source.id, priority: source.priority, liveSinceMs: source.liveSinceMs ?? memory.sources[source.id]?.confirmedAtMs ?? now }))
|
||||
.sort(byPriority);
|
||||
const liveIds = new Set(live.map((candidate) => candidate.id));
|
||||
const autoPool = live.filter((candidate) => input.sources.find((source) => source.id === candidate.id)?.autoSelect !== false);
|
||||
const healthy = (program: DirectorProgram): boolean => program.kind === "sources" && program.sourceIds.length > 0 && program.sourceIds.every((id) => liveIds.has(id));
|
||||
const fallback: DirectorProgram = { kind: "fallback", sceneId: input.fallbackSceneId ?? null };
|
||||
|
||||
// Wake for the earliest no-media timeout among eligible sources.
|
||||
const wakes: number[] = [];
|
||||
for (const source of input.sources) {
|
||||
if (liveIds.has(source.id) && source.lastMediaAtMs !== undefined) wakes.push(source.lastMediaAtMs + policy.noMediaSeconds * 1000);
|
||||
}
|
||||
const nextWake = (): number | undefined => {
|
||||
const future = wakes.filter((at) => at > now);
|
||||
return future.length === 0 ? undefined : Math.min(...future);
|
||||
};
|
||||
|
||||
// Ladder step 1: operator override / program lock.
|
||||
let override = input.override ? { ...input.override } : null;
|
||||
let overrideReleased: DirectorOverrideReleaseReason | undefined;
|
||||
if (override) {
|
||||
const permissions = input.principalPermissions?.[override.principalId] ?? [];
|
||||
const liveSet = [...liveIds].sort();
|
||||
if (!permissions.some((permission) => permission.startsWith("director."))) overrideReleased = "AUTHZ_REVOKED";
|
||||
else if (override.expiresAtMs !== undefined && now >= override.expiresAtMs) overrideReleased = "OVERRIDE_EXPIRED";
|
||||
else if (override.kind === "take" && override.release === "until_source_offline" && !healthy(override.program)) overrideReleased = "SOURCE_OFFLINE";
|
||||
else if (override.kind === "take" && override.release === "until_next_event") {
|
||||
if (override.eligibleAtTake === undefined) override.eligibleAtTake = liveSet;
|
||||
else if (override.eligibleAtTake.join(",") !== liveSet.join(",")) overrideReleased = "NEXT_EVENT";
|
||||
}
|
||||
if (overrideReleased) {
|
||||
reasons.push(`${override.kind} by ${override.principalId} released: ${overrideReleased}; returning to AUTO`);
|
||||
override = null;
|
||||
} else {
|
||||
if (override.expiresAtMs !== undefined) wakes.push(override.expiresAtMs);
|
||||
const switched = !input.lastSwitch || programKey(input.lastSwitch.program, input.lastSwitch.layout) !== programKey(override.program, override.layout);
|
||||
if (override.kind === "lock" && !healthy(override.program) && override.program.kind === "sources") reasons.push("program lock holds an unhealthy source");
|
||||
reasons.push(`${override.kind === "lock" ? "program lock" : "operator TAKE"} by ${override.principalId} wins over AUTO`);
|
||||
const lastSwitch: LastSwitch = switched
|
||||
? { atMs: now, program: override.program, layout: override.layout, heldSinceMs: mainSource(override.program) === mainSource(input.lastSwitch?.program ?? fallback) ? input.lastSwitch?.heldSinceMs ?? now : now }
|
||||
: input.lastSwitch!;
|
||||
return {
|
||||
program: override.program, layout: override.layout, reasons, mode: override.kind === "lock" ? "LOCKED" : "MANUAL",
|
||||
switched, forced: switched, memory, override, lastSwitch, ...(nextWake() === undefined ? {} : { nextWakeMs: nextWake()! }),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Ladder: ordered rules (priority desc, id asc for determinism), first match wins.
|
||||
let selection: Selection | undefined;
|
||||
const rules = input.rules.filter((rule) => input.channelId === undefined || rule.virtualChannelId === input.channelId)
|
||||
.sort((a, b) => b.priority - a.priority || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
||||
for (const rule of rules) {
|
||||
if (!matches(rule, input.sources, live, memory, reasons)) continue;
|
||||
const pool = rule.action.selection === "explicit"
|
||||
? (rule.action.sourceIds ?? []).map((id) => live.find((candidate) => candidate.id === id)).filter((candidate): candidate is Candidate => candidate !== undefined)
|
||||
: autoPool;
|
||||
const ordered = rule.action.selection === "oldest_live" ? [...pool].sort(byAge) : pool;
|
||||
const filled = fill(ordered.map((candidate) => candidate.id), rule.action.layout, reasons);
|
||||
if (!filled) { reasons.push(`rule ${rule.id} matched but has no eligible source; falling through`); continue; }
|
||||
reasons.push(`rule ${rule.id} matched (${rule.condition.field} ${rule.condition.op} ${String(rule.condition.value)})`);
|
||||
selection = { ...filled, ruleId: rule.id };
|
||||
break;
|
||||
}
|
||||
|
||||
// No rule matched: the channel's selection policy.
|
||||
if (!selection) {
|
||||
const ids = autoPool.map((candidate) => candidate.id);
|
||||
const current = input.lastSwitch;
|
||||
const currentMain = current ? mainSource(current.program) : undefined;
|
||||
let picked: { program: DirectorProgram; layout: SceneLayout } | undefined;
|
||||
switch (policy.name) {
|
||||
case "highest_priority": picked = fill(ids, "single", reasons); break;
|
||||
case "oldest_live": picked = fill([...autoPool].sort(byAge).map((candidate) => candidate.id), "single", reasons); break;
|
||||
case "newest_live": picked = fill([...autoPool].sort((a, b) => byAge(b, a)).map((candidate) => candidate.id), "single", reasons); break;
|
||||
case "round_robin": {
|
||||
const rotation = [...ids].sort();
|
||||
const index = currentMain === undefined ? -1 : rotation.indexOf(currentMain);
|
||||
if (index >= 0 && current && now - current.heldSinceMs < policy.minHoldMs) {
|
||||
picked = fill([rotation[index]!], "single", reasons);
|
||||
wakes.push(current.heldSinceMs + policy.minHoldMs);
|
||||
} else {
|
||||
picked = fill(rotation.length === 0 ? [] : [rotation[(index + 1) % rotation.length]!], "single", reasons);
|
||||
if (index >= 0 && rotation.length > 1) reasons.push("round_robin rotation");
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "operator_required":
|
||||
if (current && healthy(current.program)) picked = { program: current.program, layout: current.layout };
|
||||
else if (live.length > 0) reasons.push("operator_required: awaiting operator TAKE");
|
||||
break;
|
||||
default: picked = fill(ids, COMPOSITE_POLICY[policy.name] ?? "single", reasons);
|
||||
}
|
||||
if (picked) { selection = picked; reasons.push(`policy ${policy.name} selected ${programKey(picked.program, picked.layout)}`); }
|
||||
}
|
||||
|
||||
// Ladder terminal: fallback scene keeps output alive.
|
||||
if (!selection) {
|
||||
selection = { program: fallback, layout: "single" };
|
||||
reasons.push(live.length === 0 ? "all sources dead: fallback scene" : "no selection: fallback scene");
|
||||
}
|
||||
|
||||
// Anti-flapping against the current program.
|
||||
const current = input.lastSwitch;
|
||||
const desiredKey = programKey(selection.program, selection.layout);
|
||||
let outcome: Selection = selection;
|
||||
let switched = false;
|
||||
let forced = false;
|
||||
if (!current) {
|
||||
switched = true; forced = true;
|
||||
reasons.push("initial program");
|
||||
} else if (programKey(current.program, current.layout) !== desiredKey) {
|
||||
if (overrideReleased || !healthy(current.program)) {
|
||||
switched = true; forced = true;
|
||||
if (current.program.kind === "sources") {
|
||||
const lost = current.program.sourceIds.filter((id) => !liveIds.has(id));
|
||||
reasons.push(`failover: ${lost.map((id) => `${id} ${memory.sources[id]?.degraded ?? "not live"}`).join(", ") || "override released"}`);
|
||||
} else if (!overrideReleased) reasons.push("leaving fallback scene");
|
||||
} else if (now - current.atMs < policy.switchCooldownMs) {
|
||||
outcome = { program: current.program, layout: current.layout };
|
||||
wakes.push(current.atMs + policy.switchCooldownMs);
|
||||
reasons.push(`switch cooldown holds program until ${current.atMs + policy.switchCooldownMs}`);
|
||||
} else if (mainSource(current.program) !== mainSource(selection.program) && now - current.heldSinceMs < policy.minHoldMs) {
|
||||
outcome = { program: current.program, layout: current.layout };
|
||||
wakes.push(current.heldSinceMs + policy.minHoldMs);
|
||||
reasons.push(`min hold keeps ${mainSource(current.program) ?? "program"} until ${current.heldSinceMs + policy.minHoldMs}`);
|
||||
} else {
|
||||
switched = true;
|
||||
}
|
||||
}
|
||||
const lastSwitch: LastSwitch = switched
|
||||
? { atMs: now, program: outcome.program, layout: outcome.layout, heldSinceMs: current && mainSource(current.program) === mainSource(outcome.program) && mainSource(outcome.program) !== undefined ? current.heldSinceMs : now }
|
||||
: current!;
|
||||
const wake = nextWake();
|
||||
return {
|
||||
program: outcome.program, layout: outcome.layout, reasons, mode: outcome.program.kind === "fallback" ? "FALLBACK" : "AUTO",
|
||||
switched, forced, ...(outcome.ruleId === undefined || outcome !== selection ? {} : { ruleId: outcome.ruleId }),
|
||||
memory, override: null, ...(overrideReleased ? { overrideReleased } : {}), lastSwitch, ...(wake === undefined ? {} : { nextWakeMs: wake }),
|
||||
};
|
||||
}
|
||||
|
||||
/** Effective (debounced) state: LIVE only once confirmed and healthy. */
|
||||
function effectiveState(source: DirectorSourceInput, memory: DirectorMemory): LiveSourceStateType {
|
||||
if (source.enabled && memory.sources[source.id]?.confirmed === true) return "LIVE";
|
||||
return source.state === "LIVE" ? "PROBING" : source.state;
|
||||
}
|
||||
|
||||
function matches(rule: AutoDirectorRule, sources: readonly DirectorSourceInput[], live: readonly Candidate[], memory: DirectorMemory, reasons: string[]): boolean {
|
||||
const { field, op, value } = rule.condition;
|
||||
if (field === "source_state") {
|
||||
if (op !== "==" || typeof value !== "string") { reasons.push(`rule ${rule.id} skipped: source_state supports only == "<STATE>" or "<sourceId>:<STATE>"`); return false; }
|
||||
const split = value.lastIndexOf(":");
|
||||
const wanted = split < 0 ? value : value.slice(split + 1);
|
||||
const scope = split >= 0 ? [value.slice(0, split)] : rule.action.selection === "explicit" ? rule.action.sourceIds ?? [] : undefined;
|
||||
return sources.some((source) => (scope === undefined ? source.enabled : scope.includes(source.id)) && effectiveState(source, memory) === wanted);
|
||||
}
|
||||
const expected = typeof value === "number" ? value : Number(value);
|
||||
if (!Number.isFinite(expected)) { reasons.push(`rule ${rule.id} skipped: non-numeric ${field} value`); return false; }
|
||||
if (field === "live_source_count") return compare(live.length, op, expected);
|
||||
if (live.length === 0) return false;
|
||||
return compare(Math.max(...live.map((candidate) => candidate.priority)), op, expected);
|
||||
}
|
||||
100
apps/scheduler/src/director/evaluator.workflow.ts
Normal file
100
apps/scheduler/src/director/evaluator.workflow.ts
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
import { condition, continueAsNew, defineQuery, defineSignal, proxyActivities, setHandler, workflowInfo } from "@temporalio/workflow";
|
||||
import type { DirectorMode } from "@tactical-media-broadcast-hub/core";
|
||||
import { decide, type DirectorMemory, type LastSwitch } from "./engine.js";
|
||||
import type { ChannelRef, DirectorActivity } from "./director.activity.js";
|
||||
|
||||
const { readChannelState } = proxyActivities<Pick<DirectorActivity, "readChannelState">>({
|
||||
startToCloseTimeout: "10 seconds",
|
||||
retry: { initialInterval: "500 milliseconds", maximumInterval: "5 seconds", nonRetryableErrorTypes: ["DIRECTOR_NOT_WIRED"] },
|
||||
});
|
||||
const { applyProgramSwitch } = proxyActivities<Pick<DirectorActivity, "applyProgramSwitch">>({
|
||||
startToCloseTimeout: "15 seconds",
|
||||
retry: { initialInterval: "500 milliseconds", maximumInterval: "5 seconds", maximumAttempts: 5, nonRetryableErrorTypes: ["DIRECTOR_NOT_WIRED"] },
|
||||
});
|
||||
|
||||
/** Hint only: `(sourceId, version)`. Evaluations always re-read a fresh snapshot. */
|
||||
export interface SourceHint { sourceId: string; version: number; urgent?: boolean }
|
||||
export interface EvaluatorCarry { memory: DirectorMemory; lastSwitch?: LastSwitch; seq: number }
|
||||
export interface EvaluatorInput { ref: ChannelRef; carry?: EvaluatorCarry }
|
||||
export type EvaluatorResult = { kind: "stopped" } | { kind: "superseded"; code: "SUPERSEDED" } | { kind: "needs_reauthorization"; code: "AUTHZ_REVOKED" };
|
||||
export interface EvaluatorStatus { mode?: DirectorMode; lastSwitch?: LastSwitch; reasons: string[]; evaluations: number }
|
||||
|
||||
export const sourceChanged = defineSignal<[SourceHint]>("sourceChanged");
|
||||
export const stopDirector = defineSignal("stopDirector");
|
||||
export const directorStatus = defineQuery<EvaluatorStatus>("directorStatus");
|
||||
|
||||
export const DEBOUNCE_MS = 250;
|
||||
export const MAX_WAIT_MS = 1000;
|
||||
export const POLL_MS = 5000;
|
||||
export const CONTINUE_AS_NEW_EVENTS = 2000;
|
||||
|
||||
/** One evaluator per virtual channel: debounce hints, decide, emit fenced desired-state writes. */
|
||||
export async function autoDirectorEvaluator({ ref, carry }: EvaluatorInput): Promise<EvaluatorResult> {
|
||||
let memory: DirectorMemory = carry?.memory ?? { sources: {} };
|
||||
let lastSwitch = carry?.lastSwitch;
|
||||
let seq = carry?.seq ?? 0;
|
||||
let dirty = true;
|
||||
let urgent = false;
|
||||
let stopping = false;
|
||||
let firstPendingAt: number | undefined;
|
||||
let lastSignalAt = 0;
|
||||
let events = 0;
|
||||
let evaluations = 0;
|
||||
let nextWakeMs: number | undefined;
|
||||
let status: EvaluatorStatus = { reasons: [], evaluations: 0, ...(lastSwitch ? { lastSwitch } : {}) };
|
||||
|
||||
setHandler(sourceChanged, (hint) => {
|
||||
events += 1; dirty = true;
|
||||
if (hint.urgent === true) urgent = true;
|
||||
lastSignalAt = Date.now();
|
||||
firstPendingAt ??= lastSignalAt;
|
||||
});
|
||||
setHandler(stopDirector, () => { stopping = true; });
|
||||
setHandler(directorStatus, () => status);
|
||||
|
||||
for (;;) {
|
||||
const idle = Math.min(POLL_MS, nextWakeMs === undefined ? POLL_MS : Math.max(0, nextWakeMs - Date.now()));
|
||||
await condition(() => dirty || stopping, idle);
|
||||
if (stopping) return { kind: "stopped" };
|
||||
// Trailing debounce: 250 ms quiet window, 1 s max wait; urgent program-source failure bypasses.
|
||||
while (dirty && !urgent && firstPendingAt !== undefined) {
|
||||
const deadline = Math.min(lastSignalAt + DEBOUNCE_MS, firstPendingAt + MAX_WAIT_MS);
|
||||
if (Date.now() >= deadline) break;
|
||||
const seen = lastSignalAt;
|
||||
if (!(await condition(() => urgent || stopping || lastSignalAt !== seen, deadline - Date.now()))) break;
|
||||
if (stopping) return { kind: "stopped" };
|
||||
}
|
||||
dirty = false; urgent = false; firstPendingAt = undefined;
|
||||
|
||||
const snapshot = await readChannelState(ref);
|
||||
if (snapshot.kind === "superseded") return { kind: "superseded", code: "SUPERSEDED" };
|
||||
const decision = decide({
|
||||
sources: snapshot.sources, rules: snapshot.rules, policy: snapshot.policy, now: Date.now(), channelId: ref.channelId,
|
||||
fallbackSceneId: snapshot.fallbackSceneId, override: snapshot.override, principalPermissions: snapshot.principalPermissions,
|
||||
memory, ...(lastSwitch ? { lastSwitch } : {}),
|
||||
});
|
||||
memory = decision.memory;
|
||||
evaluations += 1;
|
||||
nextWakeMs = decision.nextWakeMs;
|
||||
if (decision.switched || decision.overrideReleased) {
|
||||
seq += 1;
|
||||
const outcome = await applyProgramSwitch({
|
||||
ref, commandId: `${ref.channelId}:${ref.generation}:${seq}`, expectedDesiredVersion: snapshot.desiredVersion,
|
||||
authzEpoch: snapshot.authzEpoch, program: decision.program, layout: decision.layout, reasons: decision.reasons,
|
||||
...(decision.overrideReleased ? { releaseOverride: decision.overrideReleased } : {}),
|
||||
});
|
||||
if (outcome.kind === "generation_superseded") return { kind: "superseded", code: "SUPERSEDED" };
|
||||
if (outcome.kind === "authz_revoked") return { kind: "needs_reauthorization", code: "AUTHZ_REVOKED" };
|
||||
// A concurrent writer (e.g. operator TAKE) won the CAS: re-read rather than re-apply.
|
||||
if (outcome.kind === "superseded") dirty = true;
|
||||
else lastSwitch = decision.lastSwitch;
|
||||
} else {
|
||||
lastSwitch = decision.lastSwitch;
|
||||
}
|
||||
status = { mode: decision.mode, reasons: decision.reasons, evaluations, ...(lastSwitch ? { lastSwitch } : {}) };
|
||||
|
||||
if (events + evaluations >= CONTINUE_AS_NEW_EVENTS || workflowInfo().continueAsNewSuggested) {
|
||||
return continueAsNew<typeof autoDirectorEvaluator>({ ref, carry: { memory, seq, ...(lastSwitch ? { lastSwitch } : {}) } });
|
||||
}
|
||||
}
|
||||
}
|
||||
128
apps/scheduler/src/director/http.client.ts
Normal file
128
apps/scheduler/src/director/http.client.ts
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
/**
|
||||
* HTTP implementation of the director ports (WP10 wiring slice).
|
||||
*
|
||||
* The scheduler worker dials the API's internal director endpoints. Every
|
||||
* response is validated before it reaches the engine: the API is a different
|
||||
* process, so a malformed or truncated reply must surface as a transport
|
||||
* error (retryable) rather than poison the evaluator's memory.
|
||||
*
|
||||
* Endpoints (apps/api/src/director/director.controller.ts):
|
||||
* GET /internal/director/channels/:workspaceId/:channelId/state
|
||||
* GET /internal/director/channels/:workspaceId/:channelId/fence
|
||||
* POST /internal/director/channels/:workspaceId/:channelId/apply
|
||||
* POST /internal/director/channels/:workspaceId/:channelId/audit
|
||||
*/
|
||||
import type { ChannelRef, DesiredFence } from "./director.activity.js";
|
||||
import type { DirectorApplyRequest, DirectorApplyResult, DirectorApi, DirectorStateRead } from "./ports.js";
|
||||
|
||||
export interface DirectorHttpClientDependencies {
|
||||
/** Base URL of the API (default http://127.0.0.1:3000). */
|
||||
baseUrl?: string;
|
||||
/** Shared-secret bearer for the internal route group. */
|
||||
token?: string;
|
||||
fetch?: typeof fetch;
|
||||
/** Default 8s; short so an unresponsive API fails the activity fast. */
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function requiredString(value: unknown, path: string): string {
|
||||
if (typeof value !== "string" || value === "") throw new Error(`director api: ${path} must be a non-empty string`);
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Shape-check an apply response without trusting the status code alone. */
|
||||
export function parseApplyResult(value: unknown): DirectorApplyResult {
|
||||
if (!isRecord(value)) throw new Error("director api: apply response must be an object");
|
||||
const kind = value.kind;
|
||||
if (kind === "conflict") return { kind: "conflict" };
|
||||
if (kind !== "applied" && kind !== "duplicate") throw new Error("director api: apply response kind is invalid");
|
||||
const desiredVersion = value.desiredVersion;
|
||||
if (typeof desiredVersion !== "number" || !Number.isSafeInteger(desiredVersion) || desiredVersion < 0) {
|
||||
throw new Error("director api: apply response desiredVersion is invalid");
|
||||
}
|
||||
return { kind, desiredVersion };
|
||||
}
|
||||
|
||||
/** Shape-check a fence response (fence may legitimately be absent). */
|
||||
export function parseFence(value: unknown): DesiredFence | undefined {
|
||||
if (value === null || value === undefined) return undefined;
|
||||
if (!isRecord(value)) throw new Error("director api: fence response must be an object or null");
|
||||
const generation = requiredString(value.generation, "fence.generation");
|
||||
const desiredVersion = value.desiredVersion;
|
||||
const authzEpoch = value.authzEpoch;
|
||||
if (typeof desiredVersion !== "number" || !Number.isSafeInteger(desiredVersion) || desiredVersion < 0) {
|
||||
throw new Error("director api: fence.desiredVersion is invalid");
|
||||
}
|
||||
if (typeof authzEpoch !== "number" || !Number.isSafeInteger(authzEpoch) || authzEpoch < 0) {
|
||||
throw new Error("director api: fence.authzEpoch is invalid");
|
||||
}
|
||||
if (typeof value.directorAuthorized !== "boolean") throw new Error("director api: fence.directorAuthorized is invalid");
|
||||
return { generation, desiredVersion, authzEpoch, directorAuthorized: value.directorAuthorized };
|
||||
}
|
||||
|
||||
export class DirectorHttpClient implements DirectorApi {
|
||||
private readonly baseUrl: string;
|
||||
private readonly token: string | undefined;
|
||||
private readonly fetchImpl: typeof fetch;
|
||||
private readonly timeoutMs: number;
|
||||
|
||||
constructor(dependencies: DirectorHttpClientDependencies = {}) {
|
||||
this.baseUrl = (dependencies.baseUrl ?? process.env.DIRECTOR_API_URL ?? "http://127.0.0.1:3000").replace(/\/+$/, "");
|
||||
this.token = dependencies.token ?? process.env.DIRECTOR_API_TOKEN;
|
||||
this.fetchImpl = dependencies.fetch ?? fetch;
|
||||
this.timeoutMs = dependencies.timeoutMs ?? 8_000;
|
||||
}
|
||||
|
||||
private path(ref: ChannelRef, suffix: string): string {
|
||||
return `${this.baseUrl}/internal/director/channels/${encodeURIComponent(ref.workspaceId)}/${encodeURIComponent(ref.channelId)}/${suffix}`;
|
||||
}
|
||||
|
||||
private async request(path: string, init?: RequestInit): Promise<unknown> {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
|
||||
try {
|
||||
const response = await this.fetchImpl(path, {
|
||||
...init,
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
...(this.token === undefined ? {} : { authorization: `Bearer ${this.token}` }),
|
||||
...init?.headers,
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
// Transport/server errors are retryable; the evaluator treats 4xx-shaped
|
||||
// outcomes as fenced results, which the callers parse explicitly.
|
||||
throw new Error(`director api: ${response.status} ${response.statusText} for ${path}`);
|
||||
}
|
||||
const text = await response.text();
|
||||
return text === "" ? null : JSON.parse(text) as unknown;
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
async channelState(ref: ChannelRef): Promise<DirectorStateRead> {
|
||||
const value = await this.request(this.path(ref, "state"));
|
||||
if (!isRecord(value)) throw new Error("director api: state response must be an object");
|
||||
if (value.kind === "superseded") return { kind: "superseded", code: "SUPERSEDED" };
|
||||
if (value.kind !== "current") throw new Error("director api: state response kind is invalid");
|
||||
return value as DirectorStateRead;
|
||||
}
|
||||
|
||||
async fence(ref: ChannelRef): Promise<DesiredFence | undefined> {
|
||||
return parseFence(await this.request(this.path(ref, "fence")));
|
||||
}
|
||||
|
||||
async applyProgramSwitch(ref: ChannelRef, body: DirectorApplyRequest): Promise<DirectorApplyResult> {
|
||||
return parseApplyResult(await this.request(this.path(ref, "apply"), { method: "POST", body: JSON.stringify(body) }));
|
||||
}
|
||||
|
||||
async audit(ref: ChannelRef, event: { commandId: string; outcome: string; reason: string }): Promise<void> {
|
||||
await this.request(this.path(ref, "audit"), { method: "POST", body: JSON.stringify(event) });
|
||||
}
|
||||
}
|
||||
104
apps/scheduler/src/director/ports.ts
Normal file
104
apps/scheduler/src/director/ports.ts
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
/**
|
||||
* Transport-agnostic port contracts for the auto-director evaluator (WP10
|
||||
* wiring slice).
|
||||
*
|
||||
* The Temporal worker hosts the evaluator workflow and proxies these ports;
|
||||
* the API process implements them. Because the worker runs in the scheduler
|
||||
* package and the implementations in apps/api, the interfaces are declared
|
||||
* with JSON-serializable shapes only (the scheduler cannot compile apps/api
|
||||
* sources), and both sides share the engine's type definitions via relative
|
||||
* imports — the same cross-package pattern apps/api/src/live-monitor/poller
|
||||
* uses for packages/connector-mock-livestream.
|
||||
*
|
||||
* `read`/`fence` return `kind: "superseded"` / `undefined` rather than throwing
|
||||
* so a dead generation never becomes a retryable activity failure loop.
|
||||
*/
|
||||
import type { AutoDirectorRule, SceneLayout, SwitchPolicy } from "@tactical-media-broadcast-hub/core";
|
||||
import type { DirectorOverride, DirectorProgram } from "./engine.js";
|
||||
import type { ChannelRef, DesiredFence } from "./director.activity.js";
|
||||
|
||||
/**
|
||||
* A channel source with its monitor state resolved for engine consumption.
|
||||
*
|
||||
* Optional fields mirror the engine's DirectorSourceInput exactly: ABSENT, not
|
||||
* null — the engine treats `lastMediaAtMs !== undefined` as "known", so a null
|
||||
* would read as epoch 0 and de-confirm every source as no-media. JSON drops
|
||||
* undefined keys, which keeps the wire form and the engine form identical.
|
||||
*/
|
||||
export interface DirectorChannelSource {
|
||||
id: string;
|
||||
priority: number;
|
||||
enabled: boolean;
|
||||
autoSelect: boolean;
|
||||
/** Debounced monitor state (LIVE only once confirmed). */
|
||||
state:
|
||||
| "OFFLINE" | "UNKNOWN" | "PROBING" | "STARTING" | "LIVE" | "UNSTABLE" | "ENDING" | "ERROR";
|
||||
/** Monitor row version; absent when no monitor row exists (every read is then a fresh sample). */
|
||||
version?: number;
|
||||
/** `source_monitor_states.state_entered_at` — when the current state was entered. */
|
||||
liveSinceMs?: number;
|
||||
/** Last media-health "media present" instant, for the no-media timeout. */
|
||||
lastMediaAtMs?: number;
|
||||
/** Media-health tier reports failure: de-confirms immediately (§5.2 step 2). */
|
||||
mediaFailed?: boolean;
|
||||
}
|
||||
|
||||
/** Result of a DirectorStatePort.read: the engine's full input for one channel. */
|
||||
export interface DirectorChannelState {
|
||||
kind: "current";
|
||||
desiredVersion: number;
|
||||
authzEpoch: number;
|
||||
generation: string;
|
||||
sources: DirectorChannelSource[];
|
||||
rules: AutoDirectorRule[];
|
||||
policy: SwitchPolicy;
|
||||
fallbackSceneId: string | null;
|
||||
override: DirectorOverride | null;
|
||||
principalPermissions: Record<string, string[]>;
|
||||
}
|
||||
|
||||
export type DirectorStateRead = DirectorChannelState | { kind: "superseded"; code: "SUPERSEDED" };
|
||||
|
||||
/** The read port: everything the evaluator needs for one decision. */
|
||||
export interface DirectorStatePort {
|
||||
read(ref: ChannelRef): Promise<DirectorStateRead>;
|
||||
}
|
||||
|
||||
/** The desired-state port contract, transport-agnostic twin of DesiredStatePort. */
|
||||
export interface DesiredStateOperations {
|
||||
fence(ref: ChannelRef): Promise<DesiredFence | undefined>;
|
||||
/** The engine's decision applied as a desired-state write; the director's commandId is the idempotency reference. */
|
||||
apply(command: {
|
||||
ref: ChannelRef;
|
||||
commandId: string;
|
||||
expectedDesiredVersion: number;
|
||||
authzEpoch: number;
|
||||
program: DirectorProgram;
|
||||
layout: SceneLayout;
|
||||
reasons: string[];
|
||||
releaseOverride?: string;
|
||||
}): Promise<{ kind: "applied" | "duplicate"; desiredVersion: number } | { kind: "conflict" }>;
|
||||
audit(ref: ChannelRef, event: { commandId: string; outcome: "SUPERSEDED" | "AUTHZ_REVOKED"; reason: string }): Promise<void>;
|
||||
}
|
||||
|
||||
/** An HTTP implementation of the API's /director endpoints (see api-side director.service.ts). */
|
||||
export interface DirectorApi {
|
||||
channelState(ref: ChannelRef): Promise<DirectorStateRead>;
|
||||
fence(ref: ChannelRef): Promise<DesiredFence | undefined>;
|
||||
applyProgramSwitch(ref: ChannelRef, body: DirectorApplyRequest): Promise<DirectorApplyResult>;
|
||||
audit(ref: ChannelRef, event: { commandId: string; outcome: string; reason: string }): Promise<void>;
|
||||
}
|
||||
|
||||
export interface DirectorApplyRequest {
|
||||
commandId: string;
|
||||
expectedDesiredVersion: number;
|
||||
authzEpoch: number;
|
||||
program: DirectorProgram;
|
||||
layout: SceneLayout;
|
||||
reasons: string[];
|
||||
releaseOverride?: string;
|
||||
}
|
||||
|
||||
export type DirectorApplyResult =
|
||||
| { kind: "applied" | "duplicate"; desiredVersion: number }
|
||||
| { kind: "conflict" };
|
||||
33
apps/scheduler/src/director/starter.ts
Normal file
33
apps/scheduler/src/director/starter.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
import { WorkflowExecutionAlreadyStartedError, type Client } from "@temporalio/client";
|
||||
import { WorkflowIdReusePolicy } from "@temporalio/common";
|
||||
import type { ChannelRef } from "./director.activity.js";
|
||||
|
||||
export function evaluatorWorkflowId(ref: Pick<ChannelRef, "workspaceId" | "channelId">): string {
|
||||
return `auto-director-${ref.workspaceId}-${ref.channelId}`;
|
||||
}
|
||||
|
||||
export type EnsureOutcome = "started" | "running" | "restarted";
|
||||
|
||||
/**
|
||||
* Starts are idempotent hints. A running evaluator is only nudged (it re-reads
|
||||
* state and fences its own generation); a closed one may be started afresh.
|
||||
*/
|
||||
export async function ensureDirector(client: Client, ref: ChannelRef, taskQueue: string): Promise<EnsureOutcome> {
|
||||
const workflowId = evaluatorWorkflowId(ref);
|
||||
const options = { taskQueue, workflowId, args: [{ ref }] as const, workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE };
|
||||
try {
|
||||
await client.workflow.start("autoDirectorEvaluator", options);
|
||||
return "started";
|
||||
} catch (error) {
|
||||
if (!(error instanceof WorkflowExecutionAlreadyStartedError)) throw error;
|
||||
const handle = client.workflow.getHandle(workflowId);
|
||||
const description = await handle.describe().catch(() => undefined);
|
||||
if (description?.status.name === "RUNNING") {
|
||||
await handle.signal("sourceChanged", { sourceId: "*", version: 0 });
|
||||
return "running";
|
||||
}
|
||||
// Closed between start and describe: one retry; a concurrent starter winning is fine.
|
||||
try { await client.workflow.start("autoDirectorEvaluator", options); return "restarted"; }
|
||||
catch (retryError) { if (retryError instanceof WorkflowExecutionAlreadyStartedError) return "running"; throw retryError; }
|
||||
}
|
||||
}
|
||||
223
apps/scheduler/src/director/wiring.test.ts
Normal file
223
apps/scheduler/src/director/wiring.test.ts
Normal file
|
|
@ -0,0 +1,223 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { parseDirectorChannels } from "./config.js";
|
||||
import { currentChannels, sweepDirectors, directorChannelsFromEnv } from "./discovery.js";
|
||||
import { DirectorHttpClient, parseApplyResult, parseFence } from "./http.client.js";
|
||||
import { HttpDirectorStatePort, HttpDesiredStatePort, engineSource, engineSnapshot } from "./adapters.js";
|
||||
import type { DirectorApi, DirectorChannelSource, DirectorStateRead } from "./ports.js";
|
||||
import type { ChannelRef } from "./director.activity.js";
|
||||
import { decide } from "./engine.js";
|
||||
|
||||
/**
|
||||
* WP10 wiring slice, scheduler side: config parsing, the HTTP port adapters
|
||||
* against a fake API, and the evaluator's discovery sweep. The port-pair
|
||||
* behaviour (state read -> decide -> desired applied) lives in
|
||||
* apps/api/src/director/director.service.test.ts, which imports this
|
||||
* package's engine directly.
|
||||
*/
|
||||
|
||||
const WS = "0b7c6d50-93a4-4a1e-8e94-93b7e0000001";
|
||||
const CH = "0b7c6d50-93a4-4a1e-8e94-93b7e0000002";
|
||||
const SRC = "0b7c6d50-93a4-4a1e-8e94-93b7e0000003";
|
||||
const RULE = "0b7c6d50-93a4-4a1e-8e94-93b7e0000004";
|
||||
|
||||
const rule = {
|
||||
id: RULE, virtualChannelId: CH, priority: 1,
|
||||
condition: { field: "live_source_count" as const, op: ">=" as const, value: 1 },
|
||||
action: { layout: "single" as const, selection: "top_priority" as const },
|
||||
};
|
||||
const policy = { name: "highest_priority" as const, liveConfirmations: 1, offlineConfirmations: 1, noMediaSeconds: 600, minHoldMs: 0, switchCooldownMs: 0 };
|
||||
const document = JSON.stringify([{ workspaceId: WS, channelId: CH, generation: "g1", rules: [rule], policy, fallbackSceneId: null }]);
|
||||
|
||||
describe("parseDirectorChannels", () => {
|
||||
it("parses the full document", () => {
|
||||
const channels = parseDirectorChannels(document);
|
||||
expect(channels).toHaveLength(1);
|
||||
expect(channels[0]).toMatchObject({ workspaceId: WS, channelId: CH, generation: "g1", fallbackSceneId: null });
|
||||
expect(channels[0]?.rules).toEqual([rule]);
|
||||
expect(channels[0]?.policy).toEqual(policy);
|
||||
});
|
||||
it("rejects malformed entries with a field path", () => {
|
||||
expect(parseDirectorChannels(undefined)).toEqual([]);
|
||||
expect(parseDirectorChannels("")).toEqual([]);
|
||||
expect(() => parseDirectorChannels("[]{}")).toThrow();
|
||||
expect(() => parseDirectorChannels('{"a":1}')).toThrow(/must be a JSON array/);
|
||||
expect(() => parseDirectorChannels('[{"channelId":"x"}]')).toThrow(/channelId must be a UUID/);
|
||||
expect(() => parseDirectorChannels(`[{"workspaceId":"${WS}","channelId":"x","generation":"g"}]`)).toThrow(/channelId must be a UUID/);
|
||||
expect(() => parseDirectorChannels(`[{"workspaceId":"${WS}","channelId":"${CH}","generation":""}]`)).toThrow(/generation/);
|
||||
expect(() => parseDirectorChannels(`[{"workspaceId":"${WS}","channelId":"${CH}","generation":"g","rules":[{"id":"bad"}]}]`)).toThrow(/rules\[0\]\.id/);
|
||||
expect(() => parseDirectorChannels(`[{"workspaceId":"${WS}","channelId":"${CH}","generation":"g","policy":{"name":"nope"}}]`)).toThrow(/policy\.name/);
|
||||
});
|
||||
it("reads from env and defaults to none", () => {
|
||||
expect(directorChannelsFromEnv({})).toEqual([]);
|
||||
expect(directorChannelsFromEnv({ DIRECTOR_CHANNELS: document })).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("discovery", () => {
|
||||
it("keeps the latest declared entry per channel", () => {
|
||||
const channels = parseDirectorChannels(`[${document.slice(1, -1)},${document.slice(1, -1).replace('"g1"', '"g2"')}]`);
|
||||
expect(channels).toHaveLength(2);
|
||||
expect(currentChannels(channels)).toHaveLength(1);
|
||||
expect(currentChannels(channels)[0]?.generation).toBe("g2");
|
||||
});
|
||||
it("ensures one evaluator per configured channel and swallows start failures", async () => {
|
||||
const channels = parseDirectorChannels(document);
|
||||
const workflowIds: string[] = [];
|
||||
const clientFor = (behaviour: () => Promise<"started" | "running" | "restarted" | "failed">): Parameters<typeof sweepDirectors>[0] =>
|
||||
({
|
||||
workflow: {
|
||||
start: async (_name: string, options: { workflowId: string }) => {
|
||||
workflowIds.push(options.workflowId);
|
||||
return behaviour();
|
||||
},
|
||||
getHandle: () => ({ describe: async () => ({ status: { name: "RUNNING" } }), signal: async () => undefined }),
|
||||
},
|
||||
}) as unknown as Parameters<typeof sweepDirectors>[0];
|
||||
// start succeeds: one evaluator ensured
|
||||
const started = await sweepDirectors(clientFor(async () => "started"), { taskQueue: "tmbh-publishing", channels });
|
||||
expect(started).toEqual({ started: 1, running: 0, restarted: 0 });
|
||||
// every Temporal failure is contained: the sweep reports zero and lives on
|
||||
const failing = await sweepDirectors(clientFor(async () => { throw new Error("temporal down"); }), { taskQueue: "q", channels });
|
||||
expect(failing).toEqual({ started: 0, running: 0, restarted: 0 });
|
||||
expect(workflowIds).toEqual([`auto-director-${WS}-${CH}`, `auto-director-${WS}-${CH}`]);
|
||||
});
|
||||
});
|
||||
|
||||
/** In-memory DirectorApi double used by the adapter tests. */
|
||||
function fakeApi(over: Partial<DirectorApi> = {}): DirectorApi & { requests: string[] } {
|
||||
const requests: string[] = [];
|
||||
const current: DirectorStateRead = {
|
||||
kind: "current", desiredVersion: 4, authzEpoch: 7, generation: "g1",
|
||||
sources: [{ id: SRC, priority: 50, enabled: true, autoSelect: true, state: "LIVE", version: 3 }],
|
||||
rules: [rule], policy, fallbackSceneId: null, override: null, principalPermissions: {},
|
||||
};
|
||||
return {
|
||||
requests,
|
||||
channelState: async (ref) => { requests.push(`state:${ref.channelId}`); return current; },
|
||||
fence: async (ref) => { requests.push(`fence:${ref.channelId}`); return { generation: "g1", desiredVersion: 4, authzEpoch: 7, directorAuthorized: true }; },
|
||||
applyProgramSwitch: async (ref, body) => { requests.push(`apply:${body.commandId}`); return { kind: "applied", desiredVersion: 5 }; },
|
||||
audit: async (ref, event) => { requests.push(`audit:${event.outcome}`); },
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
describe("port adapters", () => {
|
||||
const ref: ChannelRef = { workspaceId: WS, channelId: CH, generation: "g1" };
|
||||
|
||||
it("converts transport sources without inventing optional engine fields", () => {
|
||||
const source: DirectorChannelSource = { id: SRC, priority: 50, enabled: true, autoSelect: true, state: "LIVE" };
|
||||
const input = engineSource(source);
|
||||
expect(input).toEqual({ id: SRC, priority: 50, enabled: true, autoSelect: true, state: "LIVE" });
|
||||
expect("version" in input).toBe(false);
|
||||
expect("lastMediaAtMs" in input).toBe(false);
|
||||
expect("mediaFailed" in input).toBe(false);
|
||||
});
|
||||
|
||||
it("state read feeds decide() unchanged (engine input round-trip)", async () => {
|
||||
const api = fakeApi();
|
||||
const port = new HttpDirectorStatePort(api);
|
||||
const snapshot = await port.read(ref);
|
||||
if (snapshot.kind !== "current") throw new Error("expected current");
|
||||
expect(snapshot.sources[0]).toEqual({ id: SRC, priority: 50, enabled: true, autoSelect: true, state: "LIVE", version: 3 });
|
||||
const decision = decide({ sources: snapshot.sources, rules: snapshot.rules, policy: snapshot.policy, now: 1_000 });
|
||||
expect(decision.program).toEqual({ kind: "sources", sourceIds: [SRC] });
|
||||
});
|
||||
|
||||
it("engineSnapshot maps a transport read to the engine snapshot", () => {
|
||||
const read = { kind: "current" as const, desiredVersion: 4, authzEpoch: 7, generation: "g1",
|
||||
sources: [{ id: SRC, priority: 50, enabled: true, autoSelect: true, state: "LIVE", version: 3 } as DirectorChannelSource],
|
||||
rules: [rule], policy, fallbackSceneId: null, override: null, principalPermissions: {} };
|
||||
expect(engineSnapshot(read)).toMatchObject({ kind: "current", desiredVersion: 4, authzEpoch: 7 });
|
||||
});
|
||||
|
||||
it("desired-state port forwards fence/apply/audit with the commandId as idempotency key", async () => {
|
||||
const api = fakeApi();
|
||||
const port = new HttpDesiredStatePort(api);
|
||||
expect(await port.fence(ref)).toMatchObject({ generation: "g1", desiredVersion: 4 });
|
||||
expect(await port.findCommand(ref, "cmd")).toBeUndefined();
|
||||
const outcome = await port.writeProgram({
|
||||
ref, commandId: `${CH}:1`, expectedDesiredVersion: 4, authzEpoch: 7,
|
||||
program: { kind: "sources", sourceIds: [SRC] }, layout: "single", reasons: ["rule"],
|
||||
});
|
||||
expect(outcome).toEqual({ kind: "applied", desiredVersion: 5 });
|
||||
await port.audit(ref, { commandId: "c", outcome: "SUPERSEDED", reason: "generation" });
|
||||
expect(api.requests).toEqual([`fence:${CH}`, `apply:${CH}:1`, "audit:SUPERSEDED"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("response parsers (fail closed on malformed API replies)", () => {
|
||||
it("parseFence accepts a fence or absence and rejects junk", () => {
|
||||
expect(parseFence(null)).toBeUndefined();
|
||||
expect(parseFence({ generation: "g1", desiredVersion: 4, authzEpoch: 7, directorAuthorized: true })).toEqual({ generation: "g1", desiredVersion: 4, authzEpoch: 7, directorAuthorized: true });
|
||||
expect(() => parseFence({})).toThrow(/generation/);
|
||||
expect(() => parseFence({ generation: "g1", desiredVersion: "4", authzEpoch: 7, directorAuthorized: true })).toThrow(/desiredVersion/);
|
||||
expect(() => parseFence({ generation: "g1", desiredVersion: 4, authzEpoch: 7, directorAuthorized: "yes" })).toThrow(/directorAuthorized/);
|
||||
});
|
||||
it("parseApplyResult accepts applied/duplicate/conflict and rejects junk", () => {
|
||||
expect(parseApplyResult({ kind: "applied", desiredVersion: 5 })).toEqual({ kind: "applied", desiredVersion: 5 });
|
||||
expect(parseApplyResult({ kind: "duplicate", desiredVersion: 5 })).toEqual({ kind: "duplicate", desiredVersion: 5 });
|
||||
expect(parseApplyResult({ kind: "conflict" })).toEqual({ kind: "conflict" });
|
||||
expect(() => parseApplyResult({ kind: "applied" })).toThrow(/desiredVersion/);
|
||||
expect(() => parseApplyResult({ kind: "nope" })).toThrow(/kind/);
|
||||
expect(() => parseApplyResult("x")).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("DirectorHttpClient", () => {
|
||||
const ref: ChannelRef = { workspaceId: WS, channelId: CH, generation: "g1" };
|
||||
|
||||
function harness(): { client: DirectorHttpClient; seen: Array<{ path: string; init: RequestInit | undefined }> } {
|
||||
const seen: Array<{ path: string; init: RequestInit | undefined }> = [];
|
||||
const client = new DirectorHttpClient({
|
||||
baseUrl: "http://api.test/",
|
||||
token: "sekrit",
|
||||
fetch: async (input, init) => {
|
||||
seen.push({ path: String(input), init });
|
||||
const path = String(input);
|
||||
if (path.endsWith("/state")) return new Response(JSON.stringify({ kind: "superseded", code: "SUPERSEDED" }), { status: 200 });
|
||||
if (path.endsWith("/fence")) return new Response(JSON.stringify({ generation: "g1", desiredVersion: 4, authzEpoch: 7, directorAuthorized: true }), { status: 200 });
|
||||
if (path.endsWith("/apply")) return new Response(JSON.stringify({ kind: "applied", desiredVersion: 5 }), { status: 200 });
|
||||
return new Response(null, { status: 204 });
|
||||
},
|
||||
});
|
||||
return { client, seen };
|
||||
}
|
||||
|
||||
it("routes to the internal endpoints with the bearer token and validates replies", async () => {
|
||||
const { client, seen } = harness();
|
||||
expect(await client.channelState(ref)).toEqual({ kind: "superseded", code: "SUPERSEDED" });
|
||||
expect(await client.fence(ref)).toMatchObject({ generation: "g1" });
|
||||
expect(await client.applyProgramSwitch(ref, {
|
||||
commandId: "c1", expectedDesiredVersion: 4, authzEpoch: 7,
|
||||
program: { kind: "sources", sourceIds: [SRC] }, layout: "single", reasons: ["r"],
|
||||
})).toEqual({ kind: "applied", desiredVersion: 5 });
|
||||
await client.audit(ref, { commandId: "c1", outcome: "SUPERSEDED", reason: "gen" });
|
||||
expect(seen.map((entry) => entry.path)).toEqual([
|
||||
`http://api.test/internal/director/channels/${WS}/${CH}/state`,
|
||||
`http://api.test/internal/director/channels/${WS}/${CH}/fence`,
|
||||
`http://api.test/internal/director/channels/${WS}/${CH}/apply`,
|
||||
`http://api.test/internal/director/channels/${WS}/${CH}/audit`,
|
||||
]);
|
||||
for (const entry of seen) {
|
||||
expect((entry.init?.headers as Record<string, string>).authorization).toBe("Bearer sekrit");
|
||||
expect((entry.init?.headers as Record<string, string>)["content-type"]).toBe("application/json");
|
||||
}
|
||||
});
|
||||
|
||||
it("surfaces non-2xx as transport errors (retryable by Temporal)", async () => {
|
||||
const client = new DirectorHttpClient({
|
||||
baseUrl: "http://api.test", fetch: async () => new Response("nope", { status: 503, statusText: "Service Unavailable" }),
|
||||
});
|
||||
await expect(client.channelState(ref)).rejects.toThrow(/503/);
|
||||
});
|
||||
|
||||
it("aborts a slow API within the timeout budget", async () => {
|
||||
const client = new DirectorHttpClient({
|
||||
baseUrl: "http://api.test", timeoutMs: 20,
|
||||
fetch: async (_input, init) => new Promise((_resolve, reject) => {
|
||||
init?.signal?.addEventListener("abort", () => reject(new Error("aborted")));
|
||||
}),
|
||||
});
|
||||
await expect(client.fence(ref)).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue