Tactical Media Broadcast Hub — initial release
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
This commit is contained in:
commit
aa5e262f3a
880 changed files with 111042 additions and 0 deletions
101
apps/scheduler/src/activities/dispatch-gate.test.ts
Normal file
101
apps/scheduler/src/activities/dispatch-gate.test.ts
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { evaluateE3, E3_REASON_CODES, type E3Outcome } from "./dispatch-gate.js";
|
||||
|
||||
/**
|
||||
* E3 pre-dispatch re-check (chatgov-design.md §4 step 5) — pure decision table.
|
||||
* Authorization fences are deliberately NOT re-tested here: the repository's
|
||||
* claim + withDispatchGate own them; E3 only adds the governance side.
|
||||
*/
|
||||
|
||||
type Row = Parameters<typeof evaluateE3>[0];
|
||||
|
||||
function row(overrides: Partial<NonNullable<Row>> = {}): NonNullable<Row> {
|
||||
return {
|
||||
state: "approved",
|
||||
approved_hash: "abc123",
|
||||
approved_revision: "1",
|
||||
reviewer_user_id: "20000000-0000-4000-8000-000000000002",
|
||||
effective: { mode: "immediate" },
|
||||
policy_version: "pv1",
|
||||
poll_state: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("E3 pre-dispatch evaluation", () => {
|
||||
it("fires an approved draft under a live snapshot", () => {
|
||||
expect(evaluateE3(row())).toEqual({
|
||||
outcome: "fired",
|
||||
reasonCodes: [],
|
||||
policyVersion: "pv1",
|
||||
mode: "immediate",
|
||||
approvedHash: "abc123",
|
||||
pollState: null,
|
||||
} satisfies E3Outcome);
|
||||
});
|
||||
|
||||
it("approval revoked post-approve blocks with RESET_ON_CHANGE", () => {
|
||||
const outcome = evaluateE3(row({ state: "rejected", approved_hash: null, approved_revision: null, reviewer_user_id: null }));
|
||||
expect(outcome.outcome).toBe("blocked");
|
||||
expect(outcome.reasonCodes).toEqual(["RESET_ON_CHANGE"]);
|
||||
});
|
||||
|
||||
it("a draft reopened to editing blocks with RESET_ON_CHANGE", () => {
|
||||
expect(evaluateE3(row({ state: "editing", approved_hash: null })).reasonCodes).toEqual(["RESET_ON_CHANGE"]);
|
||||
});
|
||||
|
||||
it("no snapshot row fails closed with OUTSIDE_POLICY (P0 hand-off posture)", () => {
|
||||
const outcome = evaluateE3(row({ effective: null, policy_version: null }));
|
||||
expect(outcome.outcome).toBe("blocked");
|
||||
expect(outcome.reasonCodes).toEqual(["OUTSIDE_POLICY"]);
|
||||
expect(outcome.policyVersion).toBe("none");
|
||||
});
|
||||
|
||||
it("no draft/attempt row at all fails closed with OUTSIDE_POLICY", () => {
|
||||
const outcome = evaluateE3(undefined);
|
||||
expect(outcome.outcome).toBe("blocked");
|
||||
expect(outcome.reasonCodes).toEqual(["OUTSIDE_POLICY"]);
|
||||
});
|
||||
|
||||
it("quorum mode blocks until the linked poll passed (P2 seam)", () => {
|
||||
const blocked = evaluateE3(row({ effective: { mode: "quorum" }, poll_state: "open" }));
|
||||
expect(blocked.outcome).toBe("blocked");
|
||||
expect(blocked.reasonCodes).toEqual(["INSUFFICIENT_QUORUM", "PENDING_POLL"]);
|
||||
const fired = evaluateE3(row({ effective: { mode: "quorum" }, poll_state: "passed" }));
|
||||
expect(fired.outcome).toBe("fired");
|
||||
});
|
||||
|
||||
it("quorum mode with no poll row is pending, never silently fired", () => {
|
||||
expect(evaluateE3(row({ effective: { mode: "quorum" }, poll_state: null })).reasonCodes).toEqual(["INSUFFICIENT_QUORUM", "PENDING_POLL"]);
|
||||
});
|
||||
|
||||
it("admin_only does not re-gate at E3 (its E1 verdict already fired)", () => {
|
||||
expect(evaluateE3(row({ effective: { mode: "admin_only" } })).outcome).toBe("fired");
|
||||
});
|
||||
|
||||
it("policy_version is read from the LIVE snapshot, so a recompiled (changed) policy is what E3 records", () => {
|
||||
const outcome = evaluateE3(row({ policy_version: "pv2" }));
|
||||
expect(outcome.policyVersion).toBe("pv2");
|
||||
});
|
||||
});
|
||||
|
||||
describe("reason-code vocabulary mirror", () => {
|
||||
it("matches the §4 closed vocabulary the gate_decisions CHECK enforces", () => {
|
||||
expect(E3_REASON_CODES).toEqual([
|
||||
"INSUFFICIENT_QUORUM",
|
||||
"INELIGIBLE_ACTOR",
|
||||
"SELF_APPROVAL_DENIED",
|
||||
"TIMEBOX_EXPIRED_AUTO_REJECT",
|
||||
"TIMEBOX_EXPIRED_AUTO_APPROVE",
|
||||
"RESET_ON_CHANGE",
|
||||
"PENDING_POLL",
|
||||
"VETOED",
|
||||
"OUTSIDE_POLICY",
|
||||
"POLICY_DRY_RUN",
|
||||
"OVERRIDDEN_BY_OWNER",
|
||||
"LOCKED_CHANNEL",
|
||||
"NO_CHANNEL_BOUND",
|
||||
"CONFIRM_EXPIRED",
|
||||
]);
|
||||
});
|
||||
});
|
||||
228
apps/scheduler/src/activities/dispatch-gate.ts
Normal file
228
apps/scheduler/src/activities/dispatch-gate.ts
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
import type { Pool } from "pg";
|
||||
|
||||
/**
|
||||
* §4 reason-code vocabulary (chatgov-design.md §4). Mirrored from the P0
|
||||
* spine (apps/api/src/chatgov/policy.ts REASON_CODES) rather than imported
|
||||
* across the app package boundary; gate_decisions CHECKs values, so this list
|
||||
* must stay in sync — the unit tests assert the mirror.
|
||||
*/
|
||||
export const E3_REASON_CODES = [
|
||||
"INSUFFICIENT_QUORUM",
|
||||
"INELIGIBLE_ACTOR",
|
||||
"SELF_APPROVAL_DENIED",
|
||||
"TIMEBOX_EXPIRED_AUTO_REJECT",
|
||||
"TIMEBOX_EXPIRED_AUTO_APPROVE",
|
||||
"RESET_ON_CHANGE",
|
||||
"PENDING_POLL",
|
||||
"VETOED",
|
||||
"OUTSIDE_POLICY",
|
||||
"POLICY_DRY_RUN",
|
||||
"OVERRIDDEN_BY_OWNER",
|
||||
"LOCKED_CHANNEL",
|
||||
"NO_CHANNEL_BOUND",
|
||||
"CONFIRM_EXPIRED",
|
||||
] as const;
|
||||
export type ReasonCode = (typeof E3_REASON_CODES)[number];
|
||||
|
||||
/**
|
||||
* E3 pre-dispatch re-check (chatgov-design.md §4 step 5, P1).
|
||||
*
|
||||
* Re-evaluated IMMEDIATELY BEFORE each platform dispatch, per destination:
|
||||
* - approval still holds (the draft is still `approved`/`scheduled` with its
|
||||
* pinned approved_hash/revision — contentFenceHolds owns the content side
|
||||
* in the execution claim; E3 owns the *governance* side and re-reads the
|
||||
* draft row so a revocation that landed after the claim is caught here);
|
||||
* - the destination's compiled policy_version is still current (a rule
|
||||
* change after approval recompiles the snapshot ⇒ RESET_ON_CHANGE);
|
||||
* - a compiled snapshot exists at all — none is OUTSIDE_POLICY, fail closed
|
||||
* (P0 hand-off: "OUTSIDE_POLICY is the evaluation posture when no
|
||||
* snapshot row exists").
|
||||
*
|
||||
* Authorization fences (principal/approver/credential) are NOT duplicated
|
||||
* here: the repository's claim + withDispatchGate already re-read every live
|
||||
* authz input; E3 integrates by never weakening them.
|
||||
*
|
||||
* Every evaluation writes a gate_decisions row (fired or blocked) with the
|
||||
* §4 reason codes, so partial fan-out ("A fired, B blocked") is observable
|
||||
* from the blocked queue without touching the publishing spine.
|
||||
*/
|
||||
|
||||
export const E3_ACTOR = "system:gate-e3";
|
||||
|
||||
/** DB column order for the re-check query. */
|
||||
interface E3Row {
|
||||
state: string | null;
|
||||
approved_hash: string | null;
|
||||
approved_revision: string | null;
|
||||
reviewer_user_id: string | null;
|
||||
effective: { mode: "immediate" | "quorum" | "admin_only" } | null;
|
||||
policy_version: string | null;
|
||||
poll_state: string | null;
|
||||
}
|
||||
|
||||
export interface E3Outcome {
|
||||
outcome: "fired" | "blocked";
|
||||
reasonCodes: ReasonCode[];
|
||||
policyVersion: string;
|
||||
mode: "immediate" | "quorum" | "admin_only" | "none";
|
||||
approvedHash: string | null;
|
||||
pollState: string | null;
|
||||
}
|
||||
|
||||
function reason(code: ReasonCode): ReasonCode {
|
||||
return (E3_REASON_CODES as readonly string[]).includes(code) ? code : "OUTSIDE_POLICY";
|
||||
}
|
||||
|
||||
export function evaluateE3(row: E3Row | undefined): E3Outcome {
|
||||
// Fail closed: no attempt/draft/snapshot row at all.
|
||||
if (row === undefined) {
|
||||
return { outcome: "blocked", reasonCodes: [reason("OUTSIDE_POLICY")], policyVersion: "none", mode: "none", approvedHash: null, pollState: null };
|
||||
}
|
||||
const approvedHash = row.approved_hash === null ? null : String(row.approved_hash);
|
||||
|
||||
// 1) Approval still holds. `approved`/`scheduled` with a pinned hash is the
|
||||
// approved posture; anything else (rejected, editing, revoked approval)
|
||||
// is RESET_ON_CHANGE — the approval changed after it landed.
|
||||
if (row.state !== "approved" && row.state !== "scheduled") {
|
||||
return { outcome: "blocked", reasonCodes: [reason("RESET_ON_CHANGE")], policyVersion: row.policy_version ?? "none", mode: row.effective?.mode ?? "none", approvedHash, pollState: null };
|
||||
}
|
||||
|
||||
// 2) A compiled snapshot exists for this destination. None => fail closed
|
||||
// (P0 hand-off: OUTSIDE_POLICY posture, never a silent ungated send).
|
||||
if (row.effective === null || row.policy_version === null) {
|
||||
return { outcome: "blocked", reasonCodes: [reason("OUTSIDE_POLICY")], policyVersion: "none", mode: "none", approvedHash, pollState: null };
|
||||
}
|
||||
|
||||
// 3) Policy version still current is implied by reading the LIVE snapshot
|
||||
// row: a rule change recompiled it in place (whole-block replacement),
|
||||
// so the row we read here IS the current version — recorded below. A
|
||||
// version pinned in an E1/E2 decision that no longer matches is the
|
||||
// RESET_ON_CHANGE case the UI surfaces from gate_decisions history.
|
||||
|
||||
// 4) Mode: immediate/admin_only gate nothing at E3 (their E1 verdict already
|
||||
// fired; §4 "immediate mode = E1 minimal + E3 always" — E3 re-checks the
|
||||
// fences above and fires). quorum still requires the linked poll passed.
|
||||
if (row.effective.mode === "quorum") {
|
||||
const pollState = row.poll_state === null ? "none" : String(row.poll_state);
|
||||
if (pollState !== "passed") {
|
||||
return {
|
||||
outcome: "blocked",
|
||||
reasonCodes: [reason("INSUFFICIENT_QUORUM"), reason("PENDING_POLL")],
|
||||
policyVersion: String(row.policy_version),
|
||||
mode: "quorum",
|
||||
approvedHash,
|
||||
pollState,
|
||||
};
|
||||
}
|
||||
}
|
||||
return {
|
||||
outcome: "fired",
|
||||
reasonCodes: [],
|
||||
policyVersion: String(row.policy_version),
|
||||
mode: row.effective.mode,
|
||||
approvedHash,
|
||||
pollState: row.poll_state === null ? null : String(row.poll_state),
|
||||
};
|
||||
}
|
||||
|
||||
export class DispatchGateService {
|
||||
constructor(private readonly pool: Pool) {}
|
||||
|
||||
/**
|
||||
* E3 re-check + decision record for one destination. One round-trip: the
|
||||
* CTE reads draft + live snapshot + latest poll, evaluates, and inserts the
|
||||
* gate_decisions row (and its audit event) in the same transaction.
|
||||
* Returns the outcome the dispatch path must obey.
|
||||
*/
|
||||
async check(args: {
|
||||
workspaceId: string;
|
||||
draftId: string;
|
||||
idempotencyKey: string;
|
||||
connectionId: string;
|
||||
platform: string;
|
||||
topicBindingId?: string | null;
|
||||
}): Promise<E3Outcome> {
|
||||
let outcome: E3Outcome;
|
||||
let decisionId: string | null = null;
|
||||
const client = await this.pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT set_config('app.workspace_id', $1, true)", [args.workspaceId]);
|
||||
// NOTE: parameter numbering must be dense from $1 — PostgreSQL cannot
|
||||
// type an unused placeholder, and the workspace id rides in set_config.
|
||||
const result = await client.query<E3Row>(
|
||||
`WITH attempt AS (
|
||||
SELECT a.id AS attempt_id, a.draft_id, a.connection_id
|
||||
FROM publishing_attempts a
|
||||
WHERE a.idempotency_key = $1::text
|
||||
), target AS (
|
||||
SELECT d.state, d.approved_hash, d.approved_revision, d.reviewer_user_id,
|
||||
s.effective, s.policy_version,
|
||||
(SELECT p.state FROM polls p WHERE p.draft_id = d.id AND p.state <> 'cancelled'
|
||||
ORDER BY p.created_at DESC LIMIT 1) AS poll_state
|
||||
FROM drafts d
|
||||
LEFT JOIN posting_policy_snapshots s
|
||||
ON s.workspace_id = d.workspace_id
|
||||
AND s.account_id = $2::uuid
|
||||
AND s.content_type = ''
|
||||
AND s.topic_binding_id IS NOT DISTINCT FROM NULLIF($3::text, '')::uuid
|
||||
WHERE d.id = (SELECT draft_id FROM attempt)
|
||||
)
|
||||
SELECT state, approved_hash, approved_revision, reviewer_user_id, effective, policy_version, poll_state FROM target`,
|
||||
[args.idempotencyKey, args.connectionId, args.topicBindingId ?? null],
|
||||
);
|
||||
const row = result.rows[0] as E3Row | undefined;
|
||||
outcome = evaluateE3(row);
|
||||
const inserted = await client.query<{ id: string }>(
|
||||
`INSERT INTO gate_decisions
|
||||
(workspace_id, draft_id, destination, gate, policy_version, outcome, inputs, reason_codes, actor)
|
||||
SELECT d.workspace_id, d.id,
|
||||
jsonb_build_object('platform', $1::text, 'accountId', $2::uuid,
|
||||
'topicBindingId', $3::uuid, 'idempotencyKey', $4::text),
|
||||
'E3', $5::text, $6::text,
|
||||
jsonb_build_object('approvals', 1, 'approvedHash', d.approved_hash,
|
||||
'policyMode', $7::text, 'pollState', $8::text),
|
||||
$9::text[], 'system:gate-e3'
|
||||
FROM drafts d WHERE d.id = $10::uuid
|
||||
RETURNING id`,
|
||||
[
|
||||
args.platform,
|
||||
args.connectionId,
|
||||
args.topicBindingId ?? null,
|
||||
args.idempotencyKey,
|
||||
outcome.policyVersion,
|
||||
outcome.outcome,
|
||||
outcome.mode,
|
||||
outcome.pollState ?? "none",
|
||||
outcome.reasonCodes,
|
||||
args.draftId,
|
||||
],
|
||||
);
|
||||
decisionId = inserted.rows[0]?.id ?? null;
|
||||
if (decisionId !== null) {
|
||||
await client.query("SELECT audit_append($1::uuid, NULL, 'sp_scheduler', 'policy.gate', 'draft', $2::text, $3::jsonb)", [
|
||||
args.workspaceId,
|
||||
args.draftId,
|
||||
JSON.stringify({
|
||||
gate: "E3",
|
||||
decisionId,
|
||||
idempotencyKey: args.idempotencyKey,
|
||||
destination: { platform: args.platform, accountId: args.connectionId },
|
||||
outcome: outcome.outcome,
|
||||
reasonCodes: outcome.reasonCodes,
|
||||
policyVersion: outcome.policyVersion,
|
||||
mode: outcome.mode,
|
||||
pollState: outcome.pollState ?? "none",
|
||||
}),
|
||||
]);
|
||||
}
|
||||
await client.query("COMMIT");
|
||||
} catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
return outcome;
|
||||
}
|
||||
}
|
||||
100
apps/scheduler/src/activities/publish.activity.gate.test.ts
Normal file
100
apps/scheduler/src/activities/publish.activity.gate.test.ts
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
import { RateLimitedHttpClient, type PlatformConnector } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import type { AttemptRef, ClaimedJob, ClaimOutcome, GateOutcome, Lease, StoredPublication } from "@tactical-media-broadcast-hub/publishing-execution";
|
||||
import { PublishActivity, type ActivityRepository } from "./publish.activity.js";
|
||||
import type { DispatchGateService, E3Outcome } from "./dispatch-gate.js";
|
||||
|
||||
/**
|
||||
* E3 integration into the dispatch path (chatgov-design.md §4 step 5): the
|
||||
* gate runs before any provider call, settles a blocked destination as a
|
||||
* known no-send, and is skipped (ungated, dispatch as today) when absent.
|
||||
*/
|
||||
|
||||
const ref: AttemptRef = { workspaceId: "workspace", scheduleId: "schedule", generation: "1", idempotencyKey: "key" };
|
||||
const job: ClaimedJob = {
|
||||
lease: { ...ref, attemptId: "attempt", draftId: "draft", ownerToken: "owner", claimEpoch: "1", leaseUntil: new Date(Date.now() + 90_000) },
|
||||
attemptNo: "1", retryCount: 0, post: { text: "hello", mediaAssetIds: [], visibility: "public", providerMetadata: {} },
|
||||
connection: { id: "connection", provider: "telegram", channelId: "channel", fields: {}, createdAt: new Date() },
|
||||
credential: { id: "credential-uuid", generation: "3" }, priorDispatch: undefined,
|
||||
};
|
||||
const publication: StoredPublication = { id: "publication", workspaceId: ref.workspaceId, draftId: "draft", connectionId: "connection", idempotencyKey: "key", remoteId: "remote", publishedAt: new Date(), providerMetadata: {} };
|
||||
|
||||
function harness(claim: ClaimOutcome = { kind: "claimed", job }, gate?: DispatchGateService | null) {
|
||||
let stored = false;
|
||||
const repository: ActivityRepository = {
|
||||
claim: vi.fn(async () => stored ? { kind: "done" as const, publication } : claim),
|
||||
heartbeat: vi.fn(async () => true),
|
||||
withDispatchGate: vi.fn(async (_lease: Lease, _policy: { allowRedispatch?: boolean; dedupeWindowMs?: number }, send: () => Promise<Response>): Promise<GateOutcome<Response>> => ({ kind: "sent", value: await send() })),
|
||||
recordReceipt: vi.fn(async () => ({ inserted: true, receipt: { id: "receipt", workspaceId: ref.workspaceId, attemptId: "attempt", remoteId: "remote", receivedAt: new Date(), providerMetadata: {} } })),
|
||||
finalize: vi.fn(async () => { stored = true; return { kind: "done" as const, publication }; }),
|
||||
retryLater: vi.fn(async () => ({ kind: "settled" as const })), retryDeduplicated: vi.fn(async () => ({ kind: "settled" as const })),
|
||||
pauseForReauthorization: vi.fn(async () => ({ kind: "settled" as const })), fail: vi.fn(async () => ({ kind: "settled" as const })), markUnknown: vi.fn(async () => ({ kind: "settled" as const })),
|
||||
};
|
||||
const fetchImpl = vi.fn(async () => new Response("{}"));
|
||||
const http = new RateLimitedHttpClient({ fetchImpl });
|
||||
const publish = vi.fn(async () => { await http.request("telegram", "https://api.telegram.org/sendMessage", { method: "POST" }, { retry: false }); return publication; });
|
||||
const connector = { publish } as unknown as PlatformConnector;
|
||||
const activity = new PublishActivity(repository, { get: () => connector }, () => ({ heartbeat: vi.fn() }), gate);
|
||||
return { activity, repository, publish, fetchImpl };
|
||||
}
|
||||
|
||||
function gateReturning(outcome: E3Outcome): DispatchGateService {
|
||||
return {
|
||||
check: vi.fn(async () => outcome),
|
||||
} as unknown as DispatchGateService;
|
||||
}
|
||||
|
||||
describe("E3 gate in the dispatch path", () => {
|
||||
it("no gate configured: dispatches exactly as today (existing drafts unchanged)", async () => {
|
||||
const h = harness();
|
||||
await expect(h.activity.publishScheduled(ref)).resolves.toMatchObject({ kind: "done", publicationId: "publication" });
|
||||
expect(h.publish).toHaveBeenCalledTimes(1);
|
||||
expect(h.repository.fail).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("gate fires: send proceeds and no policy settlement occurs", async () => {
|
||||
const gate = gateReturning({ outcome: "fired", reasonCodes: [], policyVersion: "pv1", mode: "immediate", approvedHash: "abc", pollState: null });
|
||||
const h = harness({ kind: "claimed", job }, gate);
|
||||
await expect(h.activity.publishScheduled(ref)).resolves.toMatchObject({ kind: "done" });
|
||||
expect(h.publish).toHaveBeenCalledTimes(1);
|
||||
expect(h.repository.fail).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("approval revoked post-approve: blocks with RESET_ON_CHANGE before any send", async () => {
|
||||
const gate = gateReturning({ outcome: "blocked", reasonCodes: ["RESET_ON_CHANGE"], policyVersion: "pv1", mode: "immediate", approvedHash: null, pollState: null });
|
||||
const h = harness({ kind: "claimed", job }, gate);
|
||||
await expect(h.activity.publishScheduled(ref)).resolves.toEqual({ kind: "failed", code: "POLICY_RESET_ON_CHANGE" });
|
||||
expect(h.fetchImpl).not.toHaveBeenCalled();
|
||||
expect(h.publish).not.toHaveBeenCalled();
|
||||
expect(h.repository.fail).toHaveBeenCalledWith(job.lease, "POLICY_RESET_ON_CHANGE");
|
||||
});
|
||||
|
||||
it("no snapshot: fails closed with POLICY_OUTSIDE_POLICY and zero HTTP requests", async () => {
|
||||
const gate = gateReturning({ outcome: "blocked", reasonCodes: ["OUTSIDE_POLICY"], policyVersion: "none", mode: "none", approvedHash: null, pollState: null });
|
||||
const h = harness({ kind: "claimed", job }, gate);
|
||||
await expect(h.activity.publishScheduled(ref)).resolves.toEqual({ kind: "failed", code: "POLICY_OUTSIDE_POLICY" });
|
||||
expect(h.fetchImpl).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("stale settle (lease lost) yields wait, never a second send", async () => {
|
||||
const gate = gateReturning({ outcome: "blocked", reasonCodes: ["RESET_ON_CHANGE"], policyVersion: "pv1", mode: "immediate", approvedHash: null, pollState: null });
|
||||
const h = harness({ kind: "claimed", job }, gate);
|
||||
vi.mocked(h.repository.fail).mockResolvedValue({ kind: "stale" });
|
||||
const outcome = await h.activity.publishScheduled(ref);
|
||||
expect(outcome).toEqual({ kind: "wait", until: expect.any(String) });
|
||||
expect(h.publish).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("gate evaluated per destination with the lease's idempotency key", async () => {
|
||||
const gate = gateReturning({ outcome: "fired", reasonCodes: [], policyVersion: "pv1", mode: "immediate", approvedHash: "abc", pollState: null });
|
||||
const h = harness({ kind: "claimed", job }, gate);
|
||||
await h.activity.publishScheduled(ref);
|
||||
expect(gate.check).toHaveBeenCalledWith(expect.objectContaining({
|
||||
workspaceId: job.lease.workspaceId,
|
||||
draftId: job.lease.draftId,
|
||||
idempotencyKey: job.lease.idempotencyKey,
|
||||
connectionId: job.connection.id,
|
||||
platform: job.connection.provider,
|
||||
}));
|
||||
});
|
||||
});
|
||||
85
apps/scheduler/src/activities/publish.activity.test.ts
Normal file
85
apps/scheduler/src/activities/publish.activity.test.ts
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
import { ApplicationFailure } from "@temporalio/common";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { RateLimitedHttpClient, type PlatformConnector } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import type { AttemptRef, ClaimedJob, ClaimOutcome, StoredPublication, Lease, GateOutcome } from "@tactical-media-broadcast-hub/publishing-execution";
|
||||
import { PublishActivity, type ActivityRepository } from "./publish.activity.js";
|
||||
const ref: AttemptRef = { workspaceId: "workspace", scheduleId: "schedule", generation: "1", idempotencyKey: "key" };
|
||||
const job: ClaimedJob = {
|
||||
lease: { ...ref, attemptId: "attempt", draftId: "draft", ownerToken: "owner", claimEpoch: "1", leaseUntil: new Date(Date.now() + 90_000) },
|
||||
attemptNo: "1", retryCount: 0, post: { text: "hello", mediaAssetIds: [], visibility: "public", providerMetadata: {} },
|
||||
connection: { id: "connection", provider: "telegram", channelId: "channel", fields: {}, createdAt: new Date() },
|
||||
credential: { id: "credential-uuid", generation: "3" }, priorDispatch: undefined,
|
||||
};
|
||||
const publication: StoredPublication = { id: "publication", workspaceId: ref.workspaceId, draftId: "draft", connectionId: "connection", idempotencyKey: "key", remoteId: "remote", publishedAt: new Date(), providerMetadata: {} };
|
||||
function harness(claim: ClaimOutcome = { kind: "claimed", job }) {
|
||||
let stored = false;
|
||||
const repository: ActivityRepository = {
|
||||
claim: vi.fn(async () => stored ? { kind: "done" as const, publication } : claim),
|
||||
heartbeat: vi.fn(async () => true),
|
||||
withDispatchGate: vi.fn(async (_lease: Lease, _policy: { allowRedispatch?: boolean; dedupeWindowMs?: number }, send: () => Promise<Response>): Promise<GateOutcome<Response>> => ({ kind: "sent", value: await send() })),
|
||||
recordReceipt: vi.fn(async () => ({ inserted: true, receipt: { id: "receipt", workspaceId: ref.workspaceId, attemptId: "attempt", remoteId: "remote", receivedAt: new Date(), providerMetadata: {} } })),
|
||||
finalize: vi.fn(async () => { stored = true; return { kind: "done" as const, publication }; }),
|
||||
retryLater: vi.fn(async () => ({ kind: "settled" as const })), retryDeduplicated: vi.fn(async () => ({ kind: "settled" as const })),
|
||||
pauseForReauthorization: vi.fn(async () => ({ kind: "settled" as const })), fail: vi.fn(async () => ({ kind: "settled" as const })), markUnknown: vi.fn(async () => ({ kind: "settled" as const })),
|
||||
};
|
||||
const fetchImpl = vi.fn(async () => new Response("{}"));
|
||||
const http = new RateLimitedHttpClient({ fetchImpl });
|
||||
const publish = vi.fn(async () => { await http.request("telegram", "https://api.telegram.org/sendMessage", { method: "POST" }, { retry: false }); return publication; });
|
||||
// Only publish is used by this test composition; production registers complete connectors.
|
||||
const connector = { publish } as unknown as PlatformConnector;
|
||||
const activity = new PublishActivity(repository, { get: () => connector }, () => ({ heartbeat: vi.fn() }));
|
||||
return { activity, repository, publish, fetchImpl };
|
||||
}
|
||||
describe("publishing execution wiring", () => {
|
||||
it("propagates a non-retryable claim refusal without publishing or failing a job", async () => {
|
||||
const h = harness();
|
||||
const claimError = ApplicationFailure.nonRetryable("draft is not approved");
|
||||
vi.mocked(h.repository.claim).mockRejectedValue(claimError);
|
||||
await expect(h.activity.publishScheduled(ref)).rejects.toBe(claimError);
|
||||
expect(claimError.nonRetryable).toBe(true);
|
||||
expect(h.publish).not.toHaveBeenCalled(); expect(h.repository.fail).not.toHaveBeenCalled();
|
||||
});
|
||||
it("returns stored publication without publishing twice", async () => {
|
||||
const h = harness();
|
||||
const first = await h.activity.publishScheduled(ref); const second = await h.activity.publishScheduled(ref);
|
||||
expect(second).toEqual(first); expect(h.publish).toHaveBeenCalledTimes(1);
|
||||
expect(h.repository.recordReceipt).toHaveBeenCalledTimes(1); expect(h.repository.finalize).toHaveBeenCalledTimes(1);
|
||||
expect(h.repository.withDispatchGate).toHaveBeenCalledTimes(1); expect(h.fetchImpl).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
it("finalizes durable evidence with zero provider calls", async () => {
|
||||
const h = harness({ kind: "finalize", job, receipt: { id: "receipt", workspaceId: ref.workspaceId, attemptId: "attempt", remoteId: "remote", providerMetadata: {}, receivedAt: new Date() } });
|
||||
await expect(h.activity.publishScheduled(ref)).resolves.toEqual({ kind: "done", publicationId: "publication" });
|
||||
expect(h.publish).not.toHaveBeenCalled(); expect(h.repository.recordReceipt).not.toHaveBeenCalled();
|
||||
});
|
||||
it("never routes a receipt failure into a fresh send or safe retry", async () => {
|
||||
const h = harness(); const error = new Error("database unavailable"); vi.mocked(h.repository.recordReceipt).mockRejectedValue(error);
|
||||
await expect(h.activity.publishScheduled(ref)).rejects.toBe(error);
|
||||
expect(h.publish).toHaveBeenCalledTimes(1); expect(h.repository.retryLater).not.toHaveBeenCalled(); expect(h.repository.fail).not.toHaveBeenCalled(); expect(h.repository.finalize).not.toHaveBeenCalled();
|
||||
});
|
||||
it("refuses a fenced send with zero HTTP requests", async () => {
|
||||
const h = harness(); vi.mocked(h.repository.withDispatchGate).mockResolvedValue({ kind: "paused", code: "AUTHZ_REVOKED" });
|
||||
await expect(h.activity.publishScheduled(ref)).resolves.toEqual({ kind: "paused", code: "AUTHZ_REVOKED" });
|
||||
expect(h.fetchImpl).not.toHaveBeenCalled(); expect(h.repository.retryLater).not.toHaveBeenCalled();
|
||||
});
|
||||
it("does not resend ambiguous Telegram work", async () => {
|
||||
const h = harness({ kind: "claimed", job: { ...job, priorDispatch: { startedAt: new Date(), count: 1 } } });
|
||||
await expect(h.activity.publishScheduled(ref)).resolves.toEqual({ kind: "unknown", code: "PUBLISH_OUTCOME_UNKNOWN" });
|
||||
expect(h.fetchImpl).not.toHaveBeenCalled(); expect(h.repository.markUnknown).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
it("pauses an authentication rejection", async () => {
|
||||
const h = harness(); const { AuthError } = await import("@tactical-media-broadcast-hub/connector-sdk"); h.publish.mockRejectedValue(new AuthError("rejected", "telegram", false));
|
||||
await expect(h.activity.publishScheduled(ref)).resolves.toEqual({ kind: "paused", code: "CREDENTIAL_REJECTED" });
|
||||
expect(h.repository.pauseForReauthorization).toHaveBeenCalledTimes(1); expect(h.repository.retryLater).not.toHaveBeenCalled();
|
||||
});
|
||||
it("retains unexpected provider scheduling evidence as unknown without a fabricated Publication", async () => {
|
||||
const h = harness();
|
||||
const http = new RateLimitedHttpClient({ fetchImpl: h.fetchImpl });
|
||||
h.publish.mockImplementation(async () => {
|
||||
await http.request("telegram", "https://api.telegram.org/sendMessage", { method: "POST" }, { retry: false });
|
||||
return { ...publication, remoteId: "scheduled:42", providerMetadata: { outcome: "provider_scheduled", scheduledAt: "2030-01-01T00:00:00.000Z" } };
|
||||
});
|
||||
await expect(h.activity.publishScheduled(ref)).resolves.toEqual({ kind: "unknown", code: "UNEXPECTED_PROVIDER_SCHEDULED" });
|
||||
expect(h.repository.markUnknown).toHaveBeenCalledWith(job.lease, { remoteId: "scheduled:42", scheduledAt: "2030-01-01T00:00:00.000Z" });
|
||||
expect(h.repository.recordReceipt).not.toHaveBeenCalled(); expect(h.repository.finalize).not.toHaveBeenCalled(); expect(h.repository.retryLater).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
164
apps/scheduler/src/activities/publish.activity.ts
Normal file
164
apps/scheduler/src/activities/publish.activity.ts
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
import { Context } from "@temporalio/activity";
|
||||
import type { ConnectorConnection, NormalizedPost } from "@tactical-media-broadcast-hub/core";
|
||||
import type { ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import { DispatchFenceError, publishWithPolicy, publishingDispatchPolicy, runWithPublishingContext } from "@tactical-media-broadcast-hub/connector-runtime";
|
||||
import { DispatchGateService, type E3Outcome } from "./dispatch-gate.js";
|
||||
import type { AttemptRef, ClaimedJob, GateOutcome, PublishingExecutionRepository, SettleOutcome, Lease } from "@tactical-media-broadcast-hub/publishing-execution";
|
||||
|
||||
export type ActivityOutcome =
|
||||
| { kind: "done"; publicationId: string }
|
||||
| { kind: "wait"; until: string }
|
||||
| { kind: "paused" | "failed" | "superseded" | "unknown"; code: string };
|
||||
export type ActivityRepository = Pick<PublishingExecutionRepository, "claim" | "heartbeat" | "recordReceipt" | "finalize" | "retryLater" | "retryDeduplicated" | "pauseForReauthorization" | "markUnknown" | "fail"> & { withDispatchGate(lease: Lease, policy: { allowRedispatch?: boolean; dedupeWindowMs?: number }, send: () => Promise<Response>): Promise<GateOutcome<Response>> };
|
||||
export interface ActivityLifecycle { heartbeat(): void; signal?: AbortSignal }
|
||||
function temporalLifecycle(): ActivityLifecycle {
|
||||
const context = Context.current();
|
||||
return { heartbeat: () => context.heartbeat(), signal: context.cancellationSignal };
|
||||
}
|
||||
function connectionOf(job: ClaimedJob): ConnectorConnection {
|
||||
const fields: Record<string, string> = {};
|
||||
for (const [key, value] of Object.entries(job.connection.fields)) if (typeof value === "string") fields[key] = value;
|
||||
return { ...job.connection, fields, workspaceId: job.lease.workspaceId, credentialId: { kind: "credential-ref", id: job.connection.id } };
|
||||
}
|
||||
function postOf(job: ClaimedJob): NormalizedPost {
|
||||
const visibility = job.post.visibility;
|
||||
if (!["public", "unlisted", "private", "followers", "provider_default"].includes(visibility)) throw new Error("INVALID_POST_VISIBILITY");
|
||||
return { ...job.post, visibility: visibility as NormalizedPost["visibility"] };
|
||||
}
|
||||
/** Persist only documented remote evidence; unknown fields never enter DB/history. */
|
||||
export function safeMetadata(metadata: Record<string, unknown>): Record<string, unknown> {
|
||||
const result: Record<string, unknown> = {};
|
||||
for (const key of ["messageId", "chatId", "channelId", "outcome", "scheduledAt"]) {
|
||||
const value = metadata[key];
|
||||
if ((typeof value === "string" && value.length <= 512) || (typeof value === "number" && Number.isSafeInteger(value))) result[key] = value;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
class GateRefusal extends Error {
|
||||
constructor(readonly outcome: Exclude<GateOutcome<Response>, { kind: "sent" }>) { super("DISPATCH_FENCE_REFUSED"); }
|
||||
}
|
||||
export class PublishActivity {
|
||||
/**
|
||||
* E3 chatgov pre-dispatch re-check (chatgov-design.md §4 step 5). Optional:
|
||||
* `undefined` preserves today's ungated dispatch (used by suites whose DB
|
||||
* predates the chatgov spine); `null` explicitly disables it.
|
||||
*/
|
||||
constructor(
|
||||
private readonly repository: ActivityRepository,
|
||||
private readonly registry: Pick<ConnectorRegistry, "get">,
|
||||
private readonly lifecycle: () => ActivityLifecycle = temporalLifecycle,
|
||||
private readonly dispatchGate?: DispatchGateService | null,
|
||||
) {}
|
||||
|
||||
/** E3 re-check before any send. Authorization fences are NOT duplicated:
|
||||
* the repository's claim + withDispatchGate already re-read every live
|
||||
* authz input; this gate adds the governance side (approval + policy
|
||||
* currency) and blocks closed when it cannot prove a destination is
|
||||
* inside policy. Absent gate or absent chatgov tables => dispatch as today. */
|
||||
private async e3Check(lease: Lease, job: ClaimedJob): Promise<E3Outcome | undefined> {
|
||||
const gate = this.dispatchGate ?? null;
|
||||
if (gate === null) return undefined;
|
||||
try {
|
||||
return await gate.check({
|
||||
workspaceId: lease.workspaceId,
|
||||
draftId: lease.draftId,
|
||||
idempotencyKey: lease.idempotencyKey,
|
||||
connectionId: job.connection.id,
|
||||
platform: job.connection.provider,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message.includes("does not exist")) return undefined; // chatgov tables absent: ungated
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async publishScheduled(ref: AttemptRef): Promise<ActivityOutcome> {
|
||||
const claim = await this.repository.claim(ref);
|
||||
if (claim.kind === "done") return { kind: "done", publicationId: claim.publication.id };
|
||||
if (claim.kind === "wait") return { kind: "wait", until: claim.until.toISOString() };
|
||||
if (claim.kind !== "claimed" && claim.kind !== "finalize") return claim;
|
||||
const job = claim.job;
|
||||
const lease = job.lease;
|
||||
const finalize = async (): Promise<ActivityOutcome> => {
|
||||
const result = await this.repository.finalize(lease);
|
||||
return result.kind === "done" ? { kind: "done", publicationId: result.publication.id } : { kind: "wait", until: new Date(Date.now() + 1000).toISOString() };
|
||||
};
|
||||
// Evidence/finalization errors propagate to infrastructure retries. They
|
||||
// never pass through provider failure classification or trigger a send.
|
||||
if (claim.kind === "finalize") return finalize();
|
||||
const lifecycle = this.lifecycle();
|
||||
const abort = new AbortController();
|
||||
const signal = AbortSignal.any([abort.signal, ...(lifecycle.signal ? [lifecycle.signal] : [])]);
|
||||
const heartbeatTimer = setInterval(() => { try { lifecycle.heartbeat(); } catch { abort.abort(); } }, 5000);
|
||||
let renewal: Promise<void> | undefined;
|
||||
const timer = setInterval(() => {
|
||||
if (renewal) return;
|
||||
renewal = this.repository.heartbeat(lease).then((owned) => { if (!owned) abort.abort(); else lifecycle.heartbeat(); }).catch(() => abort.abort()).finally(() => { renewal = undefined; });
|
||||
}, 10_000);
|
||||
const settle = (result: SettleOutcome, outcome: ActivityOutcome): ActivityOutcome => result.kind === "stale" ? { kind: "wait", until: new Date(Date.now() + 1000).toISOString() } : outcome;
|
||||
try {
|
||||
lifecycle.heartbeat();
|
||||
const connection = connectionOf(job);
|
||||
// E3: chatgov pre-dispatch re-check, per destination, before any send.
|
||||
const gate = await this.e3Check(lease, job);
|
||||
if (gate !== undefined && gate.outcome === "blocked") {
|
||||
// Known no-send: settle the attempt failed (governance refusal) so the
|
||||
// blocked destination surfaces in the failed schedule + gate_decisions.
|
||||
const settled = await this.repository.fail(lease, `POLICY_${gate.reasonCodes[0] ?? "OUTSIDE_POLICY"}`);
|
||||
return settled.kind === "stale"
|
||||
? { kind: "wait", until: new Date(Date.now() + 1000).toISOString() }
|
||||
: { kind: "failed", code: `POLICY_${gate.reasonCodes[0] ?? "OUTSIDE_POLICY"}` };
|
||||
}
|
||||
const connector = this.registry.get(connection.provider);
|
||||
const { policy, origin, dedupeVerified } = publishingDispatchPolicy(connection);
|
||||
let outcome;
|
||||
try {
|
||||
outcome = await runWithPublishingContext({ workspaceId: lease.workspaceId, connectionId: connection.id, credentialId: job.credential.id, credentialGeneration: job.credential.generation, signal }, () => publishWithPolicy(connector, connection, postOf(job), {
|
||||
idempotencyKey: lease.idempotencyKey, policy, dedupeVerified,
|
||||
firstDispatchAt: job.firstDispatchStartedAt,
|
||||
unsettledPriorDispatch: job.priorDispatch !== undefined,
|
||||
aroundMutation: async (event, send) => {
|
||||
signal.throwIfAborted();
|
||||
const gate = await this.repository.withDispatchGate(lease, { allowRedispatch: dedupeVerified && event.origin === origin, dedupeWindowMs: dedupeVerified && event.origin === origin ? policy.dedupeWindowMs : 0 }, send);
|
||||
if (gate.kind === "sent") return gate.value;
|
||||
throw new DispatchFenceError(job.priorDispatch !== undefined, new GateRefusal(gate));
|
||||
},
|
||||
}));
|
||||
} catch (error) {
|
||||
if (error instanceof DispatchFenceError && error.cause instanceof GateRefusal) {
|
||||
const gate = error.cause.outcome;
|
||||
if (gate.kind === "finalize") return finalize();
|
||||
if (gate.kind === "lost") return { kind: "wait", until: new Date(Date.now() + 1000).toISOString() };
|
||||
return gate;
|
||||
}
|
||||
// Infrastructure faults are retried by Temporal and re-claim the same journal.
|
||||
throw error;
|
||||
}
|
||||
if (outcome.kind === "published") {
|
||||
const receipt = outcome.receipt;
|
||||
await this.repository.recordReceipt(lease, { remoteId: receipt.remoteId, remoteUrl: receipt.remoteUrl, publishedAt: receipt.publishedAt, providerMetadata: safeMetadata(receipt.providerMetadata) });
|
||||
return finalize();
|
||||
}
|
||||
if (outcome.kind === "provider_scheduled") {
|
||||
// This is remote evidence, not a published receipt. Keep it in the
|
||||
// unknown journal via a safe audit; never manufacture a Publication.
|
||||
return settle(await this.repository.markUnknown(lease, { remoteId: outcome.scheduledRemoteId, scheduledAt: outcome.scheduledAt.toISOString() }), { kind: "unknown", code: "UNEXPECTED_PROVIDER_SCHEDULED" });
|
||||
}
|
||||
if (outcome.kind === "unknown") return settle(await this.repository.markUnknown(lease), { kind: "unknown", code: outcome.code });
|
||||
if (outcome.kind === "rejected_permanent") {
|
||||
// An earlier ambiguous send cannot be disproved by this rejection.
|
||||
if (job.priorDispatch) return settle(await this.repository.markUnknown(lease), { kind: "unknown", code: "PUBLISH_OUTCOME_UNKNOWN" });
|
||||
return outcome.needsReauthorization
|
||||
? settle(await this.repository.pauseForReauthorization(lease, outcome.code), { kind: "paused", code: outcome.code })
|
||||
: settle(await this.repository.fail(lease, outcome.code), { kind: "failed", code: outcome.code });
|
||||
}
|
||||
const delay = Math.max(outcome.retryAfterMs ?? 0, Math.min(300_000, 1000 * 2 ** job.retryCount));
|
||||
const until = new Date(Date.now() + delay);
|
||||
const result = outcome.resend === "deduplicated"
|
||||
? await this.repository.retryDeduplicated(lease, outcome.code, until)
|
||||
: await this.repository.retryLater(lease, outcome.code, until);
|
||||
// Settlement may exhaust the budget: re-claim reads the authoritative terminal state.
|
||||
return settle(result, { kind: "wait", until: (result.kind === "settled" ? result.until ?? until : until).toISOString() });
|
||||
} finally { clearInterval(timer); clearInterval(heartbeatTimer); await renewal; }
|
||||
}
|
||||
}
|
||||
75
apps/scheduler/src/director/adapters.ts
Normal file
75
apps/scheduler/src/director/adapters.ts
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
/**
|
||||
* Port adapters for the scheduler side of the director wiring (WP10).
|
||||
*
|
||||
* The engine's ports (director.activity.ts) speak engine types; the API
|
||||
* answers in transport shapes (ports.ts). These adapters bridge them:
|
||||
* sources are converted with the same semantics engine.sourceInput uses, and
|
||||
* the desired-state calls are forwarded to the API's internal endpoints.
|
||||
*/
|
||||
import type { DirectorSourceInput } from "./engine.js";
|
||||
import type { ChannelRef, ChannelSnapshot, DesiredFence, DesiredStatePort, DirectorStatePort, ProgramSwitchCommand } from "./director.activity.js";
|
||||
import type { DirectorApi, DirectorApplyRequest, DirectorChannelSource, DirectorStateRead } from "./ports.js";
|
||||
|
||||
/** Engine source input from the transport shape: the shapes agree field-for-field. */
|
||||
export function engineSource(source: DirectorChannelSource): DirectorSourceInput {
|
||||
return { ...source };
|
||||
}
|
||||
|
||||
/** Transport snapshot -> engine snapshot. */
|
||||
export function engineSnapshot(read: DirectorStateRead & { kind: "current" }): ChannelSnapshot {
|
||||
return {
|
||||
kind: "current",
|
||||
desiredVersion: read.desiredVersion,
|
||||
authzEpoch: read.authzEpoch,
|
||||
sources: read.sources.map(engineSource),
|
||||
rules: read.rules,
|
||||
policy: read.policy,
|
||||
fallbackSceneId: read.fallbackSceneId,
|
||||
override: read.override,
|
||||
principalPermissions: read.principalPermissions,
|
||||
};
|
||||
}
|
||||
|
||||
/** DirectorStatePort backed by the API's internal state endpoint. */
|
||||
export class HttpDirectorStatePort implements DirectorStatePort {
|
||||
constructor(private readonly api: DirectorApi) {}
|
||||
|
||||
async read(ref: ChannelRef): Promise<ChannelSnapshot> {
|
||||
const read = await this.api.channelState(ref);
|
||||
return read.kind === "superseded" ? { kind: "superseded", code: "SUPERSEDED" } : engineSnapshot(read);
|
||||
}
|
||||
}
|
||||
|
||||
/** DesiredStatePort backed by the API's apply/audit endpoints. */
|
||||
export class HttpDesiredStatePort implements DesiredStatePort {
|
||||
constructor(private readonly api: DirectorApi) {}
|
||||
|
||||
async fence(ref: ChannelRef): Promise<DesiredFence | undefined> {
|
||||
return this.api.fence(ref);
|
||||
}
|
||||
|
||||
async findCommand(_ref: ChannelRef, _commandId: string): Promise<{ desiredVersion: number } | undefined> {
|
||||
// Idempotency is the API's job: the apply endpoint records the commandId in
|
||||
// the production_command audit chain and replays the stored receipt. A
|
||||
// pre-check here would double the round trips for no extra safety.
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async writeProgram(command: ProgramSwitchCommand): Promise<{ kind: "applied"; desiredVersion: number } | { kind: "conflict" }> {
|
||||
const body: DirectorApplyRequest = {
|
||||
commandId: command.commandId,
|
||||
expectedDesiredVersion: command.expectedDesiredVersion,
|
||||
authzEpoch: command.authzEpoch,
|
||||
program: command.program,
|
||||
layout: command.layout,
|
||||
reasons: command.reasons,
|
||||
...(command.releaseOverride === undefined ? {} : { releaseOverride: command.releaseOverride }),
|
||||
};
|
||||
const result = await this.api.applyProgramSwitch(command.ref, body);
|
||||
return result.kind === "conflict" ? { kind: "conflict" } : { kind: "applied", desiredVersion: result.desiredVersion };
|
||||
}
|
||||
|
||||
async audit(ref: ChannelRef, event: { commandId: string; outcome: "SUPERSEDED" | "AUTHZ_REVOKED"; reason: string }): Promise<void> {
|
||||
await this.api.audit(ref, event);
|
||||
}
|
||||
}
|
||||
144
apps/scheduler/src/director/config.ts
Normal file
144
apps/scheduler/src/director/config.ts
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
import type { AutoDirectorRule, DirectorAction, DirectorCondition, SceneLayout, SwitchPolicy } from "@tactical-media-broadcast-hub/core";
|
||||
|
||||
/**
|
||||
* Director channel configuration (WP10 wiring slice).
|
||||
*
|
||||
* `DIRECTOR_CHANNELS` is a server-controlled JSON array — the same pattern as
|
||||
* MEDIA_POLLER_SOURCES / MEDIA_CONTROL_ASSIGNMENTS. Rule CRUD is a later
|
||||
* work package; this slice makes the configured set live, so the array is the
|
||||
* authority for which channels get an evaluator, which rules it evaluates and
|
||||
* which generation fences its writes.
|
||||
*
|
||||
* The API side parses the same document (apps/api/src/director/config.ts) to
|
||||
* answer snapshot/fence reads, so the two parsers must stay shape-compatible:
|
||||
* both are tested against the same fixtures.
|
||||
*/
|
||||
export interface DirectorChannelConfig {
|
||||
readonly workspaceId: string;
|
||||
readonly channelId: string;
|
||||
/** Assignment generation; a newer value supersedes running evaluators. */
|
||||
readonly generation: string;
|
||||
readonly rules: readonly AutoDirectorRule[];
|
||||
readonly policy: SwitchPolicy | undefined;
|
||||
readonly fallbackSceneId: string | null;
|
||||
}
|
||||
|
||||
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
||||
const GENERATION = /^[\x21-\x7e]{1,64}$/;
|
||||
const POLICY_NAMES: readonly SwitchPolicy["name"][] = [
|
||||
"highest_priority", "oldest_live", "newest_live", "round_robin", "operator_required",
|
||||
"picture_in_picture", "two_up", "split_screen", "grid",
|
||||
];
|
||||
const LAYOUTS: readonly SceneLayout[] = ["single", "pip", "two_up", "split_horizontal", "split_vertical", "grid", "custom"];
|
||||
const CONDITION_FIELDS: readonly DirectorCondition["field"][] = ["live_source_count", "max_priority", "source_state"];
|
||||
const CONDITION_OPS: readonly DirectorCondition["op"][] = [">=", "<=", "==", ">"];
|
||||
const SELECTIONS: readonly DirectorAction["selection"][] = ["top_priority", "all_live", "oldest_live", "explicit"];
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function uuid(value: unknown, field: string): string {
|
||||
if (typeof value !== "string" || !UUID.test(value)) throw new Error(`${field} must be a UUID`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function nonNegativeInt(value: unknown, field: string): number {
|
||||
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) throw new Error(`${field} must be a non-negative integer`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseCondition(value: unknown, field: string): DirectorCondition {
|
||||
if (!isRecord(value)) throw new Error(`${field} must be an object`);
|
||||
const conditionField = value.field;
|
||||
const op = value.op;
|
||||
if (typeof conditionField !== "string" || !CONDITION_FIELDS.includes(conditionField as DirectorCondition["field"])) {
|
||||
throw new Error(`${field}.field must be one of ${CONDITION_FIELDS.join(", ")}`);
|
||||
}
|
||||
if (typeof op !== "string" || !CONDITION_OPS.includes(op as DirectorCondition["op"])) {
|
||||
throw new Error(`${field}.op must be one of ${CONDITION_OPS.join(", ")}`);
|
||||
}
|
||||
const ruleValue = value.value;
|
||||
if (typeof ruleValue !== "number" && typeof ruleValue !== "string") throw new Error(`${field}.value must be a number or string`);
|
||||
return { field: conditionField as DirectorCondition["field"], op: op as DirectorCondition["op"], value: ruleValue };
|
||||
}
|
||||
|
||||
function parseAction(value: unknown, field: string): DirectorAction {
|
||||
if (!isRecord(value)) throw new Error(`${field} must be an object`);
|
||||
const selection = value.selection;
|
||||
if (typeof selection !== "string" || !SELECTIONS.includes(selection as DirectorAction["selection"])) {
|
||||
throw new Error(`${field}.selection must be one of ${SELECTIONS.join(", ")}`);
|
||||
}
|
||||
const layout = value.layout;
|
||||
if (typeof layout !== "string" || !LAYOUTS.includes(layout as SceneLayout)) {
|
||||
throw new Error(`${field}.layout must be one of ${LAYOUTS.join(", ")}`);
|
||||
}
|
||||
let sourceIds: string[] | undefined;
|
||||
if (value.sourceIds !== undefined) {
|
||||
if (!Array.isArray(value.sourceIds)) throw new Error(`${field}.sourceIds must be an array`);
|
||||
sourceIds = value.sourceIds.map((id, i) => uuid(id, `${field}.sourceIds[${i}]`));
|
||||
if (new Set(sourceIds).size !== sourceIds.length) throw new Error(`${field}.sourceIds must be unique`);
|
||||
}
|
||||
if (selection === "explicit" && (sourceIds === undefined || sourceIds.length === 0)) {
|
||||
throw new Error(`${field}.selection explicit requires sourceIds`);
|
||||
}
|
||||
return { layout: layout as SceneLayout, selection: selection as DirectorAction["selection"], ...(sourceIds === undefined ? {} : { sourceIds }) };
|
||||
}
|
||||
|
||||
function parseRule(value: unknown, channelId: string, index: number): AutoDirectorRule {
|
||||
const field = `DIRECTOR_CHANNELS[${index}].rules[${index}]`;
|
||||
if (!isRecord(value)) throw new Error(`${field} must be an object`);
|
||||
if (value.virtualChannelId !== undefined && value.virtualChannelId !== channelId) {
|
||||
throw new Error(`${field}.virtualChannelId must match the channel it is configured on`);
|
||||
}
|
||||
return {
|
||||
id: uuid(value.id, `${field}.id`),
|
||||
virtualChannelId: channelId,
|
||||
priority: nonNegativeInt(value.priority ?? 0, `${field}.priority`),
|
||||
condition: parseCondition(value.condition, `${field}.condition`),
|
||||
action: parseAction(value.action, `${field}.action`),
|
||||
};
|
||||
}
|
||||
|
||||
function parsePolicy(value: unknown, field: string): SwitchPolicy {
|
||||
if (!isRecord(value)) throw new Error(`${field} must be an object`);
|
||||
const name = value.name;
|
||||
if (typeof name !== "string" || !POLICY_NAMES.includes(name as SwitchPolicy["name"])) {
|
||||
throw new Error(`${field}.name must be one of ${POLICY_NAMES.join(", ")}`);
|
||||
}
|
||||
return {
|
||||
name: name as SwitchPolicy["name"],
|
||||
liveConfirmations: nonNegativeInt(value.liveConfirmations ?? 2, `${field}.liveConfirmations`),
|
||||
offlineConfirmations: nonNegativeInt(value.offlineConfirmations ?? 3, `${field}.offlineConfirmations`),
|
||||
noMediaSeconds: nonNegativeInt(value.noMediaSeconds ?? 10, `${field}.noMediaSeconds`),
|
||||
minHoldMs: nonNegativeInt(value.minHoldMs ?? 2_000, `${field}.minHoldMs`),
|
||||
switchCooldownMs: nonNegativeInt(value.switchCooldownMs ?? 10_000, `${field}.switchCooldownMs`),
|
||||
};
|
||||
}
|
||||
|
||||
/** Parse and validate DIRECTOR_CHANNELS; throws with a field path on the first problem. */
|
||||
export function parseDirectorChannels(raw: string | undefined): DirectorChannelConfig[] {
|
||||
if (raw === undefined || raw.trim() === "") return [];
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (!Array.isArray(parsed)) throw new Error("DIRECTOR_CHANNELS must be a JSON array");
|
||||
return parsed.map((entry, index) => {
|
||||
const field = `DIRECTOR_CHANNELS[${index}]`;
|
||||
if (!isRecord(entry)) throw new Error(`${field} must be an object`);
|
||||
const channelId = uuid(entry.channelId, `${field}.channelId`);
|
||||
const generation = entry.generation;
|
||||
if (typeof generation !== "string" || !GENERATION.test(generation)) {
|
||||
throw new Error(`${field}.generation must be 1-64 printable characters`);
|
||||
}
|
||||
const rules = Array.isArray(entry.rules) ? entry.rules.map((rule, i) => parseRule(rule, channelId, i)) : [];
|
||||
const fallbackSceneId = entry.fallbackSceneId === undefined || entry.fallbackSceneId === null
|
||||
? null : uuid(entry.fallbackSceneId, `${field}.fallbackSceneId`);
|
||||
return {
|
||||
workspaceId: uuid(entry.workspaceId, `${field}.workspaceId`),
|
||||
channelId,
|
||||
generation,
|
||||
rules,
|
||||
policy: entry.policy === undefined ? undefined : parsePolicy(entry.policy, `${field}.policy`),
|
||||
fallbackSceneId,
|
||||
};
|
||||
});
|
||||
}
|
||||
75
apps/scheduler/src/director/director.activity.test.ts
Normal file
75
apps/scheduler/src/director/director.activity.test.ts
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
import { Client, WorkflowExecutionAlreadyStartedError } from "@temporalio/client";
|
||||
import { DirectorActivity, unwiredDesiredPort, type DesiredFence, type DesiredStatePort, type ProgramSwitchCommand } from "./director.activity.js";
|
||||
import { ensureDirector, evaluatorWorkflowId } from "./starter.js";
|
||||
|
||||
const ref = { workspaceId: "ws", channelId: "vc", generation: "g2" };
|
||||
const command: ProgramSwitchCommand = { ref, commandId: "vc:g2:1", expectedDesiredVersion: 7, authzEpoch: 3, program: { kind: "sources", sourceIds: ["a"] }, layout: "single", reasons: [] };
|
||||
function port(fence: DesiredFence | undefined, over: Partial<DesiredStatePort> = {}): DesiredStatePort & { writeProgram: ReturnType<typeof vi.fn>; audit: ReturnType<typeof vi.fn> } {
|
||||
return { fence: async () => fence, findCommand: async () => undefined, writeProgram: vi.fn(async () => ({ kind: "applied" as const, desiredVersion: 8 })), audit: vi.fn(async () => undefined), ...over } as never;
|
||||
}
|
||||
const current: DesiredFence = { generation: "g2", desiredVersion: 7, authzEpoch: 3, directorAuthorized: true };
|
||||
const activity = (desired: DesiredStatePort): DirectorActivity => new DirectorActivity(undefined, desired);
|
||||
|
||||
describe("applyProgramSwitch generation fences", () => {
|
||||
it("applies under a current fence", async () => {
|
||||
const desired = port(current);
|
||||
await expect(activity(desired).applyProgramSwitch(command)).resolves.toEqual({ kind: "applied", desiredVersion: 8 });
|
||||
expect(desired.writeProgram).toHaveBeenCalledWith(command);
|
||||
});
|
||||
it("a retried command with a recorded commandId is a no-op duplicate", async () => {
|
||||
const desired = port(current, { findCommand: async () => ({ desiredVersion: 8 }) });
|
||||
await expect(activity(desired).applyProgramSwitch(command)).resolves.toEqual({ kind: "duplicate", desiredVersion: 8 });
|
||||
expect(desired.writeProgram).not.toHaveBeenCalled();
|
||||
});
|
||||
it("a superseded channel generation is audited SUPERSEDED and never writes", async () => {
|
||||
for (const fence of [{ ...current, generation: "g3" }, undefined]) {
|
||||
const desired = port(fence);
|
||||
await expect(activity(desired).applyProgramSwitch(command)).resolves.toEqual({ kind: "generation_superseded", code: "SUPERSEDED" });
|
||||
expect(desired.writeProgram).not.toHaveBeenCalled();
|
||||
expect(desired.audit).toHaveBeenCalledWith(ref, expect.objectContaining({ outcome: "SUPERSEDED" }));
|
||||
}
|
||||
});
|
||||
it("a stale authzEpoch whose principal lost director.* is audited AUTHZ_REVOKED", async () => {
|
||||
const desired = port({ ...current, authzEpoch: 4, directorAuthorized: false });
|
||||
await expect(activity(desired).applyProgramSwitch(command)).resolves.toEqual({ kind: "authz_revoked", code: "AUTHZ_REVOKED" });
|
||||
expect(desired.writeProgram).not.toHaveBeenCalled();
|
||||
expect(desired.audit).toHaveBeenCalledWith(ref, expect.objectContaining({ outcome: "AUTHZ_REVOKED" }));
|
||||
});
|
||||
it("an unrelated epoch bump with the principal still authorized proceeds", async () => {
|
||||
await expect(activity(port({ ...current, authzEpoch: 4 })).applyProgramSwitch(command)).resolves.toMatchObject({ kind: "applied" });
|
||||
});
|
||||
it("a superseded desiredVersion or lost CAS is SUPERSEDED", async () => {
|
||||
const stale = port({ ...current, desiredVersion: 9 });
|
||||
await expect(activity(stale).applyProgramSwitch(command)).resolves.toEqual({ kind: "superseded", code: "SUPERSEDED" });
|
||||
expect(stale.writeProgram).not.toHaveBeenCalled();
|
||||
const raced = port(current, { writeProgram: vi.fn(async () => ({ kind: "conflict" as const })) });
|
||||
await expect(activity(raced).applyProgramSwitch(command)).resolves.toEqual({ kind: "superseded", code: "SUPERSEDED" });
|
||||
});
|
||||
it("is inert until the go-live slice wires the port", async () => {
|
||||
await expect(activity(unwiredDesiredPort).applyProgramSwitch(command)).rejects.toMatchObject({ type: "DIRECTOR_NOT_WIRED", nonRetryable: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe("ensureDirector", () => {
|
||||
const already = (): WorkflowExecutionAlreadyStartedError => new WorkflowExecutionAlreadyStartedError("existing", evaluatorWorkflowId(ref), "autoDirectorEvaluator");
|
||||
it("starts one evaluator per channel", async () => {
|
||||
const start = vi.fn(async () => ({}));
|
||||
await expect(ensureDirector({ workflow: { start } } as unknown as Client, ref, "q")).resolves.toBe("started");
|
||||
expect(start).toHaveBeenCalledWith("autoDirectorEvaluator", expect.objectContaining({ workflowId: "auto-director-ws-vc", args: [{ ref }] }));
|
||||
});
|
||||
it("nudges a running evaluator instead of starting a duplicate", async () => {
|
||||
const start = vi.fn().mockRejectedValue(already());
|
||||
const signal = vi.fn(async () => undefined);
|
||||
const client = { workflow: { start, getHandle: () => ({ describe: async () => ({ status: { name: "RUNNING" } }), signal }) } } as unknown as Client;
|
||||
await expect(ensureDirector(client, ref, "q")).resolves.toBe("running");
|
||||
expect(start).toHaveBeenCalledTimes(1);
|
||||
expect(signal).toHaveBeenCalledWith("sourceChanged", { sourceId: "*", version: 0 });
|
||||
});
|
||||
it("restarts once when the evaluator closed between start and describe", async () => {
|
||||
const start = vi.fn().mockRejectedValueOnce(already()).mockResolvedValue({});
|
||||
const client = { workflow: { start, getHandle: () => ({ describe: async () => ({ status: { name: "COMPLETED" } }), signal: vi.fn() }) } } as unknown as Client;
|
||||
await expect(ensureDirector(client, ref, "q")).resolves.toBe("restarted");
|
||||
expect(start).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
103
apps/scheduler/src/director/director.activity.ts
Normal file
103
apps/scheduler/src/director/director.activity.ts
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
import { ApplicationFailure } from "@temporalio/common";
|
||||
import type { AutoDirectorRule, DirectorOverrideReleaseReason, SceneLayout, SwitchPolicy } from "@tactical-media-broadcast-hub/core";
|
||||
import type { DirectorOverride, DirectorProgram, DirectorSourceInput } from "./engine.js";
|
||||
|
||||
/** Workflow identity. `generation` is the channel assignment generation; a newer one supersedes this evaluator. */
|
||||
export interface ChannelRef { workspaceId: string; channelId: string; generation: string }
|
||||
|
||||
/** Fresh monitor + desired-state snapshot; signals carry only hints, every evaluation re-reads this. */
|
||||
export type ChannelSnapshot =
|
||||
| {
|
||||
kind: "current";
|
||||
desiredVersion: number;
|
||||
authzEpoch: number;
|
||||
sources: DirectorSourceInput[];
|
||||
rules: AutoDirectorRule[];
|
||||
policy: SwitchPolicy;
|
||||
fallbackSceneId: string | null;
|
||||
override: DirectorOverride | null;
|
||||
principalPermissions: Record<string, string[]>;
|
||||
}
|
||||
| { kind: "superseded"; code: "SUPERSEDED" };
|
||||
|
||||
export interface ProgramSwitchCommand {
|
||||
ref: ChannelRef;
|
||||
/** Deterministic per workflow decision: retries and replays reuse it (idempotency key). */
|
||||
commandId: string;
|
||||
expectedDesiredVersion: number;
|
||||
authzEpoch: number;
|
||||
program: DirectorProgram;
|
||||
layout: SceneLayout;
|
||||
reasons: string[];
|
||||
releaseOverride?: DirectorOverrideReleaseReason;
|
||||
}
|
||||
|
||||
export type SwitchOutcome =
|
||||
| { kind: "applied" | "duplicate"; desiredVersion: number }
|
||||
| { kind: "superseded"; code: "SUPERSEDED" }
|
||||
| { kind: "generation_superseded"; code: "SUPERSEDED" }
|
||||
| { kind: "authz_revoked"; code: "AUTHZ_REVOKED" };
|
||||
|
||||
export interface DesiredFence {
|
||||
generation: string;
|
||||
desiredVersion: number;
|
||||
authzEpoch: number;
|
||||
/** The evaluator's recorded principal still holds `director.*` under the current epoch. */
|
||||
directorAuthorized: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ports owned by the media-control go-live slice. `writeProgram` must be one
|
||||
* transaction: CAS on `expectedDesiredVersion`, desired row + `desiredVersion`
|
||||
* bump + outbox row keyed by `commandId`.
|
||||
*/
|
||||
export interface DirectorStatePort { read(ref: ChannelRef): Promise<ChannelSnapshot> }
|
||||
export interface DesiredStatePort {
|
||||
fence(ref: ChannelRef): Promise<DesiredFence | undefined>;
|
||||
findCommand(ref: ChannelRef, commandId: string): Promise<{ desiredVersion: number } | undefined>;
|
||||
writeProgram(command: ProgramSwitchCommand): Promise<{ kind: "applied"; desiredVersion: number } | { kind: "conflict" }>;
|
||||
audit(ref: ChannelRef, event: { commandId: string; outcome: "SUPERSEDED" | "AUTHZ_REVOKED"; reason: string }): Promise<void>;
|
||||
}
|
||||
|
||||
function notWired(): never {
|
||||
throw ApplicationFailure.nonRetryable("director desired-state port is not wired", "DIRECTOR_NOT_WIRED");
|
||||
}
|
||||
/** Placeholder until the go-live slice provides the media-control desired-state adapter. */
|
||||
export const unwiredStatePort: DirectorStatePort = { read: async () => notWired() };
|
||||
export const unwiredDesiredPort: DesiredStatePort = {
|
||||
fence: async () => notWired(), findCommand: async () => notWired(), writeProgram: async () => notWired(), audit: async () => notWired(),
|
||||
};
|
||||
|
||||
export class DirectorActivity {
|
||||
constructor(private readonly state: DirectorStatePort = unwiredStatePort, private readonly desired: DesiredStatePort = unwiredDesiredPort) {}
|
||||
|
||||
async readChannelState(ref: ChannelRef): Promise<ChannelSnapshot> {
|
||||
return this.state.read(ref);
|
||||
}
|
||||
|
||||
/** Generation fences are re-checked here, at emission time, never trusted from the workflow. */
|
||||
async applyProgramSwitch(command: ProgramSwitchCommand): Promise<SwitchOutcome> {
|
||||
const { ref, commandId } = command;
|
||||
const prior = await this.desired.findCommand(ref, commandId);
|
||||
if (prior) return { kind: "duplicate", desiredVersion: prior.desiredVersion };
|
||||
const fence = await this.desired.fence(ref);
|
||||
if (!fence || fence.generation !== ref.generation) {
|
||||
await this.desired.audit(ref, { commandId, outcome: "SUPERSEDED", reason: "channel generation superseded" });
|
||||
return { kind: "generation_superseded", code: "SUPERSEDED" };
|
||||
}
|
||||
if (fence.authzEpoch !== command.authzEpoch && !fence.directorAuthorized) {
|
||||
await this.desired.audit(ref, { commandId, outcome: "AUTHZ_REVOKED", reason: "director principal lost director.*" });
|
||||
return { kind: "authz_revoked", code: "AUTHZ_REVOKED" };
|
||||
}
|
||||
if (fence.desiredVersion !== command.expectedDesiredVersion) {
|
||||
await this.desired.audit(ref, { commandId, outcome: "SUPERSEDED", reason: "desiredVersion advanced since snapshot" });
|
||||
return { kind: "superseded", code: "SUPERSEDED" };
|
||||
}
|
||||
const written = await this.desired.writeProgram(command);
|
||||
if (written.kind === "conflict") {
|
||||
await this.desired.audit(ref, { commandId, outcome: "SUPERSEDED", reason: "desiredVersion CAS lost" });
|
||||
return { kind: "superseded", code: "SUPERSEDED" };
|
||||
}
|
||||
return written;
|
||||
}
|
||||
}
|
||||
54
apps/scheduler/src/director/discovery.ts
Normal file
54
apps/scheduler/src/director/discovery.ts
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
/**
|
||||
* Director evaluator discovery (WP10 wiring slice).
|
||||
*
|
||||
* Mirrors sweepDue: starts are repair hints, and ensureDirector handles
|
||||
* AlreadyStarted itself (nudge a running evaluator, restart a closed one).
|
||||
* A discovery pass that fails must never take the scheduler down, so every
|
||||
* channel error is logged and skipped.
|
||||
*/
|
||||
import type { Client } from "@temporalio/client";
|
||||
import { ensureDirector } from "./starter.js";
|
||||
import { parseDirectorChannels, type DirectorChannelConfig } from "./config.js";
|
||||
|
||||
export interface DirectorSweepOutcome {
|
||||
started: number;
|
||||
running: number;
|
||||
restarted: number;
|
||||
}
|
||||
|
||||
/** Configured channels whose generation (and therefore evaluator identity) is still current. */
|
||||
export function currentChannels(channels: readonly DirectorChannelConfig[]): DirectorChannelConfig[] {
|
||||
const latest = new Map<string, DirectorChannelConfig>();
|
||||
for (const channel of channels) {
|
||||
const key = `${channel.workspaceId}:${channel.channelId}`;
|
||||
const existing = latest.get(key);
|
||||
// Generations are opaque tokens; the highest wins only when comparable, so
|
||||
// keep the last-declared entry for equal tokens (config is authoritative).
|
||||
if (existing === undefined || existing.generation !== channel.generation) latest.set(key, channel);
|
||||
}
|
||||
return [...latest.values()];
|
||||
}
|
||||
|
||||
/** Start (or nudge) one evaluator per configured channel. */
|
||||
export async function sweepDirectors(
|
||||
client: Client,
|
||||
options: { taskQueue: string; channels: readonly DirectorChannelConfig[] },
|
||||
): Promise<DirectorSweepOutcome> {
|
||||
const outcome: DirectorSweepOutcome = { started: 0, running: 0, restarted: 0 };
|
||||
for (const channel of currentChannels(options.channels)) {
|
||||
const ref = { workspaceId: channel.workspaceId, channelId: channel.channelId, generation: channel.generation };
|
||||
try {
|
||||
const result = await ensureDirector(client, ref, options.taskQueue);
|
||||
outcome[result] += 1;
|
||||
} catch {
|
||||
// Temporal unavailable or the start refused: the next sweep retries.
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return outcome;
|
||||
}
|
||||
|
||||
/** Parse DIRECTOR_CHANNELS, failing startup loudly (a malformed doc must not silently disable the director). */
|
||||
export function directorChannelsFromEnv(env: NodeJS.ProcessEnv = process.env): DirectorChannelConfig[] {
|
||||
return parseDirectorChannels(env.DIRECTOR_CHANNELS);
|
||||
}
|
||||
279
apps/scheduler/src/director/engine.test.ts
Normal file
279
apps/scheduler/src/director/engine.test.ts
Normal file
|
|
@ -0,0 +1,279 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import type { AutoDirectorRule, DirectorCondition, DirectorAction, SwitchPolicy } from "@tactical-media-broadcast-hub/core";
|
||||
import { decide, type DecideInput, type Decision, type DirectorMemory, type DirectorOverride, type DirectorSourceInput, type LastSwitch } from "./engine.js";
|
||||
|
||||
const policy = (over: Partial<SwitchPolicy> = {}): SwitchPolicy => ({ name: "highest_priority", liveConfirmations: 2, offlineConfirmations: 3, noMediaSeconds: 10, minHoldMs: 2_000, switchCooldownMs: 10_000, ...over });
|
||||
const src = (id: string, priority: number, state: DirectorSourceInput["state"] = "LIVE", extra: Partial<DirectorSourceInput> = {}): DirectorSourceInput => ({ id, priority, enabled: true, autoSelect: true, state, ...extra });
|
||||
const confirmed = (...ids: string[]): DirectorMemory => ({ sources: Object.fromEntries(ids.map((id) => [id, { liveStreak: 2, offlineStreak: 0, confirmed: true, confirmedAtMs: 0 }])) });
|
||||
const rule = (id: string, priority: number, condition: DirectorCondition, action: DirectorAction): AutoDirectorRule => ({ id, virtualChannelId: "vc", priority, condition, action });
|
||||
const on = (ids: string[], atMs = 0, layout: LastSwitch["layout"] = "single", heldSinceMs = atMs): LastSwitch => ({ atMs, program: { kind: "sources", sourceIds: ids }, layout, heldSinceMs });
|
||||
const program = (decision: Decision): string[] => decision.program.kind === "sources" ? decision.program.sourceIds : [`FALLBACK:${decision.program.sceneId ?? ""}`];
|
||||
const base = (over: Partial<DecideInput>): DecideInput => ({ sources: [], rules: [], policy: policy(), now: 100_000, ...over });
|
||||
|
||||
/** Feed successive snapshots through decide, carrying memory and lastSwitch like the workflow does. */
|
||||
function run(steps: Array<{ now: number; sources: DirectorSourceInput[] }>, over: Partial<DecideInput> = {}): Decision[] {
|
||||
let memory: DirectorMemory | undefined = over.memory;
|
||||
let lastSwitch: LastSwitch | undefined = over.lastSwitch;
|
||||
return steps.map(({ now, sources }) => {
|
||||
const decision = decide({ ...base(over), now, sources, ...(memory ? { memory } : {}), ...(lastSwitch ? { lastSwitch } : {}) });
|
||||
memory = decision.memory; lastSwitch = decision.lastSwitch;
|
||||
return decision;
|
||||
});
|
||||
}
|
||||
|
||||
describe("rule evaluation", () => {
|
||||
const sources = [src("a", 10), src("b", 50), src("c", 30)];
|
||||
it("evaluates by priority desc and the first matching rule wins regardless of list order", () => {
|
||||
const rules = [
|
||||
rule("low", 1, { field: "live_source_count", op: ">=", value: 1 }, { layout: "single", selection: "top_priority" }),
|
||||
rule("high", 9, { field: "live_source_count", op: ">=", value: 3 }, { layout: "grid", selection: "all_live" }),
|
||||
rule("mid", 5, { field: "live_source_count", op: ">=", value: 2 }, { layout: "two_up", selection: "top_priority" }),
|
||||
];
|
||||
const decision = decide(base({ sources, rules, memory: confirmed("a", "b", "c") }));
|
||||
expect(decision.ruleId).toBe("high");
|
||||
expect(decision.layout).toBe("grid");
|
||||
expect(program(decision)).toEqual(["b", "c", "a"]);
|
||||
});
|
||||
it("breaks priority ties by rule id and skips non-matching rules", () => {
|
||||
const rules = [
|
||||
rule("z", 5, { field: "live_source_count", op: ">=", value: 1 }, { layout: "single", selection: "top_priority" }),
|
||||
rule("y", 5, { field: "live_source_count", op: ">=", value: 1 }, { layout: "pip", selection: "top_priority" }),
|
||||
rule("never", 99, { field: "live_source_count", op: ">", value: 3 }, { layout: "grid", selection: "all_live" }),
|
||||
];
|
||||
expect(decide(base({ sources, rules, memory: confirmed("a", "b", "c") })).ruleId).toBe("y");
|
||||
});
|
||||
it("supports every condition op on live_source_count and max_priority", () => {
|
||||
const pick = (condition: DirectorCondition): string | undefined => decide(base({ sources, memory: confirmed("a", "b", "c"),
|
||||
rules: [rule("r", 1, condition, { layout: "single", selection: "oldest_live" })] })).ruleId;
|
||||
expect(pick({ field: "live_source_count", op: "==", value: 3 })).toBe("r");
|
||||
expect(pick({ field: "live_source_count", op: "<=", value: 2 })).toBeUndefined();
|
||||
expect(pick({ field: "max_priority", op: ">", value: 49 })).toBe("r");
|
||||
expect(pick({ field: "max_priority", op: ">", value: 50 })).toBeUndefined();
|
||||
expect(pick({ field: "max_priority", op: ">=", value: "50" })).toBe("r");
|
||||
expect(pick({ field: "max_priority", op: "<=", value: 10 })).toBeUndefined();
|
||||
});
|
||||
it("matches source_state on the debounced state, globally or per source", () => {
|
||||
const action: DirectorAction = { layout: "single", selection: "explicit", sourceIds: ["c"] };
|
||||
const memory = confirmed("a", "c");
|
||||
const pick = (value: string, op: DirectorCondition["op"] = "=="): string | undefined =>
|
||||
decide(base({ sources, memory, rules: [rule("r", 1, { field: "source_state", op, value }, action)] })).ruleId;
|
||||
expect(pick("c:LIVE")).toBe("r");
|
||||
// b reports LIVE but is unconfirmed: debounced state is PROBING.
|
||||
expect(pick("b:LIVE")).toBeUndefined();
|
||||
expect(pick("b:PROBING")).toBe("r");
|
||||
expect(pick("LIVE")).toBe("r");
|
||||
expect(pick("LIVE", ">=")).toBeUndefined();
|
||||
});
|
||||
it("ignores rules for other channels when channelId is given", () => {
|
||||
const foreign = { ...rule("x", 9, { field: "live_source_count", op: ">=", value: 1 }, { layout: "pip", selection: "top_priority" }), virtualChannelId: "other" };
|
||||
expect(decide(base({ channelId: "vc", sources, rules: [foreign], memory: confirmed("a", "b") })).ruleId).toBeUndefined();
|
||||
});
|
||||
it("explicit sourceIds pick the listed sources, even lower priority or non-auto ones", () => {
|
||||
const decision = decide(base({ sources: [src("a", 10, "LIVE", { autoSelect: false }), src("b", 50)], memory: confirmed("a", "b"),
|
||||
rules: [rule("r", 1, { field: "live_source_count", op: ">=", value: 1 }, { layout: "single", selection: "explicit", sourceIds: ["a"] })] }));
|
||||
expect(program(decision)).toEqual(["a"]);
|
||||
});
|
||||
it("falls through an explicit rule whose sources are all dead", () => {
|
||||
const decision = decide(base({ sources: [src("a", 10, "OFFLINE"), src("b", 50)], memory: confirmed("b"),
|
||||
rules: [rule("r", 9, { field: "live_source_count", op: ">=", value: 1 }, { layout: "single", selection: "explicit", sourceIds: ["a"] })] }));
|
||||
expect(decision.ruleId).toBeUndefined();
|
||||
expect(program(decision)).toEqual(["b"]);
|
||||
expect(decision.reasons.join("\n")).toContain("falling through");
|
||||
});
|
||||
it("oldest_live rule selection orders by liveSince", () => {
|
||||
const decision = decide(base({ sources: [src("a", 99, "LIVE", { liveSinceMs: 5 }), src("b", 1, "LIVE", { liveSinceMs: 1 })], memory: confirmed("a", "b"),
|
||||
rules: [rule("r", 1, { field: "live_source_count", op: ">=", value: 1 }, { layout: "single", selection: "oldest_live" })] }));
|
||||
expect(program(decision)).toEqual(["b"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("selection policies", () => {
|
||||
const sources = [src("a", 10, "LIVE", { liveSinceMs: 300 }), src("b", 50, "LIVE", { liveSinceMs: 100 }), src("c", 30, "LIVE", { liveSinceMs: 200 }), src("d", 20, "LIVE", { liveSinceMs: 400 }), src("e", 5, "LIVE", { liveSinceMs: 500 })];
|
||||
const memory = confirmed("a", "b", "c", "d", "e");
|
||||
const pick = (name: SwitchPolicy["name"], over: Partial<DecideInput> = {}): Decision => decide(base({ sources, memory, policy: policy({ name }), ...over }));
|
||||
it("highest_priority", () => { expect(program(pick("highest_priority"))).toEqual(["b"]); });
|
||||
it("oldest_live", () => { expect(program(pick("oldest_live"))).toEqual(["b"]); expect(program(pick("oldest_live", { sources: sources.filter((s) => s.id !== "b") }))).toEqual(["c"]); });
|
||||
it("newest_live", () => { expect(program(pick("newest_live"))).toEqual(["e"]); });
|
||||
it("round_robin holds for minHold then rotates in a stable order", () => {
|
||||
const rr = policy({ name: "round_robin", minHoldMs: 30_000, switchCooldownMs: 0 });
|
||||
const decisions = run([0, 10_000, 30_000, 60_000, 90_000, 120_000, 150_000].map((now) => ({ now, sources })), { policy: rr, memory });
|
||||
expect(decisions.map((d) => program(d)[0])).toEqual(["a", "a", "b", "c", "d", "e", "a"]);
|
||||
});
|
||||
it("operator_required never auto-selects; keeps a healthy program; dead program -> fallback", () => {
|
||||
expect(program(pick("operator_required", { fallbackSceneId: "slate" }))).toEqual(["FALLBACK:slate"]);
|
||||
expect(pick("operator_required").reasons).toContain("operator_required: awaiting operator TAKE");
|
||||
expect(program(pick("operator_required", { lastSwitch: on(["e"]) }))).toEqual(["e"]);
|
||||
const dead = pick("operator_required", { lastSwitch: on(["e"]), sources: sources.map((s) => s.id === "e" ? { ...s, state: "ERROR" as const } : s) });
|
||||
expect(program(dead)).toEqual(["FALLBACK:"]);
|
||||
expect(dead.forced).toBe(true);
|
||||
});
|
||||
it("picture_in_picture / two_up / split_screen take the top two", () => {
|
||||
expect([pick("picture_in_picture").layout, program(pick("picture_in_picture"))]).toEqual(["pip", ["b", "c"]]);
|
||||
expect([pick("two_up").layout, program(pick("two_up"))]).toEqual(["two_up", ["b", "c"]]);
|
||||
expect([pick("split_screen").layout, program(pick("split_screen"))]).toEqual(["split_horizontal", ["b", "c"]]);
|
||||
});
|
||||
it("grid caps at four slots", () => {
|
||||
expect([pick("grid").layout, program(pick("grid"))]).toEqual(["grid", ["b", "c", "d", "a"]]);
|
||||
});
|
||||
it("composites degrade to single with one live source", () => {
|
||||
for (const name of ["picture_in_picture", "two_up", "split_screen", "grid"] as const) {
|
||||
const decision = decide(base({ sources: [src("a", 1)], memory: confirmed("a"), policy: policy({ name }) }));
|
||||
expect([decision.layout, program(decision)]).toEqual(["single", ["a"]]);
|
||||
}
|
||||
});
|
||||
it("skips disabled and non-autoSelect sources", () => {
|
||||
const decision = decide(base({ sources: [src("a", 99, "LIVE", { enabled: false }), src("b", 50, "LIVE", { autoSelect: false }), src("c", 1)], memory: confirmed("a", "b", "c") }));
|
||||
expect(program(decision)).toEqual(["c"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("anti-flapping", () => {
|
||||
it("needs liveConfirmations samples before a source is eligible", () => {
|
||||
const decisions = run([{ now: 1_000, sources: [src("a", 1, "LIVE", { version: 1 })] }, { now: 2_000, sources: [src("a", 1, "LIVE", { version: 2 })] }]);
|
||||
expect(program(decisions[0]!)).toEqual(["FALLBACK:"]);
|
||||
expect(program(decisions[1]!)).toEqual(["a"]);
|
||||
});
|
||||
it("does not count a re-read of the same monitor version as a confirmation", () => {
|
||||
const decisions = run([1_000, 2_000, 3_000].map((now) => ({ now, sources: [src("a", 1, "LIVE", { version: 7 })] })));
|
||||
expect(decisions.map((d) => program(d)[0])).toEqual(["FALLBACK:", "FALLBACK:", "FALLBACK:"]);
|
||||
});
|
||||
it("keeps program through offlineConfirmations - 1 failures (UNSTABLE counts) then fails over", () => {
|
||||
const a = (state: DirectorSourceInput["state"], version: number): DirectorSourceInput => src("a", 10, state, { version });
|
||||
const decisions = run([
|
||||
{ now: 1_000, sources: [a("UNSTABLE", 1), src("b", 1)] },
|
||||
{ now: 2_000, sources: [a("OFFLINE", 2), src("b", 1)] },
|
||||
{ now: 3_000, sources: [a("OFFLINE", 3), src("b", 1)] },
|
||||
], { memory: confirmed("a", "b"), lastSwitch: on(["a"], 0) });
|
||||
expect(decisions.map((d) => program(d)[0])).toEqual(["a", "a", "b"]);
|
||||
expect(decisions[2]!.forced).toBe(true);
|
||||
expect(decisions[2]!.reasons.join("\n")).toContain("failover: a offline_confirmed");
|
||||
});
|
||||
it("a LIVE sample resets the offline streak", () => {
|
||||
const decisions = run([["OFFLINE", 1], ["OFFLINE", 2], ["LIVE", 3], ["OFFLINE", 4], ["OFFLINE", 5]].map(([state, version], i) => ({ now: 1_000 * (i + 1), sources: [src("a", 10, state as "LIVE", { version: version as number }), src("b", 1)] })),
|
||||
{ memory: confirmed("a", "b"), lastSwitch: on(["a"], 0) });
|
||||
expect(decisions.every((d) => program(d)[0] === "a")).toBe(true);
|
||||
});
|
||||
it("fails over after noMediaSeconds without media and wakes exactly at the timeout", () => {
|
||||
const sources = (now: number): DirectorSourceInput[] => [src("a", 10, "LIVE", { lastMediaAtMs: 50_000 }), src("b", 1, "LIVE", { lastMediaAtMs: now })];
|
||||
const before = decide(base({ now: 59_999, sources: sources(59_999), memory: confirmed("a", "b"), lastSwitch: on(["a"], 0) }));
|
||||
expect(program(before)).toEqual(["a"]);
|
||||
expect(before.nextWakeMs).toBe(60_000);
|
||||
const after = decide(base({ now: 60_000, sources: sources(60_000), memory: before.memory, lastSwitch: before.lastSwitch }));
|
||||
expect(program(after)).toEqual(["b"]);
|
||||
expect(after.memory.sources.a).toMatchObject({ confirmed: false, degraded: "no_media" });
|
||||
});
|
||||
it("media-health failure bypasses confirmations", () => {
|
||||
const decision = decide(base({ sources: [src("a", 10, "LIVE", { mediaFailed: true }), src("b", 1)], memory: confirmed("a", "b"), lastSwitch: on(["a"], 99_000) }));
|
||||
expect(program(decision)).toEqual(["b"]);
|
||||
expect(decision.forced).toBe(true);
|
||||
});
|
||||
it("switch cooldown blocks a non-forced preemption until it lapses", () => {
|
||||
const sources = [src("a", 10), src("hi", 99)];
|
||||
const opts = { memory: confirmed("a", "hi"), lastSwitch: on(["a"], 95_000, "single", 0) };
|
||||
const held = decide(base({ now: 100_000, sources, ...opts }));
|
||||
expect(program(held)).toEqual(["a"]);
|
||||
expect(held.switched).toBe(false);
|
||||
expect(held.nextWakeMs).toBe(105_000);
|
||||
expect(program(decide(base({ now: 105_000, sources, ...opts })))).toEqual(["hi"]);
|
||||
});
|
||||
it("minHold keeps the main source but cooldown alone does not apply when zero", () => {
|
||||
const sources = [src("a", 10), src("hi", 99)];
|
||||
const opts = { memory: confirmed("a", "hi"), policy: policy({ switchCooldownMs: 0, minHoldMs: 15_000 }), lastSwitch: on(["a"], 90_000) };
|
||||
const held = decide(base({ now: 100_000, sources, ...opts }));
|
||||
expect(program(held)).toEqual(["a"]);
|
||||
expect(held.reasons.join("\n")).toContain("min hold");
|
||||
const moved = decide(base({ now: 105_000, sources, ...opts }));
|
||||
expect(program(moved)).toEqual(["hi"]);
|
||||
expect(moved.lastSwitch).toEqual({ atMs: 105_000, program: { kind: "sources", sourceIds: ["hi"] }, layout: "single", heldSinceMs: 105_000 });
|
||||
});
|
||||
it("high-priority interrupt happens after hold and cooldown", () => {
|
||||
const decisions = run([0, 2_000, 4_000].map((now) => ({ now, sources: [src("a", 10), src("hi", 99, now === 0 ? "OFFLINE" : "LIVE", { version: now })] })),
|
||||
{ memory: confirmed("a"), policy: policy({ minHoldMs: 1_000, switchCooldownMs: 3_000 }) });
|
||||
expect(decisions.map((d) => program(d)[0])).toEqual(["a", "a", "hi"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("failover ladder and fallback", () => {
|
||||
it("degrades down the priority ladder then to the fallback scene, then recovers", () => {
|
||||
const step = (now: number, a: DirectorSourceInput["state"], b: DirectorSourceInput["state"], c: DirectorSourceInput["state"]): { now: number; sources: DirectorSourceInput[] } =>
|
||||
({ now, sources: [src("a", 30, a), src("b", 20, b), src("c", 10, c)] });
|
||||
const decisions = run([
|
||||
step(1_000, "LIVE", "LIVE", "LIVE"),
|
||||
step(1_100, "ERROR", "LIVE", "LIVE"),
|
||||
step(1_200, "ERROR", "ERROR", "LIVE"),
|
||||
step(1_300, "ERROR", "ERROR", "ERROR"),
|
||||
step(1_400, "LIVE", "ERROR", "ERROR"),
|
||||
step(1_500, "LIVE", "ERROR", "ERROR"),
|
||||
], { memory: confirmed("a", "b", "c"), fallbackSceneId: "slate" });
|
||||
expect(decisions.map((d) => program(d)[0])).toEqual(["a", "b", "c", "FALLBACK:slate", "FALLBACK:slate", "a"]);
|
||||
expect(decisions.slice(1, 4).every((d) => d.forced && d.switched)).toBe(true);
|
||||
expect(decisions[3]!.mode).toBe("FALLBACK");
|
||||
expect(decisions[5]!.reasons).toContain("leaving fallback scene");
|
||||
});
|
||||
it("a composite with one dead slot re-forms immediately", () => {
|
||||
const decision = decide(base({ policy: policy({ name: "two_up" }), sources: [src("a", 30), src("b", 20, "ERROR"), src("c", 10)], memory: confirmed("a", "b", "c"), lastSwitch: on(["a", "b"], 99_000, "two_up") }));
|
||||
expect([decision.layout, program(decision), decision.forced]).toEqual(["two_up", ["a", "c"], true]);
|
||||
});
|
||||
it("all sources OFFLINE from a cold start -> fallback with null scene", () => {
|
||||
const decision = decide(base({ sources: [src("a", 1, "OFFLINE"), src("b", 2, "OFFLINE")] }));
|
||||
expect(decision.program).toEqual({ kind: "fallback", sceneId: null });
|
||||
expect(decision.reasons).toContain("all sources dead: fallback scene");
|
||||
});
|
||||
it("no sources at all -> fallback", () => {
|
||||
expect(decide(base({ fallbackSceneId: "loop" })).program).toEqual({ kind: "fallback", sceneId: "loop" });
|
||||
});
|
||||
it("single source stays on program and is not re-switched", () => {
|
||||
const first = decide(base({ sources: [src("only", 1)], memory: confirmed("only") }));
|
||||
expect([program(first), first.switched]).toEqual([["only"], true]);
|
||||
const second = decide(base({ now: 200_000, sources: [src("only", 1)], memory: first.memory, lastSwitch: first.lastSwitch }));
|
||||
expect([program(second), second.switched]).toEqual([["only"], false]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("operator override", () => {
|
||||
const take = (over: Partial<DirectorOverride> = {}): DirectorOverride => ({ kind: "take", release: "until_released", program: { kind: "sources", sourceIds: ["low"] }, layout: "single", principalId: "op", takenAtMs: 90_000, ...over });
|
||||
const sources = [src("low", 1), src("hi", 99)];
|
||||
const permitted = { op: ["director.switch"] };
|
||||
const rules = [rule("r", 9, { field: "live_source_count", op: ">=", value: 1 }, { layout: "single", selection: "top_priority" })];
|
||||
it("operator TAKE wins over AUTO rules and is not undone by automation", () => {
|
||||
const decision = decide(base({ sources, rules, override: take(), principalPermissions: permitted, memory: confirmed("low", "hi"), lastSwitch: on(["low"], 90_000) }));
|
||||
expect([program(decision), decision.mode, decision.switched, decision.override]).toEqual([["low"], "MANUAL", false, take()]);
|
||||
});
|
||||
it("program lock holds even when its source is dead", () => {
|
||||
const decision = decide(base({ sources: [src("low", 1, "ERROR"), src("hi", 99)], override: take({ kind: "lock" }), principalPermissions: permitted, memory: confirmed("low", "hi"), lastSwitch: on(["low"], 0) }));
|
||||
expect([program(decision), decision.mode]).toEqual([["low"], "LOCKED"]);
|
||||
expect(decision.reasons).toContain("program lock holds an unhealthy source");
|
||||
});
|
||||
it("revoking director.* releases the override back to AUTO immediately (revocation matrix)", () => {
|
||||
for (const [kind, permissions] of [["take", {}], ["lock", { op: ["posts.read"] }], ["take", { other: ["director.switch"] }]] as const) {
|
||||
const decision = decide(base({ sources, override: take({ kind }), principalPermissions: permissions, memory: confirmed("low", "hi"), lastSwitch: on(["low"], 99_999) }));
|
||||
expect([decision.overrideReleased, decision.override, decision.mode, program(decision), decision.forced]).toEqual(["AUTHZ_REVOKED", null, "AUTO", ["hi"], true]);
|
||||
}
|
||||
});
|
||||
it("timed override expires; until_source_offline and until_next_event release on their triggers", () => {
|
||||
const opts = { sources, principalPermissions: permitted, lastSwitch: on(["low"], 99_000) };
|
||||
const expired = decide(base({ ...opts, override: take({ release: "timed", expiresAtMs: 100_000 }), memory: confirmed("low", "hi") }));
|
||||
expect([expired.overrideReleased, program(expired)]).toEqual(["OVERRIDE_EXPIRED", ["hi"]]);
|
||||
const pending = decide(base({ ...opts, now: 99_000, override: take({ release: "timed", expiresAtMs: 100_000 }), memory: confirmed("low", "hi") }));
|
||||
expect([pending.override?.release, pending.nextWakeMs]).toEqual(["timed", 100_000]);
|
||||
|
||||
const offline = decide(base({ ...opts, sources: [src("low", 1, "ERROR"), src("hi", 99)], override: take({ release: "until_source_offline" }), memory: confirmed("low", "hi") }));
|
||||
expect([offline.overrideReleased, program(offline)]).toEqual(["SOURCE_OFFLINE", ["hi"]]);
|
||||
|
||||
const armed = decide(base({ ...opts, override: take({ release: "until_next_event" }), memory: confirmed("low", "hi") }));
|
||||
expect(armed.override?.eligibleAtTake).toEqual(["hi", "low"]);
|
||||
const same = decide(base({ ...opts, override: armed.override, memory: armed.memory }));
|
||||
expect(same.override).not.toBeNull();
|
||||
const event = decide(base({ ...opts, sources: [...sources, src("new", 50)], override: armed.override, memory: confirmed("low", "hi", "new") }));
|
||||
expect([event.overrideReleased, program(event)]).toEqual(["NEXT_EVENT", ["hi"]]);
|
||||
});
|
||||
it("a lock ignores source-offline release modes", () => {
|
||||
const decision = decide(base({ sources: [src("low", 1, "ERROR")], override: take({ kind: "lock", release: "until_source_offline" }), principalPermissions: permitted, memory: confirmed("low") }));
|
||||
expect([decision.override?.kind, program(decision)]).toEqual(["lock", ["low"]]);
|
||||
});
|
||||
it("override applies with switch and keeps counting confirmations underneath", () => {
|
||||
const decision = decide(base({ sources: [src("low", 1), src("hi", 99, "LIVE", { version: 1 })], override: take(), principalPermissions: permitted, memory: confirmed("low"), lastSwitch: on(["hi"], 0) }));
|
||||
expect([decision.switched, program(decision)]).toEqual([true, ["low"]]);
|
||||
expect(decision.memory.sources.hi).toMatchObject({ liveStreak: 1, confirmed: false });
|
||||
});
|
||||
});
|
||||
344
apps/scheduler/src/director/engine.ts
Normal file
344
apps/scheduler/src/director/engine.ts
Normal file
|
|
@ -0,0 +1,344 @@
|
|||
/**
|
||||
* WP10 Auto Director decision engine. Pure and deterministic: no I/O, no clock,
|
||||
* no Temporal. Every timestamp is epoch milliseconds so inputs and outputs stay
|
||||
* JSON-serializable across activity boundaries and continue-as-new.
|
||||
* Imports from core are type-only so the workflow bundle stays sandbox-safe.
|
||||
*/
|
||||
import type {
|
||||
AutoDirectorRule, DirectorCondition, DirectorMode, DirectorOverrideKind, DirectorOverrideRelease,
|
||||
DirectorOverrideReleaseReason, LiveSourceState, LiveSourceStateType, SceneLayout, SwitchPolicy, VirtualChannelSource,
|
||||
} from "@tactical-media-broadcast-hub/core";
|
||||
|
||||
export interface DirectorSourceInput {
|
||||
id: string;
|
||||
priority: number;
|
||||
enabled: boolean;
|
||||
autoSelect: boolean;
|
||||
state: LiveSourceStateType;
|
||||
/** Monitor row version; a re-read of the same version is not a new confirmation sample. */
|
||||
version?: number;
|
||||
liveSinceMs?: number;
|
||||
/** Last media-health "media present" instant; drives the no-media timeout. */
|
||||
lastMediaAtMs?: number;
|
||||
/** Media-health tier reports failure: bypasses confirmations (proposal §5.2 ladder step 2). */
|
||||
mediaFailed?: boolean;
|
||||
}
|
||||
|
||||
export interface SourceTrack {
|
||||
liveStreak: number;
|
||||
offlineStreak: number;
|
||||
confirmed: boolean;
|
||||
confirmedAtMs?: number;
|
||||
lastVersion?: number;
|
||||
/** Why the source last lost (or never had) eligibility. */
|
||||
degraded?: string;
|
||||
}
|
||||
|
||||
export interface DirectorMemory { sources: Record<string, SourceTrack> }
|
||||
|
||||
export type DirectorProgram =
|
||||
| { kind: "sources"; sourceIds: string[] }
|
||||
| { kind: "fallback"; sceneId: string | null };
|
||||
|
||||
export interface LastSwitch {
|
||||
atMs: number;
|
||||
program: DirectorProgram;
|
||||
layout: SceneLayout;
|
||||
/** When the program's main (first-slot) source went on air; minHold is measured from here. */
|
||||
heldSinceMs: number;
|
||||
}
|
||||
|
||||
export interface DirectorOverride {
|
||||
kind: DirectorOverrideKind;
|
||||
release: DirectorOverrideRelease;
|
||||
program: DirectorProgram;
|
||||
layout: SceneLayout;
|
||||
principalId: string;
|
||||
takenAtMs: number;
|
||||
expiresAtMs?: number;
|
||||
/** Eligible set at TAKE time, filled by the engine for `until_next_event`. */
|
||||
eligibleAtTake?: string[];
|
||||
}
|
||||
|
||||
export interface DecideInput {
|
||||
sources: readonly DirectorSourceInput[];
|
||||
rules: readonly AutoDirectorRule[];
|
||||
policy: SwitchPolicy;
|
||||
now: number;
|
||||
lastSwitch?: LastSwitch;
|
||||
memory?: DirectorMemory;
|
||||
channelId?: string;
|
||||
fallbackSceneId?: string | null;
|
||||
override?: DirectorOverride | null;
|
||||
/** Current permissions per principal (re-read every evaluation, never the TAKE-time snapshot). */
|
||||
principalPermissions?: Readonly<Record<string, readonly string[]>>;
|
||||
}
|
||||
|
||||
export interface Decision {
|
||||
program: DirectorProgram;
|
||||
layout: SceneLayout;
|
||||
reasons: string[];
|
||||
mode: DirectorMode;
|
||||
/** Program or layout differs from `lastSwitch`; the wrapper must emit a desired-state write. */
|
||||
switched: boolean;
|
||||
/** Switch bypassed minHold/cooldown (failover, fallback exit, override end). */
|
||||
forced: boolean;
|
||||
ruleId?: string;
|
||||
memory: DirectorMemory;
|
||||
override: DirectorOverride | null;
|
||||
overrideReleased?: DirectorOverrideReleaseReason;
|
||||
lastSwitch: LastSwitch;
|
||||
/** Earliest instant a purely time-based change could alter the decision. */
|
||||
nextWakeMs?: number;
|
||||
}
|
||||
|
||||
interface Candidate { id: string; priority: number; liveSinceMs: number }
|
||||
interface Selection { program: DirectorProgram; layout: SceneLayout; ruleId?: string }
|
||||
|
||||
const CAPACITY: Record<SceneLayout, number> = { single: 1, pip: 2, two_up: 2, split_horizontal: 2, split_vertical: 2, grid: 4, custom: Number.POSITIVE_INFINITY };
|
||||
const COMPOSITE_POLICY: Partial<Record<SwitchPolicy["name"], SceneLayout>> = { picture_in_picture: "pip", two_up: "two_up", split_screen: "split_horizontal", grid: "grid" };
|
||||
|
||||
/** Adapt the persisted monitor/domain types into engine input. */
|
||||
export function sourceInput(source: Pick<VirtualChannelSource, "id" | "priority" | "enabled" | "autoSelect">, state: LiveSourceState, extra: { version?: number; mediaFailed?: boolean } = {}): DirectorSourceInput {
|
||||
return {
|
||||
id: source.id, priority: source.priority, enabled: source.enabled, autoSelect: source.autoSelect, state: state.state,
|
||||
...(extra.version === undefined ? {} : { version: extra.version }),
|
||||
...(state.liveSince ? { liveSinceMs: state.liveSince.getTime() } : {}),
|
||||
...(state.lastHealthyAt ? { lastMediaAtMs: state.lastHealthyAt.getTime() } : {}),
|
||||
...(extra.mediaFailed === undefined ? {} : { mediaFailed: extra.mediaFailed }),
|
||||
};
|
||||
}
|
||||
|
||||
/** Fold one snapshot into per-source confirmation tracking. */
|
||||
function track(previous: SourceTrack | undefined, source: DirectorSourceInput, policy: SwitchPolicy, now: number): SourceTrack {
|
||||
let next: SourceTrack = previous ? { ...previous } : { liveStreak: 0, offlineStreak: 0, confirmed: false };
|
||||
const fresh = source.version === undefined || previous?.lastVersion !== source.version;
|
||||
if (fresh) {
|
||||
if (source.version !== undefined) next.lastVersion = source.version;
|
||||
switch (source.state) {
|
||||
case "LIVE":
|
||||
next.liveStreak += 1; next.offlineStreak = 0;
|
||||
if (!next.confirmed && next.liveStreak >= policy.liveConfirmations) next = { ...next, confirmed: true, confirmedAtMs: now, degraded: undefined };
|
||||
break;
|
||||
case "OFFLINE": case "ENDING": case "UNSTABLE":
|
||||
next.offlineStreak += 1; next.liveStreak = 0;
|
||||
if (next.confirmed && next.offlineStreak >= policy.offlineConfirmations) next = { ...next, confirmed: false, degraded: "offline_confirmed" };
|
||||
break;
|
||||
case "ERROR":
|
||||
next = { ...next, liveStreak: 0, offlineStreak: 0, confirmed: false, degraded: next.confirmed ? "error" : next.degraded };
|
||||
break;
|
||||
default:
|
||||
next.liveStreak = 0;
|
||||
}
|
||||
}
|
||||
// Hard media failures override confirmations and force re-confirmation.
|
||||
if (next.confirmed && source.mediaFailed === true) next = { ...next, confirmed: false, liveStreak: 0, degraded: "media_failed" };
|
||||
if (next.confirmed && source.lastMediaAtMs !== undefined && now - source.lastMediaAtMs >= policy.noMediaSeconds * 1000) {
|
||||
next = { ...next, confirmed: false, liveStreak: 0, degraded: "no_media" };
|
||||
}
|
||||
if (!source.enabled && next.confirmed) next = { ...next, confirmed: false, degraded: "disabled" };
|
||||
if (next.degraded === undefined) delete next.degraded;
|
||||
return next;
|
||||
}
|
||||
|
||||
function byPriority(a: Candidate, b: Candidate): number {
|
||||
return b.priority - a.priority || a.liveSinceMs - b.liveSinceMs || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
|
||||
}
|
||||
function byAge(a: Candidate, b: Candidate): number {
|
||||
return a.liveSinceMs - b.liveSinceMs || byPriority(a, b);
|
||||
}
|
||||
function compare(actual: number, op: DirectorCondition["op"], expected: number): boolean {
|
||||
switch (op) {
|
||||
case ">=": return actual >= expected;
|
||||
case "<=": return actual <= expected;
|
||||
case "==": return actual === expected;
|
||||
case ">": return actual > expected;
|
||||
}
|
||||
}
|
||||
function programKey(program: DirectorProgram, layout: SceneLayout): string {
|
||||
return program.kind === "fallback" ? `fallback:${program.sceneId ?? ""}` : `${layout}:${program.sourceIds.join(",")}`;
|
||||
}
|
||||
function mainSource(program: DirectorProgram): string | undefined {
|
||||
return program.kind === "sources" ? program.sourceIds[0] : undefined;
|
||||
}
|
||||
/** Composite layouts collapse to `single` when only one source can fill them. */
|
||||
function fill(ids: string[], layout: SceneLayout, reasons: string[]): { program: DirectorProgram; layout: SceneLayout } | undefined {
|
||||
const picked = ids.slice(0, CAPACITY[layout]);
|
||||
if (picked.length === 0) return undefined;
|
||||
if (layout !== "single" && layout !== "custom" && picked.length < 2) {
|
||||
reasons.push(`layout ${layout} degraded to single: one eligible source`);
|
||||
return { program: { kind: "sources", sourceIds: picked }, layout: "single" };
|
||||
}
|
||||
return { program: { kind: "sources", sourceIds: picked }, layout };
|
||||
}
|
||||
|
||||
export function decide(input: DecideInput): Decision {
|
||||
const { policy, now } = input;
|
||||
const reasons: string[] = [];
|
||||
const memory: DirectorMemory = { sources: {} };
|
||||
for (const source of input.sources) memory.sources[source.id] = track(input.memory?.sources[source.id], source, policy, now);
|
||||
|
||||
const live: Candidate[] = input.sources
|
||||
.filter((source) => source.enabled && memory.sources[source.id]?.confirmed === true)
|
||||
.map((source) => ({ id: source.id, priority: source.priority, liveSinceMs: source.liveSinceMs ?? memory.sources[source.id]?.confirmedAtMs ?? now }))
|
||||
.sort(byPriority);
|
||||
const liveIds = new Set(live.map((candidate) => candidate.id));
|
||||
const autoPool = live.filter((candidate) => input.sources.find((source) => source.id === candidate.id)?.autoSelect !== false);
|
||||
const healthy = (program: DirectorProgram): boolean => program.kind === "sources" && program.sourceIds.length > 0 && program.sourceIds.every((id) => liveIds.has(id));
|
||||
const fallback: DirectorProgram = { kind: "fallback", sceneId: input.fallbackSceneId ?? null };
|
||||
|
||||
// Wake for the earliest no-media timeout among eligible sources.
|
||||
const wakes: number[] = [];
|
||||
for (const source of input.sources) {
|
||||
if (liveIds.has(source.id) && source.lastMediaAtMs !== undefined) wakes.push(source.lastMediaAtMs + policy.noMediaSeconds * 1000);
|
||||
}
|
||||
const nextWake = (): number | undefined => {
|
||||
const future = wakes.filter((at) => at > now);
|
||||
return future.length === 0 ? undefined : Math.min(...future);
|
||||
};
|
||||
|
||||
// Ladder step 1: operator override / program lock.
|
||||
let override = input.override ? { ...input.override } : null;
|
||||
let overrideReleased: DirectorOverrideReleaseReason | undefined;
|
||||
if (override) {
|
||||
const permissions = input.principalPermissions?.[override.principalId] ?? [];
|
||||
const liveSet = [...liveIds].sort();
|
||||
if (!permissions.some((permission) => permission.startsWith("director."))) overrideReleased = "AUTHZ_REVOKED";
|
||||
else if (override.expiresAtMs !== undefined && now >= override.expiresAtMs) overrideReleased = "OVERRIDE_EXPIRED";
|
||||
else if (override.kind === "take" && override.release === "until_source_offline" && !healthy(override.program)) overrideReleased = "SOURCE_OFFLINE";
|
||||
else if (override.kind === "take" && override.release === "until_next_event") {
|
||||
if (override.eligibleAtTake === undefined) override.eligibleAtTake = liveSet;
|
||||
else if (override.eligibleAtTake.join(",") !== liveSet.join(",")) overrideReleased = "NEXT_EVENT";
|
||||
}
|
||||
if (overrideReleased) {
|
||||
reasons.push(`${override.kind} by ${override.principalId} released: ${overrideReleased}; returning to AUTO`);
|
||||
override = null;
|
||||
} else {
|
||||
if (override.expiresAtMs !== undefined) wakes.push(override.expiresAtMs);
|
||||
const switched = !input.lastSwitch || programKey(input.lastSwitch.program, input.lastSwitch.layout) !== programKey(override.program, override.layout);
|
||||
if (override.kind === "lock" && !healthy(override.program) && override.program.kind === "sources") reasons.push("program lock holds an unhealthy source");
|
||||
reasons.push(`${override.kind === "lock" ? "program lock" : "operator TAKE"} by ${override.principalId} wins over AUTO`);
|
||||
const lastSwitch: LastSwitch = switched
|
||||
? { atMs: now, program: override.program, layout: override.layout, heldSinceMs: mainSource(override.program) === mainSource(input.lastSwitch?.program ?? fallback) ? input.lastSwitch?.heldSinceMs ?? now : now }
|
||||
: input.lastSwitch!;
|
||||
return {
|
||||
program: override.program, layout: override.layout, reasons, mode: override.kind === "lock" ? "LOCKED" : "MANUAL",
|
||||
switched, forced: switched, memory, override, lastSwitch, ...(nextWake() === undefined ? {} : { nextWakeMs: nextWake()! }),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Ladder: ordered rules (priority desc, id asc for determinism), first match wins.
|
||||
let selection: Selection | undefined;
|
||||
const rules = input.rules.filter((rule) => input.channelId === undefined || rule.virtualChannelId === input.channelId)
|
||||
.sort((a, b) => b.priority - a.priority || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
||||
for (const rule of rules) {
|
||||
if (!matches(rule, input.sources, live, memory, reasons)) continue;
|
||||
const pool = rule.action.selection === "explicit"
|
||||
? (rule.action.sourceIds ?? []).map((id) => live.find((candidate) => candidate.id === id)).filter((candidate): candidate is Candidate => candidate !== undefined)
|
||||
: autoPool;
|
||||
const ordered = rule.action.selection === "oldest_live" ? [...pool].sort(byAge) : pool;
|
||||
const filled = fill(ordered.map((candidate) => candidate.id), rule.action.layout, reasons);
|
||||
if (!filled) { reasons.push(`rule ${rule.id} matched but has no eligible source; falling through`); continue; }
|
||||
reasons.push(`rule ${rule.id} matched (${rule.condition.field} ${rule.condition.op} ${String(rule.condition.value)})`);
|
||||
selection = { ...filled, ruleId: rule.id };
|
||||
break;
|
||||
}
|
||||
|
||||
// No rule matched: the channel's selection policy.
|
||||
if (!selection) {
|
||||
const ids = autoPool.map((candidate) => candidate.id);
|
||||
const current = input.lastSwitch;
|
||||
const currentMain = current ? mainSource(current.program) : undefined;
|
||||
let picked: { program: DirectorProgram; layout: SceneLayout } | undefined;
|
||||
switch (policy.name) {
|
||||
case "highest_priority": picked = fill(ids, "single", reasons); break;
|
||||
case "oldest_live": picked = fill([...autoPool].sort(byAge).map((candidate) => candidate.id), "single", reasons); break;
|
||||
case "newest_live": picked = fill([...autoPool].sort((a, b) => byAge(b, a)).map((candidate) => candidate.id), "single", reasons); break;
|
||||
case "round_robin": {
|
||||
const rotation = [...ids].sort();
|
||||
const index = currentMain === undefined ? -1 : rotation.indexOf(currentMain);
|
||||
if (index >= 0 && current && now - current.heldSinceMs < policy.minHoldMs) {
|
||||
picked = fill([rotation[index]!], "single", reasons);
|
||||
wakes.push(current.heldSinceMs + policy.minHoldMs);
|
||||
} else {
|
||||
picked = fill(rotation.length === 0 ? [] : [rotation[(index + 1) % rotation.length]!], "single", reasons);
|
||||
if (index >= 0 && rotation.length > 1) reasons.push("round_robin rotation");
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "operator_required":
|
||||
if (current && healthy(current.program)) picked = { program: current.program, layout: current.layout };
|
||||
else if (live.length > 0) reasons.push("operator_required: awaiting operator TAKE");
|
||||
break;
|
||||
default: picked = fill(ids, COMPOSITE_POLICY[policy.name] ?? "single", reasons);
|
||||
}
|
||||
if (picked) { selection = picked; reasons.push(`policy ${policy.name} selected ${programKey(picked.program, picked.layout)}`); }
|
||||
}
|
||||
|
||||
// Ladder terminal: fallback scene keeps output alive.
|
||||
if (!selection) {
|
||||
selection = { program: fallback, layout: "single" };
|
||||
reasons.push(live.length === 0 ? "all sources dead: fallback scene" : "no selection: fallback scene");
|
||||
}
|
||||
|
||||
// Anti-flapping against the current program.
|
||||
const current = input.lastSwitch;
|
||||
const desiredKey = programKey(selection.program, selection.layout);
|
||||
let outcome: Selection = selection;
|
||||
let switched = false;
|
||||
let forced = false;
|
||||
if (!current) {
|
||||
switched = true; forced = true;
|
||||
reasons.push("initial program");
|
||||
} else if (programKey(current.program, current.layout) !== desiredKey) {
|
||||
if (overrideReleased || !healthy(current.program)) {
|
||||
switched = true; forced = true;
|
||||
if (current.program.kind === "sources") {
|
||||
const lost = current.program.sourceIds.filter((id) => !liveIds.has(id));
|
||||
reasons.push(`failover: ${lost.map((id) => `${id} ${memory.sources[id]?.degraded ?? "not live"}`).join(", ") || "override released"}`);
|
||||
} else if (!overrideReleased) reasons.push("leaving fallback scene");
|
||||
} else if (now - current.atMs < policy.switchCooldownMs) {
|
||||
outcome = { program: current.program, layout: current.layout };
|
||||
wakes.push(current.atMs + policy.switchCooldownMs);
|
||||
reasons.push(`switch cooldown holds program until ${current.atMs + policy.switchCooldownMs}`);
|
||||
} else if (mainSource(current.program) !== mainSource(selection.program) && now - current.heldSinceMs < policy.minHoldMs) {
|
||||
outcome = { program: current.program, layout: current.layout };
|
||||
wakes.push(current.heldSinceMs + policy.minHoldMs);
|
||||
reasons.push(`min hold keeps ${mainSource(current.program) ?? "program"} until ${current.heldSinceMs + policy.minHoldMs}`);
|
||||
} else {
|
||||
switched = true;
|
||||
}
|
||||
}
|
||||
const lastSwitch: LastSwitch = switched
|
||||
? { atMs: now, program: outcome.program, layout: outcome.layout, heldSinceMs: current && mainSource(current.program) === mainSource(outcome.program) && mainSource(outcome.program) !== undefined ? current.heldSinceMs : now }
|
||||
: current!;
|
||||
const wake = nextWake();
|
||||
return {
|
||||
program: outcome.program, layout: outcome.layout, reasons, mode: outcome.program.kind === "fallback" ? "FALLBACK" : "AUTO",
|
||||
switched, forced, ...(outcome.ruleId === undefined || outcome !== selection ? {} : { ruleId: outcome.ruleId }),
|
||||
memory, override: null, ...(overrideReleased ? { overrideReleased } : {}), lastSwitch, ...(wake === undefined ? {} : { nextWakeMs: wake }),
|
||||
};
|
||||
}
|
||||
|
||||
/** Effective (debounced) state: LIVE only once confirmed and healthy. */
|
||||
function effectiveState(source: DirectorSourceInput, memory: DirectorMemory): LiveSourceStateType {
|
||||
if (source.enabled && memory.sources[source.id]?.confirmed === true) return "LIVE";
|
||||
return source.state === "LIVE" ? "PROBING" : source.state;
|
||||
}
|
||||
|
||||
function matches(rule: AutoDirectorRule, sources: readonly DirectorSourceInput[], live: readonly Candidate[], memory: DirectorMemory, reasons: string[]): boolean {
|
||||
const { field, op, value } = rule.condition;
|
||||
if (field === "source_state") {
|
||||
if (op !== "==" || typeof value !== "string") { reasons.push(`rule ${rule.id} skipped: source_state supports only == "<STATE>" or "<sourceId>:<STATE>"`); return false; }
|
||||
const split = value.lastIndexOf(":");
|
||||
const wanted = split < 0 ? value : value.slice(split + 1);
|
||||
const scope = split >= 0 ? [value.slice(0, split)] : rule.action.selection === "explicit" ? rule.action.sourceIds ?? [] : undefined;
|
||||
return sources.some((source) => (scope === undefined ? source.enabled : scope.includes(source.id)) && effectiveState(source, memory) === wanted);
|
||||
}
|
||||
const expected = typeof value === "number" ? value : Number(value);
|
||||
if (!Number.isFinite(expected)) { reasons.push(`rule ${rule.id} skipped: non-numeric ${field} value`); return false; }
|
||||
if (field === "live_source_count") return compare(live.length, op, expected);
|
||||
if (live.length === 0) return false;
|
||||
return compare(Math.max(...live.map((candidate) => candidate.priority)), op, expected);
|
||||
}
|
||||
100
apps/scheduler/src/director/evaluator.workflow.ts
Normal file
100
apps/scheduler/src/director/evaluator.workflow.ts
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
import { condition, continueAsNew, defineQuery, defineSignal, proxyActivities, setHandler, workflowInfo } from "@temporalio/workflow";
|
||||
import type { DirectorMode } from "@tactical-media-broadcast-hub/core";
|
||||
import { decide, type DirectorMemory, type LastSwitch } from "./engine.js";
|
||||
import type { ChannelRef, DirectorActivity } from "./director.activity.js";
|
||||
|
||||
const { readChannelState } = proxyActivities<Pick<DirectorActivity, "readChannelState">>({
|
||||
startToCloseTimeout: "10 seconds",
|
||||
retry: { initialInterval: "500 milliseconds", maximumInterval: "5 seconds", nonRetryableErrorTypes: ["DIRECTOR_NOT_WIRED"] },
|
||||
});
|
||||
const { applyProgramSwitch } = proxyActivities<Pick<DirectorActivity, "applyProgramSwitch">>({
|
||||
startToCloseTimeout: "15 seconds",
|
||||
retry: { initialInterval: "500 milliseconds", maximumInterval: "5 seconds", maximumAttempts: 5, nonRetryableErrorTypes: ["DIRECTOR_NOT_WIRED"] },
|
||||
});
|
||||
|
||||
/** Hint only: `(sourceId, version)`. Evaluations always re-read a fresh snapshot. */
|
||||
export interface SourceHint { sourceId: string; version: number; urgent?: boolean }
|
||||
export interface EvaluatorCarry { memory: DirectorMemory; lastSwitch?: LastSwitch; seq: number }
|
||||
export interface EvaluatorInput { ref: ChannelRef; carry?: EvaluatorCarry }
|
||||
export type EvaluatorResult = { kind: "stopped" } | { kind: "superseded"; code: "SUPERSEDED" } | { kind: "needs_reauthorization"; code: "AUTHZ_REVOKED" };
|
||||
export interface EvaluatorStatus { mode?: DirectorMode; lastSwitch?: LastSwitch; reasons: string[]; evaluations: number }
|
||||
|
||||
export const sourceChanged = defineSignal<[SourceHint]>("sourceChanged");
|
||||
export const stopDirector = defineSignal("stopDirector");
|
||||
export const directorStatus = defineQuery<EvaluatorStatus>("directorStatus");
|
||||
|
||||
export const DEBOUNCE_MS = 250;
|
||||
export const MAX_WAIT_MS = 1000;
|
||||
export const POLL_MS = 5000;
|
||||
export const CONTINUE_AS_NEW_EVENTS = 2000;
|
||||
|
||||
/** One evaluator per virtual channel: debounce hints, decide, emit fenced desired-state writes. */
|
||||
export async function autoDirectorEvaluator({ ref, carry }: EvaluatorInput): Promise<EvaluatorResult> {
|
||||
let memory: DirectorMemory = carry?.memory ?? { sources: {} };
|
||||
let lastSwitch = carry?.lastSwitch;
|
||||
let seq = carry?.seq ?? 0;
|
||||
let dirty = true;
|
||||
let urgent = false;
|
||||
let stopping = false;
|
||||
let firstPendingAt: number | undefined;
|
||||
let lastSignalAt = 0;
|
||||
let events = 0;
|
||||
let evaluations = 0;
|
||||
let nextWakeMs: number | undefined;
|
||||
let status: EvaluatorStatus = { reasons: [], evaluations: 0, ...(lastSwitch ? { lastSwitch } : {}) };
|
||||
|
||||
setHandler(sourceChanged, (hint) => {
|
||||
events += 1; dirty = true;
|
||||
if (hint.urgent === true) urgent = true;
|
||||
lastSignalAt = Date.now();
|
||||
firstPendingAt ??= lastSignalAt;
|
||||
});
|
||||
setHandler(stopDirector, () => { stopping = true; });
|
||||
setHandler(directorStatus, () => status);
|
||||
|
||||
for (;;) {
|
||||
const idle = Math.min(POLL_MS, nextWakeMs === undefined ? POLL_MS : Math.max(0, nextWakeMs - Date.now()));
|
||||
await condition(() => dirty || stopping, idle);
|
||||
if (stopping) return { kind: "stopped" };
|
||||
// Trailing debounce: 250 ms quiet window, 1 s max wait; urgent program-source failure bypasses.
|
||||
while (dirty && !urgent && firstPendingAt !== undefined) {
|
||||
const deadline = Math.min(lastSignalAt + DEBOUNCE_MS, firstPendingAt + MAX_WAIT_MS);
|
||||
if (Date.now() >= deadline) break;
|
||||
const seen = lastSignalAt;
|
||||
if (!(await condition(() => urgent || stopping || lastSignalAt !== seen, deadline - Date.now()))) break;
|
||||
if (stopping) return { kind: "stopped" };
|
||||
}
|
||||
dirty = false; urgent = false; firstPendingAt = undefined;
|
||||
|
||||
const snapshot = await readChannelState(ref);
|
||||
if (snapshot.kind === "superseded") return { kind: "superseded", code: "SUPERSEDED" };
|
||||
const decision = decide({
|
||||
sources: snapshot.sources, rules: snapshot.rules, policy: snapshot.policy, now: Date.now(), channelId: ref.channelId,
|
||||
fallbackSceneId: snapshot.fallbackSceneId, override: snapshot.override, principalPermissions: snapshot.principalPermissions,
|
||||
memory, ...(lastSwitch ? { lastSwitch } : {}),
|
||||
});
|
||||
memory = decision.memory;
|
||||
evaluations += 1;
|
||||
nextWakeMs = decision.nextWakeMs;
|
||||
if (decision.switched || decision.overrideReleased) {
|
||||
seq += 1;
|
||||
const outcome = await applyProgramSwitch({
|
||||
ref, commandId: `${ref.channelId}:${ref.generation}:${seq}`, expectedDesiredVersion: snapshot.desiredVersion,
|
||||
authzEpoch: snapshot.authzEpoch, program: decision.program, layout: decision.layout, reasons: decision.reasons,
|
||||
...(decision.overrideReleased ? { releaseOverride: decision.overrideReleased } : {}),
|
||||
});
|
||||
if (outcome.kind === "generation_superseded") return { kind: "superseded", code: "SUPERSEDED" };
|
||||
if (outcome.kind === "authz_revoked") return { kind: "needs_reauthorization", code: "AUTHZ_REVOKED" };
|
||||
// A concurrent writer (e.g. operator TAKE) won the CAS: re-read rather than re-apply.
|
||||
if (outcome.kind === "superseded") dirty = true;
|
||||
else lastSwitch = decision.lastSwitch;
|
||||
} else {
|
||||
lastSwitch = decision.lastSwitch;
|
||||
}
|
||||
status = { mode: decision.mode, reasons: decision.reasons, evaluations, ...(lastSwitch ? { lastSwitch } : {}) };
|
||||
|
||||
if (events + evaluations >= CONTINUE_AS_NEW_EVENTS || workflowInfo().continueAsNewSuggested) {
|
||||
return continueAsNew<typeof autoDirectorEvaluator>({ ref, carry: { memory, seq, ...(lastSwitch ? { lastSwitch } : {}) } });
|
||||
}
|
||||
}
|
||||
}
|
||||
128
apps/scheduler/src/director/http.client.ts
Normal file
128
apps/scheduler/src/director/http.client.ts
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
/**
|
||||
* HTTP implementation of the director ports (WP10 wiring slice).
|
||||
*
|
||||
* The scheduler worker dials the API's internal director endpoints. Every
|
||||
* response is validated before it reaches the engine: the API is a different
|
||||
* process, so a malformed or truncated reply must surface as a transport
|
||||
* error (retryable) rather than poison the evaluator's memory.
|
||||
*
|
||||
* Endpoints (apps/api/src/director/director.controller.ts):
|
||||
* GET /internal/director/channels/:workspaceId/:channelId/state
|
||||
* GET /internal/director/channels/:workspaceId/:channelId/fence
|
||||
* POST /internal/director/channels/:workspaceId/:channelId/apply
|
||||
* POST /internal/director/channels/:workspaceId/:channelId/audit
|
||||
*/
|
||||
import type { ChannelRef, DesiredFence } from "./director.activity.js";
|
||||
import type { DirectorApplyRequest, DirectorApplyResult, DirectorApi, DirectorStateRead } from "./ports.js";
|
||||
|
||||
export interface DirectorHttpClientDependencies {
|
||||
/** Base URL of the API (default http://127.0.0.1:3000). */
|
||||
baseUrl?: string;
|
||||
/** Shared-secret bearer for the internal route group. */
|
||||
token?: string;
|
||||
fetch?: typeof fetch;
|
||||
/** Default 8s; short so an unresponsive API fails the activity fast. */
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function requiredString(value: unknown, path: string): string {
|
||||
if (typeof value !== "string" || value === "") throw new Error(`director api: ${path} must be a non-empty string`);
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Shape-check an apply response without trusting the status code alone. */
|
||||
export function parseApplyResult(value: unknown): DirectorApplyResult {
|
||||
if (!isRecord(value)) throw new Error("director api: apply response must be an object");
|
||||
const kind = value.kind;
|
||||
if (kind === "conflict") return { kind: "conflict" };
|
||||
if (kind !== "applied" && kind !== "duplicate") throw new Error("director api: apply response kind is invalid");
|
||||
const desiredVersion = value.desiredVersion;
|
||||
if (typeof desiredVersion !== "number" || !Number.isSafeInteger(desiredVersion) || desiredVersion < 0) {
|
||||
throw new Error("director api: apply response desiredVersion is invalid");
|
||||
}
|
||||
return { kind, desiredVersion };
|
||||
}
|
||||
|
||||
/** Shape-check a fence response (fence may legitimately be absent). */
|
||||
export function parseFence(value: unknown): DesiredFence | undefined {
|
||||
if (value === null || value === undefined) return undefined;
|
||||
if (!isRecord(value)) throw new Error("director api: fence response must be an object or null");
|
||||
const generation = requiredString(value.generation, "fence.generation");
|
||||
const desiredVersion = value.desiredVersion;
|
||||
const authzEpoch = value.authzEpoch;
|
||||
if (typeof desiredVersion !== "number" || !Number.isSafeInteger(desiredVersion) || desiredVersion < 0) {
|
||||
throw new Error("director api: fence.desiredVersion is invalid");
|
||||
}
|
||||
if (typeof authzEpoch !== "number" || !Number.isSafeInteger(authzEpoch) || authzEpoch < 0) {
|
||||
throw new Error("director api: fence.authzEpoch is invalid");
|
||||
}
|
||||
if (typeof value.directorAuthorized !== "boolean") throw new Error("director api: fence.directorAuthorized is invalid");
|
||||
return { generation, desiredVersion, authzEpoch, directorAuthorized: value.directorAuthorized };
|
||||
}
|
||||
|
||||
export class DirectorHttpClient implements DirectorApi {
|
||||
private readonly baseUrl: string;
|
||||
private readonly token: string | undefined;
|
||||
private readonly fetchImpl: typeof fetch;
|
||||
private readonly timeoutMs: number;
|
||||
|
||||
constructor(dependencies: DirectorHttpClientDependencies = {}) {
|
||||
this.baseUrl = (dependencies.baseUrl ?? process.env.DIRECTOR_API_URL ?? "http://127.0.0.1:3000").replace(/\/+$/, "");
|
||||
this.token = dependencies.token ?? process.env.DIRECTOR_API_TOKEN;
|
||||
this.fetchImpl = dependencies.fetch ?? fetch;
|
||||
this.timeoutMs = dependencies.timeoutMs ?? 8_000;
|
||||
}
|
||||
|
||||
private path(ref: ChannelRef, suffix: string): string {
|
||||
return `${this.baseUrl}/internal/director/channels/${encodeURIComponent(ref.workspaceId)}/${encodeURIComponent(ref.channelId)}/${suffix}`;
|
||||
}
|
||||
|
||||
private async request(path: string, init?: RequestInit): Promise<unknown> {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
|
||||
try {
|
||||
const response = await this.fetchImpl(path, {
|
||||
...init,
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
...(this.token === undefined ? {} : { authorization: `Bearer ${this.token}` }),
|
||||
...init?.headers,
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
// Transport/server errors are retryable; the evaluator treats 4xx-shaped
|
||||
// outcomes as fenced results, which the callers parse explicitly.
|
||||
throw new Error(`director api: ${response.status} ${response.statusText} for ${path}`);
|
||||
}
|
||||
const text = await response.text();
|
||||
return text === "" ? null : JSON.parse(text) as unknown;
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
async channelState(ref: ChannelRef): Promise<DirectorStateRead> {
|
||||
const value = await this.request(this.path(ref, "state"));
|
||||
if (!isRecord(value)) throw new Error("director api: state response must be an object");
|
||||
if (value.kind === "superseded") return { kind: "superseded", code: "SUPERSEDED" };
|
||||
if (value.kind !== "current") throw new Error("director api: state response kind is invalid");
|
||||
return value as DirectorStateRead;
|
||||
}
|
||||
|
||||
async fence(ref: ChannelRef): Promise<DesiredFence | undefined> {
|
||||
return parseFence(await this.request(this.path(ref, "fence")));
|
||||
}
|
||||
|
||||
async applyProgramSwitch(ref: ChannelRef, body: DirectorApplyRequest): Promise<DirectorApplyResult> {
|
||||
return parseApplyResult(await this.request(this.path(ref, "apply"), { method: "POST", body: JSON.stringify(body) }));
|
||||
}
|
||||
|
||||
async audit(ref: ChannelRef, event: { commandId: string; outcome: string; reason: string }): Promise<void> {
|
||||
await this.request(this.path(ref, "audit"), { method: "POST", body: JSON.stringify(event) });
|
||||
}
|
||||
}
|
||||
104
apps/scheduler/src/director/ports.ts
Normal file
104
apps/scheduler/src/director/ports.ts
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
/**
|
||||
* Transport-agnostic port contracts for the auto-director evaluator (WP10
|
||||
* wiring slice).
|
||||
*
|
||||
* The Temporal worker hosts the evaluator workflow and proxies these ports;
|
||||
* the API process implements them. Because the worker runs in the scheduler
|
||||
* package and the implementations in apps/api, the interfaces are declared
|
||||
* with JSON-serializable shapes only (the scheduler cannot compile apps/api
|
||||
* sources), and both sides share the engine's type definitions via relative
|
||||
* imports — the same cross-package pattern apps/api/src/live-monitor/poller
|
||||
* uses for packages/connector-mock-livestream.
|
||||
*
|
||||
* `read`/`fence` return `kind: "superseded"` / `undefined` rather than throwing
|
||||
* so a dead generation never becomes a retryable activity failure loop.
|
||||
*/
|
||||
import type { AutoDirectorRule, SceneLayout, SwitchPolicy } from "@tactical-media-broadcast-hub/core";
|
||||
import type { DirectorOverride, DirectorProgram } from "./engine.js";
|
||||
import type { ChannelRef, DesiredFence } from "./director.activity.js";
|
||||
|
||||
/**
|
||||
* A channel source with its monitor state resolved for engine consumption.
|
||||
*
|
||||
* Optional fields mirror the engine's DirectorSourceInput exactly: ABSENT, not
|
||||
* null — the engine treats `lastMediaAtMs !== undefined` as "known", so a null
|
||||
* would read as epoch 0 and de-confirm every source as no-media. JSON drops
|
||||
* undefined keys, which keeps the wire form and the engine form identical.
|
||||
*/
|
||||
export interface DirectorChannelSource {
|
||||
id: string;
|
||||
priority: number;
|
||||
enabled: boolean;
|
||||
autoSelect: boolean;
|
||||
/** Debounced monitor state (LIVE only once confirmed). */
|
||||
state:
|
||||
| "OFFLINE" | "UNKNOWN" | "PROBING" | "STARTING" | "LIVE" | "UNSTABLE" | "ENDING" | "ERROR";
|
||||
/** Monitor row version; absent when no monitor row exists (every read is then a fresh sample). */
|
||||
version?: number;
|
||||
/** `source_monitor_states.state_entered_at` — when the current state was entered. */
|
||||
liveSinceMs?: number;
|
||||
/** Last media-health "media present" instant, for the no-media timeout. */
|
||||
lastMediaAtMs?: number;
|
||||
/** Media-health tier reports failure: de-confirms immediately (§5.2 step 2). */
|
||||
mediaFailed?: boolean;
|
||||
}
|
||||
|
||||
/** Result of a DirectorStatePort.read: the engine's full input for one channel. */
|
||||
export interface DirectorChannelState {
|
||||
kind: "current";
|
||||
desiredVersion: number;
|
||||
authzEpoch: number;
|
||||
generation: string;
|
||||
sources: DirectorChannelSource[];
|
||||
rules: AutoDirectorRule[];
|
||||
policy: SwitchPolicy;
|
||||
fallbackSceneId: string | null;
|
||||
override: DirectorOverride | null;
|
||||
principalPermissions: Record<string, string[]>;
|
||||
}
|
||||
|
||||
export type DirectorStateRead = DirectorChannelState | { kind: "superseded"; code: "SUPERSEDED" };
|
||||
|
||||
/** The read port: everything the evaluator needs for one decision. */
|
||||
export interface DirectorStatePort {
|
||||
read(ref: ChannelRef): Promise<DirectorStateRead>;
|
||||
}
|
||||
|
||||
/** The desired-state port contract, transport-agnostic twin of DesiredStatePort. */
|
||||
export interface DesiredStateOperations {
|
||||
fence(ref: ChannelRef): Promise<DesiredFence | undefined>;
|
||||
/** The engine's decision applied as a desired-state write; the director's commandId is the idempotency reference. */
|
||||
apply(command: {
|
||||
ref: ChannelRef;
|
||||
commandId: string;
|
||||
expectedDesiredVersion: number;
|
||||
authzEpoch: number;
|
||||
program: DirectorProgram;
|
||||
layout: SceneLayout;
|
||||
reasons: string[];
|
||||
releaseOverride?: string;
|
||||
}): Promise<{ kind: "applied" | "duplicate"; desiredVersion: number } | { kind: "conflict" }>;
|
||||
audit(ref: ChannelRef, event: { commandId: string; outcome: "SUPERSEDED" | "AUTHZ_REVOKED"; reason: string }): Promise<void>;
|
||||
}
|
||||
|
||||
/** An HTTP implementation of the API's /director endpoints (see api-side director.service.ts). */
|
||||
export interface DirectorApi {
|
||||
channelState(ref: ChannelRef): Promise<DirectorStateRead>;
|
||||
fence(ref: ChannelRef): Promise<DesiredFence | undefined>;
|
||||
applyProgramSwitch(ref: ChannelRef, body: DirectorApplyRequest): Promise<DirectorApplyResult>;
|
||||
audit(ref: ChannelRef, event: { commandId: string; outcome: string; reason: string }): Promise<void>;
|
||||
}
|
||||
|
||||
export interface DirectorApplyRequest {
|
||||
commandId: string;
|
||||
expectedDesiredVersion: number;
|
||||
authzEpoch: number;
|
||||
program: DirectorProgram;
|
||||
layout: SceneLayout;
|
||||
reasons: string[];
|
||||
releaseOverride?: string;
|
||||
}
|
||||
|
||||
export type DirectorApplyResult =
|
||||
| { kind: "applied" | "duplicate"; desiredVersion: number }
|
||||
| { kind: "conflict" };
|
||||
33
apps/scheduler/src/director/starter.ts
Normal file
33
apps/scheduler/src/director/starter.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
import { WorkflowExecutionAlreadyStartedError, type Client } from "@temporalio/client";
|
||||
import { WorkflowIdReusePolicy } from "@temporalio/common";
|
||||
import type { ChannelRef } from "./director.activity.js";
|
||||
|
||||
export function evaluatorWorkflowId(ref: Pick<ChannelRef, "workspaceId" | "channelId">): string {
|
||||
return `auto-director-${ref.workspaceId}-${ref.channelId}`;
|
||||
}
|
||||
|
||||
export type EnsureOutcome = "started" | "running" | "restarted";
|
||||
|
||||
/**
|
||||
* Starts are idempotent hints. A running evaluator is only nudged (it re-reads
|
||||
* state and fences its own generation); a closed one may be started afresh.
|
||||
*/
|
||||
export async function ensureDirector(client: Client, ref: ChannelRef, taskQueue: string): Promise<EnsureOutcome> {
|
||||
const workflowId = evaluatorWorkflowId(ref);
|
||||
const options = { taskQueue, workflowId, args: [{ ref }] as const, workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE };
|
||||
try {
|
||||
await client.workflow.start("autoDirectorEvaluator", options);
|
||||
return "started";
|
||||
} catch (error) {
|
||||
if (!(error instanceof WorkflowExecutionAlreadyStartedError)) throw error;
|
||||
const handle = client.workflow.getHandle(workflowId);
|
||||
const description = await handle.describe().catch(() => undefined);
|
||||
if (description?.status.name === "RUNNING") {
|
||||
await handle.signal("sourceChanged", { sourceId: "*", version: 0 });
|
||||
return "running";
|
||||
}
|
||||
// Closed between start and describe: one retry; a concurrent starter winning is fine.
|
||||
try { await client.workflow.start("autoDirectorEvaluator", options); return "restarted"; }
|
||||
catch (retryError) { if (retryError instanceof WorkflowExecutionAlreadyStartedError) return "running"; throw retryError; }
|
||||
}
|
||||
}
|
||||
223
apps/scheduler/src/director/wiring.test.ts
Normal file
223
apps/scheduler/src/director/wiring.test.ts
Normal file
|
|
@ -0,0 +1,223 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { parseDirectorChannels } from "./config.js";
|
||||
import { currentChannels, sweepDirectors, directorChannelsFromEnv } from "./discovery.js";
|
||||
import { DirectorHttpClient, parseApplyResult, parseFence } from "./http.client.js";
|
||||
import { HttpDirectorStatePort, HttpDesiredStatePort, engineSource, engineSnapshot } from "./adapters.js";
|
||||
import type { DirectorApi, DirectorChannelSource, DirectorStateRead } from "./ports.js";
|
||||
import type { ChannelRef } from "./director.activity.js";
|
||||
import { decide } from "./engine.js";
|
||||
|
||||
/**
|
||||
* WP10 wiring slice, scheduler side: config parsing, the HTTP port adapters
|
||||
* against a fake API, and the evaluator's discovery sweep. The port-pair
|
||||
* behaviour (state read -> decide -> desired applied) lives in
|
||||
* apps/api/src/director/director.service.test.ts, which imports this
|
||||
* package's engine directly.
|
||||
*/
|
||||
|
||||
const WS = "0b7c6d50-93a4-4a1e-8e94-93b7e0000001";
|
||||
const CH = "0b7c6d50-93a4-4a1e-8e94-93b7e0000002";
|
||||
const SRC = "0b7c6d50-93a4-4a1e-8e94-93b7e0000003";
|
||||
const RULE = "0b7c6d50-93a4-4a1e-8e94-93b7e0000004";
|
||||
|
||||
const rule = {
|
||||
id: RULE, virtualChannelId: CH, priority: 1,
|
||||
condition: { field: "live_source_count" as const, op: ">=" as const, value: 1 },
|
||||
action: { layout: "single" as const, selection: "top_priority" as const },
|
||||
};
|
||||
const policy = { name: "highest_priority" as const, liveConfirmations: 1, offlineConfirmations: 1, noMediaSeconds: 600, minHoldMs: 0, switchCooldownMs: 0 };
|
||||
const document = JSON.stringify([{ workspaceId: WS, channelId: CH, generation: "g1", rules: [rule], policy, fallbackSceneId: null }]);
|
||||
|
||||
describe("parseDirectorChannels", () => {
|
||||
it("parses the full document", () => {
|
||||
const channels = parseDirectorChannels(document);
|
||||
expect(channels).toHaveLength(1);
|
||||
expect(channels[0]).toMatchObject({ workspaceId: WS, channelId: CH, generation: "g1", fallbackSceneId: null });
|
||||
expect(channels[0]?.rules).toEqual([rule]);
|
||||
expect(channels[0]?.policy).toEqual(policy);
|
||||
});
|
||||
it("rejects malformed entries with a field path", () => {
|
||||
expect(parseDirectorChannels(undefined)).toEqual([]);
|
||||
expect(parseDirectorChannels("")).toEqual([]);
|
||||
expect(() => parseDirectorChannels("[]{}")).toThrow();
|
||||
expect(() => parseDirectorChannels('{"a":1}')).toThrow(/must be a JSON array/);
|
||||
expect(() => parseDirectorChannels('[{"channelId":"x"}]')).toThrow(/channelId must be a UUID/);
|
||||
expect(() => parseDirectorChannels(`[{"workspaceId":"${WS}","channelId":"x","generation":"g"}]`)).toThrow(/channelId must be a UUID/);
|
||||
expect(() => parseDirectorChannels(`[{"workspaceId":"${WS}","channelId":"${CH}","generation":""}]`)).toThrow(/generation/);
|
||||
expect(() => parseDirectorChannels(`[{"workspaceId":"${WS}","channelId":"${CH}","generation":"g","rules":[{"id":"bad"}]}]`)).toThrow(/rules\[0\]\.id/);
|
||||
expect(() => parseDirectorChannels(`[{"workspaceId":"${WS}","channelId":"${CH}","generation":"g","policy":{"name":"nope"}}]`)).toThrow(/policy\.name/);
|
||||
});
|
||||
it("reads from env and defaults to none", () => {
|
||||
expect(directorChannelsFromEnv({})).toEqual([]);
|
||||
expect(directorChannelsFromEnv({ DIRECTOR_CHANNELS: document })).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("discovery", () => {
|
||||
it("keeps the latest declared entry per channel", () => {
|
||||
const channels = parseDirectorChannels(`[${document.slice(1, -1)},${document.slice(1, -1).replace('"g1"', '"g2"')}]`);
|
||||
expect(channels).toHaveLength(2);
|
||||
expect(currentChannels(channels)).toHaveLength(1);
|
||||
expect(currentChannels(channels)[0]?.generation).toBe("g2");
|
||||
});
|
||||
it("ensures one evaluator per configured channel and swallows start failures", async () => {
|
||||
const channels = parseDirectorChannels(document);
|
||||
const workflowIds: string[] = [];
|
||||
const clientFor = (behaviour: () => Promise<"started" | "running" | "restarted" | "failed">): Parameters<typeof sweepDirectors>[0] =>
|
||||
({
|
||||
workflow: {
|
||||
start: async (_name: string, options: { workflowId: string }) => {
|
||||
workflowIds.push(options.workflowId);
|
||||
return behaviour();
|
||||
},
|
||||
getHandle: () => ({ describe: async () => ({ status: { name: "RUNNING" } }), signal: async () => undefined }),
|
||||
},
|
||||
}) as unknown as Parameters<typeof sweepDirectors>[0];
|
||||
// start succeeds: one evaluator ensured
|
||||
const started = await sweepDirectors(clientFor(async () => "started"), { taskQueue: "tmbh-publishing", channels });
|
||||
expect(started).toEqual({ started: 1, running: 0, restarted: 0 });
|
||||
// every Temporal failure is contained: the sweep reports zero and lives on
|
||||
const failing = await sweepDirectors(clientFor(async () => { throw new Error("temporal down"); }), { taskQueue: "q", channels });
|
||||
expect(failing).toEqual({ started: 0, running: 0, restarted: 0 });
|
||||
expect(workflowIds).toEqual([`auto-director-${WS}-${CH}`, `auto-director-${WS}-${CH}`]);
|
||||
});
|
||||
});
|
||||
|
||||
/** In-memory DirectorApi double used by the adapter tests. */
|
||||
function fakeApi(over: Partial<DirectorApi> = {}): DirectorApi & { requests: string[] } {
|
||||
const requests: string[] = [];
|
||||
const current: DirectorStateRead = {
|
||||
kind: "current", desiredVersion: 4, authzEpoch: 7, generation: "g1",
|
||||
sources: [{ id: SRC, priority: 50, enabled: true, autoSelect: true, state: "LIVE", version: 3 }],
|
||||
rules: [rule], policy, fallbackSceneId: null, override: null, principalPermissions: {},
|
||||
};
|
||||
return {
|
||||
requests,
|
||||
channelState: async (ref) => { requests.push(`state:${ref.channelId}`); return current; },
|
||||
fence: async (ref) => { requests.push(`fence:${ref.channelId}`); return { generation: "g1", desiredVersion: 4, authzEpoch: 7, directorAuthorized: true }; },
|
||||
applyProgramSwitch: async (ref, body) => { requests.push(`apply:${body.commandId}`); return { kind: "applied", desiredVersion: 5 }; },
|
||||
audit: async (ref, event) => { requests.push(`audit:${event.outcome}`); },
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
describe("port adapters", () => {
|
||||
const ref: ChannelRef = { workspaceId: WS, channelId: CH, generation: "g1" };
|
||||
|
||||
it("converts transport sources without inventing optional engine fields", () => {
|
||||
const source: DirectorChannelSource = { id: SRC, priority: 50, enabled: true, autoSelect: true, state: "LIVE" };
|
||||
const input = engineSource(source);
|
||||
expect(input).toEqual({ id: SRC, priority: 50, enabled: true, autoSelect: true, state: "LIVE" });
|
||||
expect("version" in input).toBe(false);
|
||||
expect("lastMediaAtMs" in input).toBe(false);
|
||||
expect("mediaFailed" in input).toBe(false);
|
||||
});
|
||||
|
||||
it("state read feeds decide() unchanged (engine input round-trip)", async () => {
|
||||
const api = fakeApi();
|
||||
const port = new HttpDirectorStatePort(api);
|
||||
const snapshot = await port.read(ref);
|
||||
if (snapshot.kind !== "current") throw new Error("expected current");
|
||||
expect(snapshot.sources[0]).toEqual({ id: SRC, priority: 50, enabled: true, autoSelect: true, state: "LIVE", version: 3 });
|
||||
const decision = decide({ sources: snapshot.sources, rules: snapshot.rules, policy: snapshot.policy, now: 1_000 });
|
||||
expect(decision.program).toEqual({ kind: "sources", sourceIds: [SRC] });
|
||||
});
|
||||
|
||||
it("engineSnapshot maps a transport read to the engine snapshot", () => {
|
||||
const read = { kind: "current" as const, desiredVersion: 4, authzEpoch: 7, generation: "g1",
|
||||
sources: [{ id: SRC, priority: 50, enabled: true, autoSelect: true, state: "LIVE", version: 3 } as DirectorChannelSource],
|
||||
rules: [rule], policy, fallbackSceneId: null, override: null, principalPermissions: {} };
|
||||
expect(engineSnapshot(read)).toMatchObject({ kind: "current", desiredVersion: 4, authzEpoch: 7 });
|
||||
});
|
||||
|
||||
it("desired-state port forwards fence/apply/audit with the commandId as idempotency key", async () => {
|
||||
const api = fakeApi();
|
||||
const port = new HttpDesiredStatePort(api);
|
||||
expect(await port.fence(ref)).toMatchObject({ generation: "g1", desiredVersion: 4 });
|
||||
expect(await port.findCommand(ref, "cmd")).toBeUndefined();
|
||||
const outcome = await port.writeProgram({
|
||||
ref, commandId: `${CH}:1`, expectedDesiredVersion: 4, authzEpoch: 7,
|
||||
program: { kind: "sources", sourceIds: [SRC] }, layout: "single", reasons: ["rule"],
|
||||
});
|
||||
expect(outcome).toEqual({ kind: "applied", desiredVersion: 5 });
|
||||
await port.audit(ref, { commandId: "c", outcome: "SUPERSEDED", reason: "generation" });
|
||||
expect(api.requests).toEqual([`fence:${CH}`, `apply:${CH}:1`, "audit:SUPERSEDED"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("response parsers (fail closed on malformed API replies)", () => {
|
||||
it("parseFence accepts a fence or absence and rejects junk", () => {
|
||||
expect(parseFence(null)).toBeUndefined();
|
||||
expect(parseFence({ generation: "g1", desiredVersion: 4, authzEpoch: 7, directorAuthorized: true })).toEqual({ generation: "g1", desiredVersion: 4, authzEpoch: 7, directorAuthorized: true });
|
||||
expect(() => parseFence({})).toThrow(/generation/);
|
||||
expect(() => parseFence({ generation: "g1", desiredVersion: "4", authzEpoch: 7, directorAuthorized: true })).toThrow(/desiredVersion/);
|
||||
expect(() => parseFence({ generation: "g1", desiredVersion: 4, authzEpoch: 7, directorAuthorized: "yes" })).toThrow(/directorAuthorized/);
|
||||
});
|
||||
it("parseApplyResult accepts applied/duplicate/conflict and rejects junk", () => {
|
||||
expect(parseApplyResult({ kind: "applied", desiredVersion: 5 })).toEqual({ kind: "applied", desiredVersion: 5 });
|
||||
expect(parseApplyResult({ kind: "duplicate", desiredVersion: 5 })).toEqual({ kind: "duplicate", desiredVersion: 5 });
|
||||
expect(parseApplyResult({ kind: "conflict" })).toEqual({ kind: "conflict" });
|
||||
expect(() => parseApplyResult({ kind: "applied" })).toThrow(/desiredVersion/);
|
||||
expect(() => parseApplyResult({ kind: "nope" })).toThrow(/kind/);
|
||||
expect(() => parseApplyResult("x")).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("DirectorHttpClient", () => {
|
||||
const ref: ChannelRef = { workspaceId: WS, channelId: CH, generation: "g1" };
|
||||
|
||||
function harness(): { client: DirectorHttpClient; seen: Array<{ path: string; init: RequestInit | undefined }> } {
|
||||
const seen: Array<{ path: string; init: RequestInit | undefined }> = [];
|
||||
const client = new DirectorHttpClient({
|
||||
baseUrl: "http://api.test/",
|
||||
token: "sekrit",
|
||||
fetch: async (input, init) => {
|
||||
seen.push({ path: String(input), init });
|
||||
const path = String(input);
|
||||
if (path.endsWith("/state")) return new Response(JSON.stringify({ kind: "superseded", code: "SUPERSEDED" }), { status: 200 });
|
||||
if (path.endsWith("/fence")) return new Response(JSON.stringify({ generation: "g1", desiredVersion: 4, authzEpoch: 7, directorAuthorized: true }), { status: 200 });
|
||||
if (path.endsWith("/apply")) return new Response(JSON.stringify({ kind: "applied", desiredVersion: 5 }), { status: 200 });
|
||||
return new Response(null, { status: 204 });
|
||||
},
|
||||
});
|
||||
return { client, seen };
|
||||
}
|
||||
|
||||
it("routes to the internal endpoints with the bearer token and validates replies", async () => {
|
||||
const { client, seen } = harness();
|
||||
expect(await client.channelState(ref)).toEqual({ kind: "superseded", code: "SUPERSEDED" });
|
||||
expect(await client.fence(ref)).toMatchObject({ generation: "g1" });
|
||||
expect(await client.applyProgramSwitch(ref, {
|
||||
commandId: "c1", expectedDesiredVersion: 4, authzEpoch: 7,
|
||||
program: { kind: "sources", sourceIds: [SRC] }, layout: "single", reasons: ["r"],
|
||||
})).toEqual({ kind: "applied", desiredVersion: 5 });
|
||||
await client.audit(ref, { commandId: "c1", outcome: "SUPERSEDED", reason: "gen" });
|
||||
expect(seen.map((entry) => entry.path)).toEqual([
|
||||
`http://api.test/internal/director/channels/${WS}/${CH}/state`,
|
||||
`http://api.test/internal/director/channels/${WS}/${CH}/fence`,
|
||||
`http://api.test/internal/director/channels/${WS}/${CH}/apply`,
|
||||
`http://api.test/internal/director/channels/${WS}/${CH}/audit`,
|
||||
]);
|
||||
for (const entry of seen) {
|
||||
expect((entry.init?.headers as Record<string, string>).authorization).toBe("Bearer sekrit");
|
||||
expect((entry.init?.headers as Record<string, string>)["content-type"]).toBe("application/json");
|
||||
}
|
||||
});
|
||||
|
||||
it("surfaces non-2xx as transport errors (retryable by Temporal)", async () => {
|
||||
const client = new DirectorHttpClient({
|
||||
baseUrl: "http://api.test", fetch: async () => new Response("nope", { status: 503, statusText: "Service Unavailable" }),
|
||||
});
|
||||
await expect(client.channelState(ref)).rejects.toThrow(/503/);
|
||||
});
|
||||
|
||||
it("aborts a slow API within the timeout budget", async () => {
|
||||
const client = new DirectorHttpClient({
|
||||
baseUrl: "http://api.test", timeoutMs: 20,
|
||||
fetch: async (_input, init) => new Promise((_resolve, reject) => {
|
||||
init?.signal?.addEventListener("abort", () => reject(new Error("aborted")));
|
||||
}),
|
||||
});
|
||||
await expect(client.fence(ref)).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
80
apps/scheduler/src/publishing.integration.test.ts
Normal file
80
apps/scheduler/src/publishing.integration.test.ts
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { Pool } from "pg";
|
||||
import { ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import { createMastodonConnector } from "@tactical-media-broadcast-hub/connector-mastodon";
|
||||
import { createTelegramConnector } from "@tactical-media-broadcast-hub/connector-telegram";
|
||||
import { Vault, createPublishingCredentialAccessor } from "@tactical-media-broadcast-hub/connector-runtime";
|
||||
import { PublishingExecutionRepository, canonicalPostHash, assertPublishingSchema, type AttemptRef } from "@tactical-media-broadcast-hub/publishing-execution";
|
||||
import { PublishActivity } from "./activities/publish.activity.js";
|
||||
const appUrl = process.env.PUBLISHING_IT_DATABASE_URL;
|
||||
const ownerUrl = process.env.PUBLISHING_IT_SUPER_DATABASE_URL;
|
||||
describe.skipIf(!appUrl || !ownerUrl)("TEST MODE real connectors with fenced DB activity", () => {
|
||||
let pool: Pool; let owner: Pool; let repository: PublishingExecutionRepository; let workspaceId: string; let userId: string;
|
||||
beforeAll(async () => {
|
||||
pool = new Pool({ connectionString: appUrl, max: 12 }); owner = new Pool({ connectionString: ownerUrl });
|
||||
await assertPublishingSchema(pool); repository = new PublishingExecutionRepository(pool);
|
||||
workspaceId = String((await owner.query("INSERT INTO workspaces(name,slug) VALUES('Scheduler Test',$1) RETURNING id", [`scheduler-sol-${Date.now()}`])).rows[0].id);
|
||||
userId = String((await owner.query("INSERT INTO users(external_id,username,display_name) VALUES($1,$1,'Scheduler Test') RETURNING id", [`scheduler-sol-${Date.now()}`])).rows[0].id);
|
||||
await owner.query("INSERT INTO memberships(workspace_id,user_id,role,status) VALUES($1,$2,'Owner','active')", [workspaceId, userId]);
|
||||
});
|
||||
afterAll(async () => {
|
||||
if (workspaceId) await owner.query("DELETE FROM workspaces WHERE id=$1", [workspaceId]);
|
||||
if (userId) await owner.query("DELETE FROM users WHERE id=$1", [userId]);
|
||||
await pool?.end(); await owner?.end();
|
||||
});
|
||||
async function fixture(provider: "mastodon" | "telegram"): Promise<AttemptRef> {
|
||||
const fields = provider === "mastodon" ? { instanceUrl: "https://social.test" } : { channelId: "-10042" };
|
||||
const connectionId = String((await owner.query("INSERT INTO connections(workspace_id,provider,channel_id,fields) VALUES($1,$2,$3,$4) RETURNING id", [workspaceId, provider, `channel-${provider}`, fields])).rows[0].id);
|
||||
const sealed = await new Vault().seal(workspaceId, connectionId, Buffer.from(JSON.stringify(provider === "mastodon" ? { access_token: "canary-mastodon-secret" } : { botToken: "canary-telegram-secret" })));
|
||||
await owner.query("INSERT INTO credentials(workspace_id,connection_id,kind,ciphertext,key_version) VALUES($1,$2,'connector-oauth',$3,$4)", [workspaceId, connectionId, sealed.ciphertext, sealed.keyVersion]);
|
||||
const post = { text: "[TMBH TEST MODE] hello", mediaAssetIds: [], visibility: "public", providerMetadata: { connectionId } };
|
||||
const draftId = String((await owner.query("INSERT INTO drafts(workspace_id,author_user_id,post,state,reviewer_user_id,approved_revision,approved_hash,approved_at) VALUES($1,$2,$3,'approved',$2,1,$4,now()) RETURNING id", [workspaceId, userId, post, canonicalPostHash(post)])).rows[0].id);
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN"); await client.query("SELECT set_config('app.workspace_id',$1,true)", [workspaceId]);
|
||||
const attempt = await repository.scheduleAttempt(client, { workspaceId, draftId, connectionId, scheduledByUserId: userId, requiredPermission: "publication.publish", publishAt: new Date(Date.now() - 1000) });
|
||||
await client.query("COMMIT"); return { workspaceId, scheduleId: attempt.scheduleId, generation: attempt.generation, idempotencyKey: attempt.idempotencyKey };
|
||||
} catch (error) { await client.query("ROLLBACK"); throw error; } finally { client.release(); }
|
||||
}
|
||||
for (const provider of ["mastodon", "telegram"] as const) it(`publishes ${provider} through exact APV1 credentials, gate, receipt and one canonical Publication`, async () => {
|
||||
const ref = await fixture(provider); let sends = 0; let observedKey: string | null = null;
|
||||
const fetchImpl: typeof fetch = async (input, init) => {
|
||||
const url = new URL(String(input));
|
||||
if (url.pathname.endsWith("/getChat")) return new Response(JSON.stringify({ ok: true, result: { id: -10042, type: "channel" } }), { headers: { "content-type": "application/json" } });
|
||||
sends += 1;
|
||||
expect(init?.method).toBe("POST");
|
||||
if (provider === "mastodon") {
|
||||
expect(url.pathname).toBe("/api/v1/statuses"); expect(new Headers(init?.headers).get("authorization")).toBe("Bearer canary-mastodon-secret");
|
||||
observedKey = new Headers(init?.headers).get("idempotency-key");
|
||||
return new Response(JSON.stringify({ id: "10942", created_at: new Date().toISOString(), url: "https://social.test/@test/10942", visibility: "public" }), { headers: { "content-type": "application/json" } });
|
||||
}
|
||||
expect(url.pathname).toBe("/botcanary-telegram-secret/sendMessage");
|
||||
expect(JSON.parse(String(init?.body))).toMatchObject({ chat_id: "-10042" });
|
||||
return new Response(JSON.stringify({ ok: true, result: { message_id: 42, date: Math.floor(Date.now()/1000), chat: { id: -10042 } } }), { headers: { "content-type": "application/json" } });
|
||||
};
|
||||
const registry = new ConnectorRegistry(); const options = { credentials: createPublishingCredentialAccessor(pool), fetchImpl, requestsPerSecond: 1000 };
|
||||
registry.register(createMastodonConnector(options)); registry.register(createTelegramConnector(options));
|
||||
const activity = new PublishActivity(repository, registry, () => ({ heartbeat: () => undefined }));
|
||||
const first = await activity.publishScheduled(ref); const second = await activity.publishScheduled(ref);
|
||||
expect(first.kind).toBe("done"); expect(second).toEqual(first); expect(sends).toBe(1);
|
||||
if (provider === "mastodon") expect(observedKey).toBe(ref.idempotencyKey);
|
||||
const evidence = await owner.query("SELECT p.provider_metadata, a.dispatch_count, (SELECT count(*)::int FROM publications WHERE idempotency_key=$1) AS publications, (SELECT count(*)::int FROM publishing_receipts WHERE attempt_id=a.id) AS receipts FROM publishing_attempts a JOIN publications p ON p.idempotency_key=a.idempotency_key WHERE a.idempotency_key=$1", [ref.idempotencyKey]);
|
||||
expect(evidence.rows[0]).toMatchObject({ dispatch_count: 1, publications: 1, receipts: 1 });
|
||||
expect(JSON.stringify({ first, second, evidence: evidence.rows })).not.toContain("canary-");
|
||||
});
|
||||
it("retains accepted-response-loss as unknown with no Telegram resend", async () => {
|
||||
const ref = await fixture("telegram"); let sends = 0;
|
||||
const fetchImpl: typeof fetch = async (input) => {
|
||||
const url = new URL(String(input));
|
||||
if (url.pathname.endsWith("/getChat")) return new Response(JSON.stringify({ ok: true, result: { id: -10042, type: "channel" } }));
|
||||
expect(url.pathname).toBe("/botcanary-telegram-secret/sendMessage"); sends += 1;
|
||||
throw new Error("TEST_RESPONSE_LOST_AFTER_ACCEPT");
|
||||
};
|
||||
const registry = new ConnectorRegistry(); registry.register(createTelegramConnector({ credentials: createPublishingCredentialAccessor(pool), fetchImpl, requestsPerSecond: 1000 }));
|
||||
const activity = new PublishActivity(repository, registry, () => ({ heartbeat: () => undefined }));
|
||||
await expect(activity.publishScheduled(ref)).resolves.toMatchObject({ kind: "unknown", code: "PUBLISH_OUTCOME_UNKNOWN" });
|
||||
await expect(activity.publishScheduled(ref)).resolves.toMatchObject({ kind: "unknown" });
|
||||
expect(sends).toBe(1);
|
||||
expect((await owner.query("SELECT count(*)::int AS n FROM publications WHERE idempotency_key=$1", [ref.idempotencyKey])).rows[0]?.n).toBe(0);
|
||||
});
|
||||
});
|
||||
20
apps/scheduler/src/worker.test.ts
Normal file
20
apps/scheduler/src/worker.test.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
import { Client, WorkflowExecutionAlreadyStartedError } from "@temporalio/client";
|
||||
import { WorkflowIdReusePolicy } from "@temporalio/common";
|
||||
import { sweepDue } from "./worker.js";
|
||||
const due = { workspaceId: "workspace", scheduleId: "schedule", generation: "1", idempotencyKey: "key", publishAt: new Date(), evidenceOnly: false };
|
||||
describe("durable discovery", () => {
|
||||
it("passes only fenced identifiers and uses a separate receipt recovery workflow", async () => {
|
||||
const start = vi.fn(async () => ({}));
|
||||
await sweepDue({ listDue: async () => [{ ...due, evidenceOnly: true }] }, { workflow: { start } } as unknown as Client);
|
||||
expect(start).toHaveBeenCalledWith("scheduledPublish", expect.objectContaining({ workflowId: "scheduled-publication-key-receipt", args: [{ workspaceId: "workspace", scheduleId: "schedule", generation: "1", idempotencyKey: "key" }] }));
|
||||
});
|
||||
for (const status of ["COMPLETED", "RUNNING", "FAILED"]) it(`describes ${status} before allowing recovery`, async () => {
|
||||
const start = vi.fn().mockRejectedValueOnce(new WorkflowExecutionAlreadyStartedError("existing", "scheduled-publication-key", "scheduledPublish")).mockResolvedValue({});
|
||||
const describeWorkflow = vi.fn(async () => ({ status: { name: status } }));
|
||||
await sweepDue({ listDue: async () => [due] }, { workflow: { start, getHandle: () => ({ describe: describeWorkflow }) } } as unknown as Client);
|
||||
expect(describeWorkflow).toHaveBeenCalledTimes(1);
|
||||
expect(start).toHaveBeenCalledTimes(status === "FAILED" ? 2 : 1);
|
||||
if (status === "FAILED") expect(start.mock.calls[1]?.[1]).toMatchObject({ workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY });
|
||||
});
|
||||
});
|
||||
106
apps/scheduler/src/worker.ts
Normal file
106
apps/scheduler/src/worker.ts
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
import { Client, Connection, WorkflowExecutionAlreadyStartedError } from "@temporalio/client";
|
||||
import { WorkflowIdReusePolicy } from "@temporalio/common";
|
||||
import { NativeConnection, Worker } from "@temporalio/worker";
|
||||
import { Pool } from "pg";
|
||||
import { createServer } from "node:http";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { resolve } from "node:path";
|
||||
import pino from "pino";
|
||||
import { createPublishingRegistry } from "@tactical-media-broadcast-hub/connector-runtime";
|
||||
import { PublishingExecutionRepository, assertPublishingSchema, type AttemptRef } from "@tactical-media-broadcast-hub/publishing-execution";
|
||||
import { PublishActivity } from "./activities/publish.activity.js";
|
||||
import { DirectorActivity } from "./director/director.activity.js";
|
||||
import { DirectorHttpClient } from "./director/http.client.js";
|
||||
import { HttpDirectorStatePort, HttpDesiredStatePort } from "./director/adapters.js";
|
||||
import { directorChannelsFromEnv, sweepDirectors } from "./director/discovery.js";
|
||||
import { DispatchGateService } from "./activities/dispatch-gate.js";
|
||||
|
||||
const logger = pino({ level: process.env.LOG_LEVEL ?? "info" });
|
||||
const address = process.env.TEMPORAL_ADDRESS ?? "localhost:7233";
|
||||
const namespace = process.env.TEMPORAL_NAMESPACE ?? "default";
|
||||
const taskQueue = process.env.TEMPORAL_TASK_QUEUE ?? "tmbh-publishing";
|
||||
|
||||
/** Starts are repair hints. Only repository claim/gate can authorize a send. */
|
||||
export async function sweepDue(repository: Pick<PublishingExecutionRepository, "listDue">, client: Client): Promise<void> {
|
||||
const due = await repository.listDue(100);
|
||||
for (let offset = 0; offset < due.length; offset += 4) {
|
||||
await Promise.all(due.slice(offset, offset + 4).map(async (item) => {
|
||||
const ref: AttemptRef = { workspaceId: item.workspaceId, scheduleId: item.scheduleId, generation: item.generation, idempotencyKey: item.idempotencyKey };
|
||||
const workflowId = `scheduled-publication-${ref.idempotencyKey}${item.evidenceOnly ? "-receipt" : ""}`;
|
||||
try {
|
||||
await client.workflow.start("scheduledPublish", { taskQueue, workflowId, args: [ref], workflowIdReusePolicy: WorkflowIdReusePolicy.REJECT_DUPLICATE });
|
||||
} catch (error) {
|
||||
if (error instanceof WorkflowExecutionAlreadyStartedError) {
|
||||
// Closed terminal workflows stay closed. A failed workflow can be
|
||||
// repaired on the next sweep only while DB discovery still allows it.
|
||||
const description = await client.workflow.getHandle(workflowId).describe().catch(() => undefined);
|
||||
if (description?.status.name === "FAILED") {
|
||||
try { await client.workflow.start("scheduledPublish", { taskQueue, workflowId, args: [ref], workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY }); }
|
||||
catch (retryError) { if (!(retryError instanceof WorkflowExecutionAlreadyStartedError)) logger.warn({ code: "WORKFLOW_RECOVERY_FAILED", scheduleId: item.scheduleId }, "scheduler recovery failed"); }
|
||||
}
|
||||
return;
|
||||
}
|
||||
logger.warn({ code: "WORKFLOW_START_FAILED", scheduleId: item.scheduleId }, "scheduler sweep failed");
|
||||
}
|
||||
}));
|
||||
}
|
||||
}
|
||||
export async function main(): Promise<void> {
|
||||
if (!process.env.SCHEDULER_DATABASE_URL) throw new Error("SCHEDULER_DATABASE_URL is required");
|
||||
const pool = new Pool({ connectionString: process.env.SCHEDULER_DATABASE_URL, max: 20 });
|
||||
let native: NativeConnection | undefined;
|
||||
let connection: Connection | undefined;
|
||||
let worker: Worker | undefined;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
let sweeping: Promise<void> | undefined;
|
||||
let closing = false;
|
||||
let ready = false;
|
||||
const health = createServer((_request, response) => { response.writeHead(ready ? 200 : 503, { "content-type": "application/json" }); response.end(JSON.stringify({ ready })); });
|
||||
const stop = (): void => { closing = true; ready = false; clearTimeout(timer); if (worker?.getState() === "RUNNING") worker.shutdown(); };
|
||||
process.once("SIGTERM", stop); process.once("SIGINT", stop);
|
||||
try {
|
||||
await assertPublishingSchema(pool);
|
||||
const repository = new PublishingExecutionRepository(pool);
|
||||
const registry = createPublishingRegistry(pool);
|
||||
native = await NativeConnection.connect({ address });
|
||||
connection = await Connection.connect({ address });
|
||||
const temporal = new Client({ connection, namespace });
|
||||
const activity = new PublishActivity(repository, registry, undefined, new DispatchGateService(pool));
|
||||
const directorApi = new DirectorHttpClient();
|
||||
const director = new DirectorActivity(new HttpDirectorStatePort(directorApi), new HttpDesiredStatePort(directorApi));
|
||||
const channels = directorChannelsFromEnv();
|
||||
worker = await Worker.create({ connection: native, namespace, taskQueue,
|
||||
workflowsPath: fileURLToPath(new URL("./workflows/index.js", import.meta.url)),
|
||||
activities: {
|
||||
publishScheduled: activity.publishScheduled.bind(activity),
|
||||
readChannelState: director.readChannelState.bind(director),
|
||||
applyProgramSwitch: director.applyProgramSwitch.bind(director),
|
||||
}, maxConcurrentActivityTaskExecutions: 4 });
|
||||
await new Promise<void>((resolveListen, reject) => { health.once("error", reject); health.listen(Number(process.env.SCHEDULER_HEALTH_PORT ?? 8081), resolveListen); });
|
||||
const sweep = async (): Promise<void> => {
|
||||
if (closing) return;
|
||||
try {
|
||||
await assertPublishingSchema(pool); await connection?.workflowService.getSystemInfo({}); await sweepDue(repository, temporal);
|
||||
const directors = await sweepDirectors(temporal, { taskQueue, channels });
|
||||
if (directors.started + directors.restarted > 0) logger.info(directors, "director evaluators ensured");
|
||||
ready = !closing;
|
||||
}
|
||||
catch { ready = false; logger.warn({ code: "DISCOVERY_UNAVAILABLE" }, "scheduler discovery unavailable"); }
|
||||
if (!closing) timer = setTimeout(() => { sweeping = sweep(); }, 5000 + Math.floor(Math.random() * 500));
|
||||
};
|
||||
sweeping = sweep();
|
||||
await sweeping;
|
||||
if (ready) logger.info({ namespace, taskQueue }, "scheduler sweep started");
|
||||
const running = worker.run();
|
||||
if (closing) stop();
|
||||
await running;
|
||||
} finally {
|
||||
stop(); await sweeping;
|
||||
process.off("SIGTERM", stop); process.off("SIGINT", stop);
|
||||
if (health.listening) await new Promise<void>((done) => health.close(() => done()));
|
||||
await connection?.close(); await native?.close(); await pool.end();
|
||||
}
|
||||
}
|
||||
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
void main().catch(() => { logger.error({ code: "SCHEDULER_STARTUP_FAILED" }, "scheduler failed readiness"); process.exitCode = 1; });
|
||||
}
|
||||
8
apps/scheduler/src/workflows/index.ts
Normal file
8
apps/scheduler/src/workflows/index.ts
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
/**
|
||||
* Workflow entrypoints the scheduler worker registers. One file keeps the
|
||||
* worker's workflow bundle small: publish.workflow.js stays the only publishing
|
||||
* entrypoint, and the director evaluator joins it without a second Worker
|
||||
* instance. Both are pure re-exports so bundling stays stable.
|
||||
*/
|
||||
export * from "./publish.workflow.js";
|
||||
export * from "../director/evaluator.workflow.js";
|
||||
14
apps/scheduler/src/workflows/publish.workflow.ts
Normal file
14
apps/scheduler/src/workflows/publish.workflow.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
import { proxyActivities, sleep } from "@temporalio/workflow";
|
||||
import type { AttemptRef } from "@tactical-media-broadcast-hub/publishing-execution";
|
||||
import type { PublishActivity, ActivityOutcome } from "../activities/publish.activity.js";
|
||||
const { publishScheduled } = proxyActivities<Pick<PublishActivity, "publishScheduled">>({
|
||||
startToCloseTimeout: "60 seconds", heartbeatTimeout: "10 seconds",
|
||||
retry: { initialInterval: "1 second", maximumInterval: "30 seconds", maximumAttempts: 5 },
|
||||
});
|
||||
export async function scheduledPublish(ref: AttemptRef): Promise<ActivityOutcome> {
|
||||
for (;;) {
|
||||
const outcome = await publishScheduled(ref);
|
||||
if (outcome.kind !== "wait") return outcome;
|
||||
await sleep(Math.max(1000, Date.parse(outcome.until) - Date.now()));
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue