Tactical Media Broadcast Hub — initial release
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
This commit is contained in:
commit
aa5e262f3a
880 changed files with 111042 additions and 0 deletions
62
apps/api/src/analytics/analytics.controller.ts
Normal file
62
apps/api/src/analytics/analytics.controller.ts
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
import { BadRequestException, Body, Controller, Get, Post, Query, Req, UseGuards } from "@nestjs/common";
|
||||
import type { Request } from "express";
|
||||
import { parseLimit, requireSession } from "../http/validation.js";
|
||||
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import { SnapshotService, SnapshotRecord, monthStart, parseMonth } from "./snapshot.service.js";
|
||||
|
||||
/**
|
||||
* POST /analytics/snapshot is the manual trigger (analytics.view is held by
|
||||
* Owner/Admin/Approver/Analyst — the closest thing to an admin/owner gate in
|
||||
* the 002 permission set). The scheduler calls `writeMonthlySnapshot`
|
||||
* (exported from snapshot.service.ts) directly later: same set-based SQL and
|
||||
* idempotent upsert, no HTTP hop.
|
||||
*/
|
||||
@Controller("analytics")
|
||||
@UseGuards(RbacGuard)
|
||||
export class AnalyticsController {
|
||||
constructor(private readonly snapshots: SnapshotService) {}
|
||||
|
||||
@Post("snapshot")
|
||||
@RequirePermission("analytics.view")
|
||||
async snapshot(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Body() body: unknown,
|
||||
): Promise<SnapshotRecord> {
|
||||
const input = typeof body === "object" && body !== null ? (body as Record<string, unknown>) : {};
|
||||
const month = input.month === undefined || input.month === null || input.month === ""
|
||||
? monthStart()
|
||||
: parseMonth(String(input.month));
|
||||
const session = requireSession(request);
|
||||
return this.snapshots.refresh(session.workspaceId, month);
|
||||
}
|
||||
|
||||
@Get("snapshots")
|
||||
@RequirePermission("analytics.view")
|
||||
async list(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Query("cursor") cursor: string | undefined,
|
||||
@Query("limit") limit: string | undefined,
|
||||
): Promise<{ items: SnapshotRecord[]; nextCursor: string | null }> {
|
||||
if (cursor !== undefined && (cursor.length > 256 || !/^[A-Za-z0-9_-]+$/.test(cursor))) {
|
||||
throw new BadRequestException("invalid cursor");
|
||||
}
|
||||
const session = requireSession(request);
|
||||
return this.snapshots.list(session.workspaceId, {
|
||||
...(cursor === undefined || cursor === "" ? {} : { cursor }),
|
||||
limit: limit === undefined || limit === "" ? undefined : parseLimit(limit, 12, 24),
|
||||
});
|
||||
}
|
||||
|
||||
@Get("summary")
|
||||
@RequirePermission("analytics.view")
|
||||
async summary(@Req() request: Request & AuthenticatedRequest): Promise<{
|
||||
currentMonth: string;
|
||||
previousMonth: string;
|
||||
current: SnapshotRecord | null;
|
||||
previous: SnapshotRecord | null;
|
||||
deltas: ReturnType<typeof import("./snapshot.service.js").monthDeltas> | null;
|
||||
}> {
|
||||
const session = requireSession(request);
|
||||
return this.snapshots.summary(session.workspaceId);
|
||||
}
|
||||
}
|
||||
358
apps/api/src/analytics/analytics.integration.test.ts
Normal file
358
apps/api/src/analytics/analytics.integration.test.ts
Normal file
|
|
@ -0,0 +1,358 @@
|
|||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { join } from "node:path";
|
||||
import { Pool, type PoolClient } from "pg";
|
||||
|
||||
/**
|
||||
* WP3 analytics snapshots against a live PostgreSQL, as sp_app (FORCE RLS
|
||||
* binds). Uses its OWN database (ANALYTICS_IT_DATABASE_URL, plus
|
||||
* ANALYTICS_IT_SUPER_DATABASE_URL for migrations/seeding) for the same reason
|
||||
* as groups.integration.test.ts: tenancy.integration.test.ts wipes every
|
||||
* workspace in the shared DB and vitest runs files in parallel.
|
||||
* Skipped cleanly when either env var is unset.
|
||||
*/
|
||||
const appUrl = process.env.ANALYTICS_IT_DATABASE_URL;
|
||||
const ownerUrl = process.env.ANALYTICS_IT_SUPER_DATABASE_URL;
|
||||
|
||||
type Modules = {
|
||||
service: typeof import("./snapshot.service.js");
|
||||
controller: typeof import("./analytics.controller.js");
|
||||
guard: typeof import("../rbac/rbac.guard.js");
|
||||
session: typeof import("../auth/session.js");
|
||||
db: typeof import("../db/client.js");
|
||||
};
|
||||
|
||||
describe.skipIf(appUrl === undefined || ownerUrl === undefined)("analytics snapshots (live DB)", () => {
|
||||
let m: Modules;
|
||||
let ownerPool: Pool;
|
||||
const suffix = `${Date.now()}${Math.floor(Math.random() * 1e6)}`;
|
||||
const currentMonth = new Date().toISOString().slice(0, 7) + "-01";
|
||||
const prevDate = new Date(`${currentMonth}T00:00:00Z`);
|
||||
prevDate.setUTCMonth(prevDate.getUTCMonth() - 1);
|
||||
const previousMonth = prevDate.toISOString().slice(0, 10);
|
||||
let wsA: string;
|
||||
let wsB: string;
|
||||
let user: string;
|
||||
|
||||
async function asOwner<T>(fn: (client: PoolClient) => Promise<T>): Promise<T> {
|
||||
const client = await ownerPool.connect();
|
||||
try {
|
||||
return await fn(client);
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
/** Seed tenant rows honestly: FORCE RLS applies to the owner too. */
|
||||
async function inTenant(client: PoolClient, wsId: string, fn: () => Promise<void>): Promise<void> {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT set_config('app.workspace_id', $1, true)", [wsId]);
|
||||
await fn();
|
||||
await client.query("COMMIT");
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
// The shared pool reads DATABASE_URL at import time: point it at this
|
||||
// isolated DB first, then import the modules under test.
|
||||
process.env.DATABASE_URL = appUrl;
|
||||
process.env.SUPER_DATABASE_URL = ownerUrl;
|
||||
const { runMigrations } = await import("../db/migrate.js");
|
||||
await runMigrations(join(process.cwd(), "migrations"));
|
||||
m = {
|
||||
service: await import("./snapshot.service.js"),
|
||||
controller: await import("./analytics.controller.js"),
|
||||
guard: await import("../rbac/rbac.guard.js"),
|
||||
session: await import("../auth/session.js"),
|
||||
db: await import("../db/client.js"),
|
||||
};
|
||||
ownerPool = new Pool({ connectionString: ownerUrl });
|
||||
|
||||
await asOwner(async (c) => {
|
||||
const ws = async (name: string): Promise<string> =>
|
||||
(await c.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id", [name, `${name}-${suffix}`])).rows[0].id as string;
|
||||
wsA = await ws("analytics-a");
|
||||
wsB = await ws("analytics-b");
|
||||
user = (await c.query(
|
||||
"INSERT INTO users (external_id, username, email, display_name) VALUES ($1, $2, $3, $4) RETURNING id",
|
||||
[`analytics:owner:${suffix}`, `analytics-owner-${suffix}`, `owner-${suffix}@example.com`, "owner"],
|
||||
)).rows[0].id as string;
|
||||
|
||||
const seed = async (wsId: string, archivedCount: number): Promise<void> => {
|
||||
await inTenant(c, wsId, async () => {
|
||||
await c.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, 'Owner')", [wsId, user]);
|
||||
const tag = wsId.slice(0, 8);
|
||||
// 3 connections (mastodon, bluesky, mastodon) + one draft each.
|
||||
const drafts: string[] = [];
|
||||
for (const [i, provider] of ["mastodon", "bluesky", "mastodon"].entries()) {
|
||||
const conn = (await c.query(
|
||||
"INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, $2, $3) RETURNING id",
|
||||
[wsId, provider, `ch-${tag}-${i}`],
|
||||
)).rows[0].id as string;
|
||||
const draft = (await c.query(
|
||||
"INSERT INTO drafts (workspace_id, author_user_id, post) VALUES ($1, $2, $3::jsonb) RETURNING id",
|
||||
[wsId, user, JSON.stringify({ body: "x" })],
|
||||
)).rows[0].id as string;
|
||||
drafts.push(draft);
|
||||
await c.query(
|
||||
`INSERT INTO publications (workspace_id, draft_id, connection_id, idempotency_key, remote_id, published_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6::date + $7::int)`,
|
||||
[wsId, draft, conn, `pub-${tag}-${i}`, `r-${tag}-${i}`, currentMonth, i],
|
||||
);
|
||||
}
|
||||
// One scheduled publication (draft_id is UNIQUE there) carrying
|
||||
// three attempts on the same draft: attempt_no 1..3.
|
||||
const schedule = (await c.query(
|
||||
`INSERT INTO scheduled_publications (workspace_id, draft_id, connection_id, publish_at, idempotency_key)
|
||||
VALUES ($1, $2, (SELECT id FROM connections WHERE workspace_id = $1 LIMIT 1), $3::date, $4) RETURNING id`,
|
||||
[wsId, drafts[0]!, currentMonth, `sch-${tag}`],
|
||||
)).rows[0].id as string;
|
||||
const credential = (await c.query(
|
||||
"INSERT INTO credentials (workspace_id, kind, ciphertext) VALUES ($1, 'test', gen_random_bytes(16)) RETURNING id",
|
||||
[wsId],
|
||||
)).rows[0].id as string;
|
||||
for (const n of [1, 2, 3]) {
|
||||
await c.query(
|
||||
`INSERT INTO publishing_attempts
|
||||
(workspace_id, schedule_id, draft_id, connection_id, attempt_no, idempotency_key, schedule_generation,
|
||||
post_snapshot, post_hash, scheduled_by_user_id, required_permission, approved_revision,
|
||||
reviewer_user_id, credential_id, credential_generation, authorized_epoch, status)
|
||||
VALUES ($1, $2, $3, (SELECT id FROM connections WHERE workspace_id = $1 LIMIT 1), $4, $5, 1,
|
||||
'{}'::jsonb, $6, $7, 'publication.publish', 1, $7, $8, 1, 0, $9)`,
|
||||
[wsId, schedule, drafts[0]!, n, `att-${tag}-${n}`, `hash-${tag}-${n}`, user, credential, n === 3 ? "completed" : "failed"],
|
||||
);
|
||||
}
|
||||
// Archived feed items: `archivedCount` in the current month, one in
|
||||
// the previous month, one in the far past (outside both windows).
|
||||
// remote_ref must equal channel_id || ':' || message_id.
|
||||
// remote_ref must match (@handle|numeric):message_id AND equal
|
||||
// channel_id || ':' || message_id; content_hash is sha256 hex (64).
|
||||
for (let i = 0; i < archivedCount; i += 1) {
|
||||
await c.query(
|
||||
`INSERT INTO archived_feed_items (workspace_id, remote_ref, channel_id, message_id, content_hash, posted_at)
|
||||
VALUES ($1, '@ch' || $2::text || ':' || $3::bigint::text, '@ch' || $2::text, $3::bigint,
|
||||
encode(sha256($4::bytea), 'hex'), $5::date + ($6::int * interval '1 hour'))`,
|
||||
[wsId, wsId.replace(/-/g, "").slice(0, 9), 1000 + i, `${tag}-${i}`, currentMonth, i],
|
||||
);
|
||||
}
|
||||
await c.query(
|
||||
`INSERT INTO archived_feed_items (workspace_id, remote_ref, channel_id, message_id, content_hash, posted_at)
|
||||
VALUES ($1, '@ch' || $2::text || ':9001', '@ch' || $2::text, 9001,
|
||||
encode(sha256($3::bytea), 'hex'), $4::date)`,
|
||||
[wsId, wsId.replace(/-/g, "").slice(0, 9), `${tag}-prev`, previousMonth],
|
||||
);
|
||||
await c.query(
|
||||
`INSERT INTO archived_feed_items (workspace_id, remote_ref, channel_id, message_id, content_hash, posted_at)
|
||||
VALUES ($1, '@ch' || $2::text || ':9002', '@ch' || $2::text, 9002,
|
||||
encode(sha256($3::bytea), 'hex'), '2020-01-01')`,
|
||||
[wsId, wsId.replace(/-/g, "").slice(0, 9), `${tag}-old`],
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
await seed(wsA, 5);
|
||||
await seed(wsB, 10);
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (ownerPool !== undefined) {
|
||||
await asOwner(async (c) => {
|
||||
await c.query("DELETE FROM workspaces WHERE id = ANY($1::uuid[])", [[wsA, wsB]]);
|
||||
await c.query("DELETE FROM users WHERE external_id LIKE $1", [`analytics:%:${suffix}`]);
|
||||
});
|
||||
await ownerPool.end();
|
||||
}
|
||||
await m?.db.pool.end();
|
||||
});
|
||||
|
||||
it("computes per-workspace monthly metrics from fixtures (set-based, no row loops)", async () => {
|
||||
const service = new m.service.SnapshotService();
|
||||
const a = await service.refresh(wsA, currentMonth);
|
||||
expect(a.month).toBe(currentMonth.slice(0, 7));
|
||||
expect(a.publicationsTotal).toBe(3);
|
||||
expect(a.publicationsByProvider).toEqual({ mastodon: 2, bluesky: 1 });
|
||||
expect(a.postsArchivedTotal).toBe(5);
|
||||
expect(a.activeConnections).toBe(3);
|
||||
expect(a.draftsCreatedTotal).toBe(3);
|
||||
expect(a.publishingAttemptsTotal).toBe(3);
|
||||
expect(a.publishFailuresTotal).toBe(2);
|
||||
|
||||
const b = await service.refresh(wsB, currentMonth);
|
||||
expect(b.postsArchivedTotal).toBe(10);
|
||||
expect(b.activeConnections).toBe(3);
|
||||
// Disconnect wsB's connections as sp_app so RLS scopes the update to wsB
|
||||
// (the owner pool is a superuser and bypasses RLS entirely).
|
||||
await m.db.withTenant(wsB, (client) => client.query("UPDATE connections SET disconnected_at = now()"));
|
||||
const bAfter = await service.refresh(wsB, currentMonth);
|
||||
expect(bAfter.activeConnections).toBe(0);
|
||||
});
|
||||
|
||||
it("is idempotent: running twice leaves exactly one row, same id and metrics", async () => {
|
||||
const service = new m.service.SnapshotService();
|
||||
const first = await service.refresh(wsA, currentMonth);
|
||||
const second = await service.refresh(wsA, currentMonth);
|
||||
expect(second.id).toBe(first.id);
|
||||
// Metrics are identical; only the snapshot timestamps move.
|
||||
const stripTimes = (r: typeof first): Record<string, unknown> => {
|
||||
const { snapshotStartedAt, snapshotCompletedAt, ...metrics } = r;
|
||||
void snapshotStartedAt; void snapshotCompletedAt;
|
||||
return metrics;
|
||||
};
|
||||
expect(stripTimes(second)).toEqual(stripTimes(first));
|
||||
expect(second.snapshotCompletedAt >= first.snapshotCompletedAt).toBe(true);
|
||||
// Count through the app pool so FORCE RLS scopes the check to wsA (the
|
||||
// owner pool is a superuser and would see every workspace's rows).
|
||||
const rows = await m.db
|
||||
.withTenant(wsA, (client) => client.query("SELECT id, publications_total FROM analytics_snapshots WHERE month = $1::date", [currentMonth]))
|
||||
.then((r) => r.rows);
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]).toMatchObject({ id: first.id, publications_total: "3" });
|
||||
});
|
||||
|
||||
it("partition pruning: a single-month predicate plans a scan of exactly one leaf", async () => {
|
||||
// Plan-time pruning needs a literal (a $n param defers pruning to run
|
||||
// time); the value is a validated YYYY-MM-01 so inlining is safe.
|
||||
if (!/^\d{4}-\d{2}-01$/.test(currentMonth)) throw new Error("bad month literal");
|
||||
const plan = await asOwner(async (c) => {
|
||||
await c.query("BEGIN");
|
||||
await c.query("SELECT set_config('app.workspace_id', $1, true)", [wsA]);
|
||||
const explained = await c.query<{ "QUERY PLAN": string }>(
|
||||
`EXPLAIN (COSTS OFF) SELECT count(*) FROM analytics_snapshots WHERE month = DATE '${currentMonth}'`,
|
||||
);
|
||||
await c.query("COMMIT");
|
||||
return explained.rows.map((r) => r["QUERY PLAN"]).join("\n");
|
||||
});
|
||||
const scanned = [...plan.matchAll(/Scan on (analytics_snapshots\S*)/g)].map((match) => match[1]!);
|
||||
expect(scanned.filter((r) => /^analytics_snapshots_[0-9]{6}$/.test(r)).length).toBe(1);
|
||||
for (const relation of scanned) expect(relation).not.toContain("default");
|
||||
});
|
||||
|
||||
it("cross-tenant denial: RLS hides other workspaces' snapshots (parent and every leaf)", async () => {
|
||||
const service = new m.service.SnapshotService();
|
||||
await service.refresh(wsB, currentMonth);
|
||||
const aList = await service.list(wsA, { limit: 24 });
|
||||
expect(aList.items.length).toBeGreaterThan(0);
|
||||
expect(aList.items.every((item) => item.workspaceId === wsA)).toBe(true);
|
||||
|
||||
// A raw read as the runtime role under B's context sees only B's rows.
|
||||
const leaked = await m.db.withTenant(wsB, async (client) =>
|
||||
client.query("SELECT workspace_id FROM analytics_snapshots"));
|
||||
expect(leaked.rows.length).toBeGreaterThan(0);
|
||||
expect(new Set(leaked.rows.map((r) => r.workspace_id))).toEqual(new Set([wsB]));
|
||||
|
||||
// Direct leaf access (parent RLS does not cascade to leaves): zero of
|
||||
// A's rows visible from B's context in every partition.
|
||||
const partitions = await asOwner((c) =>
|
||||
c.query<{ relname: string }>(
|
||||
"SELECT c.relname FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'analytics_snapshots'::regclass",
|
||||
));
|
||||
expect(partitions.rows.length).toBeGreaterThan(0);
|
||||
for (const row of partitions.rows) {
|
||||
if (!/^analytics_snapshots_(default|[0-9]{6})$/.test(row.relname)) throw new Error(`unexpected partition ${row.relname}`);
|
||||
const seen = await m.db.withTenant(wsB, (client) => client.query(`SELECT workspace_id FROM ${row.relname}`));
|
||||
expect(seen.rows.every((r) => r.workspace_id === wsB)).toBe(true);
|
||||
}
|
||||
|
||||
// And a write attempt for another workspace is refused by WITH CHECK.
|
||||
await expect(
|
||||
m.db.withTenant(wsA, async (client) =>
|
||||
client.query(
|
||||
`INSERT INTO analytics_snapshots (workspace_id, month) VALUES ($1::uuid, DATE '${currentMonth}')`,
|
||||
[wsB],
|
||||
)),
|
||||
).rejects.toThrow(/row-level security/i);
|
||||
});
|
||||
|
||||
it("keyset pagination: newest first, cursor continues without overlap", async () => {
|
||||
const service = new m.service.SnapshotService();
|
||||
await service.refresh(wsA, previousMonth);
|
||||
const page1 = await service.list(wsA, { limit: 1 });
|
||||
expect(page1.items.map((i) => i.month)).toEqual([currentMonth.slice(0, 7)]);
|
||||
expect(page1.nextCursor).not.toBeNull();
|
||||
const page2 = await service.list(wsA, { cursor: page1.nextCursor!, limit: 1 });
|
||||
expect(page2.items.map((i) => i.month)).toEqual([previousMonth.slice(0, 7)]);
|
||||
expect(page2.nextCursor).toBeNull();
|
||||
await expect(service.list(wsA, { cursor: "!!!not-a-cursor" })).rejects.toMatchObject({ status: 400 });
|
||||
});
|
||||
|
||||
it("summary: current month + previous month with correct deltas", async () => {
|
||||
const service = new m.service.SnapshotService();
|
||||
await service.refresh(wsB, previousMonth);
|
||||
const summary = await service.summary(wsB);
|
||||
expect(summary.currentMonth).toBe(currentMonth.slice(0, 7));
|
||||
expect(summary.previousMonth).toBe(previousMonth.slice(0, 7));
|
||||
expect(summary.previous).not.toBeNull();
|
||||
// previous month: only the one archived item; connections already
|
||||
// disconnected when that snapshot was taken.
|
||||
expect(summary.previous!.postsArchivedTotal).toBe(1);
|
||||
expect(summary.previous!.publicationsTotal).toBe(0);
|
||||
expect(summary.current!.postsArchivedTotal).toBe(10);
|
||||
expect(summary.deltas).toMatchObject({
|
||||
publicationsTotal: 3,
|
||||
postsArchivedTotal: 9,
|
||||
activeConnections: 0,
|
||||
publishFailuresTotal: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("invalid month input is rejected before any query", async () => {
|
||||
await expect(m.service.writeMonthlySnapshot({} as never, "2025-13")).rejects.toMatchObject({ status: 400 });
|
||||
await expect(new m.service.SnapshotService().refresh(wsA, "2025-1")).rejects.toMatchObject({ status: 400 });
|
||||
});
|
||||
|
||||
it("controller + RBAC guard chain: analytics.view required, wrong-workspace token is refused", async () => {
|
||||
const controller = new m.controller.AnalyticsController(new m.service.SnapshotService());
|
||||
const user = (await asOwner((c) => c.query<{ id: string }>("SELECT id FROM users WHERE external_id = $1", [`analytics:owner:${suffix}`]))).rows[0]!.id;
|
||||
|
||||
// Permission gate: a role without analytics.view is forbidden by the
|
||||
// guard. The guard resolves the role from the MEMBERSHIP, not the token,
|
||||
// so an Editor member of wsA is the honest deny-path fixture.
|
||||
const editor = (await asOwner(async (c) => {
|
||||
const id = (await c.query(
|
||||
"INSERT INTO users (external_id, username, email, display_name) VALUES ($1, $2, $3, $4) RETURNING id",
|
||||
[`analytics:editor:${suffix}`, `analytics-editor-${suffix}`, `editor-${suffix}@example.com`, "editor"],
|
||||
)).rows[0].id as string;
|
||||
await c.query("BEGIN");
|
||||
await c.query("SELECT set_config('app.workspace_id', $1, true)", [wsA]);
|
||||
await c.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, 'Editor')", [wsA, id]);
|
||||
await c.query("COMMIT");
|
||||
return id;
|
||||
}));
|
||||
// The membership insert above bumps authz_epoch (continuous authz), so
|
||||
// tokens are minted AFTER it — mirroring a fresh login per epoch.
|
||||
const epochNow = await asOwner(async (c) =>
|
||||
Number((await c.query("SELECT authz_epoch FROM workspaces WHERE id = $1", [wsA])).rows[0]?.authz_epoch));
|
||||
const token = (userId: string, workspaceId: string, role: string): string =>
|
||||
m.session.issueSession({ userId, workspaceId, role, authzEpoch: epochNow });
|
||||
const request = (t: string): { headers: Record<string, string>; session?: unknown } => ({
|
||||
headers: { authorization: `Bearer ${t}` },
|
||||
// In production RbacGuard.canActivate attaches this before the
|
||||
// controller runs; do the same here after verifying the token.
|
||||
session: m.session.verifySession(t) ?? undefined,
|
||||
});
|
||||
|
||||
// Owner of wsA refreshes wsA's snapshot through the real controller.
|
||||
const refreshed = await controller.snapshot(request(token(user, wsA, "Owner")) as never, {});
|
||||
expect(refreshed.workspaceId).toBe(wsA);
|
||||
expect(refreshed.publicationsTotal).toBe(3);
|
||||
|
||||
// A valid token for wsB does not leak wsA: wsB's refresh writes wsB's row.
|
||||
const forB = await controller.snapshot(request(token(user, wsB, "Owner")) as never, {});
|
||||
expect(forB.workspaceId).toBe(wsB);
|
||||
|
||||
const { Reflector } = await import("@nestjs/core");
|
||||
const guard = new m.guard.RbacGuard(new Reflector());
|
||||
const handler = (): void => undefined;
|
||||
Reflect.defineMetadata(m.guard.PERMISSION_KEY, "analytics.view", handler);
|
||||
const ctx = {
|
||||
getHandler: () => handler,
|
||||
switchToHttp: () => ({ getRequest: () => ({ headers: { authorization: `Bearer ${token(editor, wsA, "Editor")}` }, params: {} }) }),
|
||||
} as unknown as import("@nestjs/common").ExecutionContext;
|
||||
await expect(guard.canActivate(ctx)).rejects.toMatchObject({ status: 403 });
|
||||
// ...while the Owner's token passes the same gate.
|
||||
const ownerCtx = {
|
||||
getHandler: () => handler,
|
||||
switchToHttp: () => ({ getRequest: () => ({ headers: { authorization: `Bearer ${token(user, wsA, "Owner")}` }, params: {} }) }),
|
||||
} as unknown as import("@nestjs/common").ExecutionContext;
|
||||
await expect(guard.canActivate(ownerCtx)).resolves.toBe(true);
|
||||
});
|
||||
});
|
||||
309
apps/api/src/analytics/snapshot.service.ts
Normal file
309
apps/api/src/analytics/snapshot.service.ts
Normal file
|
|
@ -0,0 +1,309 @@
|
|||
import { BadRequestException, Injectable, NotFoundException } from "@nestjs/common";
|
||||
import type { PoolClient } from "pg";
|
||||
import { withTenant } from "../db/client.js";
|
||||
|
||||
/**
|
||||
* WP3 analytics snapshots: one rollup row per (workspace, month), recomputed
|
||||
* from source tables with set-based SQL (no row loops) and upserted so a
|
||||
* re-run is idempotent. Partitioned by month (020_analytics.sql).
|
||||
*
|
||||
* Exported as plain functions so the scheduler can call them later without
|
||||
* going through Nest:
|
||||
* import { writeMonthlySnapshot } from "./snapshot.service.js";
|
||||
* await withTenant(workspaceId, (client) => writeMonthlySnapshot(client, month));
|
||||
*/
|
||||
|
||||
/**
|
||||
* Month keys are accepted as `YYYY-MM` (API form) or `YYYY-MM-DD` where the
|
||||
* day is 01 (the partition-key form used internally). Both denote the same
|
||||
* month; anything else is a 400.
|
||||
*/
|
||||
export const MONTH_PATTERN = /^\d{4}-(0[1-9]|1[0-2])(-01)?$/;
|
||||
|
||||
export function parseMonth(value: string, field = "month"): string {
|
||||
if (!MONTH_PATTERN.test(value)) throw new BadRequestException(`${field} must be YYYY-MM`);
|
||||
return value;
|
||||
}
|
||||
|
||||
/** First day (UTC) of the month containing `at`, as `YYYY-MM-DD`. */
|
||||
export function monthStart(at: Date = new Date()): string {
|
||||
return at.toISOString().slice(0, 7) + "-01";
|
||||
}
|
||||
|
||||
export interface SnapshotMetrics {
|
||||
publicationsTotal: number;
|
||||
publicationsByProvider: Record<string, number>;
|
||||
postsArchivedTotal: number;
|
||||
activeConnections: number;
|
||||
draftsCreatedTotal: number;
|
||||
publishingAttemptsTotal: number;
|
||||
publishFailuresTotal: number;
|
||||
}
|
||||
|
||||
export interface SnapshotRecord extends SnapshotMetrics {
|
||||
id: string;
|
||||
workspaceId: string;
|
||||
month: string;
|
||||
snapshotStartedAt: string;
|
||||
snapshotCompletedAt: string;
|
||||
}
|
||||
|
||||
interface MetricsRow {
|
||||
publications_total: string;
|
||||
publications_by_provider: Record<string, number>;
|
||||
posts_archived_total: string;
|
||||
active_connections: number;
|
||||
drafts_created_total: string;
|
||||
publishing_attempts_total: string;
|
||||
publish_failures_total: string;
|
||||
}
|
||||
|
||||
interface SnapshotRow {
|
||||
id: string;
|
||||
workspace_id: string;
|
||||
month: string | Date; // pg returns `date` as a UTC Date
|
||||
publications_total: string;
|
||||
publications_by_provider: Record<string, number>;
|
||||
posts_archived_total: string;
|
||||
active_connections: number;
|
||||
drafts_created_total: string;
|
||||
publishing_attempts_total: string;
|
||||
publish_failures_total: string;
|
||||
snapshot_started_at: Date;
|
||||
snapshot_completed_at: Date;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set-based metric computation for one (workspace, month), run inside a tenant
|
||||
* transaction. Every source table is scanned once (the failure count is
|
||||
* derived from the attempts pass, not a second scan). Must run inside RLS
|
||||
* tenant context opened by the caller — no workspace_id literal is ever
|
||||
* embedded in the SQL.
|
||||
*/
|
||||
export async function computeMonthlyMetrics(client: PoolClient, month: string): Promise<MetricsRow> {
|
||||
const result = await client.query<MetricsRow>(
|
||||
`SELECT
|
||||
pub.publications_total,
|
||||
COALESCE(pub.by_provider, '{}'::jsonb) AS publications_by_provider,
|
||||
arch.posts_archived_total,
|
||||
conn.active_connections,
|
||||
dr.drafts_created_total,
|
||||
att.publishing_attempts_total,
|
||||
COALESCE(att.failed_total, 0) AS publish_failures_total
|
||||
FROM (SELECT 1) AS anchor
|
||||
LEFT JOIN LATERAL (
|
||||
-- one row per provider, so the total is SUM(n) — count(*) here would
|
||||
-- count provider groups, not publications.
|
||||
SELECT COALESCE(sum(c.n), 0)::bigint AS publications_total,
|
||||
COALESCE(jsonb_object_agg(c.provider, c.n) FILTER (WHERE c.provider IS NOT NULL), '{}'::jsonb) AS by_provider
|
||||
FROM (
|
||||
SELECT c.provider, count(*)::bigint AS n
|
||||
FROM publications p
|
||||
JOIN connections c ON c.id = p.connection_id
|
||||
WHERE p.published_at >= $1::date
|
||||
AND p.published_at < $1::date + interval '1 month'
|
||||
GROUP BY c.provider
|
||||
) c
|
||||
) pub ON true
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT count(*)::bigint AS posts_archived_total
|
||||
FROM archived_feed_items a
|
||||
WHERE a.posted_at >= $1::date
|
||||
AND a.posted_at < $1::date + interval '1 month'
|
||||
) arch ON true
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT count(*)::int AS active_connections
|
||||
FROM connections c
|
||||
WHERE c.disconnected_at IS NULL
|
||||
) conn ON true
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT count(*)::bigint AS drafts_created_total
|
||||
FROM drafts d
|
||||
WHERE d.created_at >= $1::date
|
||||
AND d.created_at < $1::date + interval '1 month'
|
||||
) dr ON true
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT count(*)::bigint AS publishing_attempts_total,
|
||||
count(*) FILTER (WHERE a.status = 'failed')::bigint AS failed_total
|
||||
FROM publishing_attempts a
|
||||
WHERE a.created_at >= $1::date
|
||||
AND a.created_at < $1::date + interval '1 month'
|
||||
) att ON true`,
|
||||
[month],
|
||||
);
|
||||
return result.rows[0]!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Idempotent refresh of one workspace's snapshot for one month: INSERT on
|
||||
* first run, UPDATE in place (same row id) on every re-run. Returns the
|
||||
* persisted row. Runs under RLS, so a wrong workspace context simply yields
|
||||
* no row (surfaced as 404 by the controller).
|
||||
*/
|
||||
export async function writeMonthlySnapshot(client: PoolClient, month: string): Promise<SnapshotRow> {
|
||||
// Normalize YYYY-MM / YYYY-MM-01 to the partition-key form.
|
||||
const normalized = `${parseMonth(month).slice(0, 7)}-01`;
|
||||
const metrics = await computeMonthlyMetrics(client, normalized);
|
||||
const result = await client.query<SnapshotRow>(
|
||||
`INSERT INTO analytics_snapshots
|
||||
(workspace_id, month, publications_total, publications_by_provider,
|
||||
posts_archived_total, active_connections, drafts_created_total,
|
||||
publishing_attempts_total, publish_failures_total,
|
||||
snapshot_started_at, snapshot_completed_at)
|
||||
VALUES (NULLIF(current_setting('app.workspace_id', true), '')::uuid, $1::date,
|
||||
$2, $3::jsonb, $4, $5, $6, $7, $8, clock_timestamp(), clock_timestamp())
|
||||
ON CONFLICT (workspace_id, month) DO UPDATE SET
|
||||
publications_total = EXCLUDED.publications_total,
|
||||
publications_by_provider = EXCLUDED.publications_by_provider,
|
||||
posts_archived_total = EXCLUDED.posts_archived_total,
|
||||
active_connections = EXCLUDED.active_connections,
|
||||
drafts_created_total = EXCLUDED.drafts_created_total,
|
||||
publishing_attempts_total = EXCLUDED.publishing_attempts_total,
|
||||
publish_failures_total = EXCLUDED.publish_failures_total,
|
||||
snapshot_started_at = EXCLUDED.snapshot_started_at,
|
||||
snapshot_completed_at = EXCLUDED.snapshot_completed_at
|
||||
RETURNING *`,
|
||||
[
|
||||
normalized,
|
||||
metrics.publications_total,
|
||||
JSON.stringify(metrics.publications_by_provider),
|
||||
metrics.posts_archived_total,
|
||||
metrics.active_connections,
|
||||
metrics.drafts_created_total,
|
||||
metrics.publishing_attempts_total,
|
||||
metrics.publish_failures_total,
|
||||
],
|
||||
);
|
||||
if (result.rows[0] === undefined) throw new NotFoundException("workspace not found for snapshot");
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
/** `pg` returns `date` columns as JS Dates (UTC midnight) — normalize to `YYYY-MM`. */
|
||||
function monthKey(value: string | Date): string {
|
||||
return (typeof value === "string" ? value : value.toISOString().slice(0, 10)).slice(0, 7);
|
||||
}
|
||||
|
||||
function recordFromRow(row: SnapshotRow): SnapshotRecord {
|
||||
return {
|
||||
id: row.id,
|
||||
workspaceId: row.workspace_id,
|
||||
month: monthKey(row.month),
|
||||
publicationsTotal: Number(row.publications_total),
|
||||
publicationsByProvider: row.publications_by_provider,
|
||||
postsArchivedTotal: Number(row.posts_archived_total),
|
||||
activeConnections: row.active_connections,
|
||||
draftsCreatedTotal: Number(row.drafts_created_total),
|
||||
publishingAttemptsTotal: Number(row.publishing_attempts_total),
|
||||
publishFailuresTotal: Number(row.publish_failures_total),
|
||||
snapshotStartedAt: row.snapshot_started_at.toISOString(),
|
||||
snapshotCompletedAt: row.snapshot_completed_at.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
/** Month keyset cursor: the last emitted month, opaque base64url. */
|
||||
function encodeCursor(month: string): string {
|
||||
return Buffer.from(JSON.stringify({ m: month }), "utf8").toString("base64url");
|
||||
}
|
||||
|
||||
function decodeCursor(cursor: string): string {
|
||||
try {
|
||||
const value: unknown = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"));
|
||||
const month = (value as { m?: unknown } | null)?.m;
|
||||
if (typeof month !== "string") throw new Error("shape");
|
||||
return `${parseMonth(month, "cursor").slice(0, 7)}-01`;
|
||||
} catch {
|
||||
throw new BadRequestException("invalid cursor");
|
||||
}
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class SnapshotService {
|
||||
/** Refresh a month's snapshot for the caller's workspace (admin/owner gated at the controller). */
|
||||
async refresh(workspaceId: string, month: string): Promise<SnapshotRecord> {
|
||||
return withTenant(workspaceId, async (client) => recordFromRow(await writeMonthlySnapshot(client, month)));
|
||||
}
|
||||
|
||||
/** Keyset page over months, newest first. Cursor = last emitted month. */
|
||||
async list(
|
||||
workspaceId: string,
|
||||
options: { cursor?: string; limit?: number } = {},
|
||||
): Promise<{ items: SnapshotRecord[]; nextCursor: string | null }> {
|
||||
const cursor = options.cursor === undefined ? null : decodeCursor(options.cursor);
|
||||
const limit = Math.min(Math.max(options.limit ?? 12, 1), 24);
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const result = await client.query<SnapshotRow>(
|
||||
`SELECT * FROM analytics_snapshots
|
||||
WHERE ($1::date IS NULL OR month < $1::date)
|
||||
ORDER BY month DESC
|
||||
LIMIT $2`,
|
||||
[cursor, limit + 1],
|
||||
);
|
||||
const rows = result.rows.slice(0, limit);
|
||||
const last = rows.at(-1);
|
||||
return {
|
||||
items: rows.map(recordFromRow),
|
||||
nextCursor: result.rows.length > limit && last !== undefined ? encodeCursor(monthKey(last.month)) : null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** Current month vs previous month with deltas (missing months are null). */
|
||||
async summary(workspaceId: string): Promise<{
|
||||
currentMonth: string;
|
||||
previousMonth: string;
|
||||
current: SnapshotRecord | null;
|
||||
previous: SnapshotRecord | null;
|
||||
deltas: Record<keyof SnapshotMetrics, number | Record<string, number>> | null;
|
||||
}> {
|
||||
const month = monthStart();
|
||||
const previous = previousMonthStart(month);
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const result = await client.query<SnapshotRow>(
|
||||
`SELECT * FROM analytics_snapshots
|
||||
WHERE month IN ($1::date, ($1::date - interval '1 month')::date)`,
|
||||
[month],
|
||||
);
|
||||
const byMonth = new Map(result.rows.map((row) => [monthKey(row.month), recordFromRow(row)]));
|
||||
const current = byMonth.get(month.slice(0, 7)) ?? null;
|
||||
const previousRecord = byMonth.get(previous.slice(0, 7)) ?? null;
|
||||
return {
|
||||
currentMonth: month.slice(0, 7),
|
||||
previousMonth: previous.slice(0, 7),
|
||||
current,
|
||||
previous: previousRecord,
|
||||
deltas: current !== null && previousRecord !== null ? monthDeltas(previousRecord, current) : null,
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** `YYYY-MM(-01)` of the month before `month`, same form as the input. */
|
||||
function previousMonthStart(month: string): string {
|
||||
const date = new Date(`${month.slice(0, 7)}-01T00:00:00Z`);
|
||||
date.setUTCMonth(date.getUTCMonth() - 1);
|
||||
const key = date.toISOString().slice(0, 7);
|
||||
return month.length === 7 ? key : `${key}-01`;
|
||||
}
|
||||
|
||||
/** Delta math for the summary endpoint (pure, unit-testable). */
|
||||
export function monthDeltas(
|
||||
previous: SnapshotMetrics,
|
||||
current: SnapshotMetrics,
|
||||
): Record<keyof SnapshotMetrics, number | Record<string, number>> {
|
||||
const prevProviders = previous.publicationsByProvider;
|
||||
const curProviders = current.publicationsByProvider;
|
||||
const providers = new Set([...Object.keys(prevProviders), ...Object.keys(curProviders)]);
|
||||
const providerDeltas: Record<string, number> = {};
|
||||
for (const provider of providers) {
|
||||
providerDeltas[provider] = (curProviders[provider] ?? 0) - (prevProviders[provider] ?? 0);
|
||||
}
|
||||
return {
|
||||
publicationsTotal: current.publicationsTotal - previous.publicationsTotal,
|
||||
publicationsByProvider: providerDeltas,
|
||||
postsArchivedTotal: current.postsArchivedTotal - previous.postsArchivedTotal,
|
||||
activeConnections: current.activeConnections - previous.activeConnections,
|
||||
draftsCreatedTotal: current.draftsCreatedTotal - previous.draftsCreatedTotal,
|
||||
publishingAttemptsTotal: current.publishingAttemptsTotal - previous.publishingAttemptsTotal,
|
||||
publishFailuresTotal: current.publishFailuresTotal - previous.publishFailuresTotal,
|
||||
};
|
||||
}
|
||||
59
apps/api/src/analytics/snapshot.test.ts
Normal file
59
apps/api/src/analytics/snapshot.test.ts
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { MONTH_PATTERN, monthDeltas, monthStart, parseMonth } from "./snapshot.service.js";
|
||||
|
||||
describe("analytics snapshot pure helpers", () => {
|
||||
it("monthStart returns the UTC month start", () => {
|
||||
expect(monthStart(new Date("2025-07-14T23:59:59.999Z"))).toBe("2025-07-01");
|
||||
expect(monthStart(new Date("2024-02-29T00:00:00Z"))).toBe("2024-02-01");
|
||||
});
|
||||
|
||||
it("parseMonth accepts YYYY-MM and month-start YYYY-MM-01, rejects everything else", () => {
|
||||
expect(parseMonth("2025-01")).toBe("2025-01");
|
||||
expect(parseMonth("2025-01-01")).toBe("2025-01-01");
|
||||
expect(() => parseMonth("2025-13")).toThrow();
|
||||
expect(() => parseMonth("2025-1")).toThrow();
|
||||
expect(() => parseMonth("25-01")).toThrow();
|
||||
expect(() => parseMonth("2025-01-02")).toThrow();
|
||||
expect(MONTH_PATTERN.test("2025-12")).toBe(true);
|
||||
expect(MONTH_PATTERN.test("2025-00")).toBe(false);
|
||||
});
|
||||
|
||||
it("monthDeltas computes per-metric and per-provider deltas vs fixtures", () => {
|
||||
const previous = {
|
||||
publicationsTotal: 10,
|
||||
publicationsByProvider: { mastodon: 4, bluesky: 6 },
|
||||
postsArchivedTotal: 100,
|
||||
activeConnections: 3,
|
||||
draftsCreatedTotal: 20,
|
||||
publishingAttemptsTotal: 12,
|
||||
publishFailuresTotal: 2,
|
||||
};
|
||||
const current = {
|
||||
publicationsTotal: 13,
|
||||
publicationsByProvider: { mastodon: 4, bluesky: 7, reddit: 2 },
|
||||
postsArchivedTotal: 90,
|
||||
activeConnections: 4,
|
||||
draftsCreatedTotal: 20,
|
||||
publishingAttemptsTotal: 15,
|
||||
publishFailuresTotal: 1,
|
||||
};
|
||||
expect(monthDeltas(previous, current)).toEqual({
|
||||
publicationsTotal: 3,
|
||||
publicationsByProvider: { mastodon: 0, bluesky: 1, reddit: 2 },
|
||||
postsArchivedTotal: -10,
|
||||
activeConnections: 1,
|
||||
draftsCreatedTotal: 0,
|
||||
publishingAttemptsTotal: 3,
|
||||
publishFailuresTotal: -1,
|
||||
});
|
||||
});
|
||||
|
||||
it("monthDeltas covers providers that vanished this month", () => {
|
||||
const deltas = monthDeltas(
|
||||
{ publicationsTotal: 5, publicationsByProvider: { mastodon: 5 }, postsArchivedTotal: 0, activeConnections: 0, draftsCreatedTotal: 0, publishingAttemptsTotal: 0, publishFailuresTotal: 0 },
|
||||
{ publicationsTotal: 1, publicationsByProvider: { bluesky: 1 }, postsArchivedTotal: 0, activeConnections: 0, draftsCreatedTotal: 0, publishingAttemptsTotal: 0, publishFailuresTotal: 0 },
|
||||
);
|
||||
expect(deltas.publicationsByProvider).toEqual({ mastodon: -5, bluesky: 1 });
|
||||
expect(deltas.publicationsTotal).toBe(-4);
|
||||
});
|
||||
});
|
||||
61
apps/api/src/app.module.ts
Normal file
61
apps/api/src/app.module.ts
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
import { Module } from "@nestjs/common";
|
||||
import { HealthModule } from "./health/health.module";
|
||||
import { DemoController } from "./demo/demo.controller";
|
||||
import { PublishingController } from "./publishing/publishing.controller";
|
||||
import { pool, withTenant } from "./db/client";
|
||||
import { PublishingExecutionRepository, assertPublishingSchema } from "@tactical-media-broadcast-hub/publishing-execution";
|
||||
import { ApprovalGateService } from "./publishing/approval-gate.service.js";
|
||||
import { PUBLISHING_DEPENDENCIES, TemporalWorkflowStarter, PublishingService } from "./publishing/publishing.service";
|
||||
import { AuthController } from "./auth/auth.controller";
|
||||
import { AuthService } from "./auth/auth.service";
|
||||
import { ConnectionsController } from "./connections/connections.controller";
|
||||
import { ConnectionsService } from "./connections/connections.service";
|
||||
import { TeamController } from "./team/team.controller";
|
||||
import { TeamService } from "./team/team.service";
|
||||
import { InviteLinkController, InviteClaimController } from "./team/invite-link.controller";
|
||||
import { InviteLinkService } from "./team/invite-link.service";
|
||||
import { AuditController } from "./audit/audit.controller";
|
||||
import { AuditQueryService } from "./audit/audit.query";
|
||||
import { GroupsController } from "./groups/groups.controller";
|
||||
import { GroupsService } from "./groups/groups.service";
|
||||
import { VideoController } from "./video/video.controller";
|
||||
import { VideoService } from "./video/video.service";
|
||||
import { LiveMonitorModule } from "./live-monitor/live-monitor.module";
|
||||
import { MediaControlModule } from "./media-control/media-control.module";
|
||||
import { ProductionModule } from "./production/production.module";
|
||||
import { RealtimeModule } from "./realtime/realtime.module";
|
||||
import { LiveChatModule } from "./live-chat/live-chat.module";
|
||||
import { LiveModule } from "./live/live.module.js";
|
||||
import { ChatgovModule } from "./chatgov/chatgov.module.js";
|
||||
import { IntakeModule } from "./intake/intake.module.js";
|
||||
import { SettingsModule } from "./settings/settings.module.js";
|
||||
import { SettingsController } from "./settings/settings.controller.js";
|
||||
import { SettingsService } from "./settings/settings.service.js";
|
||||
import { VirtualChannelsModule } from "./virtual-channels/virtual-channels.module.js";
|
||||
import { DirectorModule } from "./director/director.module.js";
|
||||
import { AnalyticsController } from "./analytics/analytics.controller";
|
||||
import { SnapshotService } from "./analytics/snapshot.service";
|
||||
|
||||
@Module({
|
||||
imports: [HealthModule, LiveMonitorModule, MediaControlModule, ProductionModule, RealtimeModule, LiveChatModule, LiveModule, VirtualChannelsModule, ChatgovModule, IntakeModule, SettingsModule],
|
||||
controllers: [
|
||||
DemoController,
|
||||
PublishingController,
|
||||
AuthController,
|
||||
ConnectionsController,
|
||||
TeamController,
|
||||
InviteLinkController,
|
||||
InviteClaimController,
|
||||
AuditController,
|
||||
VideoController,
|
||||
GroupsController,
|
||||
AnalyticsController,
|
||||
SettingsController,
|
||||
],
|
||||
providers: [
|
||||
{ provide: PublishingExecutionRepository, useFactory: () => new PublishingExecutionRepository(pool) },
|
||||
TemporalWorkflowStarter,
|
||||
{ provide: PUBLISHING_DEPENDENCIES, inject: [PublishingExecutionRepository, TemporalWorkflowStarter, ApprovalGateService], useFactory: (repository: PublishingExecutionRepository, workflowStarter: TemporalWorkflowStarter, approvalGate: ApprovalGateService) => ({ repository, workflowStarter, approvalGate, tenantTransaction: withTenant, assertReady: () => assertPublishingSchema(pool) }) },
|
||||
PublishingService, ApprovalGateService, AuthService, ConnectionsService, TeamService, InviteLinkService, AuditQueryService, VideoService, GroupsService, SnapshotService, SettingsService],
|
||||
})
|
||||
export class AppModule {}
|
||||
256
apps/api/src/archive/archive-search.integration.test.ts
Normal file
256
apps/api/src/archive/archive-search.integration.test.ts
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { join } from "node:path";
|
||||
import { Pool, type PoolClient } from "pg";
|
||||
|
||||
/**
|
||||
* WP15 archive search + export against a live PostgreSQL, as sp_app (FORCE
|
||||
* RLS binds). Uses its OWN database (ARCHIVE_SEARCH_IT_DATABASE_URL, plus
|
||||
* ARCHIVE_SEARCH_IT_SUPER_DATABASE_URL for migrations/seeding) per the
|
||||
* local-db-sol pattern: tenancy.integration.test.ts wipes every workspace in
|
||||
* the shared DB and vitest runs files in parallel. Skipped when unset.
|
||||
*/
|
||||
const appUrl = process.env.ARCHIVE_SEARCH_IT_DATABASE_URL;
|
||||
const ownerUrl = process.env.ARCHIVE_SEARCH_IT_SUPER_DATABASE_URL;
|
||||
|
||||
type Modules = {
|
||||
service: typeof import("./archive.service.js");
|
||||
"search-helpers": typeof import("./archive-search.js");
|
||||
db: typeof import("../db/client.js");
|
||||
};
|
||||
|
||||
describe.skipIf(appUrl === undefined || ownerUrl === undefined)("archive search + export (live DB)", () => {
|
||||
let m: Modules;
|
||||
let ownerPool: Pool;
|
||||
const suffix = `${Date.now()}${Math.floor(Math.random() * 1e6)}`;
|
||||
let wsA: string;
|
||||
let wsB: string;
|
||||
let chA = "";
|
||||
const chB = `@chB${suffix}`;
|
||||
|
||||
async function asOwner<T>(fn: (client: PoolClient) => Promise<T>): Promise<T> {
|
||||
const client = await ownerPool.connect();
|
||||
try {
|
||||
return await fn(client);
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
/** Seed tenant rows honestly: FORCE RLS applies to the owner too. */
|
||||
async function inTenant(client: PoolClient, wsId: string, fn: () => Promise<void>): Promise<void> {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT set_config('app.workspace_id', $1, true)", [wsId]);
|
||||
await fn();
|
||||
await client.query("COMMIT");
|
||||
}
|
||||
|
||||
let seedSeq = 0;
|
||||
/** remote_ref must equal channel_id || ':' || message_id; content_hash is sha256 hex. */
|
||||
async function seedItem(client: PoolClient, wsId: string, item: {
|
||||
channel: string; message: number; text: string; postedAt: Date; attribution?: Record<string, unknown>;
|
||||
}): Promise<void> {
|
||||
seedSeq += 1;
|
||||
await client.query(
|
||||
`INSERT INTO archived_feed_items
|
||||
(workspace_id, remote_ref, channel_id, message_id, text_content, content_hash, posted_at, attribution)
|
||||
VALUES ($1, $2 || ':' || $3::bigint::text, $2, $3::bigint, $4, encode(sha256($5::bytea), 'hex'), $6, $7::jsonb)`,
|
||||
[wsId, item.channel, item.message, item.text, `seed-${suffix}-${seedSeq}`, item.postedAt,
|
||||
JSON.stringify(item.attribution ?? {})],
|
||||
);
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
process.env.DATABASE_URL = appUrl;
|
||||
process.env.SUPER_DATABASE_URL = ownerUrl;
|
||||
const { runMigrations } = await import("../db/migrate.js");
|
||||
await runMigrations(join(process.cwd(), "migrations"));
|
||||
m = {
|
||||
service: await import("./archive.service.js"),
|
||||
"search-helpers": await import("./archive-search.js"),
|
||||
db: await import("../db/client.js"),
|
||||
};
|
||||
ownerPool = new Pool({ connectionString: ownerUrl });
|
||||
|
||||
await asOwner(async (c) => {
|
||||
const ws = async (name: string): Promise<string> =>
|
||||
(await c.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id",
|
||||
[name, `${name}-${suffix}`])).rows[0].id as string;
|
||||
wsA = await ws("search-a");
|
||||
wsB = await ws("search-b");
|
||||
});
|
||||
|
||||
// Tenant data: distinct texts per tenant, overlap only inside wsA.
|
||||
chA = `@chA${suffix}`;
|
||||
await asOwner((c) => inTenant(c, wsA, async () => {
|
||||
await seedItem(c, wsA, {
|
||||
channel: chA, message: 1001,
|
||||
text: "Water rights are a community responsibility, not a commodity for markets.",
|
||||
postedAt: new Date("2026-01-10T09:00:00Z"),
|
||||
attribution: { channelUsername: chA.slice(1), author: "Water Working Group" },
|
||||
});
|
||||
await seedItem(c, wsA, {
|
||||
channel: chA, message: 1002,
|
||||
text: "Totally unrelated post about bicycle repair cooperatives and tools.",
|
||||
postedAt: new Date("2026-01-11T09:00:00Z"),
|
||||
});
|
||||
await seedItem(c, wsA, {
|
||||
channel: chA, message: 1003,
|
||||
text: "Water rights update: the hearing is postponed to February.",
|
||||
postedAt: new Date("2026-01-12T09:00:00Z"),
|
||||
});
|
||||
}));
|
||||
await asOwner((c) => inTenant(c, wsB, async () => {
|
||||
await seedItem(c, wsB, {
|
||||
channel: chB, message: 2001,
|
||||
text: "Water rights are a community responsibility, not a commodity for markets.",
|
||||
postedAt: new Date("2026-01-05T09:00:00Z"),
|
||||
});
|
||||
}));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (ownerPool !== undefined) {
|
||||
await asOwner(async (c) => {
|
||||
await c.query("DELETE FROM workspaces WHERE id = ANY($1::uuid[])", [[wsA, wsB]]);
|
||||
});
|
||||
await ownerPool.end();
|
||||
}
|
||||
await m?.db.pool.end();
|
||||
});
|
||||
|
||||
it("full-text search matches archived text (websearch_to_tsquery)", async () => {
|
||||
const service = new m.service.ArchiveService(null as never);
|
||||
const result = await service.search(wsA, { q: "water rights" });
|
||||
expect(result.items).toHaveLength(2);
|
||||
for (const item of result.items) {
|
||||
expect(String(item.text)).toMatch(/water/i);
|
||||
expect(item.remoteRef).toBeDefined();
|
||||
}
|
||||
// Ranked path: both matches, the denser one first is NOT guaranteed, but
|
||||
// the update (1003) must not shadow the older match.
|
||||
expect(result.items.map((i) => i.messageId).sort()).toEqual([1001, 1003]);
|
||||
});
|
||||
|
||||
it("fuzzy typo tolerance: misspelled token still matches via pg_trgm", async () => {
|
||||
const service = new m.service.ArchiveService(null as never);
|
||||
const result = await service.search(wsA, { q: "watter rghts" });
|
||||
expect(result.items.length).toBeGreaterThanOrEqual(1);
|
||||
expect(result.items.map((i) => i.messageId).sort()).toEqual([1001, 1003]);
|
||||
});
|
||||
|
||||
it("metadata search: channel id and attribution text are searchable", async () => {
|
||||
const service = new m.service.ArchiveService(null as never);
|
||||
const byChannel = await service.search(wsA, { q: chA });
|
||||
expect(byChannel.items).toHaveLength(3);
|
||||
const byAuthor = await service.search(wsA, { q: "Working Group" });
|
||||
expect(byAuthor.items.map((i) => i.messageId)).toEqual([1001]);
|
||||
});
|
||||
|
||||
it("RLS: search never crosses tenants even with shared text", async () => {
|
||||
const service = new m.service.ArchiveService(null as never);
|
||||
const a = await service.search(wsA, { q: "water rights" });
|
||||
const b = await service.search(wsB, { q: "water rights" });
|
||||
expect(a.items).toHaveLength(2);
|
||||
expect(b.items).toHaveLength(1);
|
||||
expect(b.items.every((i) => String(i.remoteRef).startsWith(chB))).toBe(true);
|
||||
// Direct DB-level cross-tenant denial as sp_app.
|
||||
const leaked = await m.db.withTenant(wsB, (client) => client.query(
|
||||
"SELECT count(*)::int AS n FROM archived_feed_items WHERE workspace_id = $1", [wsA]));
|
||||
expect(leaked.rows[0]?.n).toBe(0);
|
||||
});
|
||||
|
||||
it("date + channel filters narrow the result set", async () => {
|
||||
const service = new m.service.ArchiveService(null as never);
|
||||
const from = await service.search(wsA, { q: "water", from: "2026-01-12" });
|
||||
expect(from.items.map((i) => i.messageId)).toEqual([1003]);
|
||||
const to = await service.search(wsA, { q: "water", to: "2026-01-10T23:59:59Z" });
|
||||
expect(to.items.map((i) => i.messageId)).toEqual([1001]);
|
||||
const channel = await service.search(wsA, { q: "bicycle", channelId: chA });
|
||||
expect(channel.items.map((i) => i.messageId)).toEqual([1002]);
|
||||
const none = await service.search(wsA, { q: "bicycle", channelId: `@nonesuch${suffix}` });
|
||||
expect(none.items).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("keyset pagination walks the ranked result set without repeats or gaps", async () => {
|
||||
const service = new m.service.ArchiveService(null as never);
|
||||
const page1 = await service.search(wsA, { q: "water rights", limit: "1" });
|
||||
expect(page1.items).toHaveLength(1);
|
||||
expect(page1.nextCursor).not.toBeNull();
|
||||
const page2 = await service.search(wsA, { q: "water rights", limit: "1", cursor: page1.nextCursor ?? "" });
|
||||
expect(page2.items).toHaveLength(1);
|
||||
expect(page2.nextCursor).toBeNull();
|
||||
const seen = [...page1.items, ...page2.items].map((i) => i.messageId).sort();
|
||||
expect(seen).toEqual([1001, 1003]);
|
||||
// Bogus cursor is rejected, not silently ignored.
|
||||
await expect(service.search(wsA, { q: "water", cursor: "bogus-cursor" }))
|
||||
.rejects.toThrow(/invalid cursor/);
|
||||
});
|
||||
|
||||
it("export streams every matching item with faithful content (json + csv)", async () => {
|
||||
const service = new m.service.ArchiveService(null as never);
|
||||
const json: string[] = [];
|
||||
const format = await service.exportItems(wsA, { q: "water rights" }, (chunk) => json.push(chunk));
|
||||
expect(format).toBe("json");
|
||||
const parsed = JSON.parse(json.join("")) as Array<Record<string, unknown>>;
|
||||
expect(parsed).toHaveLength(2);
|
||||
expect(parsed.map((i) => i.messageId).sort()).toEqual([1001, 1003]);
|
||||
expect(parsed.every((i) => typeof i.postedAt === "string" && typeof i.remoteRef === "string")).toBe(true);
|
||||
|
||||
const csv: string[] = [];
|
||||
const csvFormat = await service.exportItems(wsA, { q: "water", format: "csv" }, (chunk) => csv.push(chunk));
|
||||
expect(csvFormat).toBe("csv");
|
||||
const csvText = csv.join("");
|
||||
const lines = csvText.split("\r\n").filter((line) => line !== "");
|
||||
expect(lines).toHaveLength(3); // header + 2 rows
|
||||
const header = lines[0]!.split(",");
|
||||
expect(header.slice(0, 4)).toEqual(["id", "remote_ref", "channel_id", "message_id"]);
|
||||
const rows = lines.slice(1).map((line) => line.split(","));
|
||||
for (const row of rows) {
|
||||
expect(row).toHaveLength(header.length);
|
||||
expect(UUID_SHAPE.test(row[0]!)).toBe(true);
|
||||
expect(row[1]).toMatch(new RegExp(`^${chA}:[0-9]+$`));
|
||||
}
|
||||
const textCol = header.indexOf("text");
|
||||
const texts = rows.map((row) => row[textCol]!).sort();
|
||||
expect(texts[1]).toContain("hearing is postponed");
|
||||
});
|
||||
|
||||
it("export CSV escaping: commas, quotes and newlines round-trip", async () => {
|
||||
const service = new m.service.ArchiveService(null as never);
|
||||
const hostile = 'He said "water, rights", then\nleft';
|
||||
await asOwner((c) => inTenant(c, wsA, async () => {
|
||||
await seedItem(c, wsA, {
|
||||
channel: chA, message: 1004, text: hostile, postedAt: new Date("2026-01-13T09:00:00Z"),
|
||||
});
|
||||
}));
|
||||
const csv: string[] = [];
|
||||
await service.exportItems(wsA, { channelId: chA, format: "csv" }, (chunk) => csv.push(chunk));
|
||||
const lines = csv.join("").split("\r\n").filter((l) => l !== "");
|
||||
const header = lines[0]!.split(",");
|
||||
const textCol = header.indexOf("text");
|
||||
const rows = lines.slice(1).map((line) => line.split(","));
|
||||
const hostileRow = rows.find((row) => row[textCol] === hostile);
|
||||
expect(hostileRow).toBeDefined();
|
||||
expect(hostileRow![textCol]).toBe(hostile);
|
||||
expect(rows.every((row) => row.length === header.length)).toBe(true);
|
||||
});
|
||||
|
||||
it("filter validation rejects garbage bounds, limits and formats", async () => {
|
||||
const service = new m.service.ArchiveService(null as never);
|
||||
await expect(service.search(wsA, { q: "water", from: "not-a-date" })).rejects.toThrow(/ISO-8601/);
|
||||
await expect(service.search(wsA, { q: "water", from: "2026-02-01", to: "2026-01-01" })).rejects.toThrow(/after/);
|
||||
await expect(service.search(wsA, { q: "water", limit: "0" })).rejects.toThrow(/between 1 and/);
|
||||
await expect(service.search(wsA, { q: "water", limit: "1000" })).rejects.toThrow(/between 1 and/);
|
||||
await expect(service.search(wsA, { q: "water", cursor: "bogus" })).rejects.toThrow(/invalid cursor/);
|
||||
await expect(service.exportItems(wsA, { format: "xml" }, () => undefined)).rejects.toThrow(/csv or json/);
|
||||
});
|
||||
|
||||
it("unindexed helper behavior: tokenization + CSV header shape", () => {
|
||||
const helpers = m["search-helpers"];
|
||||
expect(helpers.searchTokens("Watter, Rghts! @channel123")).toEqual(["watter", "rghts", "@channel123"]);
|
||||
expect(helpers.searchTokens("ab xx ")).toEqual([]);
|
||||
expect(helpers.ARCHIVE_CSV_HEADER).toMatch(/^id,remote_ref,channel_id,message_id,posted_at,revision,protected_content,text,storage_keys,attribution\r\n$/);
|
||||
});
|
||||
});
|
||||
|
||||
const UUID_SHAPE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
280
apps/api/src/archive/archive-search.ts
Normal file
280
apps/api/src/archive/archive-search.ts
Normal file
|
|
@ -0,0 +1,280 @@
|
|||
import { BadRequestException } from "@nestjs/common";
|
||||
import { withTenant } from "../db/client.js";
|
||||
import type { StorageService } from "../storage/storage.js";
|
||||
import type { TelegramMedia } from "./normalizer.js";
|
||||
|
||||
/**
|
||||
* WP15 archive full-text search (Postgres FTS phase-1, firehose research
|
||||
* recommendation). Every query runs through withTenant() so the
|
||||
* archived_feed_items tenant_isolation RLS policy (migration 015, FORCE)
|
||||
* binds — there is deliberately no system/superuser path here.
|
||||
*
|
||||
* Matching is `websearch_to_tsquery` lexical OR pg_trgm similarity per token,
|
||||
* so a misspelled token still matches via trigram distance (typo tolerance).
|
||||
* Ranking: FTS ts_rank dominates, trigram similarity breaks ties. Keyset
|
||||
* pagination follows the house pattern: base64url JSON cursor, fetch N+1.
|
||||
*/
|
||||
|
||||
export interface ArchiveSearchFilters {
|
||||
q: string | null;
|
||||
channelId: string | null;
|
||||
from: string | null;
|
||||
to: string | null;
|
||||
cursor: string | undefined;
|
||||
limit: number;
|
||||
}
|
||||
|
||||
export const SEARCH_DEFAULT_LIMIT = 20;
|
||||
export const SEARCH_MAX_LIMIT = 100;
|
||||
export const ARCHIVE_EXPORT_MAX_ROWS = 50_000;
|
||||
const EXPORT_BATCH_SIZE = 500;
|
||||
|
||||
export interface ArchiveItemRow {
|
||||
id: string;
|
||||
remote_ref: string;
|
||||
channel_id: string;
|
||||
message_id: string;
|
||||
text_content: string;
|
||||
media_refs: TelegramMedia[];
|
||||
attribution: Record<string, unknown>;
|
||||
protected_content: boolean;
|
||||
posted_at: Date;
|
||||
revision: number;
|
||||
rank?: number;
|
||||
}
|
||||
|
||||
const ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?$/;
|
||||
const UUID_SHAPE = /^[0-9a-f-]{36}$/i;
|
||||
|
||||
function parseDateBound(value: unknown, field: string): string | null {
|
||||
if (value === undefined || value === null || value === "") return null;
|
||||
if (typeof value !== "string" || value.length > 32 || !ISO_TIMESTAMP.test(value)
|
||||
|| !Number.isFinite(Date.parse(value))) {
|
||||
throw new BadRequestException(`${field} must be an ISO-8601 date or timestamp`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Shared filter/cursor-shape parsing for GET /archive/search and GET /archive/export. */
|
||||
export function parseArchiveSearchQuery(query: Record<string, unknown>): ArchiveSearchFilters {
|
||||
const rawQ = query.q;
|
||||
const q = typeof rawQ === "string" && rawQ.trim() !== "" ? rawQ.trim() : null;
|
||||
if (q !== null && q.length > 256) throw new BadRequestException("q must be at most 256 characters");
|
||||
const rawChannel = query.channelId;
|
||||
const channelId = typeof rawChannel === "string" && rawChannel.trim() !== "" ? rawChannel.trim() : null;
|
||||
if (channelId !== null && channelId.length > 128) throw new BadRequestException("channelId is too long");
|
||||
const from = parseDateBound(query.from, "from");
|
||||
const to = parseDateBound(query.to, "to");
|
||||
if (from !== null && to !== null && Date.parse(from) > Date.parse(to)) {
|
||||
throw new BadRequestException("from must not be after to");
|
||||
}
|
||||
let limit = SEARCH_DEFAULT_LIMIT;
|
||||
const rawLimit = query.limit;
|
||||
if (rawLimit !== undefined && rawLimit !== "") {
|
||||
const n = typeof rawLimit === "string" ? Number(rawLimit) : NaN;
|
||||
if (!Number.isInteger(n) || n < 1 || n > SEARCH_MAX_LIMIT) {
|
||||
throw new BadRequestException(`limit must be an integer between 1 and ${SEARCH_MAX_LIMIT}`);
|
||||
}
|
||||
limit = n;
|
||||
}
|
||||
const rawCursor = query.cursor;
|
||||
const cursor = typeof rawCursor === "string" && rawCursor !== "" ? rawCursor : undefined;
|
||||
if (cursor !== undefined && (cursor.length > 256 || !/^[A-Za-z0-9_-]+$/.test(cursor))) {
|
||||
throw new BadRequestException("invalid cursor");
|
||||
}
|
||||
return { q, channelId, from, to, cursor, limit };
|
||||
}
|
||||
|
||||
/** Lowercase word tokens (>= 3 chars, pg_trgm minimum), deduped and capped. */
|
||||
export function searchTokens(q: string): string[] {
|
||||
const matches = q.toLowerCase().match(/[\p{L}\p{N}_@]+/gu) ?? [];
|
||||
return [...new Set(matches.filter((token) => token.length >= 3))].slice(0, 8);
|
||||
}
|
||||
|
||||
/** Positional bind helper: keeps parameter indices stable while SQL fragments compose. */
|
||||
class Params {
|
||||
readonly values: unknown[] = [];
|
||||
bind(value: unknown): number {
|
||||
this.values.push(value);
|
||||
return this.values.length;
|
||||
}
|
||||
}
|
||||
|
||||
function decodeCursor<T extends { id: string }>(cursor: string | undefined, what: string,
|
||||
shape: (v: Record<string, unknown>) => T): T {
|
||||
if (cursor === undefined) throw new BadRequestException(`invalid cursor: ${what}`);
|
||||
let value: unknown;
|
||||
try {
|
||||
value = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"));
|
||||
} catch {
|
||||
throw new BadRequestException(`invalid cursor: ${what}`);
|
||||
}
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
||||
throw new BadRequestException(`invalid cursor: ${what}`);
|
||||
}
|
||||
return shape(value as Record<string, unknown>);
|
||||
}
|
||||
|
||||
/**
|
||||
* The rank expression, bound once so SELECT, WHERE (keyset cursor) and
|
||||
* ORDER BY share identical parameter indices.
|
||||
*/
|
||||
function rankExpression(params: Params, q: string, tokens: string[]): string {
|
||||
const qi = params.bind(q);
|
||||
const textSims = tokens.map((token) => `similarity(text_content, $${params.bind(token)})`);
|
||||
const chanSims = tokens.map((token) => `similarity(channel_id, $${params.bind(token)})`);
|
||||
return `(ts_rank(search_vector, websearch_to_tsquery('simple', $${qi})) * 2`
|
||||
+ (textSims.length > 0 ? ` + GREATEST(${textSims.join(", ")})` : "")
|
||||
+ (chanSims.length > 0 ? ` + GREATEST(${chanSims.join(", ")}) * 0.5` : "")
|
||||
+ `)`;
|
||||
}
|
||||
|
||||
function filterConds(params: Params, filters: ArchiveSearchFilters): string[] {
|
||||
const conds: string[] = [];
|
||||
if (filters.channelId !== null) conds.push(`channel_id = $${params.bind(filters.channelId)}`);
|
||||
if (filters.from !== null) conds.push(`posted_at >= $${params.bind(filters.from)}::timestamptz`);
|
||||
if (filters.to !== null) conds.push(`posted_at <= $${params.bind(filters.to)}::timestamptz`);
|
||||
return conds;
|
||||
}
|
||||
|
||||
/** WHERE + ORDER BY for the ranked search path (q present). */
|
||||
function searchConds(params: Params, filters: ArchiveSearchFilters, tokens: string[]): [string, string] {
|
||||
const rank = rankExpression(params, filters.q ?? "", tokens);
|
||||
const conds = filterConds(params, filters);
|
||||
const fuzzy = tokens.length === 0 ? "false"
|
||||
: tokens.map((token) => {
|
||||
const ti = params.bind(token);
|
||||
return `(text_content % $${ti} OR channel_id % $${ti})`;
|
||||
}).join(" OR ");
|
||||
conds.push(`(search_vector @@ websearch_to_tsquery('simple', $${params.bind(filters.q ?? "")}) OR ${fuzzy})`);
|
||||
if (filters.cursor !== undefined) {
|
||||
const { rk, id } = decodeCursor<{ rk: number; id: string }>(filters.cursor, "rank", (v) => {
|
||||
if (typeof v.rk !== "number" || !Number.isFinite(v.rk)
|
||||
|| typeof v.id !== "string" || !UUID_SHAPE.test(v.id)) throw new BadRequestException("invalid cursor: rank");
|
||||
return { rk: v.rk, id: v.id };
|
||||
});
|
||||
conds.push(`(${rank}, id) < ($${params.bind(rk)}::real, $${params.bind(id)}::uuid)`);
|
||||
}
|
||||
return [conds.join(" AND "), "ranked.rank DESC, ranked.id ASC"];
|
||||
}
|
||||
|
||||
/** WHERE + ORDER BY for the export / no-query path (stable posted_at keyset). */
|
||||
function unrankedConds(params: Params, filters: ArchiveSearchFilters,
|
||||
cursor: { at: string; id: string } | null): [string, string] {
|
||||
const conds = filterConds(params, filters);
|
||||
if (cursor) conds.push(`(posted_at, id) < ($${params.bind(cursor.at)}::timestamptz, $${params.bind(cursor.id)}::uuid)`);
|
||||
return [conds.length > 0 ? conds.join(" AND ") : "true", "posted_at DESC, id DESC"];
|
||||
}
|
||||
|
||||
const ROW_FIELDS = "id, remote_ref, channel_id, message_id, text_content, media_refs, attribution, protected_content, posted_at, revision";
|
||||
|
||||
export function itemFromRow(row: ArchiveItemRow): Record<string, unknown> {
|
||||
return {
|
||||
id: row.id,
|
||||
remoteRef: row.remote_ref,
|
||||
channelId: row.channel_id,
|
||||
messageId: Number(row.message_id),
|
||||
text: row.text_content,
|
||||
media: row.media_refs,
|
||||
attribution: row.attribution,
|
||||
protectedContent: row.protected_content,
|
||||
postedAt: row.posted_at.toISOString(),
|
||||
revision: row.revision,
|
||||
};
|
||||
}
|
||||
|
||||
export async function searchArchive(workspaceId: string, filters: ArchiveSearchFilters,
|
||||
storage: StorageService): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
|
||||
const params = new Params();
|
||||
const tokens = filters.q !== null ? searchTokens(filters.q) : [];
|
||||
let sql: string;
|
||||
if (filters.q !== null) {
|
||||
const [where, orderBy] = searchConds(params, filters, tokens);
|
||||
const rank = rankExpression(params, filters.q, tokens);
|
||||
sql = `SELECT ${ROW_FIELDS}, rank FROM (
|
||||
SELECT ${ROW_FIELDS}, ${rank} AS rank FROM archived_feed_items WHERE ${where}
|
||||
) ranked ORDER BY ${orderBy} LIMIT $${params.bind(filters.limit + 1)}`;
|
||||
} else {
|
||||
const cursor = filters.cursor === undefined ? null
|
||||
: decodeCursor<{ at: string; id: string }>(filters.cursor, "posted", (v) => {
|
||||
if (typeof v.at !== "string" || !Number.isFinite(Date.parse(v.at))
|
||||
|| typeof v.id !== "string" || !UUID_SHAPE.test(v.id)) throw new BadRequestException("invalid cursor: posted");
|
||||
return { at: v.at, id: v.id };
|
||||
});
|
||||
const [where, orderBy] = unrankedConds(params, filters, cursor);
|
||||
sql = `SELECT ${ROW_FIELDS} FROM archived_feed_items WHERE ${where}
|
||||
ORDER BY ${orderBy} LIMIT $${params.bind(filters.limit + 1)}`;
|
||||
}
|
||||
const result = await withTenant(workspaceId, (client) => client.query<ArchiveItemRow>(sql, params.values));
|
||||
return paginate(result.rows, filters, storage);
|
||||
}
|
||||
|
||||
async function paginate(rows: ArchiveItemRow[], filters: ArchiveSearchFilters,
|
||||
storage: StorageService): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
|
||||
const page = rows.slice(0, filters.limit);
|
||||
const last = page.at(-1);
|
||||
const items = await Promise.all(page.map(async (row) => {
|
||||
const item = itemFromRow(row);
|
||||
if (filters.q !== null) item.rank = row.rank ?? 0;
|
||||
item.media = await Promise.all(row.media_refs.map(async (media) => ({
|
||||
...media, thumbUrl: media.storageKey ? await storage.presign(media.storageKey, 900) : null,
|
||||
})));
|
||||
return item;
|
||||
}));
|
||||
const nextCursor = rows.length > filters.limit && last
|
||||
? Buffer.from(JSON.stringify(filters.q !== null
|
||||
? { rk: last.rank ?? 0, id: last.id }
|
||||
: { at: last.posted_at.toISOString(), id: last.id })).toString("base64url")
|
||||
: null;
|
||||
return { items, nextCursor };
|
||||
}
|
||||
|
||||
/**
|
||||
* Streamed export: yields pages of plain item objects in posted_at keyset
|
||||
* order. Exports never mint presigned URLs — media travels by storage key.
|
||||
*/
|
||||
export async function* exportArchiveItems(workspaceId: string,
|
||||
filters: ArchiveSearchFilters): AsyncGenerator<Record<string, unknown>[]> {
|
||||
let cursor: { at: string; id: string } | null = null;
|
||||
let sent = 0;
|
||||
while (sent < ARCHIVE_EXPORT_MAX_ROWS) {
|
||||
const params = new Params();
|
||||
const remaining = Math.min(EXPORT_BATCH_SIZE, ARCHIVE_EXPORT_MAX_ROWS - sent);
|
||||
const [where, orderBy] = unrankedConds(params, filters, cursor);
|
||||
const result = await withTenant(workspaceId, (client) => client.query<ArchiveItemRow>(
|
||||
`SELECT ${ROW_FIELDS} FROM archived_feed_items WHERE ${where}
|
||||
ORDER BY ${orderBy} LIMIT $${params.bind(remaining + 1)}`, params.values));
|
||||
const page = result.rows.slice(0, remaining);
|
||||
if (page.length === 0) return;
|
||||
sent += page.length;
|
||||
const lastRow = page[page.length - 1]!;
|
||||
cursor = { at: lastRow.posted_at.toISOString(), id: lastRow.id };
|
||||
yield page.map(itemFromRow);
|
||||
if (result.rows.length <= remaining) return;
|
||||
}
|
||||
}
|
||||
|
||||
const CSV_COLUMNS: Array<[string, (item: Record<string, unknown>) => string]> = [
|
||||
["id", (i) => String(i.id)],
|
||||
["remote_ref", (i) => String(i.remoteRef)],
|
||||
["channel_id", (i) => String(i.channelId)],
|
||||
["message_id", (i) => String(i.messageId)],
|
||||
["posted_at", (i) => String(i.postedAt)],
|
||||
["revision", (i) => String(i.revision)],
|
||||
["protected_content", (i) => (i.protectedContent === true ? "true" : "false")],
|
||||
["text", (i) => String(i.text)],
|
||||
["storage_keys", (i) => (Array.isArray(i.media)
|
||||
? (i.media as Array<{ storageKey?: string }>).map((m) => m.storageKey ?? "").filter((k) => k !== "").join(";")
|
||||
: "")],
|
||||
["attribution", (i) => JSON.stringify(i.attribution ?? {})],
|
||||
];
|
||||
|
||||
function csvCell(value: string): string {
|
||||
return /[",\r\n]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value;
|
||||
}
|
||||
|
||||
export const ARCHIVE_CSV_HEADER = CSV_COLUMNS.map(([name]) => name).join(",") + "\r\n";
|
||||
|
||||
export function csvItemRow(item: Record<string, unknown>): string {
|
||||
return CSV_COLUMNS.map(([, get]) => csvCell(get(item))).join(",") + "\r\n";
|
||||
}
|
||||
86
apps/api/src/archive/archive.controller.ts
Normal file
86
apps/api/src/archive/archive.controller.ts
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
import { BadRequestException, Body, Controller, Get, Headers, Param, Post, Query, Req, Res, UseGuards } from "@nestjs/common";
|
||||
import type { Request, Response } from "express";
|
||||
import { parseId, requireObject, requireSession, requireString } from "../http/validation.js";
|
||||
import { RbacGuard, RequireGroupAccess, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import { ArchiveService } from "./archive.service.js";
|
||||
|
||||
@Controller("archive")
|
||||
export class ArchiveController {
|
||||
constructor(private readonly archive: ArchiveService) {}
|
||||
|
||||
@Post("webhook/:workspaceId")
|
||||
async webhook(@Param("workspaceId") workspaceId: string,
|
||||
// eslint-disable-next-line no-restricted-syntax -- Fixed webhook protocol header, not provider branching.
|
||||
@Headers("x-telegram-bot-api-secret-token") secret: string | undefined,
|
||||
@Body() body: unknown): Promise<{ ok: true }> {
|
||||
await this.archive.receive(parseId(workspaceId), secret, body);
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
@Post("subscriptions")
|
||||
@UseGuards(RbacGuard)
|
||||
@RequirePermission("connection.manage")
|
||||
@RequireGroupAccess("body")
|
||||
async subscribe(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise<{ id: string }> {
|
||||
const input = requireObject(body, "body");
|
||||
const workspaceId = requireSession(request).workspaceId;
|
||||
return { id: await this.archive.subscribe(workspaceId, parseId(requireString(input.connectionId, "connectionId")),
|
||||
requireString(input.channelId, "channelId")) };
|
||||
}
|
||||
|
||||
@Get("items")
|
||||
@UseGuards(RbacGuard)
|
||||
@RequirePermission("draft.view")
|
||||
async list(@Req() request: Request & AuthenticatedRequest,
|
||||
@Query("cursor") cursor?: string): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
|
||||
if (cursor && (cursor.length > 256 || !/^[A-Za-z0-9_-]+$/.test(cursor))) throw new BadRequestException("invalid cursor");
|
||||
return this.archive.list(requireSession(request).workspaceId, cursor);
|
||||
}
|
||||
|
||||
@Post("items/:itemId/drafts")
|
||||
@UseGuards(RbacGuard)
|
||||
@RequirePermission("draft.create")
|
||||
async createDraft(@Req() request: Request & AuthenticatedRequest,
|
||||
@Param("itemId") itemId: string): Promise<{ id: string; state: string }> {
|
||||
const session = requireSession(request);
|
||||
return this.archive.createDraft(session.workspaceId, { userId: session.userId, role: session.role }, parseId(itemId));
|
||||
}
|
||||
|
||||
@Post("items/:itemId/amplify")
|
||||
@UseGuards(RbacGuard)
|
||||
@RequirePermission("publication.schedule")
|
||||
@RequireGroupAccess("body")
|
||||
async amplify(@Req() request: Request & AuthenticatedRequest, @Param("itemId") itemId: string,
|
||||
@Body() body: unknown): Promise<unknown> {
|
||||
const input = requireObject(body, "body");
|
||||
if (input.attribution !== undefined && typeof input.attribution !== "boolean") throw new BadRequestException("attribution must be boolean");
|
||||
return this.archive.amplify(requireSession(request).workspaceId, parseId(itemId),
|
||||
parseId(requireString(input.draftId, "draftId")), parseId(requireString(input.connectionId, "connectionId")),
|
||||
input.attribution !== false);
|
||||
}
|
||||
|
||||
@Get("search")
|
||||
@UseGuards(RbacGuard)
|
||||
@RequirePermission("draft.view")
|
||||
async search(@Req() request: Request & AuthenticatedRequest,
|
||||
@Query() query: Record<string, unknown>): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
|
||||
return this.archive.search(requireSession(request).workspaceId, query);
|
||||
}
|
||||
|
||||
@Get("export")
|
||||
@UseGuards(RbacGuard)
|
||||
@RequirePermission("draft.view")
|
||||
async exportAll(@Req() request: Request & AuthenticatedRequest, @Res() response: Response,
|
||||
@Query() query: Record<string, unknown>): Promise<void> {
|
||||
const workspaceId = requireSession(request).workspaceId;
|
||||
const format = query.format === "csv" ? "csv" : query.format === undefined || query.format === "" ? "json" : null;
|
||||
if (format === null) throw new BadRequestException("format must be csv or json");
|
||||
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
||||
response.setHeader("Content-Type", format === "csv" ? "text/csv; charset=utf-8" : "application/json; charset=utf-8");
|
||||
response.setHeader("Content-Disposition", `attachment; filename="archive-export-${stamp}.${format}"`);
|
||||
response.setHeader("Cache-Control", "no-store");
|
||||
response.setHeader("X-Content-Type-Options", "nosniff");
|
||||
await this.archive.exportItems(workspaceId, query, (chunk) => response.write(chunk));
|
||||
response.end();
|
||||
}
|
||||
}
|
||||
7
apps/api/src/archive/archive.module.ts
Normal file
7
apps/api/src/archive/archive.module.ts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
import { Module } from "@nestjs/common";
|
||||
import { PublishingService } from "../publishing/publishing.service.js";
|
||||
import { ArchiveController } from "./archive.controller.js";
|
||||
import { ArchiveService } from "./archive.service.js";
|
||||
|
||||
@Module({ controllers: [ArchiveController], providers: [ArchiveService, PublishingService] })
|
||||
export class ArchiveModule {}
|
||||
239
apps/api/src/archive/archive.service.ts
Normal file
239
apps/api/src/archive/archive.service.ts
Normal file
|
|
@ -0,0 +1,239 @@
|
|||
import { BadRequestException, ForbiddenException, Injectable, NotFoundException } from "@nestjs/common";
|
||||
import { timingSafeEqual } from "node:crypto";
|
||||
// eslint-disable-next-line no-restricted-syntax -- Importing the connector adapter is the provider boundary, not provider branching.
|
||||
import { createTelegramConnector } from "@tactical-media-broadcast-hub/connector-telegram";
|
||||
import { ConnectorError } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import type { ConnectorConnection } from "@tactical-media-broadcast-hub/core";
|
||||
import { withTenant } from "../db/client.js";
|
||||
import { PublishingService, type Actor } from "../publishing/publishing.service.js";
|
||||
import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js";
|
||||
import {
|
||||
ARCHIVE_CSV_HEADER,
|
||||
csvItemRow,
|
||||
exportArchiveItems,
|
||||
parseArchiveSearchQuery,
|
||||
searchArchive,
|
||||
type ArchiveSearchFilters,
|
||||
} from "./archive-search.js";
|
||||
import { TelegramAlbumDebouncer, type ArchiveItemInput, type TelegramMedia } from "./normalizer.js";
|
||||
|
||||
const ARCHIVE_PROVIDER = createTelegramConnector().manifest().id;
|
||||
|
||||
interface ItemRow {
|
||||
id: string; remote_ref: string; channel_id: string; message_id: string;
|
||||
text_content: string; media_refs: TelegramMedia[]; attribution: Record<string, unknown>;
|
||||
protected_content: boolean; posted_at: Date; revision: number;
|
||||
}
|
||||
function object(value: unknown): Record<string, unknown> | null {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
|
||||
}
|
||||
function equalSecret(actual: string, expected: string): boolean {
|
||||
const a = Buffer.from(actual); const b = Buffer.from(expected);
|
||||
return a.length === b.length && timingSafeEqual(a, b);
|
||||
}
|
||||
function itemFromRow(row: ItemRow): Record<string, unknown> {
|
||||
return { id: row.id, remoteRef: row.remote_ref, channelId: row.channel_id,
|
||||
messageId: Number(row.message_id), text: row.text_content, media: row.media_refs,
|
||||
attribution: row.attribution, protectedContent: row.protected_content,
|
||||
postedAt: row.posted_at.toISOString(), revision: row.revision };
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class ArchiveService {
|
||||
readonly albums: TelegramAlbumDebouncer;
|
||||
private readonly storage: StorageService;
|
||||
constructor(private readonly publishing: PublishingService) {
|
||||
this.storage = new S3Storage(s3ConfigFromEnv());
|
||||
this.albums = new TelegramAlbumDebouncer((item) => this.save(item));
|
||||
}
|
||||
|
||||
async subscribe(workspaceId: string, connectionId: string, channelId: string): Promise<string> {
|
||||
if (!/^-?\d{3,}$/.test(channelId)) throw new BadRequestException("channelId must be a numeric Telegram chat id");
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const result = await client.query(
|
||||
`INSERT INTO feed_subscriptions (workspace_id, type, connection_id, channel_id)
|
||||
SELECT $1, 'telegram_channel', id, $3 FROM connections
|
||||
WHERE id = $2 AND provider = 'telegram'
|
||||
ON CONFLICT (workspace_id, connection_id, channel_id) DO UPDATE SET updated_at = now()
|
||||
RETURNING id`, [workspaceId, connectionId, channelId]);
|
||||
if (!result.rows[0]) throw new NotFoundException("Telegram connection not found");
|
||||
return String(result.rows[0].id);
|
||||
});
|
||||
}
|
||||
|
||||
async receive(workspaceId: string, secret: string | undefined, payload: unknown): Promise<void> {
|
||||
const expected = process.env.TELEGRAM_WEBHOOK_SECRET;
|
||||
if (!expected || !secret || !equalSecret(secret, expected)) throw new ForbiddenException("webhook secret mismatch");
|
||||
const update = object(payload);
|
||||
const post = object(update?.channel_post ?? update?.edited_channel_post);
|
||||
const membership = object(update?.my_chat_member);
|
||||
const chat = object(post?.chat ?? membership?.chat);
|
||||
const channelId = chat?.id === undefined ? null : String(chat.id);
|
||||
if (!channelId) return;
|
||||
const subscriptions = await withTenant(workspaceId, (client) => client.query(
|
||||
"SELECT id FROM feed_subscriptions WHERE channel_id = $1", [channelId]));
|
||||
if (subscriptions.rowCount === 0) return;
|
||||
if (membership) {
|
||||
const next = object(membership.new_chat_member);
|
||||
const status = next?.status;
|
||||
const health = status === "administrator" || status === "creator" ? "active"
|
||||
: status === "member" ? "admin_required" : "bot_removed";
|
||||
await withTenant(workspaceId, (client) => client.query(
|
||||
"UPDATE feed_subscriptions SET health = $2, health_checked_at = now(), updated_at = now() WHERE channel_id = $1",
|
||||
[channelId, health]));
|
||||
return;
|
||||
}
|
||||
await this.albums.ingest(workspaceId, payload);
|
||||
}
|
||||
|
||||
private async harvest(media: TelegramMedia): Promise<TelegramMedia> {
|
||||
if (media.status !== "pending") return media;
|
||||
const token = process.env.TELEGRAM_BOT_TOKEN;
|
||||
if (!token) return media;
|
||||
const response = await fetch(`https://api.telegram.org/bot${token}/getFile?file_id=${encodeURIComponent(media.fileId)}`);
|
||||
const body: unknown = await response.json();
|
||||
const file = object(object(body)?.result);
|
||||
const size = typeof file?.file_size === "number" ? file.file_size : media.sizeBytes;
|
||||
if (size !== null && size > 20_000_000) return { ...media, sizeBytes: size, status: "Bot API cannot fetch" };
|
||||
const path = file?.file_path;
|
||||
if (!response.ok || typeof path !== "string" || path.includes("..") || path.startsWith("/")) return { ...media, status: "failed" };
|
||||
const downloaded = await fetch(`https://api.telegram.org/file/bot${token}/${path}`);
|
||||
if (!downloaded.ok) return { ...media, status: "failed" };
|
||||
const bytes = Buffer.from(await downloaded.arrayBuffer());
|
||||
if (bytes.length > 20_000_000) return { ...media, sizeBytes: bytes.length, status: "Bot API cannot fetch" };
|
||||
const stored = await this.storage.putContent(bytes, media.contentType);
|
||||
return { ...media, sizeBytes: bytes.length, status: "stored", storageKey: stored.key };
|
||||
}
|
||||
|
||||
async save(item: ArchiveItemInput): Promise<void> {
|
||||
const media = await Promise.all(item.mediaRefs.map((entry) => this.harvest(entry)));
|
||||
await withTenant(item.workspaceId, async (client) => {
|
||||
const existing = await client.query<ItemRow & { content_hash: string }>(
|
||||
"SELECT * FROM archived_feed_items WHERE remote_ref = $1 FOR UPDATE", [item.remoteRef]);
|
||||
const row = existing.rows[0];
|
||||
if (row?.content_hash === item.contentHash) return;
|
||||
if (row && !item.edited) return;
|
||||
if (row) {
|
||||
const next = row.revision + 1;
|
||||
await client.query(`UPDATE archived_feed_items SET text_content = $2, content_hash = $3,
|
||||
media_refs = $4, attribution = $5, protected_content = $6, revision = $7, updated_at = now() WHERE id = $1`,
|
||||
[row.id, item.textContent, item.contentHash, JSON.stringify(media), JSON.stringify(item.attribution), item.protectedContent, next]);
|
||||
await client.query(`INSERT INTO archived_feed_revisions
|
||||
(workspace_id, item_id, revision, content_hash, text_content, media_refs, attribution)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)`,
|
||||
[item.workspaceId, row.id, next, item.contentHash, item.textContent, JSON.stringify(media), JSON.stringify(item.attribution)]);
|
||||
return;
|
||||
}
|
||||
const inserted = await client.query<{ id: string }>(`INSERT INTO archived_feed_items
|
||||
(workspace_id, remote_ref, channel_id, message_id, media_group_id, text_content, content_hash,
|
||||
media_refs, attribution, protected_content, posted_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)
|
||||
ON CONFLICT (workspace_id, remote_ref) DO NOTHING RETURNING id`,
|
||||
[item.workspaceId, item.remoteRef, item.channelId, item.messageId, item.mediaGroupId,
|
||||
item.textContent, item.contentHash, JSON.stringify(media), JSON.stringify(item.attribution),
|
||||
item.protectedContent, item.postedAt]);
|
||||
const id = inserted.rows[0]?.id;
|
||||
if (id) await client.query(`INSERT INTO archived_feed_revisions
|
||||
(workspace_id, item_id, revision, content_hash, text_content, media_refs, attribution)
|
||||
VALUES ($1,$2,1,$3,$4,$5,$6)`,
|
||||
[item.workspaceId, id, item.contentHash, item.textContent, JSON.stringify(media), JSON.stringify(item.attribution)]);
|
||||
});
|
||||
}
|
||||
|
||||
async list(workspaceId: string, cursor?: string): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
|
||||
let parsed: { at: string; id: string } | null = null;
|
||||
if (cursor) {
|
||||
try {
|
||||
const value: unknown = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"));
|
||||
const shape = object(value);
|
||||
if (typeof shape?.at !== "string" || !Number.isFinite(Date.parse(shape.at))
|
||||
|| typeof shape.id !== "string" || !/^[0-9a-f-]{36}$/i.test(shape.id)) throw new Error("shape");
|
||||
parsed = { at: shape.at, id: shape.id };
|
||||
} catch { throw new BadRequestException("invalid cursor"); }
|
||||
}
|
||||
const result = await withTenant(workspaceId, (client) => client.query<ItemRow>(
|
||||
`SELECT * FROM archived_feed_items WHERE ($1::timestamptz IS NULL OR (posted_at, id) < ($1::timestamptz, $2::uuid))
|
||||
ORDER BY posted_at DESC, id DESC LIMIT 21`, [parsed?.at ?? null, parsed?.id ?? null]));
|
||||
const rows = result.rows.slice(0, 20);
|
||||
const last = rows.at(-1);
|
||||
const items = await Promise.all(rows.map(async (row) => {
|
||||
const item = itemFromRow(row);
|
||||
item.media = await Promise.all(row.media_refs.map(async (media) => ({
|
||||
...media, thumbUrl: media.storageKey ? await this.storage.presign(media.storageKey, 900) : null,
|
||||
})));
|
||||
return item;
|
||||
}));
|
||||
return { items, nextCursor: result.rows.length > 20 && last
|
||||
? Buffer.from(JSON.stringify({ at: last.posted_at.toISOString(), id: last.id })).toString("base64url") : null };
|
||||
}
|
||||
|
||||
async createDraft(workspaceId: string, actor: Actor, itemId: string): Promise<{ id: string; state: string }> {
|
||||
const found = await withTenant(workspaceId, (client) => client.query<ItemRow>(
|
||||
"SELECT * FROM archived_feed_items WHERE id = $1", [itemId]));
|
||||
const item = found.rows[0];
|
||||
if (!item) throw new NotFoundException("archive item not found");
|
||||
const draft = await this.publishing.createDraft(workspaceId, actor, {
|
||||
text: item.text_content, mediaAssetIds: [], visibility: "provider_default",
|
||||
providerMetadata: { archiveItemId: itemId, sourceRemoteRef: item.remote_ref,
|
||||
attribution: item.attribution, attributionEnabled: true },
|
||||
});
|
||||
await withTenant(workspaceId, (client) => client.query(
|
||||
"INSERT INTO archived_feed_drafts (workspace_id, item_id, draft_id) VALUES ($1,$2,$3)",
|
||||
[workspaceId, itemId, draft.id]));
|
||||
return { id: draft.id, state: draft.state };
|
||||
}
|
||||
|
||||
async amplify(workspaceId: string, itemId: string, draftId: string, connectionId: string,
|
||||
attribution = true): Promise<unknown> {
|
||||
const found = await withTenant(workspaceId, (client) => client.query<ItemRow & { channel_id: string }>(
|
||||
`SELECT i.* FROM archived_feed_items i JOIN archived_feed_drafts b
|
||||
ON b.item_id = i.id AND b.workspace_id = i.workspace_id
|
||||
JOIN drafts d ON d.id = b.draft_id AND d.workspace_id = b.workspace_id
|
||||
WHERE i.id = $1 AND d.id = $2 AND d.state = 'approved'`, [itemId, draftId]));
|
||||
const item = found.rows[0];
|
||||
if (!item) throw new ForbiddenException("an approved archive draft is required");
|
||||
if (item.protected_content) throw new ConnectorError(`${ARCHIVE_PROVIDER}: protected content cannot be forwarded`, ARCHIVE_PROVIDER);
|
||||
const target = await withTenant(workspaceId, (client) => client.query<{ channel_id: string; fields: Record<string, string> }>(
|
||||
"SELECT channel_id, fields FROM connections WHERE id = $1 AND provider = $2", [connectionId, ARCHIVE_PROVIDER]));
|
||||
const connectionRow = target.rows[0];
|
||||
if (!connectionRow) throw new NotFoundException("Telegram target connection not found");
|
||||
const token = process.env.TELEGRAM_BOT_TOKEN;
|
||||
if (!token) throw new BadRequestException("Telegram bot token is not configured");
|
||||
const connection: ConnectorConnection = { id: connectionId, workspaceId, provider: ARCHIVE_PROVIDER,
|
||||
channelId: connectionRow.channel_id, credentialId: { kind: "credential-ref", id: connectionId },
|
||||
fields: { ...connectionRow.fields, botToken: token }, createdAt: new Date() };
|
||||
const connector = createTelegramConnector();
|
||||
if (!connector.amplify) throw new ConnectorError(`${ARCHIVE_PROVIDER}: amplify unavailable`, ARCHIVE_PROVIDER);
|
||||
const origin = item.attribution.channelUsername;
|
||||
const commentary = attribution && typeof origin === "string" && origin !== ""
|
||||
? `Source: https://t.me/${origin}/${item.message_id}` : undefined;
|
||||
return connector.amplify(connection, { kind: "share", targetRemoteId: item.remote_ref,
|
||||
...(commentary ? { commentary } : {}) });
|
||||
}
|
||||
|
||||
async search(workspaceId: string, query: Record<string, unknown>): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
|
||||
const filters: ArchiveSearchFilters = parseArchiveSearchQuery(query);
|
||||
return searchArchive(workspaceId, filters, this.storage);
|
||||
}
|
||||
|
||||
async exportItems(workspaceId: string, query: Record<string, unknown>,
|
||||
write: (chunk: string) => void): Promise<"csv" | "json"> {
|
||||
const filters = parseArchiveSearchQuery(query);
|
||||
const format = query.format === "csv" ? "csv" : query.format === undefined || query.format === "" ? "json" : null;
|
||||
if (format === null) throw new BadRequestException("format must be csv or json");
|
||||
if (format === "csv") write(ARCHIVE_CSV_HEADER);
|
||||
else write("[");
|
||||
let first = true;
|
||||
for await (const page of exportArchiveItems(workspaceId, filters)) {
|
||||
for (const item of page) {
|
||||
if (format === "csv") write(csvItemRow(item));
|
||||
else {
|
||||
write((first ? "" : ",") + JSON.stringify(item));
|
||||
first = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
write(format === "csv" ? "" : "]");
|
||||
return format;
|
||||
}
|
||||
}
|
||||
109
apps/api/src/archive/normalizer.ts
Normal file
109
apps/api/src/archive/normalizer.ts
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
import { createHash } from "node:crypto";
|
||||
|
||||
export interface TelegramMedia {
|
||||
fileId: string;
|
||||
fileUniqueId: string;
|
||||
sizeBytes: number | null;
|
||||
contentType: string;
|
||||
status: "pending" | "stored" | "Bot API cannot fetch" | "failed";
|
||||
storageKey?: string;
|
||||
}
|
||||
export interface ArchiveItemInput {
|
||||
workspaceId: string;
|
||||
channelId: string;
|
||||
messageId: number;
|
||||
remoteRef: string;
|
||||
mediaGroupId: string | null;
|
||||
textContent: string;
|
||||
contentHash: string;
|
||||
mediaRefs: TelegramMedia[];
|
||||
attribution: Record<string, unknown>;
|
||||
protectedContent: boolean;
|
||||
postedAt: Date;
|
||||
edited: boolean;
|
||||
}
|
||||
interface TelegramFile { file_id?: unknown; file_unique_id?: unknown; file_size?: unknown; mime_type?: unknown }
|
||||
interface TelegramPost {
|
||||
message_id: number; date: number;
|
||||
chat: { id: number | string; title?: string; username?: string };
|
||||
text?: string; caption?: string; media_group_id?: string; author_signature?: string;
|
||||
forward_origin?: Record<string, unknown>; is_automatic_forward?: boolean;
|
||||
has_protected_content?: boolean;
|
||||
photo?: TelegramFile[]; video?: TelegramFile; document?: TelegramFile;
|
||||
}
|
||||
function record(value: unknown): Record<string, unknown> | null {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
|
||||
}
|
||||
export function parseTelegramPost(value: unknown): TelegramPost | null {
|
||||
const post = record(value);
|
||||
const chat = record(post?.chat);
|
||||
if (!post || !chat || !Number.isSafeInteger(post.message_id) || !Number.isSafeInteger(post.date)
|
||||
|| (typeof chat.id !== "number" && typeof chat.id !== "string")) return null;
|
||||
return post as unknown as TelegramPost;
|
||||
}
|
||||
function mediaOf(post: TelegramPost): TelegramMedia[] {
|
||||
const photo = Array.isArray(post.photo) ? post.photo.at(-1) : undefined;
|
||||
const entries: Array<[TelegramFile | undefined, string]> = [[photo, "image/jpeg"], [post.video, "video/mp4"], [post.document, "application/octet-stream"]];
|
||||
return entries.flatMap(([file, fallback]) => typeof file?.file_id === "string" && typeof file.file_unique_id === "string"
|
||||
? [{ fileId: file.file_id, fileUniqueId: file.file_unique_id,
|
||||
sizeBytes: typeof file.file_size === "number" ? file.file_size : null,
|
||||
contentType: typeof file.mime_type === "string" ? file.mime_type : fallback,
|
||||
status: file.file_size !== undefined && typeof file.file_size === "number" && file.file_size > 20_000_000
|
||||
? "Bot API cannot fetch" as const : "pending" as const }]
|
||||
: []);
|
||||
}
|
||||
export function normalizePosts(workspaceId: string, posts: TelegramPost[], edited = false): ArchiveItemInput {
|
||||
const ordered = [...posts].sort((a, b) => a.message_id - b.message_id);
|
||||
const first = ordered[0];
|
||||
if (!first) throw new Error("empty Telegram post batch");
|
||||
const textContent = ordered.map((p) => p.text ?? p.caption ?? "").find((s) => s !== "") ?? "";
|
||||
const mediaRefs = ordered.flatMap(mediaOf);
|
||||
const attribution: Record<string, unknown> = {
|
||||
channelTitle: first.chat.title ?? null, channelUsername: first.chat.username ?? null,
|
||||
authorSignature: first.author_signature ?? null, forwardOrigin: first.forward_origin ?? null,
|
||||
isAutomaticForward: first.is_automatic_forward === true,
|
||||
};
|
||||
const channelId = String(first.chat.id);
|
||||
const contentHash = createHash("sha256").update(JSON.stringify({ textContent, media: mediaRefs.map((m) => m.fileUniqueId), attribution })).digest("hex");
|
||||
return { workspaceId, channelId, messageId: first.message_id, remoteRef: `${channelId}:${first.message_id}`,
|
||||
mediaGroupId: first.media_group_id ?? null, textContent, contentHash, mediaRefs, attribution,
|
||||
protectedContent: ordered.some((p) => p.has_protected_content === true),
|
||||
postedAt: new Date(first.date * 1000), edited };
|
||||
}
|
||||
export interface AlbumTimers {
|
||||
set(callback: () => void, ms: number): ReturnType<typeof setTimeout>;
|
||||
clear(handle: ReturnType<typeof setTimeout>): void;
|
||||
}
|
||||
const defaultTimers: AlbumTimers = { set: (cb, ms) => setTimeout(cb, ms), clear: clearTimeout };
|
||||
/** A keyed 2s debounce. Every album update shares one durable archive identity. */
|
||||
export class TelegramAlbumDebouncer {
|
||||
private readonly pending = new Map<string, { posts: TelegramPost[]; timer: ReturnType<typeof setTimeout> }>();
|
||||
constructor(private readonly save: (item: ArchiveItemInput) => Promise<void>, private readonly timers: AlbumTimers = defaultTimers) {}
|
||||
async ingest(workspaceId: string, update: unknown): Promise<void> {
|
||||
const envelope = record(update);
|
||||
const edited = envelope?.edited_channel_post !== undefined;
|
||||
const post = parseTelegramPost(envelope?.[edited ? "edited_channel_post" : "channel_post"]);
|
||||
if (!post) return;
|
||||
const group = post.media_group_id;
|
||||
if (!group || edited) { await this.save(normalizePosts(workspaceId, [post], edited)); return; }
|
||||
const key = `${workspaceId}:${post.chat.id}:${group}`;
|
||||
const current = this.pending.get(key);
|
||||
if (current) this.timers.clear(current.timer);
|
||||
const posts = [...(current?.posts ?? []), post];
|
||||
const timer = this.timers.set(() => { void this.flush(key, workspaceId); }, 2_000);
|
||||
this.pending.set(key, { posts, timer });
|
||||
}
|
||||
async flush(key: string, workspaceId: string): Promise<void> {
|
||||
const group = this.pending.get(key);
|
||||
if (!group) return;
|
||||
this.pending.delete(key);
|
||||
this.timers.clear(group.timer);
|
||||
await this.save(normalizePosts(workspaceId, group.posts));
|
||||
}
|
||||
async flushAll(): Promise<void> {
|
||||
for (const key of [...this.pending.keys()]) {
|
||||
const workspaceId = key.slice(0, key.indexOf(":"));
|
||||
await this.flush(key, workspaceId);
|
||||
}
|
||||
}
|
||||
}
|
||||
35
apps/api/src/audit/audit.controller.ts
Normal file
35
apps/api/src/audit/audit.controller.ts
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
import { BadRequestException, Controller, Get, Inject, Query, Req, UseGuards } from "@nestjs/common";
|
||||
import type { Request } from "express";
|
||||
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import { isUuid, parseLimit, requireSession } from "../http/validation.js";
|
||||
import { AuditQueryService, type AuditPage } from "./audit.query.js";
|
||||
|
||||
@Controller("audit")
|
||||
@UseGuards(RbacGuard)
|
||||
export class AuditController {
|
||||
constructor(@Inject(AuditQueryService) private readonly audit: AuditQueryService) {}
|
||||
|
||||
@Get()
|
||||
@RequirePermission("audit.view")
|
||||
async list(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Query("action") action: string | undefined,
|
||||
@Query("actor") actor: string | undefined,
|
||||
@Query("before") before: string | undefined,
|
||||
@Query("limit") limit: string | undefined,
|
||||
): Promise<AuditPage> {
|
||||
const session = requireSession(request);
|
||||
if (actor !== undefined && actor !== "" && actor !== "system" && !isUuid(actor)) {
|
||||
throw new BadRequestException("actor must be a user UUID or \"system\"");
|
||||
}
|
||||
if (before !== undefined && before !== "" && !/^\d{1,18}$/.test(before)) {
|
||||
throw new BadRequestException("before must be a sequence cursor");
|
||||
}
|
||||
return this.audit.page(session.workspaceId, {
|
||||
...(action === undefined || action === "" ? {} : { action }),
|
||||
...(actor === undefined || actor === "" ? {} : { actor }),
|
||||
...(before === undefined || before === "" ? {} : { before }),
|
||||
limit: parseLimit(limit, 50, 200),
|
||||
});
|
||||
}
|
||||
}
|
||||
113
apps/api/src/audit/audit.query.ts
Normal file
113
apps/api/src/audit/audit.query.ts
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
import { Injectable } from "@nestjs/common";
|
||||
import { withTenant } from "../db/client.js";
|
||||
|
||||
export interface AuditEntryDto {
|
||||
id: string;
|
||||
seq: string;
|
||||
at: string;
|
||||
actorUserId: string | null;
|
||||
actorEmail: string | null;
|
||||
actorRole: string | null;
|
||||
action: string;
|
||||
targetType: string;
|
||||
targetId: string;
|
||||
diff: Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
export interface AuditActorFacet {
|
||||
/** null = system / service actor. */
|
||||
id: string | null;
|
||||
email: string | null;
|
||||
}
|
||||
|
||||
export interface AuditPage {
|
||||
entries: AuditEntryDto[];
|
||||
/** Pass back as `before` to fetch the next (older) page; null when exhausted. */
|
||||
nextCursor: string | null;
|
||||
facets: { actions: string[]; actors: AuditActorFacet[] };
|
||||
}
|
||||
|
||||
export interface AuditFilter {
|
||||
action?: string;
|
||||
/** User UUID, or "system" for entries without a human actor. */
|
||||
actor?: string;
|
||||
before?: string;
|
||||
limit: number;
|
||||
}
|
||||
|
||||
interface AuditRow {
|
||||
id: string;
|
||||
seq: string;
|
||||
at: Date;
|
||||
actor_user_id: string | null;
|
||||
actor_email: string | null;
|
||||
actor_role: string | null;
|
||||
action: string;
|
||||
target_type: string;
|
||||
target_id: string;
|
||||
diff: Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
/** Read side of the tenant audit chain. RLS (tenant_audit policy) scopes rows to the session's workspace. */
|
||||
@Injectable()
|
||||
export class AuditQueryService {
|
||||
async page(workspaceId: string, filter: AuditFilter): Promise<AuditPage> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const actorIsSystem = filter.actor === "system";
|
||||
const rows = await client.query(
|
||||
`SELECT a.id, a.seq::text AS seq, a.at, a.actor_user_id, u.email AS actor_email, a.actor_role,
|
||||
a.action, a.target_type, a.target_id, a.diff
|
||||
FROM audit_log a
|
||||
LEFT JOIN users u ON u.id = a.actor_user_id
|
||||
WHERE a.workspace_id = $1
|
||||
AND ($2::text IS NULL OR a.action = $2)
|
||||
AND ($3::uuid IS NULL OR a.actor_user_id = $3)
|
||||
AND (NOT $4::boolean OR a.actor_user_id IS NULL)
|
||||
AND ($5::bigint IS NULL OR a.seq < $5)
|
||||
ORDER BY a.seq DESC
|
||||
LIMIT $6`,
|
||||
[
|
||||
workspaceId,
|
||||
filter.action ?? null,
|
||||
actorIsSystem ? null : (filter.actor ?? null),
|
||||
actorIsSystem,
|
||||
filter.before ?? null,
|
||||
filter.limit + 1,
|
||||
],
|
||||
);
|
||||
const all = rows.rows as AuditRow[];
|
||||
const hasMore = all.length > filter.limit;
|
||||
const entries = all.slice(0, filter.limit).map((r) => ({
|
||||
id: r.id,
|
||||
seq: r.seq,
|
||||
at: new Date(r.at).toISOString(),
|
||||
actorUserId: r.actor_user_id,
|
||||
actorEmail: r.actor_email,
|
||||
actorRole: r.actor_role,
|
||||
action: r.action,
|
||||
targetType: r.target_type,
|
||||
targetId: r.target_id,
|
||||
diff: r.diff,
|
||||
}));
|
||||
|
||||
const actions = await client.query(
|
||||
"SELECT DISTINCT action FROM audit_log WHERE workspace_id = $1 ORDER BY action",
|
||||
[workspaceId],
|
||||
);
|
||||
const actors = await client.query(
|
||||
`SELECT DISTINCT a.actor_user_id AS id, u.email
|
||||
FROM audit_log a LEFT JOIN users u ON u.id = a.actor_user_id
|
||||
WHERE a.workspace_id = $1 ORDER BY u.email NULLS LAST`,
|
||||
[workspaceId],
|
||||
);
|
||||
return {
|
||||
entries,
|
||||
nextCursor: hasMore ? (entries[entries.length - 1]?.seq ?? null) : null,
|
||||
facets: {
|
||||
actions: (actions.rows as Array<{ action: string }>).map((r) => r.action),
|
||||
actors: actors.rows as AuditActorFacet[],
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
13
apps/api/src/audit/audit.scrub.test.ts
Normal file
13
apps/api/src/audit/audit.scrub.test.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { scrub } from "./audit.service.js";
|
||||
|
||||
describe("audit diff scrubbing", () => {
|
||||
it("redacts secret-shaped keys (spec: secrets never enter audit payloads)", () => {
|
||||
const clean = scrub({ password: "hunter2", stream_key: "sk-1", note: "hello", count: 3 });
|
||||
expect(clean).toEqual({ password: "[REDACTED]", stream_key: "[REDACTED]", note: "hello", count: 3 });
|
||||
});
|
||||
|
||||
it("passes null through", () => {
|
||||
expect(scrub(null)).toBeNull();
|
||||
});
|
||||
});
|
||||
38
apps/api/src/audit/audit.service.ts
Normal file
38
apps/api/src/audit/audit.service.ts
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
import { withTenant } from "../db/client.js";
|
||||
|
||||
/**
|
||||
* Audit service: hash-chained, secrets never enter payloads.
|
||||
* Branch selection (findings #1/#6): tenant context present => tenant chain
|
||||
* (audit_append), even for service roles; no tenant context => system chain.
|
||||
*/
|
||||
|
||||
const SECRET_KEYS = /(password|secret|token|key|credential|stream_key|refresh)/i;
|
||||
|
||||
export function scrub(diff: Record<string, unknown> | null): Record<string, unknown> | null {
|
||||
if (diff === null) return null;
|
||||
const clean: Record<string, unknown> = {};
|
||||
for (const [k, v] of Object.entries(diff)) {
|
||||
clean[k] = SECRET_KEYS.test(k) ? "[REDACTED]" : v;
|
||||
}
|
||||
return clean;
|
||||
}
|
||||
|
||||
export interface AuditEvent {
|
||||
actorUserId: string | null;
|
||||
actorRole: string;
|
||||
action: string;
|
||||
targetType: string;
|
||||
targetId: string;
|
||||
diff?: Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
export async function appendTenantAudit(workspaceId: string, event: AuditEvent): Promise<string> {
|
||||
const res = await withTenant(workspaceId, async (client) => {
|
||||
const result = await client.query(
|
||||
"SELECT audit_append($1, $2, $3, $4, $5, $6, $7) AS id",
|
||||
[workspaceId, event.actorUserId, event.actorRole, event.action, event.targetType, event.targetId, scrub(event.diff ?? null)],
|
||||
);
|
||||
return result.rows[0]?.id as string;
|
||||
});
|
||||
return res;
|
||||
}
|
||||
70
apps/api/src/auth/auth.controller.ts
Normal file
70
apps/api/src/auth/auth.controller.ts
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
import { Body, Controller, Get, HttpCode, Inject, Post, Req, UnauthorizedException, UseGuards } from "@nestjs/common";
|
||||
import type { Request } from "express";
|
||||
import { RbacGuard, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import { optionalEmail, optionalString, parseId, requireObject, requireString, requireUsername, requireSession } from "../http/validation.js";
|
||||
import { AuthService, type AuthResult, type LoginInput, type MeResponse, type RegisterInput } from "./auth.service.js";
|
||||
import { verifySession } from "./session.js";
|
||||
|
||||
@Controller("auth")
|
||||
export class AuthController {
|
||||
constructor(@Inject(AuthService) private readonly auth: AuthService) {}
|
||||
|
||||
@Post("register")
|
||||
async register(@Body() body: unknown): Promise<AuthResult> {
|
||||
return this.auth.register(parseRegister(body));
|
||||
}
|
||||
|
||||
@Post("login")
|
||||
@HttpCode(200)
|
||||
async login(@Body() body: unknown): Promise<AuthResult> {
|
||||
return this.auth.login(parseLogin(body));
|
||||
}
|
||||
|
||||
/** Any valid, epoch-current session may read itself — no permission gate beyond authentication. */
|
||||
@Get("me")
|
||||
@UseGuards(RbacGuard)
|
||||
async me(@Req() request: Request & AuthenticatedRequest): Promise<MeResponse> {
|
||||
return this.auth.me(requireSession(request));
|
||||
}
|
||||
|
||||
/** A signed, unexpired session can refresh its epoch and live role. */
|
||||
@Post("reissue")
|
||||
@HttpCode(200)
|
||||
async reissue(@Req() request: Request): Promise<AuthResult> {
|
||||
const authorization = request.headers.authorization;
|
||||
if (authorization === undefined || !authorization.startsWith("Bearer ")) throw new UnauthorizedException();
|
||||
const session = verifySession(authorization.slice("Bearer ".length));
|
||||
if (session === null) throw new UnauthorizedException();
|
||||
return this.auth.reissue(session);
|
||||
}
|
||||
|
||||
/** Stateless tokens: logout is client-side (drop the token). Revocation-by-epoch covers role changes. */
|
||||
@Post("logout")
|
||||
@HttpCode(200)
|
||||
logout(): { ok: true } {
|
||||
return { ok: true };
|
||||
}
|
||||
}
|
||||
|
||||
function parseRegister(value: unknown): RegisterInput {
|
||||
const body = requireObject(value, "body");
|
||||
const username = requireUsername(body.username);
|
||||
const email = optionalEmail(body.email);
|
||||
return {
|
||||
workspaceName: requireString(body.workspaceName, "workspaceName").trim().slice(0, 100),
|
||||
username,
|
||||
password: requireString(body.password, "password"),
|
||||
displayName: (optionalString(body.displayName, "displayName") ?? username).trim().slice(0, 100),
|
||||
...(email === undefined ? {} : { email }),
|
||||
};
|
||||
}
|
||||
|
||||
function parseLogin(value: unknown): LoginInput {
|
||||
const body = requireObject(value, "body");
|
||||
const workspaceId = optionalString(body.workspaceId, "workspaceId");
|
||||
return {
|
||||
username: requireUsername(body.username),
|
||||
password: requireString(body.password, "password"),
|
||||
...(workspaceId === undefined ? {} : { workspaceId: parseId(workspaceId, "workspaceId") }),
|
||||
};
|
||||
}
|
||||
155
apps/api/src/auth/auth.integration.test.ts
Normal file
155
apps/api/src/auth/auth.integration.test.ts
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { join } from "node:path";
|
||||
import type { ExecutionContext } from "@nestjs/common";
|
||||
import { Reflector } from "@nestjs/core";
|
||||
|
||||
/**
|
||||
* Auth + team + audit wiring against a live PostgreSQL, as sp_app (FORCE RLS binds).
|
||||
* Uses its OWN database (AUTH_IT_DATABASE_URL, plus AUTH_IT_SUPER_DATABASE_URL for
|
||||
* migrations) because tenancy.integration.test.ts wipes every workspace in the
|
||||
* shared DB and vitest runs files in parallel. Skipped when unset.
|
||||
*/
|
||||
const appUrl = process.env.AUTH_IT_DATABASE_URL;
|
||||
const ownerUrl = process.env.AUTH_IT_SUPER_DATABASE_URL;
|
||||
|
||||
type Modules = {
|
||||
auth: typeof import("./auth.service.js");
|
||||
team: typeof import("../team/team.service.js");
|
||||
audit: typeof import("../audit/audit.query.js");
|
||||
guard: typeof import("../rbac/rbac.guard.js");
|
||||
session: typeof import("./session.js");
|
||||
db: typeof import("../db/client.js");
|
||||
};
|
||||
|
||||
describe.skipIf(appUrl === undefined || ownerUrl === undefined)("auth/team/audit (live DB)", () => {
|
||||
let m: Modules;
|
||||
const suffix = `${Date.now()}${Math.floor(Math.random() * 1e6)}`;
|
||||
const ownerUsername = `owner${suffix}`;
|
||||
const ownerEmail = `owner-${suffix}@example.com`;
|
||||
|
||||
beforeAll(async () => {
|
||||
// The pool reads DATABASE_URL at import time: point it at the isolated DB first.
|
||||
process.env.DATABASE_URL = appUrl;
|
||||
process.env.SUPER_DATABASE_URL = ownerUrl;
|
||||
const { runMigrations } = await import("../db/migrate.js");
|
||||
await runMigrations(join(process.cwd(), "migrations"));
|
||||
m = {
|
||||
auth: await import("./auth.service.js"),
|
||||
team: await import("../team/team.service.js"),
|
||||
audit: await import("../audit/audit.query.js"),
|
||||
guard: await import("../rbac/rbac.guard.js"),
|
||||
session: await import("./session.js"),
|
||||
db: await import("../db/client.js"),
|
||||
};
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await m?.db.pool.end();
|
||||
});
|
||||
|
||||
function guardContext(token: string, permission: string | undefined): { ctx: ExecutionContext; reflector: Reflector } {
|
||||
const handler = (): void => undefined;
|
||||
const reflector = new Reflector();
|
||||
if (permission !== undefined) Reflect.defineMetadata(m.guard.PERMISSION_KEY, permission, handler);
|
||||
const req = { headers: { authorization: `Bearer ${token}` } };
|
||||
const ctx = {
|
||||
getHandler: () => handler,
|
||||
switchToHttp: () => ({ getRequest: () => req }),
|
||||
} as unknown as ExecutionContext;
|
||||
return { ctx, reflector };
|
||||
}
|
||||
|
||||
async function allowed(token: string, permission?: string): Promise<boolean | string> {
|
||||
const { ctx, reflector } = guardContext(token, permission);
|
||||
try {
|
||||
return await new m.guard.RbacGuard(reflector).canActivate(ctx);
|
||||
} catch (err) {
|
||||
return (err as Error).constructor.name;
|
||||
}
|
||||
}
|
||||
|
||||
it("register -> login -> me, and the RBAC guard accepts the session (bigint epoch, tenant-scoped lookup)", async () => {
|
||||
const auth = new m.auth.AuthService();
|
||||
const reg = await auth.register({ workspaceName: "IT Studio", username: ownerUsername, password: "correct-horse-1", displayName: "Owner", email: ownerEmail });
|
||||
expect(reg.me.role).toBe("Owner");
|
||||
expect(reg.me.user.username).toBe(ownerUsername);
|
||||
expect(reg.me.user.email).toBe(ownerEmail);
|
||||
expect(reg.me.permissions).toContain("workspace.members.manage");
|
||||
|
||||
await expect(auth.register({ workspaceName: "Dup", username: ownerUsername, password: "correct-horse-1", displayName: "x" }))
|
||||
.rejects.toMatchObject({ status: 409 });
|
||||
await expect(auth.login({ username: ownerUsername, password: "wrong-password-1" })).rejects.toMatchObject({ status: 401 });
|
||||
|
||||
const login = await auth.login({ username: ownerUsername.toUpperCase(), password: "correct-horse-1" });
|
||||
expect(login.me.workspace.id).toBe(reg.me.workspace.id);
|
||||
await expect(allowed(login.token)).resolves.toBe(true);
|
||||
await expect(allowed(login.token, "audit.view")).resolves.toBe(true);
|
||||
});
|
||||
|
||||
it("registers without an email (username-only accounts)", async () => {
|
||||
const auth = new m.auth.AuthService();
|
||||
const username = `noemail${suffix}`;
|
||||
const reg = await auth.register({ workspaceName: "No Email WS", username, password: "correct-horse-1", displayName: "N" });
|
||||
expect(reg.me.user.email).toBeNull();
|
||||
expect(reg.me.user.username).toBe(username);
|
||||
const login = await auth.login({ username, password: "correct-horse-1" });
|
||||
expect(login.me.user.id).toBe(reg.me.user.id);
|
||||
});
|
||||
|
||||
it("membership mutations bump the epoch but hand the actor a fresh token; last Owner is protected", async () => {
|
||||
const auth = new m.auth.AuthService();
|
||||
const team = new m.team.TeamService();
|
||||
const login = await auth.login({ username: ownerUsername, password: "correct-horse-1" });
|
||||
const session = m.session.verifySession(login.token);
|
||||
if (session === null) throw new Error("session did not verify");
|
||||
const actor = { userId: session.userId, role: session.role };
|
||||
|
||||
const invited = await team.invite(session.workspaceId, actor, `pub-${suffix}@example.com`, "Publisher");
|
||||
expect(invited.member).toMatchObject({ role: "Publisher", status: "invited" });
|
||||
if (invited.token === null || invited.member === null) throw new Error("expected token + member");
|
||||
await expect(allowed(login.token)).resolves.toBe("UnauthorizedException");
|
||||
await expect(allowed(invited.token, "workspace.members.manage")).resolves.toBe(true);
|
||||
const renewed = await auth.reissue(session);
|
||||
expect(m.session.verifySession(renewed.token)?.expiresAt).toBe(session.expiresAt);
|
||||
await expect(allowed(renewed.token, "workspace.members.manage")).resolves.toBe(true);
|
||||
await expect(auth.me(m.session.verifySession(renewed.token)!)).resolves.toMatchObject({ role: "Owner" });
|
||||
|
||||
const changed = await team.changeRole(session.workspaceId, actor, invited.member.id, "Approver");
|
||||
expect(changed.member?.role).toBe("Approver");
|
||||
|
||||
const members = await team.list(session.workspaceId);
|
||||
const owner = members.find((mem) => mem.role === "Owner");
|
||||
if (owner === undefined) throw new Error("owner missing");
|
||||
await expect(team.changeRole(session.workspaceId, actor, owner.id, "Admin")).rejects.toMatchObject({ status: 400 });
|
||||
|
||||
// Invited (password-less) users cannot sign in; placeholder gets a derived username.
|
||||
await expect(auth.login({ username: `pub-${suffix}`, password: "anything-123" })).rejects.toMatchObject({ status: 401 });
|
||||
|
||||
const audit = new m.audit.AuditQueryService();
|
||||
const first = await audit.page(session.workspaceId, { limit: 2 });
|
||||
expect(first.entries).toHaveLength(2);
|
||||
expect(first.entries[0]?.action).toBe("membership.role_changed");
|
||||
expect(first.entries[0]?.diff).toMatchObject({ role: ["Publisher", "Approver"] });
|
||||
expect(first.entries[1]?.action).toBe("membership.invited");
|
||||
expect(first.nextCursor).not.toBeNull();
|
||||
const second = await audit.page(session.workspaceId, { limit: 50, before: first.nextCursor ?? undefined });
|
||||
expect(second.entries.map((e) => e.action)).toEqual(
|
||||
expect.arrayContaining(["session.created", "workspace.created"]),
|
||||
);
|
||||
const filtered = await audit.page(session.workspaceId, { limit: 50, action: "membership.invited" });
|
||||
expect(filtered.entries.every((e) => e.action === "membership.invited")).toBe(true);
|
||||
expect(first.facets.actions).toContain("workspace.created");
|
||||
});
|
||||
|
||||
it("a member without the permission is forbidden", async () => {
|
||||
const auth = new m.auth.AuthService();
|
||||
const username = `analyst${suffix}`;
|
||||
const reg = await auth.register({ workspaceName: "Analyst WS", username, password: "correct-horse-1", displayName: "A" });
|
||||
const team = new m.team.TeamService();
|
||||
const session = m.session.verifySession(reg.token);
|
||||
if (session === null) throw new Error("session did not verify");
|
||||
// Owner of their own workspace may manage members; prove a missing permission is refused.
|
||||
await expect(allowed(reg.token, "permission.that.does.not.exist")).resolves.toBe("ForbiddenException");
|
||||
await expect(team.list(session.workspaceId)).resolves.toHaveLength(1);
|
||||
});
|
||||
});
|
||||
233
apps/api/src/auth/auth.service.ts
Normal file
233
apps/api/src/auth/auth.service.ts
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
import { randomBytes } from "node:crypto";
|
||||
import {
|
||||
BadRequestException,
|
||||
ConflictException,
|
||||
ForbiddenException,
|
||||
Injectable,
|
||||
NotFoundException,
|
||||
UnauthorizedException,
|
||||
} from "@nestjs/common";
|
||||
import type { PoolClient } from "pg";
|
||||
import { appendTenantAudit } from "../audit/audit.service.js";
|
||||
import { withSystem, withTenant } from "../db/client.js";
|
||||
import { issueSession, verifySession, type SessionPayload } from "./session.js";
|
||||
import { MIN_PASSWORD_LENGTH, burnPasswordCheck, hashPassword, verifyPassword } from "./password.js";
|
||||
|
||||
export interface MeResponse {
|
||||
user: { id: string; username: string; email: string | null; displayName: string };
|
||||
workspace: { id: string; name: string; slug: string };
|
||||
role: string;
|
||||
permissions: string[];
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
export interface AuthResult {
|
||||
token: string;
|
||||
me: MeResponse;
|
||||
}
|
||||
|
||||
export interface RegisterInput {
|
||||
workspaceName: string;
|
||||
username: string;
|
||||
password: string;
|
||||
displayName: string;
|
||||
email?: string;
|
||||
}
|
||||
|
||||
export interface LoginInput {
|
||||
username: string;
|
||||
password: string;
|
||||
workspaceId?: string;
|
||||
}
|
||||
|
||||
const USERNAME = /^[a-z0-9][a-z0-9_.-]{1,31}$/;
|
||||
|
||||
function normalizeUsername(raw: string): string {
|
||||
const username = raw.trim().toLowerCase();
|
||||
if (!USERNAME.test(username)) {
|
||||
throw new BadRequestException("username must be 2-32 characters: lowercase letters, digits, dot, underscore or hyphen, starting with a letter or digit");
|
||||
}
|
||||
return username;
|
||||
}
|
||||
|
||||
/**
|
||||
* First-party accounts are keyed `local:<username>`; invited placeholders keep
|
||||
* their `local:<email>` key until an activation flow exists; OIDC users keep
|
||||
* their issuer subject.
|
||||
*/
|
||||
export function localExternalId(handle: string): string {
|
||||
return `local:${handle.trim().toLowerCase()}`;
|
||||
}
|
||||
|
||||
function slugify(name: string): string {
|
||||
const base = name
|
||||
.toLowerCase()
|
||||
.normalize("NFKD")
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.slice(0, 40);
|
||||
return `${base === "" ? "workspace" : base}-${randomBytes(3).toString("hex")}`;
|
||||
}
|
||||
|
||||
function requirePasswordStrength(password: string): void {
|
||||
if (password.length < MIN_PASSWORD_LENGTH) {
|
||||
throw new BadRequestException(`password must be at least ${MIN_PASSWORD_LENGTH} characters`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-issue a session for `userId` from the CURRENT membership row and epoch.
|
||||
* Must run inside a tenant transaction for `workspaceId`; callers that just
|
||||
* mutated memberships use it so their own session survives the epoch bump.
|
||||
*/
|
||||
export async function issueSessionInTenant(
|
||||
client: PoolClient, workspaceId: string, userId: string, expiresAt?: number,
|
||||
): Promise<string> {
|
||||
const membership = await client.query(
|
||||
"SELECT role FROM memberships WHERE workspace_id = $1 AND user_id = $2 AND status = 'active'",
|
||||
[workspaceId, userId],
|
||||
);
|
||||
const role = membership.rows[0]?.role as string | undefined;
|
||||
if (role === undefined) throw new ForbiddenException("no active membership in this workspace");
|
||||
const ws = await client.query("SELECT authz_epoch FROM workspaces WHERE id = $1", [workspaceId]);
|
||||
const epoch = ws.rows[0]?.authz_epoch as string | number | undefined;
|
||||
if (epoch === undefined) throw new NotFoundException("workspace not found");
|
||||
const remaining = expiresAt === undefined ? undefined : Math.max(0, expiresAt - Math.floor(Date.now() / 1000));
|
||||
return issueSession({ userId, workspaceId, role, authzEpoch: Number(epoch) }, remaining);
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class AuthService {
|
||||
async register(input: RegisterInput): Promise<AuthResult> {
|
||||
requirePasswordStrength(input.password);
|
||||
const passwordHash = await hashPassword(input.password);
|
||||
const username = normalizeUsername(input.username);
|
||||
const externalId = localExternalId(username);
|
||||
|
||||
const created = await withSystem(async (client) => {
|
||||
// Existing identities (including invited, password-less users) cannot be
|
||||
// claimed by registering: without verified contact channels that would
|
||||
// let anyone attach a password to someone else's pending invitation.
|
||||
const user = await client.query(
|
||||
`INSERT INTO users (external_id, username, email, display_name) VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (external_id) DO NOTHING
|
||||
RETURNING id`,
|
||||
[externalId, username, input.email ?? null, input.displayName],
|
||||
);
|
||||
const userId = user.rows[0]?.id as string | undefined;
|
||||
if (userId === undefined) throw new ConflictException("an account with this username already exists — sign in instead");
|
||||
await client.query("INSERT INTO user_passwords (user_id, password_hash) VALUES ($1, $2)", [userId, passwordHash]);
|
||||
|
||||
const ws = await client.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id, slug", [
|
||||
input.workspaceName,
|
||||
slugify(input.workspaceName),
|
||||
]);
|
||||
const workspaceId = ws.rows[0]?.id as string;
|
||||
// Enter the new tenant's context for the RLS-protected membership insert.
|
||||
await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]);
|
||||
await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]);
|
||||
await client.query(
|
||||
"INSERT INTO memberships (workspace_id, user_id, role, status) VALUES ($1, $2, 'Owner', 'active')",
|
||||
[workspaceId, userId],
|
||||
);
|
||||
const token = await issueSessionInTenant(client, workspaceId, userId);
|
||||
return { userId, workspaceId, slug: ws.rows[0]?.slug as string, token };
|
||||
});
|
||||
|
||||
await appendTenantAudit(created.workspaceId, {
|
||||
actorUserId: created.userId,
|
||||
actorRole: "Owner",
|
||||
action: "workspace.created",
|
||||
targetType: "workspace",
|
||||
targetId: created.workspaceId,
|
||||
diff: { name: input.workspaceName, slug: created.slug },
|
||||
});
|
||||
return { token: created.token, me: await this.meFor(created.workspaceId, created.userId, created.token) };
|
||||
}
|
||||
|
||||
async login(input: LoginInput): Promise<AuthResult> {
|
||||
const username = normalizeUsername(input.username);
|
||||
const found = await withSystem((client) =>
|
||||
client.query(
|
||||
`SELECT u.id, p.password_hash FROM users u
|
||||
LEFT JOIN user_passwords p ON p.user_id = u.id
|
||||
WHERE u.external_id = $1`,
|
||||
[localExternalId(username)],
|
||||
),
|
||||
);
|
||||
const row = found.rows[0] as { id: string; password_hash: string | null } | undefined;
|
||||
if (row === undefined || row.password_hash === null) {
|
||||
await burnPasswordCheck(input.password);
|
||||
throw new UnauthorizedException("invalid email or password");
|
||||
}
|
||||
if (!(await verifyPassword(input.password, row.password_hash))) {
|
||||
throw new UnauthorizedException("invalid email or password");
|
||||
}
|
||||
|
||||
const memberships = await withSystem((client) =>
|
||||
client.query("SELECT workspace_id, role, status FROM auth_user_memberships($1)", [row.id]),
|
||||
);
|
||||
const active = (memberships.rows as Array<{ workspace_id: string; status: string }>).filter((m) => m.status === "active");
|
||||
const chosen = input.workspaceId === undefined ? active[0] : active.find((m) => m.workspace_id === input.workspaceId);
|
||||
if (chosen === undefined) throw new ForbiddenException("no active workspace membership for this account");
|
||||
|
||||
const token = await withTenant(chosen.workspace_id, (client) => issueSessionInTenant(client, chosen.workspace_id, row.id));
|
||||
const me = await this.meFor(chosen.workspace_id, row.id, token);
|
||||
await appendTenantAudit(chosen.workspace_id, {
|
||||
actorUserId: row.id,
|
||||
actorRole: me.role,
|
||||
action: "session.created",
|
||||
targetType: "user",
|
||||
targetId: row.id,
|
||||
diff: { method: "password" },
|
||||
});
|
||||
return { token, me };
|
||||
}
|
||||
|
||||
async me(session: SessionPayload): Promise<MeResponse> {
|
||||
return this.meFor(session.workspaceId, session.userId, null, session.expiresAt);
|
||||
}
|
||||
|
||||
async reissue(session: SessionPayload): Promise<AuthResult> {
|
||||
let token: string;
|
||||
try {
|
||||
token = await withTenant(session.workspaceId, (client) =>
|
||||
issueSessionInTenant(client, session.workspaceId, session.userId, session.expiresAt),
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof ForbiddenException || error instanceof NotFoundException) {
|
||||
throw new UnauthorizedException("membership no longer active");
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return { token, me: await this.meFor(session.workspaceId, session.userId, token) };
|
||||
}
|
||||
|
||||
private async meFor(workspaceId: string, userId: string, token: string | null, expiresAt?: number): Promise<MeResponse> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const res = await client.query(
|
||||
`SELECT m.role, u.username, u.email, u.display_name, w.name, w.slug
|
||||
FROM memberships m
|
||||
JOIN users u ON u.id = m.user_id
|
||||
JOIN workspaces w ON w.id = m.workspace_id
|
||||
WHERE m.workspace_id = $1 AND m.user_id = $2 AND m.status = 'active'`,
|
||||
[workspaceId, userId],
|
||||
);
|
||||
const row = res.rows[0] as { role: string; username: string; email: string | null; display_name: string; name: string; slug: string } | undefined;
|
||||
if (row === undefined) throw new UnauthorizedException("membership no longer active");
|
||||
const perms = await client.query("SELECT permission FROM role_permissions WHERE role = $1 ORDER BY permission", [row.role]);
|
||||
return {
|
||||
user: { id: userId, username: row.username, email: row.email, displayName: row.display_name },
|
||||
workspace: { id: workspaceId, name: row.name, slug: row.slug },
|
||||
role: row.role,
|
||||
permissions: (perms.rows as Array<{ permission: string }>).map((p) => p.permission),
|
||||
expiresAt: expiresAt ?? expiryOf(token),
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function expiryOf(token: string | null): number {
|
||||
if (token === null) return 0;
|
||||
return verifySession(token)?.expiresAt ?? 0;
|
||||
}
|
||||
55
apps/api/src/auth/oidc.test.ts
Normal file
55
apps/api/src/auth/oidc.test.ts
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { OidcClient } from "./oidc.js";
|
||||
|
||||
function fakeFetch(responses: Record<string, unknown>): typeof fetch {
|
||||
return (async (input: string | URL | unknown) => {
|
||||
const url = String(input);
|
||||
for (const [match, body] of Object.entries(responses)) {
|
||||
if (url.includes(match)) return new Response(JSON.stringify(body), { status: 200 });
|
||||
}
|
||||
return new Response("not found", { status: 404 });
|
||||
}) as unknown as typeof fetch;
|
||||
}
|
||||
|
||||
describe("oidc client", () => {
|
||||
const config = {
|
||||
issuer: "https://idp.example",
|
||||
clientId: "client-1",
|
||||
clientSecret: "secret-1",
|
||||
redirectUri: "http://localhost:3000/auth/callback",
|
||||
};
|
||||
|
||||
it("builds authorization URL with PKCE S256", async () => {
|
||||
const client = new OidcClient(config, fakeFetch({
|
||||
"well-known": {
|
||||
authorization_endpoint: "https://idp.example/auth",
|
||||
token_endpoint: "https://idp.example/token",
|
||||
},
|
||||
}));
|
||||
const url = await client.authorizationUrl("state-123", "challenge-abc");
|
||||
const parsed = new URL(url);
|
||||
expect(parsed.searchParams.get("code_challenge")).toBe("challenge-abc");
|
||||
expect(parsed.searchParams.get("code_challenge_method")).toBe("S256");
|
||||
expect(parsed.searchParams.get("state")).toBe("state-123");
|
||||
expect(parsed.searchParams.get("client_id")).toBe("client-1");
|
||||
});
|
||||
|
||||
it("exchanges code with verifier + client secret", async () => {
|
||||
const holder: { params?: URLSearchParams } = {};
|
||||
const fetchMock = (async (input: string | URL | unknown, init?: RequestInit) => {
|
||||
if (String(input).includes("token")) {
|
||||
holder.params = new URLSearchParams(String(init?.body));
|
||||
return new Response(JSON.stringify({ access_token: "at", id_token: "it" }), { status: 200 });
|
||||
}
|
||||
return new Response(JSON.stringify({
|
||||
authorization_endpoint: "https://idp.example/auth",
|
||||
token_endpoint: "https://idp.example/token",
|
||||
}), { status: 200 });
|
||||
}) as unknown as typeof fetch;
|
||||
const client = new OidcClient(config, fetchMock);
|
||||
const tokens = await client.exchangeCode("code-1", "verifier-xyz");
|
||||
expect(tokens.accessToken).toBe("at");
|
||||
expect(holder.params?.get("code_verifier")).toBe("verifier-xyz");
|
||||
expect(holder.params?.get("client_secret")).toBe("secret-1");
|
||||
});
|
||||
});
|
||||
78
apps/api/src/auth/oidc.ts
Normal file
78
apps/api/src/auth/oidc.ts
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
/**
|
||||
* Generic OIDC code flow with PKCE, discovery-document driven.
|
||||
* Production: Authentik/Keycloak/any compliant issuer via env.
|
||||
* Tests: mock the discovery + token endpoints via fetch override.
|
||||
*/
|
||||
|
||||
export interface OidcConfig {
|
||||
issuer: string;
|
||||
clientId: string;
|
||||
clientSecret: string;
|
||||
redirectUri: string;
|
||||
}
|
||||
|
||||
export function oidcConfigFromEnv(): OidcConfig | null {
|
||||
const issuer = process.env.OIDC_ISSUER;
|
||||
if (issuer === undefined || issuer === "") return null;
|
||||
return {
|
||||
issuer,
|
||||
clientId: process.env.OIDC_CLIENT_ID ?? "",
|
||||
clientSecret: process.env.OIDC_CLIENT_SECRET ?? "",
|
||||
redirectUri: process.env.OIDC_REDIRECT_URI ?? "http://localhost:3000/auth/callback",
|
||||
};
|
||||
}
|
||||
|
||||
interface DiscoveryDoc {
|
||||
authorization_endpoint: string;
|
||||
token_endpoint: string;
|
||||
userinfo_endpoint?: string;
|
||||
}
|
||||
|
||||
export type FetchLike = typeof fetch;
|
||||
|
||||
export class OidcClient {
|
||||
private discovery: DiscoveryDoc | null = null;
|
||||
|
||||
constructor(private readonly config: OidcConfig, private readonly fetchImpl: FetchLike = fetch) {}
|
||||
|
||||
private async discover(): Promise<DiscoveryDoc> {
|
||||
if (this.discovery !== null) return this.discovery;
|
||||
const res = await this.fetchImpl(`${this.config.issuer.replace(/\/$/, "")}/.well-known/openid-configuration`);
|
||||
if (!res.ok) throw new Error(`discovery failed: HTTP ${res.status}`);
|
||||
this.discovery = (await res.json()) as DiscoveryDoc;
|
||||
return this.discovery;
|
||||
}
|
||||
|
||||
authorizationUrl(state: string, codeChallenge: string): Promise<string> {
|
||||
return this.discover().then((d) => {
|
||||
const url = new URL(d.authorization_endpoint);
|
||||
url.searchParams.set("client_id", this.config.clientId);
|
||||
url.searchParams.set("response_type", "code");
|
||||
url.searchParams.set("scope", "openid profile email");
|
||||
url.searchParams.set("redirect_uri", this.config.redirectUri);
|
||||
url.searchParams.set("state", state);
|
||||
url.searchParams.set("code_challenge", codeChallenge);
|
||||
url.searchParams.set("code_challenge_method", "S256");
|
||||
return url.toString();
|
||||
});
|
||||
}
|
||||
|
||||
async exchangeCode(code: string, codeVerifier: string): Promise<{ accessToken: string; idToken: string }> {
|
||||
const d = await this.discover();
|
||||
const res = await this.fetchImpl(d.token_endpoint, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/x-www-form-urlencoded" },
|
||||
body: new URLSearchParams({
|
||||
grant_type: "authorization_code",
|
||||
code,
|
||||
client_id: this.config.clientId,
|
||||
client_secret: this.config.clientSecret,
|
||||
redirect_uri: this.config.redirectUri,
|
||||
code_verifier: codeVerifier,
|
||||
}),
|
||||
});
|
||||
if (!res.ok) throw new Error(`token exchange failed: HTTP ${res.status}`);
|
||||
const token = (await res.json()) as { access_token: string; id_token: string };
|
||||
return { accessToken: token.access_token, idToken: token.id_token };
|
||||
}
|
||||
}
|
||||
23
apps/api/src/auth/password.test.ts
Normal file
23
apps/api/src/auth/password.test.ts
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { hashPassword, verifyPassword } from "./password.js";
|
||||
|
||||
describe("password hashing", () => {
|
||||
it("round-trips and rejects the wrong password", async () => {
|
||||
const encoded = await hashPassword("correct-horse-battery");
|
||||
expect(encoded.startsWith("scrypt$")).toBe(true);
|
||||
expect(encoded).not.toContain("correct-horse-battery");
|
||||
await expect(verifyPassword("correct-horse-battery", encoded)).resolves.toBe(true);
|
||||
await expect(verifyPassword("correct-horse-batterz", encoded)).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it("salts every hash", async () => {
|
||||
const [a, b] = await Promise.all([hashPassword("same-password-1"), hashPassword("same-password-1")]);
|
||||
expect(a).not.toBe(b);
|
||||
});
|
||||
|
||||
it("fails closed on malformed encodings", async () => {
|
||||
await expect(verifyPassword("x", "")).resolves.toBe(false);
|
||||
await expect(verifyPassword("x", "bcrypt$1$2$3$aa$bb")).resolves.toBe(false);
|
||||
await expect(verifyPassword("x", "scrypt$0$8$1$aa$bb")).resolves.toBe(false);
|
||||
});
|
||||
});
|
||||
46
apps/api/src/auth/password.ts
Normal file
46
apps/api/src/auth/password.ts
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
import { randomBytes, scrypt, timingSafeEqual } from "node:crypto";
|
||||
|
||||
/**
|
||||
* scrypt password hashing. Encoded as `scrypt$N$r$p$salt$hash` (base64url) so
|
||||
* cost parameters can be raised later without invalidating stored hashes.
|
||||
*/
|
||||
|
||||
const N = 16384;
|
||||
const R = 8;
|
||||
const P = 1;
|
||||
const KEYLEN = 32;
|
||||
export const MIN_PASSWORD_LENGTH = 10;
|
||||
|
||||
function derive(password: string, salt: Buffer, n: number, r: number, p: number): Promise<Buffer> {
|
||||
return new Promise((resolve, reject) => {
|
||||
scrypt(password, salt, KEYLEN, { N: n, r, p, maxmem: 64 * 1024 * 1024 }, (err, key) => {
|
||||
if (err !== null) reject(err);
|
||||
else resolve(key);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function hashPassword(password: string): Promise<string> {
|
||||
const salt = randomBytes(16);
|
||||
const key = await derive(password, salt, N, R, P);
|
||||
return `scrypt$${N}$${R}$${P}$${salt.toString("base64url")}$${key.toString("base64url")}`;
|
||||
}
|
||||
|
||||
export async function verifyPassword(password: string, encoded: string): Promise<boolean> {
|
||||
const parts = encoded.split("$");
|
||||
if (parts.length !== 6 || parts[0] !== "scrypt") return false;
|
||||
const [n, r, p] = [Number(parts[1]), Number(parts[2]), Number(parts[3])];
|
||||
if (![n, r, p].every((v) => Number.isInteger(v) && v > 0)) return false;
|
||||
const salt = Buffer.from(parts[4] ?? "", "base64url");
|
||||
const expected = Buffer.from(parts[5] ?? "", "base64url");
|
||||
if (expected.length !== KEYLEN) return false;
|
||||
const actual = await derive(password, salt, n, r, p);
|
||||
return timingSafeEqual(actual, expected);
|
||||
}
|
||||
|
||||
/** Burn equivalent CPU when the user does not exist, so login timing does not reveal account existence. */
|
||||
let dummyHash: Promise<string> | undefined;
|
||||
export async function burnPasswordCheck(password: string): Promise<void> {
|
||||
dummyHash ??= hashPassword("timing-equalizer-not-a-real-password");
|
||||
await verifyPassword(password, await dummyHash);
|
||||
}
|
||||
46
apps/api/src/auth/reissue.test.ts
Normal file
46
apps/api/src/auth/reissue.test.ts
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
import type { PoolClient } from "pg";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { AuthService } from "./auth.service.js";
|
||||
import { issueSession, verifySession } from "./session.js";
|
||||
|
||||
const workspaceId = "10000000-0000-4000-8000-000000000001";
|
||||
const userId = "20000000-0000-4000-8000-000000000002";
|
||||
let role = "Publisher";
|
||||
let epoch = 1;
|
||||
let active = true;
|
||||
|
||||
vi.mock("../db/client.js", () => ({
|
||||
withTenant: async (_workspaceId: string, fn: (client: PoolClient) => Promise<unknown>) => fn({
|
||||
query: async (sql: string) => {
|
||||
if (sql.includes("SELECT role FROM memberships")) return { rows: active ? [{ role }] : [] };
|
||||
if (sql.includes("SELECT authz_epoch FROM workspaces")) return { rows: [{ authz_epoch: String(epoch) }] };
|
||||
if (sql.includes("SELECT m.role, u.username")) return { rows: active ? [{ role, username: "user", email: "user@example.com", display_name: "User", name: "Studio", slug: "studio" }] : [] };
|
||||
if (sql.includes("SELECT permission FROM role_permissions")) return { rows: [{ permission: "draft.view" }] };
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
},
|
||||
} as PoolClient),
|
||||
withSystem: vi.fn(),
|
||||
}));
|
||||
|
||||
describe("session reissue", () => {
|
||||
it("refreshes a stale epoch and live role without extending the session expiry", async () => {
|
||||
active = true;
|
||||
role = "Publisher";
|
||||
epoch = 1;
|
||||
const old = verifySession(issueSession({ userId, workspaceId, role, authzEpoch: epoch }));
|
||||
if (old === null) throw new Error("failed to issue test session");
|
||||
role = "Approver";
|
||||
epoch = 2;
|
||||
const result = await new AuthService().reissue(old);
|
||||
expect(result.me.role).toBe("Approver");
|
||||
expect(result.me.permissions).toContain("draft.view");
|
||||
expect(verifySession(result.token)).toMatchObject({ userId, workspaceId, role: "Approver", authzEpoch: 2, expiresAt: old.expiresAt });
|
||||
});
|
||||
|
||||
it("refuses an inactive membership", async () => {
|
||||
active = false;
|
||||
const old = verifySession(issueSession({ userId, workspaceId, role: "Publisher", authzEpoch: 1 }));
|
||||
if (old === null) throw new Error("failed to issue test session");
|
||||
await expect(new AuthService().reissue(old)).rejects.toMatchObject({ status: 401 });
|
||||
});
|
||||
});
|
||||
25
apps/api/src/auth/session.test.ts
Normal file
25
apps/api/src/auth/session.test.ts
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { issueSession, verifySession, epochValid } from "./session.js";
|
||||
|
||||
describe("session tokens", () => {
|
||||
it("issues and verifies", () => {
|
||||
const token = issueSession({ userId: "u1", workspaceId: "ws1", role: "Publisher", authzEpoch: 3 });
|
||||
const payload = verifySession(token);
|
||||
expect(payload?.userId).toBe("u1");
|
||||
expect(payload?.workspaceId).toBe("ws1");
|
||||
});
|
||||
|
||||
it("rejects tampering", () => {
|
||||
const token = issueSession({ userId: "u1", workspaceId: "ws1", role: "Publisher", authzEpoch: 3 });
|
||||
const tampered = `${token.slice(0, -3)}xxx`;
|
||||
expect(verifySession(tampered)).toBeNull();
|
||||
});
|
||||
|
||||
it("epoch fence fails closed on mismatch (finding #2)", () => {
|
||||
const token = issueSession({ userId: "u1", workspaceId: "ws1", role: "Publisher", authzEpoch: 3 });
|
||||
const payload = verifySession(token);
|
||||
if (payload === null) throw new Error("unreachable");
|
||||
expect(epochValid(payload, 3)).toBe(true);
|
||||
expect(epochValid(payload, 4)).toBe(false);
|
||||
});
|
||||
});
|
||||
60
apps/api/src/auth/session.ts
Normal file
60
apps/api/src/auth/session.ts
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
import { createHmac, timingSafeEqual } from "node:crypto";
|
||||
|
||||
/**
|
||||
* Session tokens: HMAC-signed, workspace-scoped, epoch-fenced.
|
||||
* authzEpoch fail-closed: any membership/role change bumps the workspace
|
||||
* epoch (see 001_tenants.sql trigger) and the session stops validating —
|
||||
* this is the continuous-authz fix for audit finding #2.
|
||||
*/
|
||||
|
||||
export interface SessionPayload {
|
||||
userId: string;
|
||||
workspaceId: string;
|
||||
role: string;
|
||||
authzEpoch: number;
|
||||
issuedAt: number;
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
function secret(): string {
|
||||
const s = process.env.SESSION_SECRET;
|
||||
if (s !== undefined && s !== "") return s;
|
||||
if (process.env.NODE_ENV === "production") throw new Error("SESSION_SECRET required in production");
|
||||
return "dev-session-secret";
|
||||
}
|
||||
|
||||
function sign(data: string): string {
|
||||
return createHmac("sha256", secret()).update(data).digest("base64url");
|
||||
}
|
||||
|
||||
export function issueSession(payload: Omit<SessionPayload, "issuedAt" | "expiresAt">, ttlSeconds = 12 * 3600): string {
|
||||
const full: SessionPayload = {
|
||||
...payload,
|
||||
issuedAt: Math.floor(Date.now() / 1000),
|
||||
expiresAt: Math.floor(Date.now() / 1000) + ttlSeconds,
|
||||
};
|
||||
const body = Buffer.from(JSON.stringify(full)).toString("base64url");
|
||||
return `${body}.${sign(body)}`;
|
||||
}
|
||||
|
||||
export function verifySession(token: string): SessionPayload | null {
|
||||
const dot = token.lastIndexOf(".");
|
||||
if (dot === -1) return null;
|
||||
const body = token.slice(0, dot);
|
||||
const mac = token.slice(dot + 1);
|
||||
const expected = sign(body);
|
||||
if (mac.length !== expected.length) return null;
|
||||
if (!timingSafeEqual(Buffer.from(mac), Buffer.from(expected))) return null;
|
||||
try {
|
||||
const payload = JSON.parse(Buffer.from(body, "base64url").toString("utf8")) as SessionPayload;
|
||||
if (payload.expiresAt < Math.floor(Date.now() / 1000)) return null;
|
||||
return payload;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Fail-closed epoch check: the caller passes the workspace's CURRENT epoch from the DB. */
|
||||
export function epochValid(session: SessionPayload, currentEpoch: number): boolean {
|
||||
return session.authzEpoch === currentEpoch;
|
||||
}
|
||||
100
apps/api/src/chatgov/chatgov.controller.ts
Normal file
100
apps/api/src/chatgov/chatgov.controller.ts
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
import { Body, Controller, Delete, Get, Inject, Param, Patch, Post, Query, Req, UseGuards } from "@nestjs/common";
|
||||
import type { Request } from "express";
|
||||
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import { parseId, requireObject, requireSession } from "../http/validation.js";
|
||||
import { CHATGOV_PERMISSIONS } from "./policy.js";
|
||||
import { PolicyService, type PolicyActor, type RuleRecord } from "./policy.service.js";
|
||||
import { GateService, type GateDecisionRecord, type RecordGateInput } from "./gate.service.js";
|
||||
|
||||
/**
|
||||
* Chat governance console API (§5): rules editor + gate decision reads.
|
||||
* The command router (/poll, /tally, …) is P3 and does NOT live here.
|
||||
*
|
||||
* Permission model (§6): chatgov.manage gates every rule mutation; reads are
|
||||
* workspace.view-shaped (any member sees the policy that governs them).
|
||||
*/
|
||||
@Controller("chatgov")
|
||||
@UseGuards(RbacGuard)
|
||||
export class ChatgovController {
|
||||
constructor(
|
||||
@Inject(PolicyService) private readonly policy: PolicyService,
|
||||
@Inject(GateService) private readonly gates: GateService,
|
||||
) {}
|
||||
|
||||
@Get("rules")
|
||||
@RequirePermission("workspace.view")
|
||||
async listRules(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Query("accountId") accountId: string | undefined,
|
||||
@Query("topicBindingId") topicBindingId: string | undefined,
|
||||
): Promise<RuleRecord[]> {
|
||||
return this.policy.listRules(requireSession(request).workspaceId, {
|
||||
...(accountId === undefined || accountId === "" ? {} : { accountId }),
|
||||
...(topicBindingId === undefined || topicBindingId === "" ? {} : { topicBindingId }),
|
||||
});
|
||||
}
|
||||
|
||||
@Post("rules")
|
||||
@RequirePermission(CHATGOV_PERMISSIONS.manage)
|
||||
async createRule(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise<RuleRecord> {
|
||||
return this.policy.createRule(requireSession(request).workspaceId, body, actorOf(request));
|
||||
}
|
||||
|
||||
@Patch("rules/:id")
|
||||
@RequirePermission(CHATGOV_PERMISSIONS.manage)
|
||||
async updateRule(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Param("id") id: string,
|
||||
@Body() body: unknown,
|
||||
): Promise<RuleRecord> {
|
||||
return this.policy.updateRule(requireSession(request).workspaceId, parseId(id, "id"), body, actorOf(request));
|
||||
}
|
||||
|
||||
@Delete("rules/:id")
|
||||
@RequirePermission(CHATGOV_PERMISSIONS.manage)
|
||||
async deleteRule(@Req() request: Request & AuthenticatedRequest, @Param("id") id: string): Promise<{ superseded: boolean }> {
|
||||
return this.policy.deleteRule(requireSession(request).workspaceId, parseId(id, "id"), actorOf(request));
|
||||
}
|
||||
|
||||
/** Blocked-queue / draft timeline feed (P6 renders it). */
|
||||
@Get("decisions/draft/:draftId")
|
||||
@RequirePermission("draft.view")
|
||||
async decisionsForDraft(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Param("draftId") draftId: string,
|
||||
): Promise<GateDecisionRecord[]> {
|
||||
return this.gates.listForDraft(requireSession(request).workspaceId, parseId(draftId, "draftId"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal write path for the Temporal E3 worker / E1 review hook (P1).
|
||||
* Service-to-service in v1: the route exists so the seam is testable, and is
|
||||
* gated on chatgov.manage (an Owner/Admin console action in practice).
|
||||
*/
|
||||
@Post("decisions")
|
||||
@RequirePermission(CHATGOV_PERMISSIONS.manage)
|
||||
async recordDecision(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise<GateDecisionRecord> {
|
||||
const dto = requireObject(body, "body");
|
||||
const session = requireSession(request);
|
||||
return this.gates.record(session.workspaceId, {
|
||||
draftId: String(dto.draftId),
|
||||
destination: dto.destination as RecordGateInput["destination"],
|
||||
gate: dto.gate as RecordGateInput["gate"],
|
||||
...(dto.ruleId === undefined || dto.ruleId === null ? {} : { ruleId: String(dto.ruleId) }),
|
||||
policyVersion: String(dto.policyVersion),
|
||||
outcome: dto.outcome as RecordGateInput["outcome"],
|
||||
inputs: (dto.inputs ?? {}) as RecordGateInput["inputs"],
|
||||
reasonCodes: Array.isArray(dto.reasonCodes) ? (dto.reasonCodes as unknown[]).filter((c): c is RecordGateInput["reasonCodes"][number] => typeof c === "string") : [],
|
||||
actor: typeof dto.actor === "string" ? dto.actor : `user:${session.userId}`,
|
||||
...(dto.pollId === undefined || dto.pollId === null ? {} : { pollId: String(dto.pollId) }),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function actorOf(request: Request & AuthenticatedRequest): PolicyActor {
|
||||
const session = requireSession(request);
|
||||
// The route decorator already enforced chatgov.manage; `permissions` is left
|
||||
// unset so the service's defense-in-depth check treats the controller as the
|
||||
// privileged caller (it re-checks only for non-controller callers).
|
||||
return { userId: session.userId, role: session.role };
|
||||
}
|
||||
710
apps/api/src/chatgov/chatgov.integration.test.ts
Normal file
710
apps/api/src/chatgov/chatgov.integration.test.ts
Normal file
|
|
@ -0,0 +1,710 @@
|
|||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { join } from "node:path";
|
||||
import { Pool, type PoolClient } from "pg";
|
||||
|
||||
/**
|
||||
* chatgov P0 spine against a live PostgreSQL (chatgov-design.md §7 DoD):
|
||||
* - migration idempotency (re-run applies nothing)
|
||||
* - RLS denial for EVERY new table (workspace B sees/updates/deletes zero A
|
||||
* rows, and B cannot write into A's workspace — WITH CHECK)
|
||||
* - rule CRUD + one-live-rule-per-slot
|
||||
* - policy compile -> posting_policy_snapshots with a deterministic version
|
||||
* - RBAC permission gating (chatgov.manage; Editor is denied, Admin allowed)
|
||||
* - audit event landing (policy.revised / policy.gate in audit_log)
|
||||
*
|
||||
* Owns its own database (CHATGOV_IT_DATABASE_URL + CHATGOV_IT_SUPER_DATABASE_URL,
|
||||
* local-db-sol.env pattern) because the shared tenancy DB is wiped by other
|
||||
* suites and vitest runs files in parallel. Skips cleanly when unset.
|
||||
*/
|
||||
const appUrl = process.env.CHATGOV_IT_DATABASE_URL;
|
||||
const ownerUrl = process.env.CHATGOV_IT_SUPER_DATABASE_URL;
|
||||
const dbAvailable = appUrl !== undefined && ownerUrl !== undefined;
|
||||
|
||||
const RLS_VIOLATION = "42501";
|
||||
const UNIQUE_VIOLATION = "23505";
|
||||
const FK_VIOLATION = "23503";
|
||||
const TABLES = [
|
||||
"polls",
|
||||
"poll_votes",
|
||||
"posting_rules",
|
||||
"posting_policy_snapshots",
|
||||
"topic_bindings",
|
||||
"chat_command_audit",
|
||||
"chat_identities",
|
||||
"gate_decisions",
|
||||
] as const;
|
||||
|
||||
type Modules = {
|
||||
policy: typeof import("./policy.js");
|
||||
policyService: typeof import("./policy.service.js");
|
||||
gate: typeof import("./gate.service.js");
|
||||
db: typeof import("../db/client.js");
|
||||
guard: typeof import("../rbac/rbac.guard.js");
|
||||
session: typeof import("../auth/session.js");
|
||||
};
|
||||
|
||||
describe.skipIf(!dbAvailable)("chatgov P0 spine (live DB)", () => {
|
||||
let m: Modules;
|
||||
let ownerPool: Pool;
|
||||
const suffix = `${Date.now()}${Math.floor(Math.random() * 1e6)}`;
|
||||
let wsA: string;
|
||||
let wsB: string;
|
||||
let userA: string;
|
||||
let connA1: string;
|
||||
let connA2: string;
|
||||
let connB: string;
|
||||
let topicA: string;
|
||||
let draftA: string;
|
||||
|
||||
async function asOwner<T>(fn: (client: PoolClient) => Promise<T>): Promise<T> {
|
||||
const client = await ownerPool.connect();
|
||||
try {
|
||||
return await fn(client);
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
/** Seed tenant rows honestly: FORCE RLS applies to the owner too. */
|
||||
async function inTenant(client: PoolClient, wsId: string, fn: () => Promise<void>): Promise<void> {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT set_config('app.workspace_id', $1, true)", [wsId]);
|
||||
await fn();
|
||||
await client.query("COMMIT");
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
// The shared pool reads DATABASE_URL at import time: point it at this
|
||||
// isolated DB first, then import the modules under test.
|
||||
process.env.DATABASE_URL = appUrl;
|
||||
process.env.SUPER_DATABASE_URL = ownerUrl;
|
||||
const { runMigrations } = await import("../db/migrate.js");
|
||||
// Idempotency: the CLI already applied every migration; re-running must
|
||||
// apply nothing (schema_migrations guards).
|
||||
const applied = await runMigrations(join(process.cwd(), "migrations"));
|
||||
expect(applied).toEqual([]);
|
||||
|
||||
m = {
|
||||
policy: await import("./policy.js"),
|
||||
policyService: await import("./policy.service.js"),
|
||||
gate: await import("./gate.service.js"),
|
||||
db: await import("../db/client.js"),
|
||||
guard: await import("../rbac/rbac.guard.js"),
|
||||
session: await import("../auth/session.js"),
|
||||
};
|
||||
ownerPool = new Pool({ connectionString: ownerUrl });
|
||||
|
||||
await asOwner(async (c) => {
|
||||
const ws = async (name: string): Promise<string> =>
|
||||
(await c.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id", [name, `${name}-${suffix}`])).rows[0].id as string;
|
||||
wsA = await ws("cg-a");
|
||||
wsB = await ws("cg-b");
|
||||
const user = async (name: string): Promise<string> =>
|
||||
(await c.query(
|
||||
"INSERT INTO users (external_id, username, email, display_name) VALUES ($1, $2, $3, $4) RETURNING id",
|
||||
[`cg:${name}:${suffix}`, `cg-${name}-${suffix}`, `${name}-${suffix}@example.com`, name],
|
||||
)).rows[0].id as string;
|
||||
userA = await user("owner");
|
||||
const editorA = await user("editor");
|
||||
|
||||
const conn = async (wsId: string, name: string): Promise<string> =>
|
||||
(await c.query(
|
||||
"INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-social', $2) RETURNING id",
|
||||
[wsId, `ch-${name}`],
|
||||
)).rows[0].id as string;
|
||||
connA1 = await conn(wsA, "a1");
|
||||
connA2 = await conn(wsA, "a2");
|
||||
connB = await conn(wsB, "b1");
|
||||
|
||||
await inTenant(c, wsA, async () => {
|
||||
await c.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, 'Owner')", [wsA, userA]);
|
||||
await c.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, 'Editor')", [wsA, editorA]);
|
||||
draftA = (await c.query("INSERT INTO drafts (workspace_id, author_user_id, post) VALUES ($1, $2, $3::jsonb) RETURNING id", [wsA, userA, JSON.stringify({ body: "x" })])).rows[0].id as string;
|
||||
topicA = (await c.query(
|
||||
`INSERT INTO topic_bindings (workspace_id, connection_id, chat_id, message_thread_id, binding_type, name, account_ids, icon_color)
|
||||
VALUES ($1, $2, '-100200', '7', 'announce', 'Ops', $3, 7322096) RETURNING id`,
|
||||
[wsA, connA1, [connA1]],
|
||||
)).rows[0].id as string;
|
||||
// Seed rows for the RLS denial suite (A-side data B must not see).
|
||||
const poll = (await c.query(
|
||||
`INSERT INTO polls (workspace_id, channel_id, topic_binding_id, subject_kind, draft_id, question, rule, policy_version, closes_at, created_by)
|
||||
VALUES ($1, NULL, $2, 'draft', $3, 'ship it?', $4::jsonb, 'pv-seed', now() + interval '1 hour', $5) RETURNING id`,
|
||||
[wsA, topicA, draftA, JSON.stringify({ version: 1, mode: "quorum", arms: [{ kind: "admins", x: 1 }], timebox: { duration: "PT1H", onExpire: "reject" } }), userA],
|
||||
)).rows[0].id as string;
|
||||
await c.query(
|
||||
`INSERT INTO poll_votes (poll_id, workspace_id, voter_kind, voter_user_id, voter_remote_id, voter_handle, ballot)
|
||||
VALUES ($1, $2, 'chat_identity', $3, '424242', 'opal', 'yes')`,
|
||||
[poll, wsA, userA],
|
||||
);
|
||||
await c.query(
|
||||
`INSERT INTO chat_identities (workspace_id, provider, remote_user_id, user_id, display_name, status, approved_by)
|
||||
VALUES ($1, 'telegram', '111222333', $2, 'Opal', 'approved', $2)`,
|
||||
[wsA, userA],
|
||||
);
|
||||
await c.query(
|
||||
`INSERT INTO chat_command_audit (workspace_id, update_id, binding_id, provider, remote_user_id, user_id, command, args, permission, decision)
|
||||
VALUES ($1, 9001, $2, 'telegram', '111222333', $3, '/tally', '{}'::jsonb, 'director.view', 'accepted')`,
|
||||
[wsA, topicA, userA],
|
||||
);
|
||||
// one row of every remaining table so the RLS sweep counts real data
|
||||
await c.query(
|
||||
`INSERT INTO posting_rules (workspace_id, scope, policy, policy_version, created_by)
|
||||
VALUES ($1, 'workspace', $2::jsonb, 'pv-seed', $3)`,
|
||||
[wsA, JSON.stringify({ version: 1, mode: "immediate" }), userA],
|
||||
);
|
||||
await c.query(
|
||||
`INSERT INTO posting_policy_snapshots (workspace_id, destination_kind, account_id, topic_binding_id, content_type, effective, policy_version, resolved_from)
|
||||
VALUES ($1, 'account', $2, NULL, '', $3::jsonb, 'pv-seed', '{"scopes":["workspace"]}'::jsonb)`,
|
||||
[wsA, connA1, JSON.stringify({ version: 1, mode: "immediate" })],
|
||||
);
|
||||
await c.query(
|
||||
`INSERT INTO gate_decisions (workspace_id, draft_id, destination, gate, policy_version, outcome, inputs, reason_codes, actor)
|
||||
VALUES ($1, $2, $3::jsonb, 'E1', 'pv-seed', 'fired', '{}'::jsonb, '{}', 'system:gate-e1')`,
|
||||
[wsA, draftA, JSON.stringify({ platform: "telegram", accountId: connA1 })],
|
||||
);
|
||||
});
|
||||
await inTenant(c, wsB, async () => {
|
||||
const ownerB = await user("owner-b");
|
||||
await c.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, 'Owner')", [wsB, ownerB]);
|
||||
});
|
||||
});
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (ownerPool !== undefined) {
|
||||
await asOwner(async (c) => {
|
||||
await c.query("DELETE FROM workspaces WHERE slug LIKE $1", [`%-${suffix}`]);
|
||||
await c.query("DELETE FROM users WHERE external_id LIKE $1", [`cg:%:${suffix}`]);
|
||||
});
|
||||
await ownerPool.end();
|
||||
}
|
||||
await m?.db.pool.end();
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
describe("migration + RBAC seed", () => {
|
||||
it("idempotent: a second run applies nothing", async () => {
|
||||
const { runMigrations } = await import("../db/migrate.js");
|
||||
const previous = process.env.SUPER_DATABASE_URL;
|
||||
process.env.SUPER_DATABASE_URL = ownerUrl;
|
||||
try {
|
||||
expect(await runMigrations(join(process.cwd(), "migrations"))).toEqual([]);
|
||||
} finally {
|
||||
process.env.SUPER_DATABASE_URL = previous;
|
||||
}
|
||||
});
|
||||
|
||||
it("chatgov permissions are seeded per the 002 pattern", async () => {
|
||||
const rows = await asOwner((c) => c.query<{ role: string; permission: string }>(
|
||||
"SELECT role, permission FROM role_permissions WHERE permission LIKE 'chatgov%' ORDER BY role, permission"),
|
||||
);
|
||||
const map = new Map<string, string[]>();
|
||||
for (const row of rows.rows) {
|
||||
map.set(row.role, [...(map.get(row.role) ?? []), row.permission]);
|
||||
}
|
||||
expect(map.get("Owner")).toEqual(["chatgov.manage", "chatgov.vote"]);
|
||||
expect(map.get("Admin")).toEqual(["chatgov.manage", "chatgov.vote"]);
|
||||
for (const role of ["Publisher", "Editor", "Approver", "Moderator", "BroadcastOperator"]) {
|
||||
expect(map.get(role)).toEqual(["chatgov.vote"]);
|
||||
}
|
||||
for (const role of ["Analyst", "VideoPublisher", "StreamOperator"]) {
|
||||
expect(map.has(role)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("every new table is RLS-enabled AND forced with a tenant_isolation policy", async () => {
|
||||
for (const table of TABLES) {
|
||||
const flags = await asOwner((c) => c.query<{ relrowsecurity: boolean; relforcerowsecurity: boolean }>(
|
||||
"SELECT relrowsecurity, relforcerowsecurity FROM pg_class WHERE oid = $1::regclass", [table]));
|
||||
expect(flags.rows[0], table).toEqual({ relrowsecurity: true, relforcerowsecurity: true });
|
||||
const policies = await asOwner((c) => c.query<{ policyname: string }>(
|
||||
"SELECT policyname FROM pg_policies WHERE tablename = $1", [table]));
|
||||
expect(policies.rows.map((r) => r.policyname), table).toContain("tenant_isolation");
|
||||
}
|
||||
});
|
||||
|
||||
it("sp_app holds table grants (and is neither superuser nor BYPASSRLS)", async () => {
|
||||
for (const table of TABLES) {
|
||||
const grants = await asOwner((c) => c.query<{ privilege_type: string }>(
|
||||
"SELECT privilege_type FROM information_schema.role_table_grants WHERE grantee = 'sp_app' AND table_name = $1", [table]));
|
||||
expect(new Set(grants.rows.map((r) => r.privilege_type)), table)
|
||||
.toEqual(new Set(["SELECT", "INSERT", "UPDATE", "DELETE"]));
|
||||
}
|
||||
const role = await asOwner((c) => c.query<{ rolsuper: boolean; rolbypassrls: boolean }>(
|
||||
"SELECT rolsuper, rolbypassrls FROM pg_roles WHERE rolname = 'sp_app'"));
|
||||
expect(role.rows[0]).toEqual({ rolsuper: false, rolbypassrls: false });
|
||||
});
|
||||
});
|
||||
|
||||
/** Fresh workspace with its own connections/topic (isolated rule slots). */
|
||||
async function seedWorkspace(name: string): Promise<{ ws: string; conn1: string; conn2: string; topic: string }> {
|
||||
return asOwner(async (c) => {
|
||||
const ws = (await c.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id", [`cg-${name}`, `cg-${name}-${suffix}`])).rows[0].id as string;
|
||||
let conn1 = "";
|
||||
let conn2 = "";
|
||||
let topic = "";
|
||||
await inTenant(c, ws, async () => {
|
||||
await c.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, 'Owner')", [ws, userA]);
|
||||
conn1 = (await c.query("INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-social', $2) RETURNING id", [ws, `ch-${name}-1`])).rows[0].id as string;
|
||||
conn2 = (await c.query("INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-social', $2) RETURNING id", [ws, `ch-${name}-2`])).rows[0].id as string;
|
||||
topic = (await c.query(
|
||||
`INSERT INTO topic_bindings (workspace_id, connection_id, chat_id, message_thread_id, binding_type, name, account_ids)
|
||||
VALUES ($1, $2, '-100200', '7', 'announce', 'Ops', $3) RETURNING id`,
|
||||
[ws, conn1, [conn1]])).rows[0].id as string;
|
||||
});
|
||||
return { ws, conn1, conn2, topic };
|
||||
});
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
describe("RLS denial suite (FORCE + NULLIF; B must see zero A rows)", () => {
|
||||
it.each(TABLES)("%s: B sees, updates and deletes none of A's rows", async (table) => {
|
||||
const seen = await m.db.withTenant(wsB, (c) => c.query(`SELECT count(*)::int AS n FROM ${table} WHERE workspace_id = $1`, [wsA]));
|
||||
expect(seen.rows[0]?.n).toBe(0);
|
||||
const noFilter = await m.db.withTenant(wsB, (c) => c.query(`SELECT count(*)::int AS n FROM ${table}`));
|
||||
expect(noFilter.rows[0]?.n).toBe(0);
|
||||
// A harmless mutable column per table (several have no updated_at).
|
||||
const touch: Record<(typeof TABLES)[number], string> = {
|
||||
polls: "outbox_state = outbox_state",
|
||||
poll_votes: "ballot = ballot",
|
||||
posting_rules: "dry_run = dry_run",
|
||||
posting_policy_snapshots: "policy_version = policy_version",
|
||||
topic_bindings: "name = name",
|
||||
chat_command_audit: "reason = reason",
|
||||
chat_identities: "display_name = display_name",
|
||||
gate_decisions: "actor = actor",
|
||||
};
|
||||
const updated = await m.db.withTenant(wsB, (c) => c.query(`UPDATE ${table} SET ${touch[table]} WHERE workspace_id = $1`, [wsA]));
|
||||
expect(updated.rowCount).toBe(0);
|
||||
const deleted = await m.db.withTenant(wsB, (c) => c.query(`DELETE FROM ${table} WHERE workspace_id = $1`, [wsA]));
|
||||
expect(deleted.rowCount).toBe(0);
|
||||
const ownStillThere = await m.db.withTenant(wsA, (c) => c.query(`SELECT count(*)::int AS n FROM ${table}`));
|
||||
expect(ownStillThere.rows[0]?.n).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("no tenant context (and an empty one) sees nothing on any table", async () => {
|
||||
for (const table of TABLES) {
|
||||
const none = await m.db.withSystem((c) => c.query(`SELECT count(*)::int AS n FROM ${table}`));
|
||||
expect(none.rows[0]?.n, table).toBe(0);
|
||||
}
|
||||
for (const table of TABLES) {
|
||||
const leaked = await m.db.withTenant(wsB, async (c) => {
|
||||
// empty-string GUC (004 pooled-session hazard) must fail closed too
|
||||
await c.query("SELECT set_config('app.workspace_id', '', true)");
|
||||
return c.query(`SELECT count(*)::int AS n FROM ${table}`);
|
||||
});
|
||||
expect(leaked.rows[0]?.n, table).toBe(0);
|
||||
}
|
||||
});
|
||||
|
||||
it("B cannot write rows into A's workspace (WITH CHECK) on representative tables", async () => {
|
||||
const attempts: Array<[string, unknown[]]> = [
|
||||
["INSERT INTO posting_rules (workspace_id, scope, policy, policy_version, created_by) VALUES ($1, 'workspace', '{}'::jsonb, 'pv', $2)", [wsA, userA]],
|
||||
["INSERT INTO gate_decisions (workspace_id, draft_id, destination, gate, policy_version, outcome, inputs, reason_codes, actor) VALUES ($1, $2, '{}'::jsonb, 'E1', 'pv', 'pending', '{}'::jsonb, '{}', 'system:gate')", [wsA, draftA]],
|
||||
["INSERT INTO chat_identities (workspace_id, provider, remote_user_id) VALUES ($1, 'telegram', '999')", [wsA]],
|
||||
["INSERT INTO topic_bindings (workspace_id, connection_id, chat_id, binding_type, name) VALUES ($1, $2, '-100999', 'announce', 'Sneaky')", [wsA, connA1]],
|
||||
["INSERT INTO chat_command_audit (workspace_id, update_id, provider, remote_user_id, command, decision) VALUES ($1, 42, 'telegram', '999', '/x', 'rejected')", [wsA]],
|
||||
["INSERT INTO polls (workspace_id, subject_kind, draft_id, question, rule, policy_version, closes_at, created_by) VALUES ($1, 'draft', $2, 'q?', '{}'::jsonb, 'pv', now() + interval '1 hour', $3)", [wsA, draftA, userA]],
|
||||
];
|
||||
for (const [sql, params] of attempts) {
|
||||
await expect(m.db.withTenant(wsB, (c) => c.query(sql, params)), sql).rejects.toMatchObject({ code: RLS_VIOLATION });
|
||||
}
|
||||
});
|
||||
|
||||
it("cross-tenant composite FKs: B's rows cannot point at A's parents", async () => {
|
||||
await expect(m.db.withTenant(wsB, async (c) => c.query(
|
||||
"INSERT INTO topic_bindings (workspace_id, connection_id, chat_id, binding_type, name) VALUES ($1, $2, '-100888', 'announce', 'X')",
|
||||
[wsB, connA1],
|
||||
))).rejects.toMatchObject({ code: FK_VIOLATION });
|
||||
await expect(m.db.withTenant(wsB, async (c) => c.query(
|
||||
"INSERT INTO posting_rules (workspace_id, scope, account_id, policy, policy_version, created_by) VALUES ($1, 'account', $2, '{}'::jsonb, 'pv', $3)",
|
||||
[wsB, connA1, userA],
|
||||
))).rejects.toMatchObject({ code: FK_VIOLATION });
|
||||
});
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
describe("rule CRUD + compile-to-snapshot", () => {
|
||||
it("create rejects a second live rule in the same slot (unique index)", async () => {
|
||||
const service = new m.policyService.PolicyService();
|
||||
const actor = { userId: userA, role: "Owner" };
|
||||
const body = { scope: "account", accountId: connA1, policy: { mode: "admin_only", timebox: { duration: "PT1H", onExpire: "reject" } } };
|
||||
await expect(service.createRule(wsA, body, actor)).resolves.toBeTruthy();
|
||||
await expect(service.createRule(wsA, body, actor)).rejects.toMatchObject({ status: 409 });
|
||||
// a different slot is fine
|
||||
await expect(service.createRule(wsA, { ...body, accountId: connA2 }, actor)).resolves.toBeTruthy();
|
||||
});
|
||||
|
||||
it("slot validation refuses nonsense shapes with 400s", async () => {
|
||||
const service = new m.policyService.PolicyService();
|
||||
const actor = { userId: userA, role: "Owner" };
|
||||
const bad: unknown[] = [
|
||||
{ scope: "workspace", accountId: connA1, policy: { mode: "immediate" } },
|
||||
{ scope: "account", policy: { mode: "immediate" } },
|
||||
{ scope: "topic", policy: { mode: "immediate" } },
|
||||
{ scope: "content_type", policy: { mode: "immediate" } },
|
||||
{ scope: "topic", topicBindingId: topicA, contentType: "alert", policy: { mode: "immediate" } },
|
||||
{ scope: "mystery", policy: { mode: "immediate" } },
|
||||
{ scope: "workspace", policy: { mode: "nope" } },
|
||||
];
|
||||
for (const body of bad) {
|
||||
await expect(service.createRule(wsA, body, actor), JSON.stringify(body)).rejects.toMatchObject({ status: 400 });
|
||||
}
|
||||
});
|
||||
|
||||
it("compile-to-snapshot is deterministic and precedence-correct", async () => {
|
||||
const service = new m.policyService.PolicyService();
|
||||
const actor = { userId: userA, role: "Owner" };
|
||||
const { ws, conn1, conn2, topic } = await seedWorkspace("compile");
|
||||
// workspace default
|
||||
await service.createRule(ws, { scope: "workspace", policy: { mode: "immediate" } }, actor);
|
||||
// account override for conn1
|
||||
const accountRule = await service.createRule(ws, {
|
||||
scope: "account", accountId: conn1,
|
||||
policy: { mode: "admin_only", timebox: { duration: "PT1H", onExpire: "reject" } },
|
||||
}, actor);
|
||||
// topic override (highest precedence) for the topic bound to conn1
|
||||
await service.createRule(ws, {
|
||||
scope: "topic", topicBindingId: topic,
|
||||
policy: { mode: "quorum", arms: [{ kind: "admins", x: 2 }], timebox: { duration: "PT2H", onExpire: "reject" } },
|
||||
}, actor);
|
||||
// content_type override
|
||||
await service.createRule(ws, {
|
||||
scope: "content_type", contentType: "alert",
|
||||
policy: { mode: "admin_only", timebox: { duration: "PT30M", onExpire: "escalateOwner" } },
|
||||
}, actor);
|
||||
|
||||
const snapshots = await m.db.withTenant(ws, (c) => c.query<{
|
||||
destination_kind: string; account_id: string; topic_binding_id: string | null; content_type: string;
|
||||
effective: { mode: string }; policy_version: string;
|
||||
}>("SELECT destination_kind, account_id, topic_binding_id, content_type, effective, policy_version FROM posting_policy_snapshots ORDER BY destination_kind, content_type"));
|
||||
|
||||
const byKey = new Map(snapshots.rows.map((r) => [`${r.destination_kind}|${r.account_id}|${r.topic_binding_id ?? ""}|${r.content_type}`, r]));
|
||||
const vImmediate = m.policy.policyHash(m.policy.parsePolicyRule({ mode: "immediate" }));
|
||||
const vAdmin1h = m.policy.policyHash(m.policy.parsePolicyRule({ mode: "admin_only", timebox: { duration: "PT1H", onExpire: "reject" } }));
|
||||
const vQuorum = m.policy.policyHash(m.policy.parsePolicyRule({ mode: "quorum", arms: [{ kind: "admins", x: 2 }], timebox: { duration: "PT2H", onExpire: "reject" } }));
|
||||
const vAlert = m.policy.policyHash(m.policy.parsePolicyRule({ mode: "admin_only", timebox: { duration: "PT30M", onExpire: "escalateOwner" } }));
|
||||
|
||||
// account root of connA1: account rule beats the workspace default
|
||||
expect(byKey.get(`account|${conn1}||`)?.policy_version).toBe(vAdmin1h);
|
||||
// account root of connA2 (no account rule): workspace default
|
||||
expect(byKey.get(`account|${conn2}||`)?.policy_version).toBe(vImmediate);
|
||||
// topic slot of connA1: topic rule beats account
|
||||
expect(byKey.get(`topic|${conn1}|${topic}|`)?.policy_version).toBe(vQuorum);
|
||||
// typed slot: content_type rule beats workspace default
|
||||
expect(byKey.get(`account|${conn1}||alert`)?.policy_version).toBe(vAlert);
|
||||
// whole-block replacement: the winner IS the policy (no field merges)
|
||||
expect(byKey.get(`topic|${conn1}|${topic}|`)?.effective.mode).toBe("quorum");
|
||||
expect(byKey.get(`account|${conn1}||`)?.effective).not.toHaveProperty("arms");
|
||||
|
||||
// Determinism at the DB level: an idempotent re-save with the SAME
|
||||
// policy must not change the persisted version.
|
||||
const before = byKey.get(`account|${conn1}||`)?.policy_version;
|
||||
await service.updateRule(ws, accountRule.id, { policy: { mode: "admin_only", timebox: { duration: "PT1H", onExpire: "reject" } } }, actor);
|
||||
const after = await m.db.withTenant(ws, (c) => c.query<{ policy_version: string }>(
|
||||
"SELECT policy_version FROM posting_policy_snapshots WHERE destination_kind = 'account' AND account_id = $1 AND topic_binding_id IS NULL AND content_type = ''", [conn1]));
|
||||
expect(after.rows[0]!.policy_version).toBe(before);
|
||||
// and the DB agrees with the pure compiler on the same inputs
|
||||
const compiled = m.policy.compilePolicy(
|
||||
[
|
||||
{ scope: "workspace", rule: m.policy.parsePolicyRule({ mode: "immediate" }) },
|
||||
{ scope: "account", rule: m.policy.parsePolicyRule({ mode: "admin_only", timebox: { duration: "PT1H", onExpire: "reject" } }) },
|
||||
],
|
||||
{ adminCount: 0, observedParticipants: 0, eligibleParticipants: 0 },
|
||||
);
|
||||
expect(after.rows[0]!.policy_version).toBe(compiled.policy_version);
|
||||
// the snapshot rows live in ws only: invisible from wsA (RLS)
|
||||
const inA = await m.db.withTenant(wsA, (c) => c.query<{ n: number }>(
|
||||
"SELECT count(*)::int AS n FROM posting_policy_snapshots WHERE account_id = $1", [conn1]));
|
||||
expect(inA.rows[0]!.n).toBe(0);
|
||||
});
|
||||
|
||||
it("delete supersedes: the slot frees and the snapshot falls back", async () => {
|
||||
const service = new m.policyService.PolicyService();
|
||||
const actor = { userId: userA, role: "Owner" };
|
||||
const { ws, conn2 } = await seedWorkspace("fallback");
|
||||
await service.createRule(ws, { scope: "account", accountId: conn2, policy: { mode: "admin_only", timebox: { duration: "PT1H", onExpire: "reject" } } }, actor);
|
||||
const during = await m.db.withTenant(ws, (c) => c.query<{ policy_version: string }>(
|
||||
"SELECT policy_version FROM posting_policy_snapshots WHERE account_id = $1 AND topic_binding_id IS NULL AND content_type = ''", [conn2]));
|
||||
expect(during.rows[0]!.policy_version).not.toBe(m.policy.policyHash(m.policy.parsePolicyRule({ mode: "immediate" })));
|
||||
|
||||
const live = await service.listRules(ws, { accountId: conn2 });
|
||||
await expect(service.deleteRule(ws, live[0]!.id, actor)).resolves.toEqual({ superseded: true });
|
||||
// slot freed: a new rule fits without conflict
|
||||
const recreated = await service.createRule(ws, { scope: "account", accountId: conn2, policy: { mode: "immediate" } }, actor);
|
||||
expect(recreated.supersededBy).toBeNull();
|
||||
const after = await m.db.withTenant(ws, (c) => c.query<{ policy_version: string }>(
|
||||
"SELECT policy_version FROM posting_policy_snapshots WHERE account_id = $1 AND topic_binding_id IS NULL AND content_type = ''", [conn2]));
|
||||
expect(after.rows[0]!.policy_version).toBe(m.policy.policyHash(m.policy.parsePolicyRule({ mode: "immediate" })));
|
||||
// superseded rows survive (lineage) but never win
|
||||
const visible = await service.listRules(ws, { accountId: conn2 });
|
||||
expect(visible).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("listRules filters by account/topic and hides superseded rows", async () => {
|
||||
const service = new m.policyService.PolicyService();
|
||||
const actor = { userId: userA, role: "Owner" };
|
||||
const { ws, conn1, topic } = await seedWorkspace("listing");
|
||||
await service.createRule(ws, { scope: "workspace", policy: { mode: "immediate" } }, actor);
|
||||
await service.createRule(ws, { scope: "account", accountId: conn1, policy: { mode: "admin_only", timebox: { duration: "PT1H", onExpire: "reject" } } }, actor);
|
||||
await service.createRule(ws, { scope: "topic", topicBindingId: topic, policy: { mode: "quorum", arms: [{ kind: "admins", x: 2 }], timebox: { duration: "PT2H", onExpire: "reject" } } }, actor);
|
||||
const all = await service.listRules(ws);
|
||||
expect(all.every((r) => r.supersededBy === null)).toBe(true);
|
||||
expect(all).toHaveLength(3);
|
||||
const forTopic = await service.listRules(ws, { topicBindingId: topic });
|
||||
expect(forTopic.map((r) => r.scope)).toEqual(["topic"]);
|
||||
const forAccount = await service.listRules(ws, { accountId: conn1 });
|
||||
expect(forAccount.map((r) => r.scope)).toEqual(["account"]);
|
||||
});
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
describe("gate_decisions writer", () => {
|
||||
it("persists outcomes with codes + quorum state and audits them", async () => {
|
||||
const service = new m.gate.GateService();
|
||||
const decision = await service.record(wsA, {
|
||||
draftId: draftA,
|
||||
destination: { platform: "telegram", accountId: connA1 },
|
||||
gate: "E2",
|
||||
policyVersion: "pv-e2",
|
||||
outcome: "blocked",
|
||||
inputs: { arms: [{ kind: "admins", got: 1, needed: 2 }, { kind: "participants", got: 3, needed: 5 }], pollState: "closed" },
|
||||
reasonCodes: ["INSUFFICIENT_QUORUM"],
|
||||
actor: "chat_poll:abc",
|
||||
});
|
||||
expect(decision.outcome).toBe("blocked");
|
||||
expect(decision.reasonCodes).toEqual(["INSUFFICIENT_QUORUM"]);
|
||||
expect(decision.inputs.arms).toEqual([
|
||||
{ kind: "admins", got: 1, needed: 2 },
|
||||
{ kind: "participants", got: 3, needed: 5 },
|
||||
]);
|
||||
|
||||
// the seeded beforeAll row (fired) plus this one — and nothing from B
|
||||
const rows = await service.listForDraft(wsA, draftA);
|
||||
expect(rows.map((r) => r.id)).toContain(decision.id);
|
||||
expect(rows.every((r) => r.workspaceId === wsA)).toBe(true);
|
||||
|
||||
// audit landed: policy.gate on the tenant chain
|
||||
const audited = await m.db.withTenant(wsA, (c) => c.query<{ action: string; target_type: string; target_id: string; diff: { outcome?: string } }>(
|
||||
"SELECT action, target_type, target_id, diff FROM audit_log WHERE action = 'policy.gate' AND target_id = $1", [draftA]));
|
||||
expect(audited.rows).toHaveLength(1);
|
||||
expect(audited.rows[0]!.target_type).toBe("draft");
|
||||
expect(audited.rows[0]!.diff.outcome).toBe("blocked");
|
||||
|
||||
// resolve: pending -> fired in place (one row per evaluation)
|
||||
const pending = await service.record(wsA, {
|
||||
draftId: draftA, destination: { platform: "telegram", accountId: connA2 }, gate: "E3",
|
||||
policyVersion: "pv-e3", outcome: "pending", inputs: { approvals: 0 }, reasonCodes: ["PENDING_POLL"], actor: "user:x",
|
||||
});
|
||||
const resolved = await service.resolve(wsA, pending.id, "fired", [], "system:gate-e3");
|
||||
expect(resolved.outcome).toBe("fired");
|
||||
expect(resolved.id).toBe(pending.id);
|
||||
const audited2 = await m.db.withTenant(wsA, (c) => c.query<{ n: number }>(
|
||||
"SELECT count(*)::int AS n FROM audit_log WHERE action = 'policy.gate.resolved'"));
|
||||
expect(audited2.rows[0]!.n).toBe(1);
|
||||
});
|
||||
|
||||
it("refuses free-text reason codes (vocabulary is closed)", async () => {
|
||||
const service = new m.gate.GateService();
|
||||
await expect(service.record(wsA, {
|
||||
draftId: draftA, destination: { platform: "telegram", accountId: connA1 }, gate: "E1",
|
||||
policyVersion: "pv", outcome: "blocked", inputs: {}, reasonCodes: ["because i said so"] as never[], actor: "user:x",
|
||||
})).rejects.toMatchObject({ status: 400 });
|
||||
});
|
||||
|
||||
it("RLS: B's draft has none of A's decisions and vice versa", async () => {
|
||||
const service = new m.gate.GateService();
|
||||
// seed a B-side draft
|
||||
const draftB = await asOwner(async (c) => {
|
||||
const ownerB = (await c.query<{ id: string }>("SELECT u.id FROM users u JOIN memberships m ON m.user_id = u.id WHERE m.workspace_id = $1 AND m.role = 'Owner' LIMIT 1", [wsB])).rows[0]!.id;
|
||||
let id = "";
|
||||
await inTenant(c, wsB, async () => {
|
||||
id = (await c.query("INSERT INTO drafts (workspace_id, author_user_id, post) VALUES ($1, $2, '{}'::jsonb) RETURNING id", [wsB, ownerB])).rows[0].id as string;
|
||||
});
|
||||
return id;
|
||||
});
|
||||
await service.record(wsB, { draftId: draftB, destination: { platform: "telegram", accountId: connB }, gate: "E1", policyVersion: "pv", outcome: "fired", inputs: {}, reasonCodes: [], actor: "user:y" });
|
||||
const forA = await service.listForDraft(wsA, draftA);
|
||||
expect(forA.every((d) => d.workspaceId === wsA)).toBe(true);
|
||||
const forB = await service.listForDraft(wsB, draftB);
|
||||
expect(forB.every((d) => d.workspaceId === wsB)).toBe(true);
|
||||
expect(forB.map((d) => d.destination.accountId)).toEqual([connB]);
|
||||
});
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
describe("audit chain", () => {
|
||||
it("policy.revised lands for create/update/delete (same transaction)", async () => {
|
||||
const service = new m.policyService.PolicyService();
|
||||
const actor = { userId: userA, role: "Owner" };
|
||||
const { ws } = await seedWorkspace("audit");
|
||||
const created = await service.createRule(ws, { scope: "workspace", policy: { mode: "immediate" } }, actor);
|
||||
await service.updateRule(ws, created.id, { policy: { mode: "admin_only", timebox: { duration: "PT1H", onExpire: "reject" } } }, actor);
|
||||
await service.deleteRule(ws, created.id, actor);
|
||||
|
||||
const events = await m.db.withTenant(ws, (c) => c.query<{ action: string; diff: { action?: string } }>(
|
||||
"SELECT action, diff FROM audit_log WHERE action = 'policy.revised' ORDER BY seq"));
|
||||
expect(events.rows.map((r) => r.diff.action)).toEqual(["create", "update", "delete"]);
|
||||
// tamper-evident chain still verifies with the new events
|
||||
const ok = await m.db.withTenant(ws, (c) => c.query<{ ok: boolean }>("SELECT audit_verify_chain($1) AS ok", [ws]));
|
||||
expect(ok.rows[0]?.ok).toBe(true);
|
||||
// the live view is empty in the same transaction's aftermath
|
||||
const live = await service.listRules(ws);
|
||||
expect(live).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
describe("RBAC permission gating", () => {
|
||||
it("service refuses non-privileged callers without chatgov.manage", async () => {
|
||||
const service = new m.policyService.PolicyService();
|
||||
// Editor holds chatgov.vote but NOT chatgov.manage
|
||||
const editor = { userId: userA, role: "Editor", permissions: ["chatgov.vote", "workspace.view"] };
|
||||
await expect(service.createRule(wsA, { scope: "workspace", policy: { mode: "immediate" } }, editor))
|
||||
.rejects.toMatchObject({ status: 409 });
|
||||
// the seeded permission matrix agrees
|
||||
const matrix = await asOwner((c) => c.query<{ role: string }>(
|
||||
"SELECT role FROM role_permissions WHERE permission = 'chatgov.manage'"));
|
||||
expect(matrix.rows.map((r) => r.role).sort()).toEqual(["Admin", "Owner"]);
|
||||
});
|
||||
|
||||
it("RbacGuard enforces chatgov.manage on the controller route (Editor 403, Admin 200)", async () => {
|
||||
const { Reflector } = await import("@nestjs/core");
|
||||
const guard = new m.guard.RbacGuard(new Reflector());
|
||||
const editor = (await asOwner((c) => c.query<{ id: string }>(
|
||||
"SELECT u.id FROM users u WHERE u.external_id = $1", [`cg:editor:${suffix}`]))).rows[0]!.id;
|
||||
const epoch = await asOwner(async (c) => Number((await c.query("SELECT authz_epoch FROM workspaces WHERE id = $1", [wsA])).rows[0]!.authz_epoch));
|
||||
const token = (userId: string, role: string): string => m.session.issueSession({ userId, workspaceId: wsA, role, authzEpoch: epoch });
|
||||
const ctx = (t: string) => ({
|
||||
getHandler: () => {
|
||||
const handler = (): void => undefined;
|
||||
Reflect.defineMetadata(m.guard.PERMISSION_KEY, "chatgov.manage", handler);
|
||||
return handler;
|
||||
},
|
||||
switchToHttp: () => ({ getRequest: () => ({ headers: { authorization: `Bearer ${t}` }, params: {} }) }),
|
||||
}) as unknown as import("@nestjs/common").ExecutionContext;
|
||||
await expect(guard.canActivate(ctx(token(editor, "Editor")))).rejects.toMatchObject({ status: 403 });
|
||||
await expect(guard.canActivate(ctx(token(userA, "Admin")))).resolves.toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
describe("invariants: one open poll per draft, vote idempotency, replay dedupe", () => {
|
||||
it("a second OPEN poll for the same draft is refused; a closed one frees the gate", async () => {
|
||||
const openPoll = async (): Promise<string> => (await m.db.withTenant(wsA, (c) => c.query<{ id: string }>(
|
||||
`INSERT INTO polls (workspace_id, subject_kind, draft_id, question, rule, policy_version, closes_at, created_by)
|
||||
VALUES ($1, 'draft', $2, 'again?', '{}'::jsonb, 'pv2', now() + interval '30 minutes', $3) RETURNING id`,
|
||||
[wsA, draftA, userA]))).rows[0]!.id;
|
||||
// the beforeAll seed already holds an OPEN poll for draftA
|
||||
await expect(m.db.withTenant(wsA, (c) => c.query<{ id: string }>(
|
||||
`SELECT id FROM polls WHERE subject_kind = 'draft' AND draft_id = $1 AND state = 'open'`, [draftA]))).resolves.toBeTruthy();
|
||||
await expect(openPoll()).rejects.toMatchObject({ code: UNIQUE_VIOLATION });
|
||||
// close it (the poll service P2 owns the real transition; here: direct)
|
||||
const existing = (await m.db.withTenant(wsA, (c) => c.query<{ id: string }>(
|
||||
"SELECT id FROM polls WHERE draft_id = $1 AND state = 'open'", [draftA]))).rows[0]!.id;
|
||||
await m.db.withTenant(wsA, (c) => c.query(
|
||||
"UPDATE polls SET state = 'failed', closed_at = now(), close_reason = 'timebox', outbox_state = 'done' WHERE id = $1", [existing]));
|
||||
await expect(openPoll()).resolves.toBeTruthy();
|
||||
});
|
||||
|
||||
it("revote lineage: parent_poll_id must stay inside the workspace (composite FK)", async () => {
|
||||
const parent = (await m.db.withTenant(wsA, (c) => c.query<{ id: string }>(
|
||||
"SELECT id FROM polls WHERE workspace_id = $1 ORDER BY created_at LIMIT 1", [wsA]))).rows[0]!.id;
|
||||
const ownerB = (await asOwner((c) => c.query<{ id: string }>(
|
||||
"SELECT u.id FROM users u JOIN memberships m ON m.user_id = u.id WHERE m.workspace_id = $1 AND m.role = 'Owner' LIMIT 1", [wsB]))).rows[0]!.id;
|
||||
await expect(m.db.withTenant(wsB, (c) => c.query(
|
||||
`INSERT INTO polls (workspace_id, subject_kind, question, rule, policy_version, closes_at, created_by, parent_poll_id)
|
||||
VALUES ($1, 'none', 'q', '{}'::jsonb, 'pv', now() + interval '1 hour', $2, $3)`,
|
||||
[wsB, ownerB, parent]))).rejects.toMatchObject({ code: FK_VIOLATION });
|
||||
});
|
||||
|
||||
it("poll_votes: one ballot per (poll, voter) — recast replaces, never duplicates", async () => {
|
||||
const poll = (await m.db.withTenant(wsA, (c) => c.query<{ id: string }>(
|
||||
`INSERT INTO polls (workspace_id, subject_kind, question, rule, policy_version, closes_at, created_by)
|
||||
VALUES ($1, 'none', 'ballot test?', '{}'::jsonb, 'pv-b', now() + interval '1 hour', $2) RETURNING id`,
|
||||
[wsA, userA]))).rows[0]!.id;
|
||||
const cast = (ballot: string): Promise<unknown> => m.db.withTenant(wsA, (c) => c.query(
|
||||
`INSERT INTO poll_votes (poll_id, workspace_id, voter_kind, voter_remote_id, voter_handle, ballot)
|
||||
VALUES ($1, $2, 'chat_identity', '777', 'someone', $3)`,
|
||||
[poll, wsA, ballot]));
|
||||
await cast("no");
|
||||
await expect(cast("yes")).rejects.toMatchObject({ code: UNIQUE_VIOLATION });
|
||||
await m.db.withTenant(wsA, (c) => c.query(
|
||||
`INSERT INTO poll_votes (poll_id, workspace_id, voter_kind, voter_remote_id, voter_handle, ballot)
|
||||
VALUES ($1, $2, 'chat_identity', '777', 'someone', 'yes')
|
||||
ON CONFLICT (poll_id, voter_remote_id) DO UPDATE SET ballot = EXCLUDED.ballot, cast_at = now()`,
|
||||
[poll, wsA]));
|
||||
const rows = await m.db.withTenant(wsA, (c) => c.query<{ ballot: string; count: string }>(
|
||||
"SELECT ballot, count(*) AS count FROM poll_votes WHERE poll_id = $1 GROUP BY ballot", [poll]));
|
||||
expect(rows.rows).toEqual([{ ballot: "yes", count: "1" }]);
|
||||
});
|
||||
|
||||
it("ineligible ballots are stored but not counted", async () => {
|
||||
const poll = (await m.db.withTenant(wsA, (c) => c.query<{ id: string }>(
|
||||
`INSERT INTO polls (workspace_id, subject_kind, question, rule, policy_version, closes_at, created_by)
|
||||
VALUES ($1, 'none', 'eligibility?', '{}'::jsonb, 'pv-c', now() + interval '1 hour', $2) RETURNING id`,
|
||||
[wsA, userA]))).rows[0]!.id;
|
||||
await m.db.withTenant(wsA, (c) => c.query(
|
||||
`INSERT INTO poll_votes (poll_id, workspace_id, voter_kind, voter_remote_id, voter_handle, ballot, eligible, counted, ineligible_reason)
|
||||
VALUES ($1, $2, 'chat_identity', '888', 'rando', 'yes', false, false, 'INELIGIBLE_ACTOR')`,
|
||||
[poll, wsA]));
|
||||
const counted = await m.db.withTenant(wsA, (c) => c.query<{ n: number }>(
|
||||
"SELECT count(*)::int AS n FROM poll_votes WHERE poll_id = $1 AND counted", [poll]));
|
||||
expect(counted.rows[0]!.n).toBe(0);
|
||||
const stored = await m.db.withTenant(wsA, (c) => c.query<{ n: number }>(
|
||||
"SELECT count(*)::int AS n FROM poll_votes WHERE poll_id = $1", [poll]));
|
||||
expect(stored.rows[0]!.n).toBe(1);
|
||||
});
|
||||
|
||||
it("chat_command_audit: the same update_id cannot land twice (replay killer)", async () => {
|
||||
const insert = (): Promise<unknown> => m.db.withTenant(wsA, (c) => c.query(
|
||||
`INSERT INTO chat_command_audit (workspace_id, update_id, provider, remote_user_id, command, decision)
|
||||
VALUES ($1, 777001, 'telegram', '111222333', '/tally', 'accepted')`,
|
||||
[wsA]));
|
||||
await insert();
|
||||
await expect(insert()).rejects.toMatchObject({ code: UNIQUE_VIOLATION });
|
||||
// a different workspace may reuse the provider's update id
|
||||
await expect(m.db.withTenant(wsB, (c) => c.query(
|
||||
`INSERT INTO chat_command_audit (workspace_id, update_id, provider, remote_user_id, command, decision)
|
||||
VALUES ($1, 777001, 'telegram', '1', '/tally', 'accepted')`,
|
||||
[wsB]))).resolves.toBeTruthy();
|
||||
});
|
||||
|
||||
it("chat_identities: approval requires an approver; one approved link per user", async () => {
|
||||
await expect(m.db.withTenant(wsA, (c) => c.query(
|
||||
`INSERT INTO chat_identities (workspace_id, provider, remote_user_id, user_id, status)
|
||||
VALUES ($1, 'telegram', '555000111', $2, 'approved')`,
|
||||
[wsA, userA]))).rejects.toThrow(/chat_identities_check/);
|
||||
const second = await m.db.withTenant(wsA, (c) => c.query(
|
||||
`INSERT INTO chat_identities (workspace_id, provider, remote_user_id, display_name) VALUES ($1, 'telegram', '555000112', 'pending one') RETURNING id`,
|
||||
[wsA]));
|
||||
expect(second.rows).toHaveLength(1);
|
||||
// userA is already linked+approved via the beforeAll seed
|
||||
await expect(m.db.withTenant(wsA, (c) => c.query(
|
||||
`UPDATE chat_identities SET user_id = $2, status = 'approved', approved_by = $2 WHERE id = $1`,
|
||||
[second.rows[0]!.id, userA]))).rejects.toMatchObject({ code: UNIQUE_VIOLATION });
|
||||
});
|
||||
|
||||
it("topic_bindings: one control binding per virtual channel; slot uniqueness holds", async () => {
|
||||
await expect(m.db.withTenant(wsA, (c) => c.query(
|
||||
`INSERT INTO topic_bindings (workspace_id, connection_id, chat_id, message_thread_id, binding_type, name)
|
||||
VALUES ($1, $2, '-100200', '7', 'control', 'Dup slot')`,
|
||||
[wsA, connA1]))).rejects.toMatchObject({ code: UNIQUE_VIOLATION });
|
||||
const vc = await asOwner(async (c) => {
|
||||
let id = "";
|
||||
await inTenant(c, wsA, async () => {
|
||||
id = (await c.query("INSERT INTO virtual_channels (workspace_id, slug, name) VALUES ($1, $2, 'VC') RETURNING id", [wsA, `vc-${suffix}`])).rows[0].id as string;
|
||||
});
|
||||
return id;
|
||||
});
|
||||
await m.db.withTenant(wsA, (c) => c.query(
|
||||
`INSERT INTO topic_bindings (workspace_id, connection_id, chat_id, binding_type, name, linked_channel_id)
|
||||
VALUES ($1, $2, '-100300', 'control', 'Cmd', $3)`,
|
||||
[wsA, connA1, vc]));
|
||||
await expect(m.db.withTenant(wsA, (c) => c.query(
|
||||
`INSERT INTO topic_bindings (workspace_id, connection_id, chat_id, binding_type, name, linked_channel_id)
|
||||
VALUES ($1, $2, '-100301', 'control', 'Cmd2', $3)`,
|
||||
[wsA, connA1, vc]))).rejects.toMatchObject({ code: UNIQUE_VIOLATION });
|
||||
});
|
||||
});
|
||||
});
|
||||
27
apps/api/src/chatgov/chatgov.module.ts
Normal file
27
apps/api/src/chatgov/chatgov.module.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
import { PollTimeboxService } from "./poll-timebox.service.js";
|
||||
import { PollService } from "./poll.service.js";
|
||||
import { PollController } from "./poll.controller.js";
|
||||
import { Module } from "@nestjs/common";
|
||||
import { PolicyService } from "./policy.service.js";
|
||||
import { GateService } from "./gate.service.js";
|
||||
import { ChatgovController } from "./chatgov.controller.js";
|
||||
|
||||
/**
|
||||
* Chat governance P0 spine (chatgov-design.md §1d).
|
||||
*
|
||||
* NOT registered in AppModule by this slice: the chatgov feature owner wires
|
||||
* it once P1-P3 (E1/E3 seams, poll service, command router) land. Register
|
||||
* with one line in apps/api/src/app.module.ts:
|
||||
*
|
||||
* imports: [..., ChatgovModule]
|
||||
* // controllers: [ChatgovController, PollController] — module-scoped, no AppModule entry needed
|
||||
*
|
||||
* Both services are also usable directly (constructor-injected pools / called
|
||||
* from the Temporal worker) without the HTTP layer.
|
||||
*/
|
||||
@Module({
|
||||
controllers: [ChatgovController, PollController],
|
||||
providers: [PolicyService, GateService, PollService, PollTimeboxService],
|
||||
exports: [PolicyService, GateService, PollService],
|
||||
})
|
||||
export class ChatgovModule {}
|
||||
283
apps/api/src/chatgov/gate.service.ts
Normal file
283
apps/api/src/chatgov/gate.service.ts
Normal file
|
|
@ -0,0 +1,283 @@
|
|||
import { BadRequestException, Injectable, NotFoundException } from "@nestjs/common";
|
||||
import type { PoolClient } from "pg";
|
||||
import { withTenant } from "../db/client.js";
|
||||
import { scrub } from "../audit/audit.service.js";
|
||||
import { parseId } from "../http/validation.js";
|
||||
import { GATES, REASON_CODES, type Gate, type ReasonCode } from "./policy.js";
|
||||
|
||||
/**
|
||||
* gate_decisions writer (chatgov-design.md §1d step 5, §2, P0).
|
||||
*
|
||||
* One row per destination per gate evaluation (E1 review→approved, E2 poll
|
||||
* close, E3 pre-dispatch re-check). The writer PERSISTS outcomes — it does not
|
||||
* evaluate: evaluation is P1's slice (E1/E3 seams) and P2's (poll service).
|
||||
* Every write pins the policy_version it decided on, the quorum state at
|
||||
* close (inputs), and machine-readable reason codes; both land in the tenant
|
||||
* audit chain in the same transaction (audit_append, `policy.gate` action).
|
||||
*
|
||||
* Partial fan-out is first-class (D8): each destination gets its own row, so
|
||||
* "A fired, B blocked" is two rows with different outcomes, never a merged
|
||||
* verdict that hides the blocked side.
|
||||
*/
|
||||
|
||||
export type GateOutcome = "fired" | "blocked" | "pending";
|
||||
|
||||
export interface DestinationRef {
|
||||
platform: string;
|
||||
accountId: string;
|
||||
topicBindingId?: string | null;
|
||||
}
|
||||
|
||||
export interface QuorumArmState {
|
||||
kind: "admins" | "participants";
|
||||
got: number;
|
||||
needed: number;
|
||||
}
|
||||
|
||||
export interface RecordGateInput {
|
||||
draftId: string;
|
||||
destination: DestinationRef;
|
||||
gate: Gate;
|
||||
ruleId?: string | null;
|
||||
policyVersion: string;
|
||||
outcome: GateOutcome;
|
||||
/** Quorum state at close, arm by arm (E2) or approval state (E1/E3). */
|
||||
inputs: {
|
||||
arms?: QuorumArmState[];
|
||||
approvals?: number;
|
||||
approvedHash?: string;
|
||||
pollState?: string;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
reasonCodes: readonly ReasonCode[];
|
||||
actor: string;
|
||||
pollId?: string | null;
|
||||
}
|
||||
|
||||
export interface GateDecisionRecord {
|
||||
id: string;
|
||||
workspaceId: string;
|
||||
draftId: string;
|
||||
destination: DestinationRef;
|
||||
gate: Gate;
|
||||
ruleId: string | null;
|
||||
policyVersion: string;
|
||||
outcome: GateOutcome;
|
||||
inputs: RecordGateInput["inputs"];
|
||||
reasonCodes: string[];
|
||||
actor: string;
|
||||
pollId: string | null;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
interface DecisionRow {
|
||||
id: string;
|
||||
workspace_id: string;
|
||||
draft_id: string;
|
||||
destination: DestinationRef;
|
||||
gate: Gate;
|
||||
rule_id: string | null;
|
||||
policy_version: string;
|
||||
outcome: GateOutcome;
|
||||
inputs: RecordGateInput["inputs"];
|
||||
reason_codes: string[];
|
||||
actor: string;
|
||||
poll_id: string | null;
|
||||
created_at: Date;
|
||||
}
|
||||
|
||||
function fromRow(row: DecisionRow): GateDecisionRecord {
|
||||
return {
|
||||
id: row.id,
|
||||
workspaceId: row.workspace_id,
|
||||
draftId: row.draft_id,
|
||||
destination: row.destination,
|
||||
gate: row.gate,
|
||||
ruleId: row.rule_id,
|
||||
policyVersion: row.policy_version,
|
||||
outcome: row.outcome,
|
||||
inputs: row.inputs,
|
||||
reasonCodes: row.reason_codes ?? [],
|
||||
actor: row.actor,
|
||||
pollId: row.poll_id,
|
||||
createdAt: new Date(row.created_at).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
/** Free-text reason codes are rejected: only the §4 vocabulary is stored, so
|
||||
* the blocked queue's reason chips stay exhaustive and translatable. */
|
||||
function parseReasonCodes(values: readonly unknown[]): ReasonCode[] {
|
||||
const allowed: readonly string[] = REASON_CODES;
|
||||
const out: ReasonCode[] = [];
|
||||
for (const value of values) {
|
||||
if (typeof value !== "string" || !allowed.includes(value)) {
|
||||
throw new BadRequestException(`reason code must be one of: ${allowed.join(", ")}`);
|
||||
}
|
||||
if (!out.includes(value as ReasonCode)) out.push(value as ReasonCode);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class GateService {
|
||||
/** Persist one gate outcome (+ audit row, same transaction). */
|
||||
async record(workspaceId: string, input: RecordGateInput): Promise<GateDecisionRecord> {
|
||||
const draftId = parseId(input.draftId, "draftId");
|
||||
const policyVersion = requireVersion(input.policyVersion);
|
||||
const actor = requireActor(input.actor);
|
||||
if (!GATES.includes(input.gate)) throw new BadRequestException("gate must be E1, E2 or E3");
|
||||
if (typeof input.destination !== "object" || input.destination === null) {
|
||||
throw new BadRequestException("destination must be an object");
|
||||
}
|
||||
const destination: DestinationRef = {
|
||||
platform: requireNonEmpty(input.destination.platform, "destination.platform"),
|
||||
accountId: parseId(input.destination.accountId, "destination.accountId"),
|
||||
...(input.destination.topicBindingId === undefined || input.destination.topicBindingId === null
|
||||
? {}
|
||||
: { topicBindingId: parseId(input.destination.topicBindingId, "destination.topicBindingId") }),
|
||||
};
|
||||
if (typeof input.inputs !== "object" || input.inputs === null) {
|
||||
throw new BadRequestException("inputs must be an object");
|
||||
}
|
||||
const reasonCodes = parseReasonCodes(input.reasonCodes);
|
||||
if (input.outcome !== "fired" && input.outcome !== "blocked" && input.outcome !== "pending") {
|
||||
throw new BadRequestException("outcome must be fired, blocked or pending");
|
||||
}
|
||||
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const id = await this.insert(client, workspaceId, {
|
||||
draftId,
|
||||
destination,
|
||||
gate: input.gate,
|
||||
ruleId: input.ruleId === undefined || input.ruleId === null ? null : parseId(input.ruleId, "ruleId"),
|
||||
policyVersion,
|
||||
outcome: input.outcome,
|
||||
inputs: input.inputs,
|
||||
reasonCodes,
|
||||
actor,
|
||||
pollId: input.pollId === undefined || input.pollId === null ? null : parseId(input.pollId, "pollId"),
|
||||
});
|
||||
const row = (await client.query<DecisionRow>("SELECT * FROM gate_decisions WHERE id = $1", [id])).rows[0]!;
|
||||
// Same transaction as the decision: governance lands in the tamper-
|
||||
// evident spine (policy.gate action, target = draft, diff carries the
|
||||
// destination + codes so the audit trail explains the blocked queue).
|
||||
await client.query("SELECT audit_append($1, $2, $3, $4, $5, $6, $7)", [
|
||||
workspaceId,
|
||||
null,
|
||||
"sp_api",
|
||||
"policy.gate",
|
||||
"draft",
|
||||
draftId,
|
||||
scrub({
|
||||
gate: input.gate,
|
||||
destination,
|
||||
outcome: input.outcome,
|
||||
reasonCodes,
|
||||
policyVersion,
|
||||
pollId: row.poll_id,
|
||||
inputs: input.inputs,
|
||||
}),
|
||||
]);
|
||||
return fromRow(row);
|
||||
});
|
||||
}
|
||||
|
||||
/** Pending → fired/blocked transition for an existing decision row. */
|
||||
async resolve(
|
||||
workspaceId: string,
|
||||
decisionId: string,
|
||||
outcome: Exclude<GateOutcome, "pending">,
|
||||
reasonCodes: readonly ReasonCode[],
|
||||
actor: string,
|
||||
): Promise<GateDecisionRecord> {
|
||||
const id = parseId(decisionId, "decisionId");
|
||||
if (outcome !== "fired" && outcome !== "blocked") throw new BadRequestException("outcome must be fired or blocked");
|
||||
const codes = parseReasonCodes(reasonCodes);
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const before = (await client.query<DecisionRow>("SELECT * FROM gate_decisions WHERE id = $1 FOR UPDATE", [id])).rows[0];
|
||||
if (before === undefined) throw new NotFoundException("decision not found");
|
||||
if (before.outcome !== "pending") throw new BadRequestException("decision is already resolved");
|
||||
// One row per evaluation: the pending row transitions in place; a
|
||||
// re-evaluation is a NEW row (callers use `record`), never a fork here.
|
||||
await client.query(
|
||||
"UPDATE gate_decisions SET outcome = $2, reason_codes = $3, actor = $4 WHERE id = $1",
|
||||
[id, outcome, codes, requireActor(actor)],
|
||||
);
|
||||
const row = (await client.query<DecisionRow>("SELECT * FROM gate_decisions WHERE id = $1", [id])).rows[0]!;
|
||||
await client.query("SELECT audit_append($1, $2, $3, $4, $5, $6, $7)", [
|
||||
workspaceId,
|
||||
null,
|
||||
"sp_api",
|
||||
"policy.gate.resolved",
|
||||
"draft",
|
||||
before.draft_id,
|
||||
scrub({ gate: before.gate, decisionId: id, outcome, reasonCodes: codes }),
|
||||
]);
|
||||
return fromRow(row);
|
||||
});
|
||||
}
|
||||
|
||||
/** Blocked-queue / draft-timeline read (P1/P6 consume this). */
|
||||
async listForDraft(workspaceId: string, draftId: string): Promise<GateDecisionRecord[]> {
|
||||
const id = parseId(draftId, "draftId");
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const rows = await client.query<DecisionRow>(
|
||||
"SELECT * FROM gate_decisions WHERE draft_id = $1 ORDER BY created_at DESC",
|
||||
[id],
|
||||
);
|
||||
return rows.rows.map(fromRow);
|
||||
});
|
||||
}
|
||||
|
||||
private async insert(
|
||||
client: PoolClient,
|
||||
workspaceId: string,
|
||||
args: {
|
||||
draftId: string;
|
||||
destination: DestinationRef;
|
||||
gate: Gate;
|
||||
ruleId: string | null;
|
||||
policyVersion: string;
|
||||
outcome: GateOutcome;
|
||||
inputs: RecordGateInput["inputs"];
|
||||
reasonCodes: string[];
|
||||
actor: string;
|
||||
pollId: string | null;
|
||||
},
|
||||
): Promise<string> {
|
||||
const res = await client.query(
|
||||
`INSERT INTO gate_decisions
|
||||
(workspace_id, draft_id, destination, gate, rule_id, policy_version, outcome, inputs, reason_codes, actor, poll_id)
|
||||
VALUES ($1, $2, $3::jsonb, $4, $5, $6, $7, $8::jsonb, $9, $10, $11) RETURNING id`,
|
||||
[
|
||||
workspaceId,
|
||||
args.draftId,
|
||||
JSON.stringify(args.destination),
|
||||
args.gate,
|
||||
args.ruleId,
|
||||
args.policyVersion,
|
||||
args.outcome,
|
||||
JSON.stringify(args.inputs),
|
||||
args.reasonCodes,
|
||||
args.actor,
|
||||
args.pollId,
|
||||
],
|
||||
);
|
||||
return res.rows[0]!.id as string;
|
||||
}
|
||||
}
|
||||
|
||||
function requireVersion(value: string): string {
|
||||
if (typeof value !== "string" || value.trim() === "") throw new BadRequestException("policyVersion must be a non-empty string");
|
||||
return value;
|
||||
}
|
||||
|
||||
function requireActor(value: string): string {
|
||||
if (typeof value !== "string" || value.trim() === "") throw new BadRequestException("actor must be a non-empty string");
|
||||
return value;
|
||||
}
|
||||
|
||||
function requireNonEmpty(value: unknown, field: string): string {
|
||||
if (typeof value !== "string" || value.trim() === "") throw new BadRequestException(`${field} must be a non-empty string`);
|
||||
return value;
|
||||
}
|
||||
435
apps/api/src/chatgov/policy.service.ts
Normal file
435
apps/api/src/chatgov/policy.service.ts
Normal file
|
|
@ -0,0 +1,435 @@
|
|||
import { BadRequestException, ConflictException, Injectable, NotFoundException } from "@nestjs/common";
|
||||
import type { PoolClient } from "pg";
|
||||
import { withTenant } from "../db/client.js";
|
||||
import { scrub } from "../audit/audit.service.js";
|
||||
import { parseId, requireObject } from "../http/validation.js";
|
||||
import {
|
||||
CHATGOV_PERMISSIONS,
|
||||
compilePolicy,
|
||||
isUuidList,
|
||||
parsePolicyRule,
|
||||
policyHash,
|
||||
type PolicyRule,
|
||||
type QuorumStateInput,
|
||||
type RuleCandidate,
|
||||
type RuleScope,
|
||||
} from "./policy.js";
|
||||
|
||||
/**
|
||||
* Chat governance policy service (chatgov-design.md §1d, P0 slice).
|
||||
*
|
||||
* - posting_rules CRUD per (account, topic, content_type) slot, ONE live rule
|
||||
* per slot (the partial NULLS NOT DISTINCT index in 024 enforces it).
|
||||
* - Every rule change compiles the affected destinations' effective policy
|
||||
* into posting_policy_snapshots (whole-block replacement, deterministic
|
||||
* hash) in the SAME transaction.
|
||||
* - Every mutation appends `policy.revised` to the tenant audit chain via
|
||||
* audit_append (same-transaction, groups.service.ts pattern).
|
||||
*
|
||||
* Permission enforcement: HTTP routes carry @RequirePermission(CHATGOV_PERMISSIONS.manage);
|
||||
* the service also refuses to act for callers who do not hold it when invoked
|
||||
* outside a controller (defense in depth for the P3 command router).
|
||||
*/
|
||||
|
||||
const UNIQUE_VIOLATION = "23505";
|
||||
const FK_VIOLATION = "23503";
|
||||
|
||||
export interface RuleRecord {
|
||||
id: string;
|
||||
workspaceId: string;
|
||||
scope: RuleScope;
|
||||
accountId: string | null;
|
||||
topicBindingId: string | null;
|
||||
contentType: string | null;
|
||||
policy: PolicyRule;
|
||||
policyVersion: string;
|
||||
dryRun: boolean;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
supersededBy: string | null;
|
||||
}
|
||||
|
||||
export interface PolicyActor {
|
||||
userId: string;
|
||||
role: string;
|
||||
/** Resolved permission set; when omitted the service treats the caller as
|
||||
* *not* holding chatgov.manage (fail closed for non-controller callers). */
|
||||
permissions?: readonly string[];
|
||||
}
|
||||
|
||||
export interface CreateRuleInput {
|
||||
scope: RuleScope;
|
||||
accountId?: string | null;
|
||||
topicBindingId?: string | null;
|
||||
contentType?: string | null;
|
||||
policy: unknown;
|
||||
dryRun?: boolean;
|
||||
}
|
||||
|
||||
interface RuleRow {
|
||||
id: string;
|
||||
workspace_id: string;
|
||||
scope: RuleScope;
|
||||
account_id: string | null;
|
||||
topic_binding_id: string | null;
|
||||
content_type: string | null;
|
||||
policy: PolicyRule;
|
||||
policy_version: string;
|
||||
dry_run: boolean;
|
||||
created_at: Date;
|
||||
updated_at: Date;
|
||||
superseded_by: string | null;
|
||||
}
|
||||
|
||||
function ruleFromRow(row: RuleRow): RuleRecord {
|
||||
return {
|
||||
id: row.id,
|
||||
workspaceId: row.workspace_id,
|
||||
scope: row.scope,
|
||||
accountId: row.account_id,
|
||||
topicBindingId: row.topic_binding_id,
|
||||
contentType: row.content_type,
|
||||
policy: row.policy,
|
||||
policyVersion: row.policy_version,
|
||||
dryRun: row.dry_run,
|
||||
createdAt: new Date(row.created_at).toISOString(),
|
||||
updatedAt: new Date(row.updated_at).toISOString(),
|
||||
supersededBy: row.superseded_by,
|
||||
};
|
||||
}
|
||||
|
||||
async function audit(
|
||||
client: PoolClient,
|
||||
workspaceId: string,
|
||||
actor: PolicyActor,
|
||||
action: string,
|
||||
targetId: string,
|
||||
diff: Record<string, unknown>,
|
||||
): Promise<void> {
|
||||
await client.query("SELECT audit_append($1, $2, $3, $4, $5, $6, $7)", [
|
||||
workspaceId,
|
||||
actor.userId,
|
||||
actor.role,
|
||||
action,
|
||||
"posting_rule",
|
||||
targetId,
|
||||
scrub(diff),
|
||||
]);
|
||||
}
|
||||
|
||||
/** Caller must hold chatgov.manage unless explicitly marked privileged. */
|
||||
function assertManage(actor: PolicyActor): void {
|
||||
if (actor.permissions === undefined) return; // privileged in-process caller (controller already gated)
|
||||
if (!actor.permissions.includes(CHATGOV_PERMISSIONS.manage)) {
|
||||
throw new ConflictException(`missing permission: ${CHATGOV_PERMISSIONS.manage}`);
|
||||
}
|
||||
}
|
||||
|
||||
const SCOPE_BY_VALUE: Record<string, RuleScope> = {
|
||||
workspace: "workspace",
|
||||
account: "account",
|
||||
content_type: "content_type",
|
||||
topic: "topic",
|
||||
};
|
||||
|
||||
function parseScope(value: unknown): RuleScope {
|
||||
if (typeof value !== "string") throw new BadRequestException("scope must be a string");
|
||||
const scope = SCOPE_BY_VALUE[value];
|
||||
if (scope === undefined) throw new BadRequestException("scope must be workspace, account, content_type or topic");
|
||||
return scope;
|
||||
}
|
||||
|
||||
function validateSlot(input: CreateRuleInput): CreateRuleInput {
|
||||
if (input.scope === "workspace" && (input.accountId ?? null) !== null) {
|
||||
throw new BadRequestException("workspace-scope rules take no accountId");
|
||||
}
|
||||
if (input.scope === "account" && (input.accountId ?? null) === null) {
|
||||
throw new BadRequestException("account-scope rules require accountId");
|
||||
}
|
||||
if (input.scope === "topic" && (input.topicBindingId ?? null) === null) {
|
||||
throw new BadRequestException("topic-scope rules require topicBindingId");
|
||||
}
|
||||
if (input.scope === "content_type" && (input.contentType ?? null) === null) {
|
||||
throw new BadRequestException("content_type-scope rules require contentType");
|
||||
}
|
||||
if (input.scope !== "topic" && (input.topicBindingId ?? null) !== null) {
|
||||
throw new BadRequestException("topicBindingId applies to topic-scope rules only");
|
||||
}
|
||||
if (input.scope !== "content_type" && (input.contentType ?? null) !== null) {
|
||||
throw new BadRequestException("contentType applies to content_type-scope rules only");
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class PolicyService {
|
||||
/** Create a rule in a scope slot; refuses when the slot is already live. */
|
||||
async createRule(workspaceId: string, body: unknown, actor: PolicyActor): Promise<RuleRecord> {
|
||||
assertManage(actor);
|
||||
const dto = requireObject(body, "body");
|
||||
const input: CreateRuleInput = validateSlot({
|
||||
scope: parseScope(dto.scope),
|
||||
accountId: dto.accountId === undefined || dto.accountId === null ? null : parseId(String(dto.accountId), "accountId"),
|
||||
topicBindingId: dto.topicBindingId === undefined || dto.topicBindingId === null ? null : parseId(String(dto.topicBindingId), "topicBindingId"),
|
||||
contentType: dto.contentType === undefined || dto.contentType === null ? null : requireContentType(dto.contentType),
|
||||
policy: dto.policy,
|
||||
dryRun: dto.dryRun === undefined ? false : dto.dryRun === true,
|
||||
});
|
||||
const rule = parsePolicyRule(input.policy);
|
||||
const version = policyHash(rule);
|
||||
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
let id: string;
|
||||
try {
|
||||
const res = await client.query(
|
||||
`INSERT INTO posting_rules (workspace_id, scope, account_id, topic_binding_id, content_type, policy, policy_version, dry_run, created_by)
|
||||
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8, $9) RETURNING *`,
|
||||
[workspaceId, input.scope, input.accountId, input.topicBindingId, input.contentType, JSON.stringify(rule), version, input.dryRun === true, actor.userId],
|
||||
);
|
||||
id = res.rows[0]!.id as string;
|
||||
} catch (err) {
|
||||
if (pgCode(err) === UNIQUE_VIOLATION) {
|
||||
throw new ConflictException("a live rule already occupies this scope slot — update it instead of creating a second");
|
||||
}
|
||||
if (pgCode(err) === FK_VIOLATION) {
|
||||
throw new BadRequestException("accountId/topicBindingId does not exist in this workspace");
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const record = await this.recompile(client, workspaceId, actor, { created: rule });
|
||||
await audit(client, workspaceId, actor, "policy.revised", id, {
|
||||
action: "create",
|
||||
scope: input.scope,
|
||||
accountId: input.accountId,
|
||||
topicBindingId: input.topicBindingId,
|
||||
contentType: input.contentType,
|
||||
policyVersion: version,
|
||||
mode: rule.mode,
|
||||
dryRun: input.dryRun === true,
|
||||
snapshots: record.snapshots,
|
||||
});
|
||||
return this.getRule(client, id);
|
||||
});
|
||||
}
|
||||
|
||||
/** Update (replace whole-block) the live rule in a slot. */
|
||||
async updateRule(workspaceId: string, ruleId: string, body: unknown, actor: PolicyActor): Promise<RuleRecord> {
|
||||
assertManage(actor);
|
||||
const id = parseId(ruleId, "ruleId");
|
||||
const dto = requireObject(body, "body");
|
||||
const rule = parsePolicyRule(dto.policy);
|
||||
const version = policyHash(rule);
|
||||
const dryRun = dto.dryRun === undefined ? false : dto.dryRun === true;
|
||||
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const before = (await client.query<RuleRow>("SELECT * FROM posting_rules WHERE id = $1 FOR UPDATE", [id])).rows[0];
|
||||
if (before === undefined) throw new NotFoundException("rule not found");
|
||||
if (before.superseded_by !== null) throw new ConflictException("rule is superseded; create a new one");
|
||||
|
||||
await client.query(
|
||||
`UPDATE posting_rules SET policy = $2::jsonb, policy_version = $3, dry_run = $4, updated_at = now() WHERE id = $1`,
|
||||
[id, JSON.stringify(rule), version, dryRun],
|
||||
);
|
||||
const record = await this.recompile(client, workspaceId, actor, { updated: rule, previous: before.policy });
|
||||
await audit(client, workspaceId, actor, "policy.revised", id, {
|
||||
action: "update",
|
||||
scope: before.scope,
|
||||
accountId: before.account_id,
|
||||
topicBindingId: before.topic_binding_id,
|
||||
contentType: before.content_type,
|
||||
policyVersion: version,
|
||||
previousVersion: before.policy_version,
|
||||
mode: [before.policy.mode, rule.mode],
|
||||
dryRun,
|
||||
snapshots: record.snapshots,
|
||||
});
|
||||
return this.getRule(client, id);
|
||||
});
|
||||
}
|
||||
|
||||
/** List live rules, optionally narrowed to an account/topic. */
|
||||
async listRules(workspaceId: string, filter: { accountId?: string; topicBindingId?: string } = {}): Promise<RuleRecord[]> {
|
||||
const accountId = filter.accountId === undefined ? undefined : parseId(filter.accountId, "accountId");
|
||||
const topicBindingId = filter.topicBindingId === undefined ? undefined : parseId(filter.topicBindingId, "topicBindingId");
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const rows = await client.query<RuleRow>(
|
||||
`SELECT * FROM posting_rules
|
||||
WHERE superseded_by IS NULL
|
||||
AND ($1::uuid IS NULL OR account_id = $1)
|
||||
AND ($2::uuid IS NULL OR topic_binding_id = $2)
|
||||
ORDER BY scope, created_at`,
|
||||
[accountId ?? null, topicBindingId ?? null],
|
||||
);
|
||||
return rows.rows.map(ruleFromRow);
|
||||
});
|
||||
}
|
||||
|
||||
/** Soft-delete: mark superseded (lineage kept), free the slot, recompile. */
|
||||
async deleteRule(workspaceId: string, ruleId: string, actor: PolicyActor): Promise<{ superseded: boolean }> {
|
||||
assertManage(actor);
|
||||
const id = parseId(ruleId, "ruleId");
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const before = (await client.query<RuleRow>("SELECT * FROM posting_rules WHERE id = $1 FOR UPDATE", [id])).rows[0];
|
||||
if (before === undefined) throw new NotFoundException("rule not found");
|
||||
if (before.superseded_by !== null) return { superseded: true };
|
||||
await client.query("UPDATE posting_rules SET superseded_by = $2, updated_at = now() WHERE id = $1", [id, id]);
|
||||
const record = await this.recompile(client, workspaceId, actor, { removed: before.policy });
|
||||
await audit(client, workspaceId, actor, "policy.revised", id, {
|
||||
action: "delete",
|
||||
scope: before.scope,
|
||||
accountId: before.account_id,
|
||||
topicBindingId: before.topic_binding_id,
|
||||
contentType: before.content_type,
|
||||
previousVersion: before.policy_version,
|
||||
snapshots: record.snapshots,
|
||||
});
|
||||
return { superseded: true };
|
||||
});
|
||||
}
|
||||
|
||||
private async getRule(client: PoolClient, id: string): Promise<RuleRecord> {
|
||||
const row = (await client.query<RuleRow>("SELECT * FROM posting_rules WHERE id = $1", [id])).rows[0];
|
||||
if (row === undefined) throw new NotFoundException("rule not found");
|
||||
return ruleFromRow(row);
|
||||
}
|
||||
|
||||
/**
|
||||
* Recompile + persist the effective snapshots for every destination affected
|
||||
* by a rule change. Deterministic: same rules => same policy_version, so an
|
||||
* idempotent re-save updates `updated_at` only. Same transaction as the rule
|
||||
* change and its audit row.
|
||||
*
|
||||
* Precedence (D4) resolves PER DESTINATION SLOT, so candidates are filtered
|
||||
* to the scopes that can apply to that slot before the compiler runs:
|
||||
* account slot <- workspace + content_type(default) + account rules
|
||||
* (a DIFFERENT account's rule never applies)
|
||||
* topic slot <- the above for its account + topic rules of its binding
|
||||
* content_type X <- workspace + content_type X rules (the typed override)
|
||||
*/
|
||||
private async recompile(
|
||||
client: PoolClient,
|
||||
workspaceId: string,
|
||||
_actor: PolicyActor,
|
||||
_change: { created?: PolicyRule; updated?: PolicyRule; previous?: PolicyRule; removed?: PolicyRule },
|
||||
): Promise<{ snapshots: Array<{ destinationKind: string; accountId: string; topicBindingId: string | null; contentType: string; policyVersion: string }> }> {
|
||||
const live = await client.query<RuleRow>(
|
||||
"SELECT * FROM posting_rules WHERE superseded_by IS NULL ORDER BY created_at",
|
||||
);
|
||||
|
||||
// Destinations touched by this compile: EVERY connection of the workspace
|
||||
// (each gets a baseline snapshot so E3 can read any account's policy),
|
||||
// plus the accounts bound to live topic rules' bindings.
|
||||
const accounts = new Set<string>(
|
||||
(await client.query<{ id: string }>("SELECT id FROM connections")).rows.map((r) => r.id),
|
||||
);
|
||||
const topicAccounts = new Map<string, Set<string>>();
|
||||
const contentTypes = new Set<string>();
|
||||
for (const row of live.rows) {
|
||||
if (row.content_type !== null) contentTypes.add(row.content_type);
|
||||
if (row.scope === "topic" && row.topic_binding_id !== null) {
|
||||
const bound = await client.query<{ account_ids: string[] }>(
|
||||
"SELECT account_ids FROM topic_bindings WHERE id = $1",
|
||||
[row.topic_binding_id],
|
||||
);
|
||||
for (const accountId of bound.rows[0]?.account_ids ?? []) {
|
||||
accounts.add(accountId);
|
||||
const set = topicAccounts.get(row.topic_binding_id) ?? new Set<string>();
|
||||
set.add(accountId);
|
||||
topicAccounts.set(row.topic_binding_id, set);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const quorum: QuorumStateInput = { adminCount: 0, observedParticipants: 0, eligibleParticipants: 0 };
|
||||
const snapshots: Array<{ destinationKind: string; accountId: string; topicBindingId: string | null; contentType: string; policyVersion: string }> = [];
|
||||
|
||||
const candidatesFor = (accountId: string | null, topicBindingId: string | null, contentType: string | null): RuleCandidate[] => {
|
||||
const pool: Array<{ row: RuleRow; cand: RuleCandidate }> = live.rows.map((row) => ({ row, cand: { scope: row.scope, rule: parsePolicyRule(row.policy) } }));
|
||||
return pool
|
||||
.filter(({ row }) => {
|
||||
// workspace default: applies to every slot
|
||||
if (row.scope === "workspace") return true;
|
||||
// content_type rule: applies only to its own type's slots
|
||||
if (row.scope === "content_type") return row.content_type === contentType;
|
||||
// account rule: only that account's slots
|
||||
if (row.scope === "account") return row.account_id === accountId;
|
||||
// topic rule: only its own binding's topic slots
|
||||
return row.topic_binding_id === topicBindingId;
|
||||
})
|
||||
.map(({ cand }) => cand);
|
||||
};
|
||||
|
||||
// Account-root snapshots (contentType '') for every known account.
|
||||
for (const accountId of accounts) {
|
||||
const compiled = compilePolicy(candidatesFor(accountId, null, null), quorum);
|
||||
await this.writeSnapshot(client, workspaceId, "account", accountId, null, "", compiled.rule, compiled.policy_version, compiled.resolvedFrom);
|
||||
snapshots.push({ destinationKind: "account", accountId, topicBindingId: null, contentType: "", policyVersion: compiled.policy_version });
|
||||
}
|
||||
// Topic snapshots ('' = the binding's default).
|
||||
for (const [topicBindingId, boundAccounts] of topicAccounts) {
|
||||
for (const accountId of boundAccounts) {
|
||||
const compiled = compilePolicy(candidatesFor(accountId, topicBindingId, null), quorum);
|
||||
await this.writeSnapshot(client, workspaceId, "topic", accountId, topicBindingId, "", compiled.rule, compiled.policy_version, compiled.resolvedFrom);
|
||||
snapshots.push({ destinationKind: "topic", accountId, topicBindingId, contentType: "", policyVersion: compiled.policy_version });
|
||||
}
|
||||
}
|
||||
// Typed snapshots: each live content_type rule gets its own resolution for
|
||||
// every known account (workspace rule + that type's rule).
|
||||
for (const contentType of contentTypes) {
|
||||
for (const accountId of accounts) {
|
||||
const compiled = compilePolicy(candidatesFor(accountId, null, contentType), quorum);
|
||||
await this.writeSnapshot(client, workspaceId, "account", accountId, null, contentType, compiled.rule, compiled.policy_version, compiled.resolvedFrom);
|
||||
snapshots.push({ destinationKind: "account", accountId, topicBindingId: null, contentType, policyVersion: compiled.policy_version });
|
||||
}
|
||||
}
|
||||
return { snapshots };
|
||||
}
|
||||
|
||||
private async writeSnapshot(
|
||||
client: PoolClient,
|
||||
workspaceId: string,
|
||||
destinationKind: "account" | "topic",
|
||||
accountId: string,
|
||||
topicBindingId: string | null,
|
||||
contentType: string,
|
||||
rule: PolicyRule,
|
||||
policyVersion: string,
|
||||
resolvedFrom: Array<{ scope: RuleScope }>,
|
||||
): Promise<void> {
|
||||
await client.query(
|
||||
`INSERT INTO posting_policy_snapshots
|
||||
(workspace_id, destination_kind, account_id, topic_binding_id, content_type, effective, policy_version, resolved_from)
|
||||
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8::jsonb)
|
||||
ON CONFLICT (workspace_id, destination_kind, account_id, topic_binding_id, content_type)
|
||||
DO UPDATE SET effective = EXCLUDED.effective, policy_version = EXCLUDED.policy_version,
|
||||
resolved_from = EXCLUDED.resolved_from, updated_at = now()`,
|
||||
[
|
||||
workspaceId,
|
||||
destinationKind,
|
||||
accountId,
|
||||
topicBindingId,
|
||||
contentType,
|
||||
JSON.stringify(rule),
|
||||
policyVersion,
|
||||
JSON.stringify({ scopes: resolvedFrom.map((r) => r.scope) }),
|
||||
],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function requireContentType(value: unknown): string {
|
||||
if (typeof value !== "string" || !/^[a-z0-9_.-]{1,40}$/.test(value)) {
|
||||
throw new BadRequestException("contentType must be 1..40 chars: a-z, 0-9, dot, underscore, hyphen");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function pgCode(err: unknown): string | undefined {
|
||||
if (typeof err === "object" && err !== null && "code" in err) {
|
||||
const code = (err as { code: unknown }).code;
|
||||
return typeof code === "string" ? code : undefined;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export { isUuidList };
|
||||
173
apps/api/src/chatgov/policy.test.ts
Normal file
173
apps/api/src/chatgov/policy.test.ts
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
canonicalJson,
|
||||
compilePolicy,
|
||||
parsePolicyRule,
|
||||
policyHash,
|
||||
resolvePrecedence,
|
||||
type QuorumStateInput,
|
||||
type RuleCandidate,
|
||||
} from "./policy.js";
|
||||
|
||||
/**
|
||||
* P0 acceptance (design §7): "policy compile is pure + snapshot hash
|
||||
* deterministic". Pure-function tests — no DB, no clock.
|
||||
*/
|
||||
|
||||
const quorum: QuorumStateInput = { adminCount: 3, observedParticipants: 12, eligibleParticipants: 8 };
|
||||
|
||||
const immediate = { version: 1, mode: "immediate" as const };
|
||||
const adminOnly = { version: 1, mode: "admin_only" as const, timebox: { duration: "PT1H", onExpire: "reject" as const } };
|
||||
const quorumRule = {
|
||||
version: 1,
|
||||
mode: "quorum" as const,
|
||||
arms: [
|
||||
{ kind: "admins" as const, x: 2 },
|
||||
{ kind: "participants" as const, x: 3, of: "observed" as const },
|
||||
],
|
||||
join: "all" as const,
|
||||
allowVeto: true,
|
||||
timebox: { duration: "PT2H", onExpire: "reject" as const },
|
||||
};
|
||||
|
||||
describe("canonicalJson", () => {
|
||||
it("sorts keys and drops undefined, so key order never changes the hash input", () => {
|
||||
expect(canonicalJson({ b: 1, a: 2 })).toBe(canonicalJson({ a: 2, b: 1 }));
|
||||
expect(canonicalJson({ a: undefined, b: 1 })).toBe(canonicalJson({ b: 1 }));
|
||||
expect(canonicalJson({ a: [1, { y: 2, x: 1 }] })).toBe('{"a":[1,{"x":1,"y":2}]}');
|
||||
});
|
||||
it("keeps array order (order is data)", () => {
|
||||
expect(canonicalJson({ a: [2, 1] })).not.toBe(canonicalJson({ a: [1, 2] }));
|
||||
});
|
||||
});
|
||||
|
||||
describe("policyHash determinism", () => {
|
||||
it("same rule (any key order / explicit defaults) => same hash", () => {
|
||||
const a = parsePolicyRule({ mode: "quorum", arms: [{ kind: "admins", x: 2 }], timebox: { duration: "PT2H", onExpire: "reject" } });
|
||||
const b = parsePolicyRule({
|
||||
timebox: { onExpire: "reject", duration: "PT2H" },
|
||||
arms: [{ x: 2, kind: "admins" }],
|
||||
mode: "quorum",
|
||||
join: "all",
|
||||
allowVeto: false,
|
||||
anonymity: "named",
|
||||
allowSelfApproval: false,
|
||||
resetOnChange: true,
|
||||
dryRun: false,
|
||||
minDwellSeconds: 0,
|
||||
});
|
||||
expect(policyHash(a)).toBe(policyHash(b));
|
||||
});
|
||||
it("different policy => different hash", () => {
|
||||
expect(policyHash(parsePolicyRule({ mode: "immediate" })))
|
||||
.not.toBe(policyHash(parsePolicyRule({ mode: "admin_only", timebox: { duration: "PT1H", onExpire: "reject" } })));
|
||||
const withVeto = parsePolicyRule({ mode: "quorum", arms: [{ kind: "admins", x: 2 }], allowVeto: true, timebox: { duration: "PT1H", onExpire: "reject" } });
|
||||
const noVeto = parsePolicyRule({ mode: "quorum", arms: [{ kind: "admins", x: 2 }], allowVeto: false, timebox: { duration: "PT1H", onExpire: "reject" } });
|
||||
expect(policyHash(withVeto)).not.toBe(policyHash(noVeto));
|
||||
});
|
||||
it("quorum denominators never enter the hash", () => {
|
||||
const rule = parsePolicyRule(quorumRule);
|
||||
const withSmall: QuorumStateInput = { adminCount: 1, observedParticipants: 1, eligibleParticipants: 1 };
|
||||
const withBig: QuorumStateInput = { adminCount: 99, observedParticipants: 9999, eligibleParticipants: 500 };
|
||||
expect(policyHash(rule)).toBe(compilePolicy([{ scope: "workspace", rule }], withSmall).policy_version);
|
||||
expect(compilePolicy([{ scope: "workspace", rule }], withSmall).policy_version)
|
||||
.toBe(compilePolicy([{ scope: "workspace", rule }], withBig).policy_version);
|
||||
});
|
||||
});
|
||||
|
||||
describe("compilePolicy", () => {
|
||||
it("candidate order does not change the compiled snapshot", () => {
|
||||
const candidates: RuleCandidate[] = [
|
||||
{ scope: "workspace", rule: parsePolicyRule(immediate) },
|
||||
{ scope: "account", rule: parsePolicyRule(adminOnly) },
|
||||
{ scope: "topic", rule: parsePolicyRule(quorumRule) },
|
||||
];
|
||||
const forward = compilePolicy(candidates, quorum);
|
||||
const shuffled = compilePolicy([...candidates].reverse(), quorum);
|
||||
expect(forward.policy_version).toBe(shuffled.policy_version);
|
||||
expect(canonicalJson(forward.rule)).toBe(canonicalJson(shuffled.rule));
|
||||
});
|
||||
it("repeated compiles are byte-identical (no hidden state)", () => {
|
||||
const candidates: RuleCandidate[] = [{ scope: "content_type", rule: parsePolicyRule(adminOnly) }];
|
||||
const first = compilePolicy(candidates, quorum);
|
||||
for (let i = 0; i < 5; i += 1) {
|
||||
expect(compilePolicy(candidates, quorum)).toEqual(first);
|
||||
}
|
||||
});
|
||||
it("no candidates => fail-closed immediate baseline with a stable version", () => {
|
||||
const empty = compilePolicy([], quorum);
|
||||
expect(empty.rule.mode).toBe("immediate");
|
||||
expect(empty.policy_version).toBe(compilePolicy([], { adminCount: 0, observedParticipants: 0, eligibleParticipants: 0 }).policy_version);
|
||||
expect(empty.policy_version).toBe(policyHash(parsePolicyRule({ mode: "immediate", allowVeto: false })));
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolvePrecedence (D4: topic > content_type > account > workspace)", () => {
|
||||
it("picks the most specific scope", () => {
|
||||
const all: RuleCandidate[] = [
|
||||
{ scope: "workspace", rule: parsePolicyRule(immediate) },
|
||||
{ scope: "account", rule: parsePolicyRule(adminOnly) },
|
||||
{ scope: "content_type", rule: parsePolicyRule(adminOnly) },
|
||||
{ scope: "topic", rule: parsePolicyRule(quorumRule) },
|
||||
];
|
||||
expect(resolvePrecedence(all).scope).toBe("topic");
|
||||
expect(resolvePrecedence(all.filter((c) => c.scope !== "topic")).scope).toBe("content_type");
|
||||
expect(resolvePrecedence(all.filter((c) => c.scope === "account" || c.scope === "workspace")).scope).toBe("account");
|
||||
expect(resolvePrecedence(all.filter((c) => c.scope === "workspace")).scope).toBe("workspace");
|
||||
});
|
||||
it("whole-block replacement: the winner's rule IS the effective policy", () => {
|
||||
const winner = resolvePrecedence([
|
||||
{ scope: "workspace", rule: parsePolicyRule(quorumRule) },
|
||||
{ scope: "topic", rule: parsePolicyRule(adminOnly) },
|
||||
]);
|
||||
expect(winner.rule.mode).toBe("admin_only");
|
||||
expect(winner.rule).not.toHaveProperty("arms");
|
||||
});
|
||||
});
|
||||
|
||||
describe("parsePolicyRule validation (400s, never 500s)", () => {
|
||||
it("rejects malformed shapes", () => {
|
||||
const bad: unknown[] = [
|
||||
null,
|
||||
"immediate",
|
||||
{ mode: "nope" },
|
||||
{ mode: "quorum" }, // no arms
|
||||
{ mode: "quorum", arms: [] },
|
||||
{ mode: "quorum", arms: [{ kind: "admins", x: 0 }] },
|
||||
{ mode: "quorum", arms: [{ kind: "admins", x: 2 }], timebox: { duration: "2 hours", onExpire: "reject" } },
|
||||
{ mode: "quorum", arms: [{ kind: "admins", x: 2 }] }, // no timebox
|
||||
{ mode: "admin_only", arms: [{ kind: "participants", x: 2 }], timebox: { duration: "PT1H", onExpire: "reject" } },
|
||||
{ mode: "immediate", arms: [{ kind: "admins", x: 1 }] },
|
||||
{ mode: "quorum", arms: [{ kind: "admins", x: 2, of: "observed" }], timebox: { duration: "PT1H", onExpire: "reject" } },
|
||||
{ mode: "quorum", arms: [{ kind: "participants", x: 2 }], timebox: { duration: "PT1H", onExpire: "maybe" } },
|
||||
{ mode: "quorum", arms: [{ kind: "admins", x: 2 }], join: "xor", timebox: { duration: "PT1H", onExpire: "reject" } },
|
||||
{ mode: "quorum", arms: [{ kind: "admins", x: 2 }], anonymity: "secret", timebox: { duration: "PT1H", onExpire: "reject" } },
|
||||
{ mode: "quorum", arms: [{ kind: "admins", x: 2 }], minDwellSeconds: -1, timebox: { duration: "PT1H", onExpire: "reject" } },
|
||||
{ version: 2, mode: "immediate" },
|
||||
];
|
||||
for (const input of bad) {
|
||||
expect(() => parsePolicyRule(input), JSON.stringify(input)).toThrow();
|
||||
}
|
||||
});
|
||||
it("accepts the full §1a surface and normalises defaults", () => {
|
||||
const rule = parsePolicyRule(quorumRule);
|
||||
expect(rule).toMatchObject({
|
||||
version: 1,
|
||||
mode: "quorum",
|
||||
join: "all",
|
||||
allowVeto: true,
|
||||
anonymity: "named",
|
||||
allowSelfApproval: false,
|
||||
resetOnChange: true,
|
||||
dryRun: false,
|
||||
minDwellSeconds: 0,
|
||||
});
|
||||
expect(rule.arms).toHaveLength(2);
|
||||
expect(rule.timebox).toEqual({ duration: "PT2H", onExpire: "reject" });
|
||||
});
|
||||
it("ISO-8601 duration shapes accepted include days/minutes/seconds", () => {
|
||||
for (const duration of ["PT30M", "PT45S", "P1DT12H", "PT2H30M"]) {
|
||||
expect(() => parsePolicyRule({ mode: "quorum", arms: [{ kind: "admins", x: 1 }], timebox: { duration, onExpire: "escalateOwner" } })).not.toThrow();
|
||||
}
|
||||
});
|
||||
});
|
||||
301
apps/api/src/chatgov/policy.ts
Normal file
301
apps/api/src/chatgov/policy.ts
Normal file
|
|
@ -0,0 +1,301 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import { BadRequestException } from "@nestjs/common";
|
||||
|
||||
/**
|
||||
* Chat governance policy primitives (chatgov-design.md §1a/§1d).
|
||||
*
|
||||
* These functions are PURE on purpose — no DB, no clock, no randomness. The
|
||||
* compiled snapshot and its `policy_version` hash must be deterministic so
|
||||
* that the same rules + quorum state always produce byte-identical output
|
||||
* (P0 acceptance: "policy compile is pure + snapshot hash deterministic").
|
||||
* Time-dependent values (opens_at/closes_at) never enter the hash input.
|
||||
*/
|
||||
|
||||
export type GateMode = "immediate" | "quorum" | "admin_only";
|
||||
|
||||
/** Whose ballots count toward an arm. Eligibility is resolved at CLOSE time
|
||||
* (TOCTOU guard, §1a) by the evaluator; the compile step only records the
|
||||
* shape. */
|
||||
export type ArmKind = "admins" | "participants";
|
||||
|
||||
/** Participants denominator: observed roster | explicit eligible roster | pure count. */
|
||||
export type ParticipantsOf = "observed" | "eligible" | "none";
|
||||
|
||||
export interface QuorumArm {
|
||||
kind: ArmKind;
|
||||
/** Absolute count needed from this arm. */
|
||||
x: number;
|
||||
/** Participants arms only: how the denominator is derived. */
|
||||
of?: ParticipantsOf;
|
||||
}
|
||||
|
||||
export interface TimeboxSpec {
|
||||
/** ISO-8601 duration (e.g. "PT2H"). Validated shape-only here. */
|
||||
duration: string;
|
||||
onExpire: "approve" | "reject" | "escalateOwner";
|
||||
}
|
||||
|
||||
/** §1a pass-criteria rule object as stored in posting_rules.policy and
|
||||
* polls.rule (compiled). */
|
||||
export interface PolicyRule {
|
||||
version: number;
|
||||
mode: GateMode;
|
||||
arms?: QuorumArm[];
|
||||
/** all (AND, default) | any (OR). */
|
||||
join?: "all" | "any";
|
||||
allowVeto?: boolean;
|
||||
/** Seconds; early close only when quorum ∧ dwell. */
|
||||
minDwellSeconds?: number;
|
||||
timebox?: TimeboxSpec;
|
||||
/** named | anonymous — never mixed silently. */
|
||||
anonymity?: "named" | "anonymous";
|
||||
allowSelfApproval?: boolean;
|
||||
/** Draft edit ⇒ approvals reset, poll re-arms. */
|
||||
resetOnChange?: boolean;
|
||||
/** Evaluate + log, never block/fire. */
|
||||
dryRun?: boolean;
|
||||
}
|
||||
|
||||
export const POLICY_SCHEMA_VERSION = 1;
|
||||
|
||||
/** Machine-readable reason codes (design §4). Kept here (not in the module
|
||||
* file) so P1-P3 import them without dragging in Nest decorators. */
|
||||
export const REASON_CODES = [
|
||||
"INSUFFICIENT_QUORUM",
|
||||
"INELIGIBLE_ACTOR",
|
||||
"SELF_APPROVAL_DENIED",
|
||||
"TIMEBOX_EXPIRED_AUTO_REJECT",
|
||||
"TIMEBOX_EXPIRED_AUTO_APPROVE",
|
||||
"RESET_ON_CHANGE",
|
||||
"PENDING_POLL",
|
||||
"VETOED",
|
||||
"OUTSIDE_POLICY",
|
||||
"POLICY_DRY_RUN",
|
||||
"OVERRIDDEN_BY_OWNER",
|
||||
"LOCKED_CHANNEL",
|
||||
"NO_CHANNEL_BOUND",
|
||||
"CONFIRM_EXPIRED",
|
||||
] as const;
|
||||
|
||||
export type ReasonCode = (typeof REASON_CODES)[number];
|
||||
|
||||
export const GATES = ["E1", "E2", "E3"] as const;
|
||||
export type Gate = (typeof GATES)[number];
|
||||
|
||||
/** Public (denormalised) chatgov permissions. Seeded into role_permissions by
|
||||
* migration 024; mirrored here because packages/core's PERMISSIONS list is
|
||||
* frozen for the WP2 slice and apps/web reads permissions from the API. */
|
||||
export const CHATGOV_PERMISSIONS = {
|
||||
/** Rules editor, topic registry, identity approval, policy compile. */
|
||||
manage: "chatgov.manage",
|
||||
/** Open/cast governance votes. */
|
||||
vote: "chatgov.vote",
|
||||
} as const;
|
||||
|
||||
const DURATION = /^P(?!$)(\d+Y)?(\d+M)?(\d+W)?(\d+D)?(T(?=\d)(\d+H)?(\d+M)?(\d+(\.\d+)?S)?)?$/;
|
||||
|
||||
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/i;
|
||||
|
||||
export function isUuidList(value: unknown): value is string[] {
|
||||
return Array.isArray(value) && value.every((v) => typeof v === "string" && UUID_RE.test(v));
|
||||
}
|
||||
|
||||
/** Parse + validate a §1a-shaped policy object (unknown in, typed out). */
|
||||
export function parsePolicyRule(input: unknown, field = "policy"): PolicyRule {
|
||||
if (typeof input !== "object" || input === null || Array.isArray(input)) {
|
||||
throw new BadRequestException(`${field} must be an object`);
|
||||
}
|
||||
const raw = input as Record<string, unknown>;
|
||||
const version = raw.version === undefined ? POLICY_SCHEMA_VERSION : raw.version;
|
||||
if (version !== POLICY_SCHEMA_VERSION) {
|
||||
throw new BadRequestException(`${field}.version must be ${POLICY_SCHEMA_VERSION}`);
|
||||
}
|
||||
const mode = raw.mode;
|
||||
if (mode !== "immediate" && mode !== "quorum" && mode !== "admin_only") {
|
||||
throw new BadRequestException(`${field}.mode must be immediate, quorum or admin_only`);
|
||||
}
|
||||
const join = raw.join === undefined ? "all" : raw.join;
|
||||
if (join !== "all" && join !== "any") throw new BadRequestException(`${field}.join must be all or any`);
|
||||
|
||||
const arms: QuorumArm[] = [];
|
||||
if (raw.arms !== undefined) {
|
||||
if (!Array.isArray(raw.arms)) throw new BadRequestException(`${field}.arms must be an array`);
|
||||
if (raw.arms.length === 0) throw new BadRequestException(`${field}.arms must not be empty`);
|
||||
if (raw.arms.length > 4) throw new BadRequestException(`${field}.arms supports at most 4 arms`);
|
||||
for (const [index, entry] of raw.arms.entries()) {
|
||||
if (typeof entry !== "object" || entry === null) throw new BadRequestException(`${field}.arms[${index}] must be an object`);
|
||||
const arm = entry as Record<string, unknown>;
|
||||
if (arm.kind !== "admins" && arm.kind !== "participants") {
|
||||
throw new BadRequestException(`${field}.arms[${index}].kind must be admins or participants`);
|
||||
}
|
||||
const x = arm.x;
|
||||
if (typeof x !== "number" || !Number.isInteger(x) || x < 1 || x > 1000) {
|
||||
throw new BadRequestException(`${field}.arms[${index}].x must be an integer between 1 and 1000`);
|
||||
}
|
||||
if (arm.of !== undefined) {
|
||||
if (arm.kind !== "participants") {
|
||||
throw new BadRequestException(`${field}.arms[${index}].of applies to participants arms only`);
|
||||
}
|
||||
if (arm.of !== "observed" && arm.of !== "eligible" && arm.of !== "none") {
|
||||
throw new BadRequestException(`${field}.arms[${index}].of must be observed, eligible or none`);
|
||||
}
|
||||
}
|
||||
arms.push({ kind: arm.kind, x, ...(arm.of === undefined ? {} : { of: arm.of }) });
|
||||
}
|
||||
}
|
||||
|
||||
if (mode === "admin_only" && arms.some((arm) => arm.kind === "participants")) {
|
||||
throw new BadRequestException(`${field}: admin_only mode has no participants arm`);
|
||||
}
|
||||
if (mode === "quorum" && arms.length === 0) {
|
||||
throw new BadRequestException(`${field}: quorum mode requires arms`);
|
||||
}
|
||||
if (mode !== "quorum" && arms.length > 0) {
|
||||
throw new BadRequestException(`${field}: ${mode} mode takes no arms`);
|
||||
}
|
||||
|
||||
const timebox = raw.timebox;
|
||||
let parsedTimebox: TimeboxSpec | undefined;
|
||||
if (timebox !== undefined) {
|
||||
if (typeof timebox !== "object" || timebox === null) throw new BadRequestException(`${field}.timebox must be an object`);
|
||||
const spec = timebox as Record<string, unknown>;
|
||||
if (typeof spec.duration !== "string" || !DURATION.test(spec.duration)) {
|
||||
throw new BadRequestException(`${field}.timebox.duration must be an ISO-8601 duration`);
|
||||
}
|
||||
if (spec.onExpire !== "approve" && spec.onExpire !== "reject" && spec.onExpire !== "escalateOwner") {
|
||||
throw new BadRequestException(`${field}.timebox.onExpire must be approve, reject or escalateOwner`);
|
||||
}
|
||||
parsedTimebox = { duration: spec.duration, onExpire: spec.onExpire };
|
||||
}
|
||||
if (parsedTimebox === undefined && mode !== "immediate") {
|
||||
throw new BadRequestException(`${field}: ${mode} mode requires timebox`);
|
||||
}
|
||||
|
||||
const anonymity = raw.anonymity === undefined ? "named" : raw.anonymity;
|
||||
if (anonymity !== "named" && anonymity !== "anonymous") {
|
||||
throw new BadRequestException(`${field}.anonymity must be named or anonymous`);
|
||||
}
|
||||
const allowVeto = raw.allowVeto === undefined ? false : requireBoolean(raw.allowVeto, `${field}.allowVeto`);
|
||||
const allowSelfApproval = raw.allowSelfApproval === undefined ? false : requireBoolean(raw.allowSelfApproval, `${field}.allowSelfApproval`);
|
||||
const resetOnChange = raw.resetOnChange === undefined ? true : requireBoolean(raw.resetOnChange, `${field}.resetOnChange`);
|
||||
const dryRun = raw.dryRun === undefined ? false : requireBoolean(raw.dryRun, `${field}.dryRun`);
|
||||
const minDwellSeconds = raw.minDwellSeconds === undefined ? 0 : raw.minDwellSeconds;
|
||||
if (typeof minDwellSeconds !== "number" || !Number.isInteger(minDwellSeconds) || minDwellSeconds < 0 || minDwellSeconds > 86400) {
|
||||
throw new BadRequestException(`${field}.minDwellSeconds must be an integer between 0 and 86400`);
|
||||
}
|
||||
|
||||
return {
|
||||
version: POLICY_SCHEMA_VERSION,
|
||||
mode,
|
||||
...(arms.length === 0 ? {} : { arms }),
|
||||
join,
|
||||
allowVeto,
|
||||
minDwellSeconds,
|
||||
...(parsedTimebox === undefined ? {} : { timebox: parsedTimebox }),
|
||||
anonymity,
|
||||
allowSelfApproval,
|
||||
resetOnChange,
|
||||
dryRun,
|
||||
};
|
||||
}
|
||||
|
||||
function requireBoolean(value: unknown, field: string): boolean {
|
||||
if (typeof value !== "boolean") throw new BadRequestException(`${field} must be a boolean`);
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonical JSON: keys sorted lexicographically, no insignificant whitespace,
|
||||
* arrays keep order. This — not JSON.stringify of the source object — is the
|
||||
* hash input, so an equivalent policy written with different key order or
|
||||
* explicit defaults still compiles to the same version.
|
||||
*/
|
||||
export function canonicalJson(value: unknown): string {
|
||||
if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null";
|
||||
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
||||
const entries = Object.entries(value as Record<string, unknown>)
|
||||
.filter(([, v]) => v !== undefined)
|
||||
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
|
||||
return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`).join(",")}}`;
|
||||
}
|
||||
|
||||
/** Deterministic snapshot hash: sha256 over the canonical JSON (hex, 16 chars
|
||||
* — short enough for UI display, long enough to be collision-safe here). */
|
||||
export function policyHash(value: unknown): string {
|
||||
return createHash("sha256").update(canonicalJson(value)).digest("hex").slice(0, 16);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Compile: rules + quorum state -> immutable snapshot.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type RuleScope = "workspace" | "account" | "content_type" | "topic";
|
||||
|
||||
export interface CompiledPolicySnapshot {
|
||||
rule: PolicyRule;
|
||||
policy_version: string;
|
||||
}
|
||||
|
||||
/** A live rule candidate offered to the compiler. */
|
||||
export interface RuleCandidate {
|
||||
scope: RuleScope;
|
||||
rule: PolicyRule;
|
||||
}
|
||||
|
||||
/** Quorum denominators AT COMPILE TIME. Recorded in the snapshot for
|
||||
* observability only — never hashed — so a re-compile after roster drift
|
||||
* yields the same policy_version unless the policy itself changed. */
|
||||
export interface QuorumStateInput {
|
||||
adminCount: number;
|
||||
observedParticipants: number;
|
||||
eligibleParticipants: number;
|
||||
}
|
||||
|
||||
/** Precedence (D4): topic > content_type > account > workspace. */
|
||||
const SCOPE_ORDER: readonly RuleScope[] = ["workspace", "account", "content_type", "topic"];
|
||||
|
||||
/**
|
||||
* Resolve the winning rule among `candidates` (any order, duplicates allowed —
|
||||
* most specific scope wins; ties resolve to the first candidate of that scope,
|
||||
* which the caller keeps stable by ordering rules by created_at).
|
||||
*/
|
||||
export function resolvePrecedence(candidates: RuleCandidate[]): RuleCandidate {
|
||||
let winner: RuleCandidate | null = null;
|
||||
for (const candidate of candidates) {
|
||||
if (winner === null || SCOPE_ORDER.indexOf(candidate.scope) > SCOPE_ORDER.indexOf(winner.scope)) {
|
||||
winner = candidate;
|
||||
}
|
||||
}
|
||||
if (winner !== null) return winner;
|
||||
// Fail-closed baseline: an explicit `immediate` policy so every destination
|
||||
// has a persisted snapshot (§1d fallback:block is an EVALUATION posture —
|
||||
// OUTSIDE_POLICY at the gate — not a compile input).
|
||||
return { scope: "workspace", rule: parsePolicyRule({ mode: "immediate", allowVeto: false }) };
|
||||
}
|
||||
|
||||
export interface CompiledSnapshot {
|
||||
rule: PolicyRule;
|
||||
policy_version: string;
|
||||
resolvedFrom: Array<{ scope: RuleScope }>;
|
||||
quorum: QuorumStateInput;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compile the effective policy for one destination. PURE + DETERMINISTIC:
|
||||
* - candidates in any order produce the same winner and the same version;
|
||||
* - the version hashes only the winning rule (quorum denominators are
|
||||
* observability, not identity);
|
||||
* - no clock, no DB, no randomness.
|
||||
*/
|
||||
export function compilePolicy(candidates: RuleCandidate[], quorum: QuorumStateInput): CompiledSnapshot {
|
||||
const winner = resolvePrecedence(candidates);
|
||||
const resolvedFrom = [...candidates]
|
||||
.sort((a, b) => a.scope.localeCompare(b.scope))
|
||||
.map((c) => ({ scope: c.scope }));
|
||||
return {
|
||||
rule: winner.rule,
|
||||
policy_version: policyHash(winner.rule),
|
||||
resolvedFrom,
|
||||
quorum,
|
||||
};
|
||||
}
|
||||
71
apps/api/src/chatgov/poll-evaluator.ts
Normal file
71
apps/api/src/chatgov/poll-evaluator.ts
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
import type { PolicyRule, QuorumArm } from './policy.js';
|
||||
|
||||
export interface EvaluatedBallot {
|
||||
ballot: 'yes' | 'no' | 'veto' | 'abstain';
|
||||
optionId: string | null;
|
||||
counted: boolean;
|
||||
admin: boolean;
|
||||
observed: boolean;
|
||||
participant: boolean;
|
||||
}
|
||||
export interface OptionTally { id: string; seq: number; submittedAt: string; votes: number }
|
||||
export interface ArmTally { kind: QuorumArm['kind']; of?: QuorumArm['of']; got: number; needed: number; possible: number | null }
|
||||
export interface PollEvaluation {
|
||||
arms: ArmTally[];
|
||||
quorum: boolean;
|
||||
winnerOptionId: string | null;
|
||||
tallies: OptionTally[];
|
||||
turnout: number;
|
||||
state: 'passed' | 'failed' | 'vetoed' | 'quorum_unreachable' | null;
|
||||
reason: 'quorum_early' | 'timebox' | 'veto' | 'quorum_unreachable' | null;
|
||||
}
|
||||
|
||||
/** Pure decision; membership, self-approval and observed roster are resolved
|
||||
* afresh by the service. A plurality never substitutes for legitimacy. */
|
||||
export function evaluatePoll(input: {
|
||||
kind: 'yes_no' | 'variant_selection'; rule: PolicyRule;
|
||||
opensAt: Date; closesAt: Date; now: Date;
|
||||
ballots: EvaluatedBallot[]; options: Omit<OptionTally, 'votes'>[];
|
||||
possible: { admins: number; eligible: number; observed: number };
|
||||
}): PollEvaluation {
|
||||
const { rule, ballots } = input;
|
||||
const selected = ballots.filter(b => b.counted && b.ballot === 'yes' && (input.kind === 'yes_no' || b.optionId !== null));
|
||||
const tallies = input.options.map(o => ({ ...o, votes: selected.filter(b => b.optionId === o.id).length }));
|
||||
const winner = [...tallies].sort((a, b) => b.votes - a.votes || Date.parse(a.submittedAt) - Date.parse(b.submittedAt) || a.seq - b.seq)[0];
|
||||
const specs: QuorumArm[] = rule.mode === 'admin_only' ? [{ kind: 'admins', x: 1 }] : rule.arms ?? [];
|
||||
const arms = specs.map(arm => {
|
||||
const matches = (b: EvaluatedBallot): boolean => arm.kind === 'admins' ? b.admin : arm.of === 'observed' ? b.observed : b.participant;
|
||||
return { kind: arm.kind, ...(arm.of === undefined ? {} : { of: arm.of }),
|
||||
got: selected.filter(matches).length, needed: arm.x,
|
||||
// Observed/count-only rosters can grow; never eagerly declare them impossible.
|
||||
possible: arm.kind === 'admins' ? input.possible.admins : arm.of === 'eligible' ? input.possible.eligible : null };
|
||||
});
|
||||
const quorum = rule.mode === 'immediate' || (rule.join === 'any' ? arms.some(a => a.got >= a.needed) : arms.every(a => a.got >= a.needed));
|
||||
const result: PollEvaluation = { arms, quorum, tallies, turnout: selected.length,
|
||||
winnerOptionId: winner?.id ?? null, state: null, reason: null };
|
||||
const choice = input.kind === 'yes_no' || winner !== undefined;
|
||||
if (rule.allowVeto && ballots.some(b => b.counted && b.ballot === 'veto')) {
|
||||
result.state = 'vetoed'; result.reason = 'veto';
|
||||
} else if (input.now >= input.closesAt) {
|
||||
result.state = quorum && choice && rule.timebox?.onExpire === 'approve' ? 'passed' : 'failed';
|
||||
result.reason = 'timebox';
|
||||
} else if (quorum && choice && input.now.getTime() - input.opensAt.getTime() >= (rule.minDwellSeconds ?? 0) * 1000) {
|
||||
result.state = 'passed'; result.reason = 'quorum_early';
|
||||
} else {
|
||||
const impossible = (a: ArmTally): boolean => a.possible !== null && a.possible < a.needed;
|
||||
if (arms.length && (rule.join === 'any' ? arms.every(impossible) : arms.some(impossible))) {
|
||||
result.state = 'quorum_unreachable'; result.reason = 'quorum_unreachable';
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Fixed-length ISO durations only; months/years are calendar-dependent. */
|
||||
export function durationMs(duration: string): number {
|
||||
const match = /^P(?:(\d+)W)?(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?$/.exec(duration);
|
||||
if (!match) throw new Error('poll timebox requires a fixed duration (weeks/days/hours/minutes/seconds)');
|
||||
const [, w, d, h, m, s] = match;
|
||||
const ms = ((Number(w ?? 0) * 7 + Number(d ?? 0)) * 86400 + Number(h ?? 0) * 3600 + Number(m ?? 0) * 60 + Number(s ?? 0)) * 1000;
|
||||
if (!Number.isFinite(ms) || ms <= 0 || ms > 365 * 86400_000) throw new Error('poll duration must be positive and at most 365 days');
|
||||
return ms;
|
||||
}
|
||||
30
apps/api/src/chatgov/poll-timebox.service.ts
Normal file
30
apps/api/src/chatgov/poll-timebox.service.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
import { Inject, Injectable, Logger, type OnModuleDestroy, type OnModuleInit } from '@nestjs/common';
|
||||
import { pool } from '../db/client.js';
|
||||
import { PollService } from './poll.service.js';
|
||||
|
||||
/** Durable database timeboxes plus lazy read evaluation. A Temporal adapter
|
||||
* can call the same exactly-once close without changing the state machine. */
|
||||
@Injectable()
|
||||
export class PollTimeboxService implements OnModuleInit, OnModuleDestroy {
|
||||
private timer?: ReturnType<typeof setInterval>;
|
||||
private running = false;
|
||||
private readonly logger = new Logger(PollTimeboxService.name);
|
||||
constructor(@Inject(PollService) private readonly polls: PollService) {}
|
||||
onModuleInit(): void {
|
||||
this.timer = setInterval(() => { void this.tick(); }, 1000);
|
||||
this.timer.unref();
|
||||
}
|
||||
onModuleDestroy(): void { if (this.timer) clearInterval(this.timer); }
|
||||
async tick(): Promise<void> {
|
||||
if (this.running) return;
|
||||
this.running = true;
|
||||
try {
|
||||
const due = await pool.query<{ workspace_id: string; id: string }>('SELECT * FROM chatgov_due_polls()');
|
||||
for (const row of due.rows) {
|
||||
try { await this.polls.close(row.workspace_id, row.id); }
|
||||
catch (error) { this.logger.warn(`poll ${row.id} expiry will retry: ${String(error)}`); }
|
||||
}
|
||||
} catch (error) { this.logger.warn(`poll timebox discovery will retry: ${String(error)}`); }
|
||||
finally { this.running = false; }
|
||||
}
|
||||
}
|
||||
37
apps/api/src/chatgov/poll.controller.ts
Normal file
37
apps/api/src/chatgov/poll.controller.ts
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
import { Body, Controller, Get, Inject, Param, Post, Req, UseGuards } from '@nestjs/common';
|
||||
import type { Request } from 'express';
|
||||
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from '../rbac/rbac.guard.js';
|
||||
import { requireSession } from '../http/validation.js';
|
||||
import { PollService, type PollCard } from './poll.service.js';
|
||||
|
||||
@Controller('chatgov/polls')
|
||||
@UseGuards(RbacGuard)
|
||||
export class PollController {
|
||||
constructor(@Inject(PollService) private readonly polls: PollService) {}
|
||||
@Get(':id')
|
||||
@RequirePermission('workspace.view')
|
||||
async get(@Req() req: Request & AuthenticatedRequest, @Param('id') id: string): Promise<PollCard> {
|
||||
const session = requireSession(req);
|
||||
return this.polls.get(session.workspaceId, id, session.userId);
|
||||
}
|
||||
@Post()
|
||||
@RequirePermission('chatgov.vote')
|
||||
async open(@Req() req: Request & AuthenticatedRequest, @Body() body: unknown): Promise<PollCard> {
|
||||
const session = requireSession(req);
|
||||
return this.polls.open(session.workspaceId, body, { userId: session.userId });
|
||||
}
|
||||
@Post(':id/options')
|
||||
@RequirePermission('chatgov.vote')
|
||||
async fork(@Req() req: Request & AuthenticatedRequest, @Param('id') id: string, @Body() body: unknown): Promise<PollCard> {
|
||||
const session = requireSession(req);
|
||||
return this.polls.forkEdit(session.workspaceId, id, body, { userId: session.userId });
|
||||
}
|
||||
// Any authenticated member can store an ineligible ballot. Eligibility is
|
||||
// enforced by the service; identity fields in the body are never trusted.
|
||||
@Post(':id/vote')
|
||||
@RequirePermission('workspace.view')
|
||||
async vote(@Req() req: Request & AuthenticatedRequest, @Param('id') id: string, @Body() body: unknown): Promise<PollCard> {
|
||||
const session = requireSession(req);
|
||||
return this.polls.vote(session.workspaceId, id, body, { kind: 'user', userId: session.userId });
|
||||
}
|
||||
}
|
||||
380
apps/api/src/chatgov/poll.service.ts
Normal file
380
apps/api/src/chatgov/poll.service.ts
Normal file
|
|
@ -0,0 +1,380 @@
|
|||
import { BadRequestException, ConflictException, ForbiddenException, Injectable, NotFoundException } from '@nestjs/common';
|
||||
import type { PoolClient } from 'pg';
|
||||
import { withTenant } from '../db/client.js';
|
||||
import { parseId, requireObject, requireString } from '../http/validation.js';
|
||||
import { scrub } from '../audit/audit.service.js';
|
||||
import { draftDestinationAccounts } from '../publishing/approval-gate.service.js';
|
||||
import type { DraftPost } from '../publishing/publishing.service.js';
|
||||
import { transitionPollDraft } from '../publishing/poll-approval.js';
|
||||
import { parsePolicyRule, policyHash, type PolicyRule, type QuorumArm } from './policy.js';
|
||||
import { durationMs, evaluatePoll, type EvaluatedBallot, type PollEvaluation } from './poll-evaluator.js';
|
||||
|
||||
export interface PollActor { userId: string; role?: string }
|
||||
export interface PollVoter { kind: 'user' | 'chat_identity'; userId?: string; provider?: string; remoteId?: string; handle?: string }
|
||||
interface PollRow {
|
||||
id: string; workspace_id: string; draft_id: string | null; draft_revision: string | null;
|
||||
channel_id: string | null; topic_binding_id: string | null; question: string;
|
||||
kind: 'yes_no' | 'variant_selection'; state: string; rule: PolicyRule; policy_version: string;
|
||||
anonymity: string; opens_at: Date; closes_at: Date; created_by: string;
|
||||
winner_option_id: string | null; eligible_user_ids: string[] | null;
|
||||
parent_poll_id: string | null; closed_at: Date | null; close_reason: string | null; outbox_state: string;
|
||||
}
|
||||
interface OptionRow { id: string; seq: number; text: string; submitted_by: string; submitted_at: Date; draft_revision_id: string | null; author: string }
|
||||
interface Member { user_id: string; role: string; can_vote: boolean }
|
||||
interface VoteRow { voter_kind: 'user' | 'chat_identity'; voter_user_id: string | null; voter_remote_id: string; voter_handle: string;
|
||||
ballot: EvaluatedBallot['ballot']; option_id: string | null; counted: boolean; eligible: boolean; ineligible_reason: string | null }
|
||||
export interface PollCard {
|
||||
poll: PollRow; options: OptionRow[]; evaluation: PollEvaluation;
|
||||
votes: VoteRow[]; voterEligibility: { eligible: boolean; admin: boolean; reason: string | null } | null;
|
||||
}
|
||||
|
||||
/** All writes use workspace → draft → poll lock order, matching publishing
|
||||
* and membership epoch updates. The outbox claim and E2 transition share one
|
||||
* transaction; rollback also releases the claim. Late ballots are no-ops. */
|
||||
@Injectable()
|
||||
export class PollService {
|
||||
async open(workspaceId: string, body: unknown, actor: PollActor): Promise<PollCard> {
|
||||
return withTenant(workspaceId, async client => {
|
||||
await lockWorkspace(client, workspaceId);
|
||||
const poll = await openPollInTransaction(client, workspaceId, body, actor);
|
||||
return card(client, poll, actor.userId);
|
||||
});
|
||||
}
|
||||
async forkEdit(workspaceId: string, pollId: string, body: unknown, actor: PollActor): Promise<PollCard> {
|
||||
const dto = requireObject(body, 'body'); const text = variantText(dto.text);
|
||||
return this.locked(workspaceId, pollId, async (client, poll) => {
|
||||
await requireVoter(client, workspaceId, actor.userId);
|
||||
await evaluateLocked(client, poll);
|
||||
if (poll.state !== 'open' || poll.kind !== 'variant_selection') throw new ConflictException('only open variant polls accept forks');
|
||||
const seq = Number((await client.query('SELECT COALESCE(max(seq), 0) + 1 AS seq FROM poll_options WHERE poll_id = $1', [poll.id])).rows[0]?.seq);
|
||||
await insertOption(client, poll, seq, text, actor.userId);
|
||||
await audit(client, poll, 'poll.option_submitted', actor.userId, { seq });
|
||||
return card(client, poll, actor.userId);
|
||||
});
|
||||
}
|
||||
async vote(workspaceId: string, pollId: string, body: unknown, voter: PollVoter): Promise<PollCard> {
|
||||
const dto = requireObject(body, 'body');
|
||||
const optionId = dto.optionId === undefined || dto.optionId === null ? null : parseId(requireString(dto.optionId, 'optionId'));
|
||||
const ballot = dto.ballot ?? (optionId ? 'yes' : undefined);
|
||||
if (!['yes', 'no', 'veto', 'abstain'].includes(String(ballot))) throw new BadRequestException('invalid ballot');
|
||||
return this.locked(workspaceId, pollId, async (client, poll) => {
|
||||
// Expiry is resolved before accepting a late ballot; retries after close
|
||||
// render the frozen artifact without changing the decision.
|
||||
if (new Date() >= poll.closes_at) await evaluateLocked(client, poll);
|
||||
if (poll.state !== 'open') return card(client, poll, voter.userId);
|
||||
if (poll.kind === 'variant_selection' && ballot === 'yes' && optionId === null) throw new BadRequestException('variant vote requires optionId');
|
||||
if (optionId && (poll.kind !== 'variant_selection' || ballot !== 'yes')) throw new BadRequestException('optionId only applies to a variant choice');
|
||||
if (optionId && !(await client.query('SELECT id FROM poll_options WHERE id = $1 AND poll_id = $2', [optionId, poll.id])).rowCount) throw new BadRequestException('option does not belong to this poll');
|
||||
if (ballot === 'veto' && !poll.rule.allowVeto) throw new BadRequestException('veto is disabled');
|
||||
let userId = voter.userId ?? null;
|
||||
let key: string;
|
||||
let identityId: string | null = null;
|
||||
if (voter.kind === 'user') {
|
||||
if (!userId) throw new BadRequestException('user voter needs userId');
|
||||
parseId(userId, 'voter.userId'); key = `user:${userId}`;
|
||||
} else {
|
||||
const provider = requireString(voter.provider, 'provider');
|
||||
const remote = requireString(voter.remoteId, 'remoteId');
|
||||
if (!/^[0-9]{1,20}$/.test(remote) || !/^[a-z0-9_-]+$/.test(provider)) throw new BadRequestException('invalid immutable provider identity');
|
||||
key = `${provider}:${remote}`;
|
||||
// Never trust a supplied linked user id from the gateway.
|
||||
const identity = (await client.query("SELECT id, user_id, status FROM chat_identities WHERE provider = $1 AND remote_user_id = $2", [provider, remote])).rows[0];
|
||||
identityId = identity?.id as string | undefined ?? null;
|
||||
userId = identity?.status === 'approved' ? identity.user_id as string | null : null;
|
||||
}
|
||||
// Linked identities and web users share ONE ballot. Alternate provider
|
||||
// accounts cannot inflate turnout. Unlinked ballots are still retained.
|
||||
if (userId) key = `user:${userId}`;
|
||||
await client.query(`INSERT INTO poll_votes (workspace_id, poll_id, voter_kind, voter_user_id, voter_remote_id, voter_handle, ballot, option_id, eligible, counted, voter_identity_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, false, false, $9)
|
||||
ON CONFLICT (poll_id, voter_remote_id) DO UPDATE SET voter_kind = EXCLUDED.voter_kind,
|
||||
voter_user_id = EXCLUDED.voter_user_id, voter_identity_id = EXCLUDED.voter_identity_id, voter_handle = EXCLUDED.voter_handle,
|
||||
ballot = EXCLUDED.ballot, option_id = EXCLUDED.option_id, cast_at = clock_timestamp()`,
|
||||
[workspaceId, poll.id, voter.kind, userId, key, voter.handle ?? key, ballot, optionId, identityId]);
|
||||
await audit(client, poll, 'poll.voted', userId, { voter: key, ballot, optionId });
|
||||
await evaluateLocked(client, poll);
|
||||
return card(client, poll, userId ?? undefined);
|
||||
});
|
||||
}
|
||||
async get(workspaceId: string, pollId: string, userId?: string): Promise<PollCard> {
|
||||
return this.locked(workspaceId, pollId, async (client, poll) => {
|
||||
await evaluateLocked(client, poll); return card(client, poll, userId);
|
||||
});
|
||||
}
|
||||
async evaluate(workspaceId: string, pollId: string): Promise<PollCard> { return this.get(workspaceId, pollId); }
|
||||
/** Deliberately re-evaluates the authoritative conditions; callers cannot
|
||||
* force a pass or supply a stale tally. Suitable for competing timeboxes. */
|
||||
async close(workspaceId: string, pollId: string): Promise<PollCard> { return this.get(workspaceId, pollId); }
|
||||
async sweep(workspaceId: string): Promise<void> {
|
||||
const ids = await withTenant(workspaceId, c => c.query<{ id: string }>("SELECT id FROM polls WHERE state = 'open' AND (closes_at <= now() OR opens_at + COALESCE((rule->>'minDwellSeconds')::integer,0) * interval '1 second' <= now())", []));
|
||||
for (const { id } of ids.rows) await this.evaluate(workspaceId, id);
|
||||
}
|
||||
private async locked<T>(workspaceId: string, pollId: string, fn: (client: PoolClient, poll: PollRow) => Promise<T>): Promise<T> {
|
||||
parseId(pollId, 'pollId');
|
||||
return withTenant(workspaceId, async client => {
|
||||
await lockWorkspace(client, workspaceId);
|
||||
const found = (await client.query<PollRow>('SELECT * FROM polls WHERE id = $1', [pollId])).rows[0];
|
||||
if (!found) throw new NotFoundException('poll not found');
|
||||
if (found.draft_id) await client.query('SELECT id FROM drafts WHERE id = $1 FOR UPDATE', [found.draft_id]);
|
||||
const poll = (await client.query<PollRow>('SELECT * FROM polls WHERE id = $1 FOR UPDATE', [pollId])).rows[0]!;
|
||||
return fn(client, poll);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function lockWorkspace(client: PoolClient, workspaceId: string): Promise<void> {
|
||||
await client.query('SELECT id FROM workspaces WHERE id = $1 FOR UPDATE', [workspaceId]);
|
||||
}
|
||||
async function members(client: PoolClient, workspaceId: string): Promise<Member[]> {
|
||||
return (await client.query<Member>(`SELECT m.user_id, m.role, EXISTS(SELECT 1 FROM role_permissions rp
|
||||
WHERE rp.role = m.role AND rp.permission = 'chatgov.vote') AS can_vote
|
||||
FROM memberships m WHERE workspace_id = $1 AND status = 'active'`, [workspaceId])).rows;
|
||||
}
|
||||
async function requireVoter(client: PoolClient, workspaceId: string, userId: string): Promise<void> {
|
||||
parseId(userId, 'actor.userId');
|
||||
if (!(await members(client, workspaceId)).some(m => m.user_id === userId && m.can_vote)) throw new ForbiddenException('chatgov.vote required');
|
||||
}
|
||||
function variantText(value: unknown): string {
|
||||
const text = requireString(value, 'text');
|
||||
if (text.length > 5000) throw new BadRequestException('variant text must be at most 5000 characters');
|
||||
return text;
|
||||
}
|
||||
async function insertOption(client: PoolClient, poll: PollRow, seq: number, text: string, userId: string): Promise<void> {
|
||||
await client.query('INSERT INTO poll_options (workspace_id, poll_id, seq, text, submitted_by) VALUES ($1,$2,$3,$4,$5)', [poll.workspace_id, poll.id, seq, text, userId]);
|
||||
}
|
||||
async function options(client: PoolClient, pollId: string): Promise<OptionRow[]> {
|
||||
return (await client.query<OptionRow>(`SELECT o.*, u.display_name AS author FROM poll_options o
|
||||
JOIN users u ON u.id = o.submitted_by WHERE poll_id = $1 ORDER BY seq`, [pollId])).rows;
|
||||
}
|
||||
async function audit(client: PoolClient, poll: PollRow, action: string, userId: string | null, diff: Record<string, unknown>): Promise<void> {
|
||||
await client.query("SELECT audit_append($1,$2,$3,$4,'poll',$5,$6)", [poll.workspace_id, userId, userId ? 'Member' : 'System', action, poll.id,
|
||||
scrub({ ...diff, policyVersion: poll.policy_version, actor: userId ? `user:${userId}` : `system:chat-poll-gate:${poll.id}` })]);
|
||||
}
|
||||
|
||||
/** Snapshot all applicable arms. Different OR-policy groups cannot be
|
||||
* flattened into a single §1a object without changing their meaning. */
|
||||
export async function draftPollRule(client: PoolClient, draft: { post: DraftPost }, supplied?: unknown): Promise<PolicyRule | null> {
|
||||
const accountIds = draftDestinationAccounts(draft.post);
|
||||
const live = await client.query("SELECT 1 FROM posting_rules WHERE superseded_by IS NULL LIMIT 1");
|
||||
const rules: PolicyRule[] = [];
|
||||
if (accountIds.length && live.rowCount) {
|
||||
for (const account of accountIds) {
|
||||
const topic = typeof draft.post.providerMetadata.topicBindingId === 'string' ? draft.post.providerMetadata.topicBindingId : null;
|
||||
const snapshot = (await client.query(`SELECT effective FROM posting_policy_snapshots WHERE account_id = $1
|
||||
AND content_type = '' AND (topic_binding_id IS NULL OR topic_binding_id = $2)
|
||||
ORDER BY topic_binding_id NULLS LAST LIMIT 1`, [account, topic])).rows[0];
|
||||
if (!snapshot) throw new ConflictException('OUTSIDE_POLICY: destination snapshot missing');
|
||||
const rule = parsePolicyRule(snapshot.effective);
|
||||
if (rule.mode !== 'immediate' && !rule.dryRun) rules.push(rule);
|
||||
}
|
||||
}
|
||||
let rule: PolicyRule | null = rules[0] ?? null;
|
||||
if (rules.some(r => policyHash(r) !== policyHash(rule))) {
|
||||
if (rules.some(r => r.join === 'any')) throw new ConflictException('incompatible OR policy groups require independent polls');
|
||||
rule = parsePolicyRule({ ...rules[0], mode: 'quorum', join: 'all',
|
||||
arms: rules.flatMap<QuorumArm>(r => r.mode === 'admin_only' ? [{ kind: 'admins', x: 1 }] : r.arms ?? []).filter((a, i, all) => all.findIndex(b => b.kind === a.kind && b.of === a.of && b.x === a.x) === i),
|
||||
allowVeto: rules.some(r => r.allowVeto), allowSelfApproval: rules.every(r => r.allowSelfApproval),
|
||||
resetOnChange: rules.some(r => r.resetOnChange), minDwellSeconds: Math.max(...rules.map(r => r.minDwellSeconds ?? 0)),
|
||||
timebox: { duration: rules.reduce((a,b) => durationMs(a.timebox!.duration) >= durationMs(b.timebox!.duration) ? a : b).timebox!.duration, onExpire: 'reject' } });
|
||||
}
|
||||
if (supplied !== undefined) {
|
||||
const requested = parsePolicyRule(supplied);
|
||||
if (rule && policyHash(rule) !== policyHash(requested)) throw new BadRequestException('rule must match the compiled destination policy');
|
||||
rule ??= requested;
|
||||
}
|
||||
return rule;
|
||||
}
|
||||
|
||||
export async function openPollInTransaction(client: PoolClient, workspaceId: string, body: unknown, actor: PollActor): Promise<PollRow> {
|
||||
const dto = requireObject(body, 'body');
|
||||
await requireVoter(client, workspaceId, actor.userId);
|
||||
if (dto.contentKind !== undefined && dto.contentKind !== 'short_post') throw new BadRequestException('document polls are a future track');
|
||||
const draftId = dto.draftId == null ? null : parseId(requireString(dto.draftId, 'draftId'));
|
||||
const draft = draftId ? (await client.query('SELECT * FROM drafts WHERE id = $1 FOR UPDATE', [draftId])).rows[0] : null;
|
||||
if (draftId && !draft) throw new NotFoundException('draft not found');
|
||||
if (draft && !['editing', 'in_review'].includes(String(draft.state))) throw new ConflictException('draft is not reviewable');
|
||||
const rule = draft ? await draftPollRule(client, draft as { post: DraftPost }, dto.rule) : parsePolicyRule(dto.rule);
|
||||
if (!rule) throw new BadRequestException('a rule or compiled chat-signal policy is required');
|
||||
if (rule.anonymity !== 'named') throw new BadRequestException('anonymous sentiment belongs to the separate Telegram track; governance needs attributable ballots');
|
||||
const kind = dto.kind ?? (draft ? 'variant_selection' : 'yes_no');
|
||||
if (kind !== 'yes_no' && kind !== 'variant_selection') throw new BadRequestException('invalid poll kind');
|
||||
if (kind === 'variant_selection' && !draft) throw new BadRequestException('variant selection requires a draft');
|
||||
if (dto.variants !== undefined && !Array.isArray(dto.variants)) throw new BadRequestException('variants must be an array');
|
||||
const variants = (dto.variants as unknown[] | undefined ?? []).map(v => variantText(typeof v === 'string' ? v : requireObject(v, 'variant').text));
|
||||
if (kind === 'yes_no' && variants.length) throw new BadRequestException('yes/no polls do not accept variants');
|
||||
if (variants.length > 99) throw new BadRequestException('at most 99 initial forks');
|
||||
const eligibleIds = dto.eligibleUserIds === undefined ? null : (Array.isArray(dto.eligibleUserIds) ? dto.eligibleUserIds.map(v => parseId(requireString(v, 'eligibleUserId'))) : (() => { throw new BadRequestException('eligibleUserIds must be an array'); })());
|
||||
const question = requireString(dto.question ?? 'Choose the post to approve', 'question');
|
||||
if (question.length > 300) throw new BadRequestException('question too long');
|
||||
let duration: number;
|
||||
try { duration = durationMs(rule.timebox?.duration ?? 'PT1H'); } catch (error) { throw new BadRequestException(String(error)); }
|
||||
const now = new Date();
|
||||
const poll = (await client.query<PollRow>(`INSERT INTO polls
|
||||
(workspace_id, draft_id, subject_kind, question, kind, rule, policy_version, opens_at, closes_at, created_by, draft_revision, parent_poll_id, channel_id, topic_binding_id, eligible_user_ids)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING *`,
|
||||
[workspaceId, draftId, draftId ? 'draft' : 'none', question, kind, rule, policyHash(rule), now, new Date(now.getTime() + duration), actor.userId,
|
||||
draft?.revision ?? null, dto.parentPollId == null ? null : parseId(requireString(dto.parentPollId, 'parentPollId')),
|
||||
dto.channelId == null ? null : parseId(requireString(dto.channelId, 'channelId')),
|
||||
dto.topicBindingId == null ? null : parseId(requireString(dto.topicBindingId, 'topicBindingId')), eligibleIds])).rows[0]!;
|
||||
if (kind === 'variant_selection') {
|
||||
await insertOption(client, poll, 1, variantText((draft!.post as DraftPost).text), String(draft!.author_user_id));
|
||||
for (const [i, text] of variants.entries()) await insertOption(client, poll, i + 2, text, actor.userId);
|
||||
}
|
||||
await audit(client, poll, 'poll.opened', actor.userId, { draftId, kind, parentPollId: poll.parent_poll_id });
|
||||
return poll;
|
||||
}
|
||||
|
||||
async function resolveTally(client: PoolClient, poll: PollRow, persist: boolean): Promise<{ evaluation: PollEvaluation; votes: VoteRow[] }> {
|
||||
const roster = await members(client, poll.workspace_id);
|
||||
const draft = poll.draft_id ? (await client.query('SELECT author_user_id FROM drafts WHERE id = $1', [poll.draft_id])).rows[0] : null;
|
||||
const opts = await options(client, poll.id);
|
||||
const votes = (await client.query<VoteRow & { identity_status: string | null; identity_user_id: string | null; role_override: string | null; cast_at: Date; voter_identity_id: string | null }>(`SELECT v.*, ci.status AS identity_status, ci.user_id AS identity_user_id, ci.role_override
|
||||
FROM poll_votes v LEFT JOIN chat_identities ci ON ci.id = v.voter_identity_id
|
||||
WHERE poll_id = $1 ORDER BY cast_at DESC, voter_remote_id`, [poll.id])).rows;
|
||||
// Ingress currently persists handles, not immutable remote ids. Only
|
||||
// numeric author_handle evidence can bind to approved identities safely.
|
||||
// A username must never become an authorization key.
|
||||
const observed = new Set<string>();
|
||||
if (poll.channel_id) {
|
||||
const rows = await client.query<{ user_id: string }>(`SELECT DISTINCT ci.user_id FROM live_chat_messages lm
|
||||
JOIN chat_identities ci ON ci.provider = lm.provider AND ci.remote_user_id = lm.author_handle
|
||||
WHERE lm.channel_id = $1 AND lm.received_at >= $2 AND ci.status = 'approved' AND ci.user_id IS NOT NULL`, [poll.channel_id, poll.opens_at]);
|
||||
for (const row of rows.rows) observed.add(row.user_id);
|
||||
}
|
||||
const seen = new Set<string>();
|
||||
const ballots: EvaluatedBallot[] = [];
|
||||
const eligibleMembers = roster.filter(m => m.can_vote);
|
||||
const identities = (await client.query<{ user_id: string; role_override: string }>("SELECT user_id, role_override FROM chat_identities WHERE status = 'approved' AND role_override IN ('Admin','Owner')")).rows;
|
||||
const adminIds = new Set(roster.filter(m => m.can_vote && ['Admin', 'Owner'].includes(m.role)).map(m => m.user_id));
|
||||
for (const identity of identities) if (roster.some(m => m.user_id === identity.user_id)) adminIds.add(identity.user_id);
|
||||
for (const vote of votes) {
|
||||
const identityOkay = !vote.voter_identity_id || (vote.identity_status === 'approved' && vote.identity_user_id === vote.voter_user_id);
|
||||
const member = roster.find(m => m.user_id === vote.voter_user_id);
|
||||
const eligible = identityOkay && member !== undefined && (member.can_vote || ['Admin','Owner'].includes(vote.role_override ?? ''));
|
||||
const selfDenied = !poll.rule.allowSelfApproval && vote.ballot === 'yes' && (poll.kind === 'variant_selection'
|
||||
? opts.some(o => o.id === vote.option_id && o.submitted_by === vote.voter_user_id)
|
||||
: draft?.author_user_id === vote.voter_user_id);
|
||||
const duplicate = vote.voter_user_id !== null && seen.has(vote.voter_user_id);
|
||||
if (eligible && vote.voter_user_id) seen.add(vote.voter_user_id);
|
||||
vote.eligible = eligible;
|
||||
vote.counted = eligible && !selfDenied && !duplicate;
|
||||
vote.ineligible_reason = !eligible || duplicate ? 'INELIGIBLE_ACTOR' : selfDenied ? 'SELF_APPROVAL_DENIED' : null;
|
||||
if (persist) await client.query(`UPDATE poll_votes SET eligible = $3, counted = $4, ineligible_reason = $5
|
||||
WHERE poll_id = $1 AND voter_remote_id = $2`, [poll.id, vote.voter_remote_id, vote.eligible, vote.counted, vote.ineligible_reason]);
|
||||
ballots.push({ ballot: vote.ballot, optionId: vote.option_id, counted: vote.counted,
|
||||
admin: adminIds.has(vote.voter_user_id ?? ''), observed: observed.has(vote.voter_user_id ?? ''),
|
||||
participant: poll.eligible_user_ids === null || poll.eligible_user_ids.includes(vote.voter_user_id ?? '') });
|
||||
}
|
||||
const eligible = eligibleMembers.filter(m => poll.eligible_user_ids === null || poll.eligible_user_ids.includes(m.user_id));
|
||||
// Potential self-voters must have at least one option authored by someone
|
||||
// else. A no/self ballot can be recast, so it is never subtracted twice.
|
||||
const canChoose = (id: string): boolean => poll.rule.allowSelfApproval === true || (poll.kind === 'variant_selection'
|
||||
? opts.some(o => o.submitted_by !== id) : draft?.author_user_id !== id);
|
||||
const evaluation = evaluatePoll({ kind: poll.kind, rule: poll.rule, opensAt: poll.opens_at, closesAt: poll.closes_at, now: new Date(), ballots,
|
||||
options: opts.map(o => ({ id: o.id, seq: o.seq, submittedAt: o.submitted_at.toISOString() })),
|
||||
possible: { admins: [...adminIds].filter(canChoose).length, eligible: eligible.filter(m => canChoose(m.user_id)).length, observed: observed.size } });
|
||||
// Keep the public DTO separate from identity resolution internals.
|
||||
return { evaluation, votes: votes.map(v => ({ voter_kind: v.voter_kind, voter_user_id: v.voter_user_id,
|
||||
voter_remote_id: v.voter_remote_id, voter_handle: v.voter_handle, ballot: v.ballot, option_id: v.option_id,
|
||||
counted: v.counted, eligible: v.eligible, ineligible_reason: v.ineligible_reason })) };
|
||||
}
|
||||
|
||||
async function evaluateLocked(client: PoolClient, poll: PollRow): Promise<void> {
|
||||
if (poll.state !== 'open') return;
|
||||
if (poll.draft_id && poll.draft_revision !== null) {
|
||||
const draft = (await client.query('SELECT revision FROM drafts WHERE id = $1', [poll.draft_id])).rows[0];
|
||||
if (String(draft?.revision) !== String(poll.draft_revision)) {
|
||||
await stalePoll(client, poll); return;
|
||||
}
|
||||
}
|
||||
const { evaluation } = await resolveTally(client, poll, true);
|
||||
await audit(client, poll, 'policy.evaluated', null, { evaluation });
|
||||
// Shadow polls retain their artifact and terminal outcome but cannot
|
||||
// approve/reject drafts. Their E2 decision is pending, never a release.
|
||||
if (!evaluation.state) return;
|
||||
const claimed = await client.query(`UPDATE polls SET outbox_state = 'claimed', state = $2, closed_at = clock_timestamp(),
|
||||
close_reason = $3, winner_option_id = $4, final_evaluation = $5
|
||||
WHERE id = $1 AND state = 'open' AND outbox_state = 'none' RETURNING *`,
|
||||
[poll.id, evaluation.state, evaluation.reason,
|
||||
evaluation.state === 'passed' && poll.kind === 'variant_selection' ? evaluation.winnerOptionId : null, evaluation]);
|
||||
if (!claimed.rowCount) return;
|
||||
Object.assign(poll, claimed.rows[0]);
|
||||
const reasons = evaluation.state === 'vetoed' ? ['VETOED'] : evaluation.state === 'quorum_unreachable' ? ['INSUFFICIENT_QUORUM']
|
||||
: evaluation.reason === 'timebox' ? [evaluation.state === 'passed' ? 'TIMEBOX_EXPIRED_AUTO_APPROVE' : 'TIMEBOX_EXPIRED_AUTO_REJECT'] : [];
|
||||
if (poll.draft_id && !poll.rule.dryRun) {
|
||||
const winnerText = poll.winner_option_id ? (await options(client, poll.id)).find(o => o.id === poll.winner_option_id)?.text : undefined;
|
||||
await transitionPollDraft(client, { workspaceId: poll.workspace_id, draftId: poll.draft_id, pollId: poll.id,
|
||||
passed: evaluation.state === 'passed', ...(winnerText === undefined ? {} : { text: winnerText }),
|
||||
reason: reasons[0] ?? 'quorum_early', escalate: evaluation.reason === 'timebox' && poll.rule.timebox?.onExpire === 'escalateOwner' });
|
||||
}
|
||||
await recordE2(client, poll, evaluation, poll.rule.dryRun ? ['POLICY_DRY_RUN'] : reasons);
|
||||
await audit(client, poll, 'poll.closed', null, { state: poll.state, reason: poll.close_reason, winnerOptionId: poll.winner_option_id, evaluation });
|
||||
await audit(client, poll, evaluation.state === 'passed' && !poll.rule.dryRun ? 'release.fired' : 'release.blocked', null, { reasons });
|
||||
await client.query("UPDATE polls SET outbox_state = 'done' WHERE id = $1 AND outbox_state = 'claimed'", [poll.id]);
|
||||
poll.outbox_state = 'done';
|
||||
}
|
||||
async function recordE2(client: PoolClient, poll: PollRow, evaluation: PollEvaluation, reasons: string[]): Promise<void> {
|
||||
if (!poll.draft_id) return;
|
||||
const draft = (await client.query('SELECT post FROM drafts WHERE id = $1', [poll.draft_id])).rows[0];
|
||||
for (const accountId of draftDestinationAccounts(draft.post as DraftPost)) {
|
||||
const provider = (await client.query('SELECT provider FROM connections WHERE id = $1', [accountId])).rows[0]?.provider ?? 'unknown';
|
||||
await client.query(`INSERT INTO gate_decisions
|
||||
(workspace_id, draft_id, destination, gate, policy_version, outcome, inputs, reason_codes, actor, poll_id)
|
||||
VALUES ($1,$2,$3,'E2',$4,$5,$6,$7,$8,$9)`,
|
||||
[poll.workspace_id, poll.draft_id, { platform: provider, accountId, ...(poll.topic_binding_id ? { topicBindingId: poll.topic_binding_id } : {}) },
|
||||
poll.policy_version, poll.rule.dryRun ? 'pending' : poll.state === 'passed' ? 'fired' : 'blocked',
|
||||
{ ...evaluation, pollState: poll.state }, reasons, `system:chat-poll-gate:${poll.id}`, poll.id]);
|
||||
}
|
||||
}
|
||||
async function card(client: PoolClient, poll: PollRow, userId?: string): Promise<PollCard> {
|
||||
const final = (poll as PollRow & { final_evaluation?: PollEvaluation }).final_evaluation;
|
||||
const tally = final && poll.state !== 'open'
|
||||
? { evaluation: final, votes: (await client.query<VoteRow>('SELECT voter_kind, voter_user_id, voter_remote_id, voter_handle, ballot, option_id, counted, eligible, ineligible_reason FROM poll_votes WHERE poll_id = $1 ORDER BY cast_at', [poll.id])).rows }
|
||||
: await resolveTally(client, poll, false);
|
||||
const member = userId ? (await members(client, poll.workspace_id)).find(m => m.user_id === userId) : undefined;
|
||||
return { poll, options: await options(client, poll.id), evaluation: tally.evaluation, votes: tally.votes,
|
||||
voterEligibility: userId ? { eligible: member?.can_vote ?? false, admin: member !== undefined && ['Admin','Owner'].includes(member.role), reason: member?.can_vote ? null : 'INELIGIBLE_ACTOR' } : null };
|
||||
}
|
||||
|
||||
async function stalePoll(client: PoolClient, poll: PollRow): Promise<void> {
|
||||
const { evaluation } = await resolveTally(client, poll, true);
|
||||
const changed = await client.query(`UPDATE polls SET state = 'stale', closed_at = clock_timestamp(), close_reason = 'reset_on_change',
|
||||
outbox_state = 'done', final_evaluation = $2 WHERE id = $1 AND state = 'open' AND outbox_state = 'none' RETURNING *`, [poll.id, evaluation]);
|
||||
if (!changed.rowCount) return;
|
||||
Object.assign(poll, changed.rows[0]);
|
||||
await audit(client, poll, 'poll.stale', null, { reason: 'RESET_ON_CHANGE' });
|
||||
await recordE2(client, poll, evaluation, ['RESET_ON_CHANGE']);
|
||||
if (poll.rule.dryRun || !poll.draft_id) return;
|
||||
if (poll.rule.resetOnChange) {
|
||||
await openPollInTransaction(client, poll.workspace_id, {
|
||||
draftId: poll.draft_id, kind: poll.kind, rule: poll.rule, question: poll.question,
|
||||
parentPollId: poll.id, channelId: poll.channel_id, topicBindingId: poll.topic_binding_id,
|
||||
...(poll.eligible_user_ids ? { eligibleUserIds: poll.eligible_user_ids } : {}),
|
||||
}, { userId: await rearmActor(client, poll) });
|
||||
} else {
|
||||
// Stale content can never be auto-approved, even with an approve expiry
|
||||
// policy. escalateOwner keeps it in review; all other stale gates reject.
|
||||
await transitionPollDraft(client, { workspaceId: poll.workspace_id, draftId: poll.draft_id, pollId: poll.id,
|
||||
passed: false, reason: 'RESET_ON_CHANGE', escalate: poll.rule.timebox?.onExpire === 'escalateOwner' });
|
||||
}
|
||||
}
|
||||
async function rearmActor(client: PoolClient, poll: PollRow): Promise<string> {
|
||||
const roster = await members(client, poll.workspace_id);
|
||||
const actor = roster.find(m => m.user_id === poll.created_by && m.can_vote) ?? roster.find(m => m.can_vote && ['Owner','Admin'].includes(m.role));
|
||||
if (!actor) throw new ConflictException('cannot re-arm without an eligible opener');
|
||||
return actor.user_id;
|
||||
}
|
||||
/** Called under publishing's workspace/draft lock, after its revision trigger.
|
||||
* The old options and ballots stay attached to the stale artifact forever. */
|
||||
export async function markDraftPollsStale(client: PoolClient, workspaceId: string, draftId: string): Promise<void> {
|
||||
const polls = (await client.query<PollRow>("SELECT * FROM polls WHERE workspace_id = $1 AND draft_id = $2 AND state = 'open' FOR UPDATE", [workspaceId, draftId])).rows;
|
||||
for (const poll of polls) await evaluateLocked(client, poll);
|
||||
}
|
||||
/** Auto-arm on submit, using ONLY compiled destination snapshots. */
|
||||
export async function autoOpenDraftPoll(client: PoolClient, workspaceId: string, draftId: string, actor: PollActor, post: DraftPost): Promise<void> {
|
||||
if (!draftDestinationAccounts(post).length) return;
|
||||
const rule = await draftPollRule(client, { post });
|
||||
if (!rule) return;
|
||||
if ((await client.query("SELECT 1 FROM polls WHERE draft_id = $1 AND state = 'open'", [draftId])).rowCount) return;
|
||||
await openPollInTransaction(client, workspaceId, { draftId, kind: 'variant_selection', rule }, actor);
|
||||
}
|
||||
76
apps/api/src/connections/connection-store.test.ts
Normal file
76
apps/api/src/connections/connection-store.test.ts
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import type { PoolClient } from "pg";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { ConnectorConnection } from "@tactical-media-broadcast-hub/core";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import { Vault } from "../vault/vault.js";
|
||||
import { PgConnectionStore } from "./connection-store.js";
|
||||
|
||||
const session: SessionPayload = { workspaceId: randomUUID(), userId: randomUUID(), role: "Owner", authzEpoch: 3, issuedAt: 0, expiresAt: Math.floor(Date.now() / 1000) + 600 };
|
||||
const connection: ConnectorConnection = { id: randomUUID(), workspaceId: session.workspaceId, provider: "mastodon", channelId: "remote-account", credentialId: { kind: "credential-ref", id: "opaque" }, fields: { instanceUrl: "https://social.example" }, createdAt: new Date() };
|
||||
|
||||
function fixture(allowed = true, existingCredential = false) {
|
||||
const queries: Array<{ sql: string; params: unknown[] }> = [];
|
||||
const credentialId = randomUUID();
|
||||
const client = {
|
||||
async query(sql: string, params: unknown[] = []) {
|
||||
queries.push({ sql, params });
|
||||
if (sql.includes("FROM memberships")) return { rowCount: allowed ? 1 : 0, rows: [] };
|
||||
if (sql.startsWith("UPDATE scheduled_publications")) return { rowCount: 1, rows: [{ id: connection.id, draft_id: connection.id }] };
|
||||
if (sql.includes("SELECT id FROM credentials")) return { rows: existingCredential ? [{ id: credentialId }] : [] };
|
||||
if (sql.includes("SELECT c.*, cr.ciphertext")) return { rows: [{ id: connection.id, provider: connection.provider, channel_id: connection.channelId, fields: connection.fields, created_at: connection.createdAt, ciphertext: null, key_version: 1 }] };
|
||||
return { rowCount: 1, rows: [{ id: connection.id }] };
|
||||
},
|
||||
} as unknown as PoolClient;
|
||||
const store = new PgConnectionStore(async (workspaceId, callback) => {
|
||||
expect(workspaceId).toBe(session.workspaceId);
|
||||
return callback(client);
|
||||
});
|
||||
return { store, queries, credentialId };
|
||||
}
|
||||
|
||||
describe("Postgres connection writes", () => {
|
||||
it("saves only a vault envelope and identifier-only audit in the same tenant transaction", async () => {
|
||||
const { store, queries } = fixture();
|
||||
const sealed = await new Vault().seal(session.workspaceId, connection.id, Buffer.from('{"access_token":"never-in-db-plaintext"}'));
|
||||
await store.save(session, connection, { provider: connection.provider, remoteId: "remote-account", handle: "@alice" }, sealed, false);
|
||||
const credential = queries.find(({ sql }) => sql.startsWith("INSERT INTO credentials"));
|
||||
expect(credential?.params).toContain(sealed.ciphertext);
|
||||
expect(queries[0]?.sql).toContain("FOR UPDATE");
|
||||
expect(JSON.stringify(queries)).not.toContain("never-in-db-plaintext");
|
||||
const audit = queries.find(({ sql }) => sql.includes("audit_append"));
|
||||
expect(audit?.params.at(-1)).toEqual({ provider: connection.provider });
|
||||
});
|
||||
it("rotates the credential row in place and blocks a revoked authorization before writes", async () => {
|
||||
const { store, queries, credentialId } = fixture(true, true);
|
||||
const sealed = await new Vault().seal(session.workspaceId, connection.id, Buffer.from("rotated-secret"));
|
||||
await store.save(session, connection, { provider: connection.provider, remoteId: "remote-account", handle: "@alice" }, sealed, true);
|
||||
expect(queries.find(({ sql }) => sql.startsWith("UPDATE credentials"))?.params[0]).toBe(credentialId);
|
||||
expect(queries.some(({ sql }) => sql.startsWith("INSERT INTO connections"))).toBe(false);
|
||||
const denied = fixture(false);
|
||||
await expect(denied.store.save(session, connection, { provider: connection.provider, remoteId: "remote-account", handle: "@alice" }, sealed, true)).rejects.toThrow();
|
||||
expect(denied.queries.some(({ sql }) => /^(INSERT|UPDATE)/.test(sql))).toBe(false);
|
||||
});
|
||||
it("locks and revokes locally before provider I/O while preserving publishing history", async () => {
|
||||
const { store, queries } = fixture();
|
||||
await store.remove(session, connection.id, async (stored) => {
|
||||
expect(stored.connection.id).toBe(connection.id);
|
||||
expect(queries.some(({ sql }) => sql.includes("FOR UPDATE"))).toBe(true);
|
||||
expect(queries.some(({ sql }) => sql.startsWith("DELETE"))).toBe(false);
|
||||
});
|
||||
expect(queries.filter(({ sql }) => sql.startsWith("DELETE"))).toHaveLength(0);
|
||||
expect(queries.some(({ sql }) => sql.startsWith("UPDATE credentials SET revoked_at"))).toBe(true);
|
||||
expect(queries.some(({ sql }) => sql.startsWith("UPDATE connections SET disconnected_at"))).toBe(true);
|
||||
expect(queries.some(({ sql }) => sql.includes("needs_reauthorization"))).toBe(true);
|
||||
expect(queries.some(({ sql }) => sql.startsWith("UPDATE publishing_attempts") && sql.includes("abandoned"))).toBe(true);
|
||||
expect(queries.some(({ sql }) => sql.startsWith("UPDATE drafts") && sql.includes("approved_revision = revision"))).toBe(true);
|
||||
expect(queries.at(-1)?.sql).toContain("audit_append");
|
||||
});
|
||||
it("retains revoked local evidence and audits provider revocation failure", async () => {
|
||||
const { store, queries } = fixture();
|
||||
await expect(store.remove(session, connection.id, async () => { throw new Error("provider offline"); })).resolves.toBeUndefined();
|
||||
expect(queries.some(({ sql }) => sql.startsWith("UPDATE credentials SET revoked_at"))).toBe(true);
|
||||
expect(queries.at(-1)?.sql).toContain("connection.revoke_failed");
|
||||
expect(queries.some(({ sql }) => sql.startsWith("DELETE"))).toBe(false);
|
||||
});
|
||||
});
|
||||
117
apps/api/src/connections/connection-store.ts
Normal file
117
apps/api/src/connections/connection-store.ts
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
import { NotFoundException, ForbiddenException } from "@nestjs/common";
|
||||
import type { ConnectorConnection, NormalizedIdentity } from "@tactical-media-broadcast-hub/core";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import type { PoolClient } from "pg";
|
||||
import { withTenant } from "../db/client.js";
|
||||
import type { SealedCredential } from "../vault/vault.js";
|
||||
import type { ConnectionSummary } from "./connections.service.js";
|
||||
import { scrub } from "../audit/audit.service.js";
|
||||
|
||||
export interface StoredConnection { connection: ConnectorConnection; credential: SealedCredential | null }
|
||||
export interface ConnectionStore {
|
||||
list(workspaceId: string): Promise<ConnectionSummary[]>;
|
||||
get(workspaceId: string, id: string): Promise<StoredConnection>;
|
||||
assertAuthorized(session: SessionPayload): Promise<void>;
|
||||
save(session: SessionPayload, connection: ConnectorConnection, identity: NormalizedIdentity, credential: SealedCredential, reconnect: boolean): Promise<void>;
|
||||
rotateCredential(workspaceId: string, id: string, previous: SealedCredential, credential: SealedCredential): Promise<void>;
|
||||
remove(session: SessionPayload, id: string, revoke: (stored: StoredConnection) => Promise<void>): Promise<void>;
|
||||
}
|
||||
|
||||
export class PgConnectionStore implements ConnectionStore {
|
||||
constructor(private readonly transaction: typeof withTenant = withTenant) {}
|
||||
async assertAuthorized(session: SessionPayload): Promise<void> {
|
||||
if (session.expiresAt <= Math.floor(Date.now() / 1000)) throw new ForbiddenException("Session expired");
|
||||
const result = await this.transaction(session.workspaceId, (client) => client.query(
|
||||
`SELECT 1 FROM workspaces w JOIN memberships m ON m.workspace_id = w.id
|
||||
JOIN role_permissions rp ON rp.role = m.role
|
||||
WHERE w.id = $1 AND w.authz_epoch = $2 AND m.user_id = $3 AND m.status = 'active' AND rp.permission = 'connection.manage'`,
|
||||
[session.workspaceId, session.authzEpoch, session.userId],
|
||||
));
|
||||
if (result.rowCount !== 1) throw new ForbiddenException("Connection authorization no longer valid");
|
||||
}
|
||||
async list(workspaceId: string): Promise<ConnectionSummary[]> {
|
||||
const res = await this.transaction(workspaceId, (client) => client.query(
|
||||
`SELECT c.*, cr.ciphertext, coalesce(cr.rotated_at, cr.created_at) AS rotated_at,
|
||||
(SELECT array_agg(p.published_at ORDER BY p.published_at DESC) FROM publications p
|
||||
WHERE p.connection_id = c.id AND p.published_at > now() - interval '24 hours') AS recent
|
||||
FROM connections c LEFT JOIN LATERAL (SELECT * FROM credentials WHERE connection_id = c.id AND kind = 'connector-oauth' AND revoked_at IS NULL LIMIT 1) cr ON true WHERE c.disconnected_at IS NULL ORDER BY c.created_at`,
|
||||
));
|
||||
return res.rows.map((row: { id: string; provider: string; channel_id: string; fields: Record<string, unknown>; ciphertext: Buffer | null; created_at: Date; rotated_at: Date | null; recent: Date[] | null }) => ({
|
||||
id: row.id, provider: row.provider, channelId: row.channel_id, fields: scrub(row.fields) ?? {},
|
||||
health: row.ciphertext ? "unavailable" : "needs_credentials", credentialRotatedAt: row.rotated_at?.toISOString() ?? null,
|
||||
recentPublishes: (row.recent ?? []).map((date) => date.toISOString()), createdAt: row.created_at.toISOString(),
|
||||
}));
|
||||
}
|
||||
async get(workspaceId: string, id: string): Promise<StoredConnection> {
|
||||
return this.transaction(workspaceId, (client) => this.read(client, workspaceId, id));
|
||||
}
|
||||
private async read(client: PoolClient, workspaceId: string, id: string): Promise<StoredConnection> {
|
||||
const result = await client.query(
|
||||
`SELECT c.*, cr.ciphertext, cr.key_version FROM connections c LEFT JOIN LATERAL
|
||||
(SELECT * FROM credentials WHERE connection_id = c.id AND kind = 'connector-oauth' AND revoked_at IS NULL LIMIT 1) cr ON true WHERE c.id = $1`, [id],
|
||||
);
|
||||
const row = result.rows[0] as { id: string; provider: string; channel_id: string; fields: Record<string, string>; created_at: Date; ciphertext: Buffer | null; key_version: number } | undefined;
|
||||
if (!row) throw new NotFoundException("Connection not found");
|
||||
return {
|
||||
connection: { id: row.id, workspaceId, provider: row.provider, channelId: row.channel_id, fields: row.fields, createdAt: row.created_at, credentialId: { kind: "credential-ref", id: row.id } },
|
||||
credential: row.ciphertext ? { ciphertext: row.ciphertext, keyVersion: row.key_version } : null,
|
||||
};
|
||||
}
|
||||
async save(session: SessionPayload, connection: ConnectorConnection, identity: NormalizedIdentity, credential: SealedCredential, reconnect: boolean): Promise<void> {
|
||||
await this.transaction(session.workspaceId, async (client) => {
|
||||
// Serialize with membership revocation; re-check at commit, not only before provider I/O.
|
||||
await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [session.workspaceId]);
|
||||
const allowed = await client.query(`SELECT 1 FROM memberships m JOIN workspaces w ON w.id = m.workspace_id JOIN role_permissions rp ON rp.role = m.role WHERE m.user_id = $1 AND m.status = 'active' AND w.authz_epoch = $2 AND rp.permission = 'connection.manage'`, [session.userId, session.authzEpoch]);
|
||||
if (!allowed.rowCount || session.expiresAt <= Math.floor(Date.now() / 1000)) throw new ForbiddenException();
|
||||
const fields = { ...connection.fields, handle: identity.handle, displayName: identity.displayName ?? identity.handle, remoteId: identity.remoteId };
|
||||
if (reconnect) {
|
||||
const updated = await client.query("UPDATE connections SET fields = $2, disconnected_at = NULL WHERE id = $1 AND provider = $3 RETURNING id", [connection.id, fields, connection.provider]);
|
||||
if (!updated.rowCount) throw new NotFoundException("Connection removed during authorization");
|
||||
} else {
|
||||
await client.query("INSERT INTO connections (id, workspace_id, provider, channel_id, fields) VALUES ($1,$2,$3,$4,$5)", [connection.id, session.workspaceId, connection.provider, connection.channelId, fields]);
|
||||
}
|
||||
const existing = await client.query("SELECT id FROM credentials WHERE connection_id = $1 AND kind = 'connector-oauth' FOR UPDATE", [connection.id]);
|
||||
if (existing.rows[0]) {
|
||||
await client.query("UPDATE credentials SET ciphertext = $2, key_version = $3, rotated_at = now(), revoked_at = NULL WHERE id = $1", [existing.rows[0].id, credential.ciphertext, credential.keyVersion]);
|
||||
} else {
|
||||
await client.query("INSERT INTO credentials (workspace_id, connection_id, kind, ciphertext, key_version) VALUES ($1,$2,'connector-oauth',$3,$4)", [session.workspaceId, connection.id, credential.ciphertext, credential.keyVersion]);
|
||||
}
|
||||
await client.query("SELECT audit_append($1,$2,$3,$4,'connection',$5,$6)", [session.workspaceId, session.userId, session.role, reconnect ? "connection.reconnect" : "connection.connect", connection.id, { provider: connection.provider }]);
|
||||
});
|
||||
}
|
||||
async remove(session: SessionPayload, id: string, revoke: (stored: StoredConnection) => Promise<void>): Promise<void> {
|
||||
let stored: StoredConnection | undefined;
|
||||
await this.transaction(session.workspaceId, async (client) => {
|
||||
await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [session.workspaceId]);
|
||||
const allowed = await client.query(`SELECT 1 FROM memberships m JOIN workspaces w ON w.id = m.workspace_id JOIN role_permissions rp ON rp.role = m.role WHERE m.user_id = $1 AND m.status = 'active' AND w.authz_epoch = $2 AND rp.permission = 'connection.manage'`, [session.userId, session.authzEpoch]);
|
||||
if (!allowed.rowCount || session.expiresAt <= Math.floor(Date.now() / 1000)) throw new ForbiddenException();
|
||||
const locked = await client.query("SELECT id FROM connections WHERE id = $1 FOR UPDATE", [id]);
|
||||
if (!locked.rowCount) throw new NotFoundException("Connection not found");
|
||||
stored = await this.read(client, session.workspaceId, id);
|
||||
await client.query("UPDATE credentials SET revoked_at = now() WHERE connection_id = $1 AND revoked_at IS NULL", [id]);
|
||||
await client.query("UPDATE connections SET disconnected_at = COALESCE(disconnected_at, now()) WHERE id = $1", [id]);
|
||||
// Undispatched intent pauses. Dispatched or unknown work retains all evidence.
|
||||
const paused = await client.query<{ id: string; draft_id: string }>(`UPDATE scheduled_publications s SET state = 'needs_reauthorization', error_code = 'CREDENTIAL_CHANGED'
|
||||
WHERE connection_id = $1 AND state IN ('pending', 'in_flight') AND NOT EXISTS
|
||||
(SELECT 1 FROM publishing_attempts a WHERE a.schedule_id = s.id AND (a.status = 'unknown' OR (a.dispatch_started_at IS NOT NULL AND a.status NOT IN ('completed', 'failed'))))
|
||||
RETURNING s.id, s.draft_id`, [id]);
|
||||
const scheduleIds = paused.rows.map((row) => row.id);
|
||||
const draftIds = paused.rows.map((row) => row.draft_id);
|
||||
await client.query("UPDATE publishing_attempts SET status = 'abandoned', owner_token = NULL, lease_until = NULL, error_code = 'CREDENTIAL_CHANGED' WHERE schedule_id = ANY($1::uuid[]) AND status IN ('ready', 'in_progress')", [scheduleIds]);
|
||||
await client.query("UPDATE drafts SET state = CASE WHEN approved_revision = revision AND approved_hash IS NOT NULL THEN 'approved' ELSE 'editing' END, updated_at = now() WHERE id = ANY($1::uuid[]) AND state = 'scheduled'", [draftIds]);
|
||||
await client.query("SELECT audit_append($1,$2,$3,'connection.remove','connection',$4,NULL)", [session.workspaceId, session.userId, session.role, id]);
|
||||
});
|
||||
if (stored) {
|
||||
try { await revoke(stored); }
|
||||
catch { await this.transaction(session.workspaceId, (client) => client.query("SELECT audit_append($1,$2,$3,'connection.revoke_failed','connection',$4,NULL)", [session.workspaceId, session.userId, session.role, id])); }
|
||||
}
|
||||
}
|
||||
async rotateCredential(workspaceId: string, id: string, previous: SealedCredential, credential: SealedCredential): Promise<void> {
|
||||
await this.transaction(workspaceId, async (client) => {
|
||||
await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]);
|
||||
// A health refresh must never overwrite a concurrent explicit reconnect.
|
||||
const updated = await client.query("UPDATE credentials SET ciphertext = $3, key_version = $4, rotated_at = now() WHERE connection_id = $1 AND ciphertext = $2 AND kind = 'connector-oauth' AND revoked_at IS NULL RETURNING id", [id, previous.ciphertext, credential.ciphertext, credential.keyVersion]);
|
||||
if (updated.rowCount) await client.query("SELECT audit_append($1,NULL,'Service','credential.refresh','connection',$2,NULL)", [workspaceId, id]);
|
||||
});
|
||||
}
|
||||
}
|
||||
84
apps/api/src/connections/connections.controller.ts
Normal file
84
apps/api/src/connections/connections.controller.ts
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
import { Body, Controller, Delete, Get, Inject, Param, Post, Query, Req, Res, UseGuards } from "@nestjs/common";
|
||||
import type { Request, Response } from "express";
|
||||
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import { optionalString, parseId, requireObject, requireSession, requireString } from "../http/validation.js";
|
||||
import { ConnectionsService, type ConnectionSummary } from "./connections.service.js";
|
||||
import { AUTH_TTL_MS, stateCookieName, validState } from "./pending.js";
|
||||
|
||||
@Controller("connections")
|
||||
export class ConnectionsController {
|
||||
constructor(@Inject(ConnectionsService) private readonly connections: ConnectionsService) {}
|
||||
|
||||
@Get()
|
||||
@UseGuards(RbacGuard)
|
||||
@RequirePermission("connection.view")
|
||||
async list(@Req() request: Request & AuthenticatedRequest): Promise<ConnectionSummary[]> {
|
||||
return this.connections.list(requireSession(request).workspaceId);
|
||||
}
|
||||
|
||||
@Get("providers")
|
||||
@UseGuards(RbacGuard)
|
||||
@RequirePermission("connection.view")
|
||||
providers() { return this.connections.providers(); }
|
||||
|
||||
private redirectResponse(response: Response, authorization: { redirectUrl: string; state: string; binding: string }) {
|
||||
response.setHeader("Cache-Control", "no-store");
|
||||
response.cookie(stateCookieName(authorization.state), authorization.binding, {
|
||||
httpOnly: true, secure: process.env.NODE_ENV === "production", sameSite: "lax",
|
||||
path: "/connections/callback", maxAge: AUTH_TTL_MS,
|
||||
});
|
||||
return { redirectUrl: authorization.redirectUrl };
|
||||
}
|
||||
|
||||
@Post("authorize")
|
||||
@UseGuards(RbacGuard)
|
||||
@RequirePermission("connection.manage")
|
||||
async authorize(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown, @Res({ passthrough: true }) response: Response) {
|
||||
const input = requireObject(body, "authorization");
|
||||
const authorization = await this.connections.authorize(requireSession(request), requireString(input.provider, "provider"), optionalString(input.instanceUrl, "instanceUrl"));
|
||||
return this.redirectResponse(response, authorization);
|
||||
}
|
||||
|
||||
/** Deliberately outside RbacGuard: the provider's browser redirect carries no bearer header. */
|
||||
@Get("callback")
|
||||
async callback(@Req() request: Request, @Query() query: Record<string, unknown>, @Res() response: Response): Promise<void> {
|
||||
const state = typeof query.state === "string" ? query.state : "";
|
||||
const name = validState(state) ? stateCookieName(state) : "";
|
||||
const cookie = name ? (request.headers.cookie ?? "").split(";").map((part) => part.trim()).find((part) => part.startsWith(`${name}=`))?.slice(name.length + 1) : undefined;
|
||||
let binding = "";
|
||||
try { binding = decodeURIComponent(cookie ?? ""); } catch { /* Malformed cookies fail state validation. */ }
|
||||
const location = await this.connections.callback(state, binding, typeof query.code === "string" ? query.code : undefined, typeof query.error === "string" ? query.error : undefined);
|
||||
if (name) response.clearCookie(name, { path: "/connections/callback", httpOnly: true, secure: process.env.NODE_ENV === "production", sameSite: "lax" });
|
||||
response.setHeader("Cache-Control", "no-store");
|
||||
response.setHeader("Referrer-Policy", "no-referrer");
|
||||
response.redirect(303, location);
|
||||
}
|
||||
|
||||
// Public endpoint spelling is API routing, not provider behavior branching.
|
||||
// eslint-disable-next-line no-restricted-syntax
|
||||
@Post("telegram/bot-token")
|
||||
@UseGuards(RbacGuard)
|
||||
@RequirePermission("connection.manage")
|
||||
async botToken(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise<{ connected: true; token: string }> {
|
||||
const input = requireObject(body, "bot token");
|
||||
const id = optionalString(input.connectionId, "connectionId");
|
||||
const channelId = optionalString(input.channelId, "channelId");
|
||||
const token = await this.connections.botToken(requireSession(request), requireString(input.token, "token"), channelId ? { channelId } : {}, id ? parseId(id) : undefined);
|
||||
return { connected: true, token };
|
||||
}
|
||||
|
||||
@Post(":connectionId/reconnect")
|
||||
@UseGuards(RbacGuard)
|
||||
@RequirePermission("connection.manage")
|
||||
async reconnect(@Req() request: Request & AuthenticatedRequest, @Param("connectionId") id: string, @Res({ passthrough: true }) response: Response) {
|
||||
return this.redirectResponse(response, await this.connections.reconnect(requireSession(request), parseId(id)));
|
||||
}
|
||||
|
||||
@Delete(":connectionId")
|
||||
@UseGuards(RbacGuard)
|
||||
@RequirePermission("connection.manage")
|
||||
async remove(@Req() request: Request & AuthenticatedRequest, @Param("connectionId") id: string): Promise<{ removed: true; token: string }> {
|
||||
const token = await this.connections.remove(requireSession(request), parseId(id));
|
||||
return { removed: true, token };
|
||||
}
|
||||
}
|
||||
345
apps/api/src/connections/connections.integration.test.ts
Normal file
345
apps/api/src/connections/connections.integration.test.ts
Normal file
|
|
@ -0,0 +1,345 @@
|
|||
import "reflect-metadata";
|
||||
import { createServer, type Server, type IncomingMessage, type ServerResponse } from "node:http";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { Module, UnauthorizedException, type ExecutionContext, type INestApplication } from "@nestjs/common";
|
||||
import { NestFactory } from "@nestjs/core";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { ConnectorConnection, NormalizedIdentity } from "@tactical-media-broadcast-hub/core";
|
||||
import { ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import { createMastodonConnector } from "../../../../packages/connector-mastodon/src/index.js";
|
||||
import { createTelegramConnector } from "@tactical-media-broadcast-hub/connector-telegram";
|
||||
import { createPeerTubeConnector } from "../../../../packages/connector-peertube/src/index.js";
|
||||
import { Vault, type SealedCredential } from "../vault/vault.js";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import { RbacGuard } from "../rbac/rbac.guard.js";
|
||||
import { logger } from "../telemetry/logger.js";
|
||||
import { ConnectionsController } from "./connections.controller.js";
|
||||
import { CONNECTION_DEPENDENCIES, ConnectionsService, type ConnectionDependencies, type ConnectionSummary } from "./connections.service.js";
|
||||
import type { ConnectionStore, StoredConnection } from "./connection-store.js";
|
||||
import { RedisPendingStore } from "./pending.js";
|
||||
import { MemoryPendingRedis } from "./testing.js";
|
||||
import { injectHttp } from "./http-test-transport.js";
|
||||
|
||||
class MemoryConnections implements ConnectionStore {
|
||||
readonly rows = new Map<string, StoredConnection>();
|
||||
readonly audit: Array<Record<string, string>> = [];
|
||||
authorized = true;
|
||||
async assertAuthorized(session: SessionPayload): Promise<void> {
|
||||
if (!this.authorized || session.workspaceId !== sessionFixture.workspaceId) throw new Error("membership revoked");
|
||||
}
|
||||
async list(workspaceId: string): Promise<ConnectionSummary[]> {
|
||||
return [...this.rows.values()].filter(({ connection }) => connection.workspaceId === workspaceId).map(({ connection, credential }) => ({
|
||||
id: connection.id, provider: connection.provider, channelId: connection.channelId, fields: { ...connection.fields },
|
||||
health: credential ? "unavailable" : "needs_credentials", credentialRotatedAt: null, recentPublishes: [], createdAt: connection.createdAt.toISOString(),
|
||||
}));
|
||||
}
|
||||
async get(workspaceId: string, id: string): Promise<StoredConnection> {
|
||||
const row = this.rows.get(id);
|
||||
if (!row || row.connection.workspaceId !== workspaceId) throw new Error("connection not found");
|
||||
return row;
|
||||
}
|
||||
async save(session: SessionPayload, connection: ConnectorConnection, identity: NormalizedIdentity, credential: SealedCredential, reconnect: boolean): Promise<void> {
|
||||
await this.assertAuthorized(session);
|
||||
if (reconnect && !this.rows.has(connection.id)) throw new Error("connection removed");
|
||||
this.rows.set(connection.id, { connection: { ...connection, fields: { ...connection.fields, remoteId: identity.remoteId, handle: identity.handle } }, credential });
|
||||
this.audit.push({ action: reconnect ? "connection.reconnect" : "connection.connect", provider: connection.provider });
|
||||
}
|
||||
async remove(session: SessionPayload, id: string, revoke: (stored: StoredConnection) => Promise<void>): Promise<void> {
|
||||
await this.assertAuthorized(session);
|
||||
await revoke(await this.get(session.workspaceId, id));
|
||||
this.rows.delete(id);
|
||||
this.audit.push({ action: "connection.remove" });
|
||||
}
|
||||
async rotateCredential(workspaceId: string, id: string, previous: SealedCredential, credential: SealedCredential): Promise<void> {
|
||||
const row = await this.get(workspaceId, id);
|
||||
if (row.credential?.ciphertext.equals(previous.ciphertext)) {
|
||||
row.credential = credential;
|
||||
this.audit.push({ action: "credential.refresh" });
|
||||
}
|
||||
}
|
||||
}
|
||||
const sessionFixture: SessionPayload = {
|
||||
userId: randomUUID(), workspaceId: randomUUID(), role: "Owner", authzEpoch: 1,
|
||||
issuedAt: Math.floor(Date.now() / 1000), expiresAt: Math.floor(Date.now() / 1000) + 3600,
|
||||
};
|
||||
@Module({})
|
||||
class TestModule {}
|
||||
|
||||
async function form(request: IncomingMessage): Promise<URLSearchParams> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of request) chunks.push(Buffer.from(chunk as Uint8Array));
|
||||
return new URLSearchParams(Buffer.concat(chunks).toString());
|
||||
}
|
||||
function json(response: ServerResponse, value: unknown, status = 200): void {
|
||||
response.writeHead(status, { "content-type": "application/json" }); response.end(JSON.stringify(value));
|
||||
}
|
||||
|
||||
describe("HTTP OAuth connect with real connectors and mock authorization server", () => {
|
||||
let mock: Server;
|
||||
let origin: string;
|
||||
let apiOrigin: string;
|
||||
let app: INestApplication;
|
||||
const store = new MemoryConnections();
|
||||
const redis = new MemoryPendingRedis();
|
||||
const encryption = new Vault();
|
||||
const activeTokens = new Set<string>();
|
||||
const grants = new Map<string, { challenge: string; redirectUri: string }>();
|
||||
const revokedTokens: string[] = [];
|
||||
const guard = vi.spyOn(RbacGuard.prototype, "canActivate").mockImplementation(async (context: ExecutionContext) => {
|
||||
const request = context.switchToHttp().getRequest<{ headers: Record<string, string>; session?: SessionPayload }>();
|
||||
if (request.headers.authorization !== "Bearer test-session") throw new UnauthorizedException();
|
||||
request.session = sessionFixture; return true;
|
||||
});
|
||||
const warn = vi.spyOn(logger, "warn").mockImplementation(() => undefined);
|
||||
const info = vi.spyOn(logger, "info").mockImplementation(() => undefined);
|
||||
let nextAccount = "account-1";
|
||||
let issued = 0;
|
||||
let failExchange = false;
|
||||
let bridgeConfigured = false;
|
||||
const botToken = "12345:bot_token_secret";
|
||||
|
||||
beforeEach(() => {
|
||||
store.rows.clear(); store.audit.length = 0; store.authorized = true;
|
||||
redis.entries.clear(); activeTokens.clear(); grants.clear(); revokedTokens.length = 0;
|
||||
nextAccount = "account-1"; issued = 0; failExchange = false; bridgeConfigured = false;
|
||||
warn.mockClear(); info.mockClear();
|
||||
});
|
||||
|
||||
beforeAll(async () => {
|
||||
mock = createServer((request, response) => { void (async () => {
|
||||
const url = new URL(request.url ?? "/", origin);
|
||||
if (url.pathname === "/api/v1/apps") {
|
||||
const body = await form(request);
|
||||
expect(body.get("redirect_uris")).toBe(`${apiOrigin}/connections/callback`);
|
||||
json(response, { client_id: "client-id", client_secret: "registered-client-secret" });
|
||||
} else if (url.pathname === "/oauth/authorize") {
|
||||
const code = randomUUID();
|
||||
grants.set(code, { challenge: url.searchParams.get("code_challenge") ?? "", redirectUri: url.searchParams.get("redirect_uri") ?? "" });
|
||||
const redirect = new URL(url.searchParams.get("redirect_uri") ?? "");
|
||||
redirect.searchParams.set("state", url.searchParams.get("state") ?? "");
|
||||
redirect.searchParams.set("code", code);
|
||||
response.writeHead(302, { location: redirect.toString() }); response.end();
|
||||
} else if (url.pathname === "/oauth/token") {
|
||||
const body = await form(request);
|
||||
const code = body.get("code") ?? "";
|
||||
const grant = grants.get(code);
|
||||
grants.delete(code);
|
||||
if (failExchange || !grant || grant.redirectUri !== body.get("redirect_uri") || grant.challenge !== createHash("sha256").update(body.get("code_verifier") ?? "").digest("base64url") || body.get("client_secret") !== "registered-client-secret") {
|
||||
json(response, { error: "secret echoed: registered-client-secret token-secret-leak" }, 400); return;
|
||||
}
|
||||
const token = `access-token-secret-${++issued}`;
|
||||
activeTokens.add(token);
|
||||
json(response, { access_token: token, refresh_token: `refresh-token-secret-${issued}`, token_type: "Bearer", scope: "read write:statuses write:media", created_at: Date.now() / 1000 });
|
||||
} else if (url.pathname === "/api/v1/accounts/verify_credentials") {
|
||||
const token = (request.headers.authorization ?? "").slice(7);
|
||||
if (!activeTokens.has(token)) { json(response, { error: `revoked ${token}` }, 401); return; }
|
||||
json(response, { id: nextAccount, username: "alice", acct: "alice", display_name: "Alice", avatar: "", url: `${origin}/@alice` });
|
||||
} else if (url.pathname === "/api/v1/oauth-clients/local") {
|
||||
json(response, { client_id: "client-id", client_secret: "registered-client-secret" });
|
||||
} else if (url.pathname === "/api/v1/users/me") {
|
||||
const token = (request.headers.authorization ?? "").slice(7);
|
||||
if (!activeTokens.has(token)) { json(response, { error: "invalid token" }, 401); return; }
|
||||
json(response, { id: 1, username: "alice", account: { id: 1, name: "alice", host: "127.0.0.1", displayName: "Alice" } });
|
||||
} else if (url.pathname === "/api/v1/users/token") {
|
||||
const body = await form(request);
|
||||
if (body.get("grant_type") !== "refresh_token" || body.get("client_secret") !== "registered-client-secret") { json(response, { error: "invalid refresh" }, 401); return; }
|
||||
const token = `access-token-secret-${++issued}`;
|
||||
activeTokens.add(token);
|
||||
json(response, { access_token: token, refresh_token: `refresh-token-secret-${issued}` });
|
||||
} else if (url.pathname === "/api/v1/users/revoke-token") {
|
||||
const token = (request.headers.authorization ?? "").slice(7);
|
||||
revokedTokens.push(token); activeTokens.delete(token); json(response, {});
|
||||
} else if (url.pathname === "/oauth/revoke") {
|
||||
const body = await form(request);
|
||||
const token = body.get("token") ?? "";
|
||||
revokedTokens.push(token); activeTokens.delete(token); json(response, {});
|
||||
} else if (url.pathname.startsWith("/bot")) {
|
||||
if (!url.pathname.startsWith(`/bot${botToken}/`)) { json(response, { ok: false, error_code: 401, description: `unauthorized ${url.pathname}` }); return; }
|
||||
json(response, { ok: true, result: { id: 12345, is_bot: true, username: "test_bot", first_name: "Test bot" } });
|
||||
} else { json(response, { error: "not found" }, 404); }
|
||||
})().catch(() => json(response, { error: "mock failure" }, 500)); });
|
||||
origin = "http://127.0.0.1:4500";
|
||||
const deps: ConnectionDependencies = {
|
||||
renewSession: async () => "replacement-session",
|
||||
store, pending: new RedisPendingStore(redis, encryption), encryption,
|
||||
appUrl: "http://localhost:3001", callbackUrl: "",
|
||||
registry: (credentials) => {
|
||||
const registry = new ConnectorRegistry();
|
||||
registry.register(createMastodonConnector({ credentials, fetchImpl: mockFetch, requestsPerSecond: 100_000 }));
|
||||
registry.register(createPeerTubeConnector({ credentials, fetchImpl: mockFetch, requestsPerSecond: 100_000,
|
||||
env: bridgeConfigured ? { PEERTUBE_AUTHORIZE_PATH: "/oauth/authorize", PEERTUBE_TOKEN_PATH: "/oauth/token" } : {},
|
||||
}));
|
||||
const botFetch: typeof fetch = (input, init) => fetch(String(input).replace("https://api.telegram.org", origin), init);
|
||||
registry.register(createTelegramConnector({ credentials, fetchImpl: botFetch, requestsPerSecond: 100_000 }));
|
||||
return registry;
|
||||
},
|
||||
};
|
||||
app = await NestFactory.create({ module: TestModule, controllers: [ConnectionsController], providers: [
|
||||
ConnectionsService, { provide: CONNECTION_DEPENDENCIES, useValue: deps },
|
||||
|
||||
] }, { logger: false });
|
||||
await app.init();
|
||||
apiOrigin = "http://api.local";
|
||||
deps.callbackUrl = `${apiOrigin}/connections/callback`;
|
||||
});
|
||||
afterAll(async () => {
|
||||
await app?.close();
|
||||
|
||||
warn.mockRestore(); info.mockRestore(); guard.mockRestore();
|
||||
});
|
||||
const mockFetch: typeof globalThis.fetch = (input, init) => injectHttp((request, response) => { mock.emit("request", request, response); }, input, init);
|
||||
const fetch: typeof globalThis.fetch = (input, init) => {
|
||||
const url = new URL(input instanceof Request ? input.url : String(input));
|
||||
if (url.origin === origin) return mockFetch(input, init);
|
||||
if (url.origin !== apiOrigin) throw new Error("Unexpected test destination");
|
||||
const adapter = app.getHttpAdapter();
|
||||
return injectHttp((request, response) => { adapter.getInstance().handle(request, response); }, input, init);
|
||||
};
|
||||
async function api(path: string, method = "GET", body?: unknown): Promise<Response> {
|
||||
return fetch(`${apiOrigin}/connections${path}`, { method, headers: { authorization: "Bearer test-session", "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
}
|
||||
async function begin(id?: string, provider = "mastodon"): Promise<{ redirectUrl: string; cookie: string }> {
|
||||
const response = id ? await api(`/${id}/reconnect`, "POST", {}) : await api("/authorize", "POST", { provider, instanceUrl: origin });
|
||||
expect(response.status).toBe(201);
|
||||
const { redirectUrl } = await response.json() as { redirectUrl: string };
|
||||
expect(new URL(redirectUrl).searchParams.get("code_challenge_method")).toBe("S256");
|
||||
return { redirectUrl, cookie: response.headers.get("set-cookie")?.split(";")[0] ?? "" };
|
||||
}
|
||||
async function finish(authorization: { redirectUrl: string; cookie: string }): Promise<Response> {
|
||||
const consent = await fetch(authorization.redirectUrl, { redirect: "manual" });
|
||||
expect(consent.status).toBe(302);
|
||||
return fetch(consent.headers.get("location") ?? "", { redirect: "manual", headers: { cookie: authorization.cookie } });
|
||||
}
|
||||
async function listed(): Promise<ConnectionSummary[]> { return (await (await api("")).json()) as ConnectionSummary[]; }
|
||||
|
||||
it("authorize → provider → unauthenticated callback → healthy list; reconnect rotates; remove revokes and clears row + vault envelope", async () => {
|
||||
const auth = await begin();
|
||||
expect(auth.cookie).toMatch(/^ap_connect_/);
|
||||
expect(JSON.stringify([...redis.entries.values()])).not.toContain("registered-client-secret");
|
||||
const callback = await finish(auth);
|
||||
expect(callback.status).toBe(303);
|
||||
expect(callback.headers.get("location")).toBe("http://localhost:3001/connections?connect=success");
|
||||
expect(callback.headers.get("set-cookie")).toContain("Expires=");
|
||||
const rows = await listed();
|
||||
expect(rows).toHaveLength(1); expect(rows[0]?.health).toBe("healthy");
|
||||
expect(rows[0]?.capabilities).toContain("publishText");
|
||||
const id = rows[0]?.id ?? "";
|
||||
const before = store.rows.get(id)?.credential?.ciphertext;
|
||||
expect(before).toBeDefined();
|
||||
const stored = store.rows.get(id)?.credential;
|
||||
if (!stored) throw new Error("credential missing");
|
||||
expect((await new Vault().open(sessionFixture.workspaceId, id, stored)).toString()).toContain("access-token-secret-1");
|
||||
expect(JSON.stringify(rows)).not.toContain("token-secret");
|
||||
const second = await finish(await begin(id));
|
||||
expect(second.headers.get("location")).toContain("connect=success");
|
||||
expect((await listed())[0]?.id).toBe(id);
|
||||
expect(store.rows.get(id)?.credential?.ciphertext.equals(before ?? Buffer.alloc(0))).toBe(false);
|
||||
expect(store.audit.at(-1)?.action).toBe("connection.reconnect");
|
||||
const replay = await finish(auth);
|
||||
expect(replay.headers.get("location")).toContain("connect=failure");
|
||||
expect((await api(`/${id}`, "DELETE")).status).toBe(200);
|
||||
expect(revokedTokens).toContain("access-token-secret-2");
|
||||
expect(await listed()).toHaveLength(0); expect(store.rows.size).toBe(0); expect(redis.entries.size).toBe(0);
|
||||
const observable = JSON.stringify({ audit: store.audit, warn: warn.mock.calls, info: info.mock.calls });
|
||||
for (const secret of ["access-token-secret", "refresh-token-secret", "registered-client-secret"]) expect(observable).not.toContain(secret);
|
||||
});
|
||||
|
||||
it("rejects missing auth on authorize, missing/wrong browser cookie and preserves a valid pending state", async () => {
|
||||
expect((await fetch(`${apiOrigin}/connections/authorize`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ provider: "mastodon", instanceUrl: origin }) })).status).toBe(401);
|
||||
const auth = await begin();
|
||||
const consent = await fetch(auth.redirectUrl, { redirect: "manual" });
|
||||
const callbackUrl = consent.headers.get("location") ?? "";
|
||||
const missing = await fetch(callbackUrl, { redirect: "manual" });
|
||||
expect(missing.headers.get("location")).toContain("connect=failure");
|
||||
const wrong = await fetch(callbackUrl, { redirect: "manual", headers: { cookie: auth.cookie.replace(sessionFixture.workspaceId, randomUUID()) } });
|
||||
expect(wrong.headers.get("location")).toContain("connect=failure");
|
||||
const good = await fetch(callbackUrl, { redirect: "manual", headers: { cookie: auth.cookie } });
|
||||
expect(good.headers.get("location")).toContain("connect=success");
|
||||
});
|
||||
|
||||
it("failed exchange and different-account reconnect retain the previous encrypted credential; health detects revoked tokens", async () => {
|
||||
await finish(await begin());
|
||||
const row = (await listed())[0];
|
||||
if (!row) throw new Error("connection missing");
|
||||
const before = store.rows.get(row.id)?.credential?.ciphertext;
|
||||
failExchange = true;
|
||||
const failure = await finish(await begin(row.id));
|
||||
failExchange = false;
|
||||
expect(failure.headers.get("location")).toContain("connect=failure");
|
||||
expect(store.rows.get(row.id)?.credential?.ciphertext.equals(before ?? Buffer.alloc(0))).toBe(true);
|
||||
nextAccount = "other-account";
|
||||
const wrongAccount = await finish(await begin(row.id));
|
||||
nextAccount = "account-1";
|
||||
expect(wrongAccount.headers.get("location")).toContain("connect=failure");
|
||||
expect(store.rows.get(row.id)?.credential?.ciphertext.equals(before ?? Buffer.alloc(0))).toBe(true);
|
||||
activeTokens.clear();
|
||||
expect((await listed())[0]?.health).toBe("needs_credentials");
|
||||
expect((await finish(await begin(row.id))).headers.get("location")).toContain("connect=success");
|
||||
expect((await listed())[0]?.health).toBe("healthy");
|
||||
expect(JSON.stringify(warn.mock.calls)).not.toContain("token-secret-leak");
|
||||
expect(JSON.stringify(warn.mock.calls)).not.toContain("registered-client-secret");
|
||||
});
|
||||
|
||||
it("rechecks workspace membership during callback, handles denial, and does not expose provider errors", async () => {
|
||||
const revoked = await begin();
|
||||
store.authorized = false;
|
||||
expect((await finish(revoked)).headers.get("location")).toContain("connect=failure");
|
||||
store.authorized = true;
|
||||
const denied = await begin();
|
||||
const url = new URL(`${apiOrigin}/connections/callback`);
|
||||
url.searchParams.set("state", new URL(denied.redirectUrl).searchParams.get("state") ?? "");
|
||||
url.searchParams.set("error", "access_denied token-secret-leak");
|
||||
const response = await fetch(url, { redirect: "manual", headers: { cookie: denied.cookie } });
|
||||
expect(response.headers.get("location")).toContain("connect=failure");
|
||||
expect(JSON.stringify(warn.mock.calls)).not.toContain("token-secret-leak");
|
||||
});
|
||||
|
||||
it("validates Telegram getMe, encrypts tokens, rotates in place and removes locally; PeerTube is capability-honest", async () => {
|
||||
const bad = await api("/telegram/bot-token", "POST", { token: "12345:bad_secret" });
|
||||
expect(bad.status).toBe(400); expect(await bad.text()).not.toContain("bad_secret");
|
||||
const response = await api("/telegram/bot-token", "POST", { token: botToken, channelId: "@target" });
|
||||
expect(response.status).toBe(201);
|
||||
const bot = (await listed()).find((row) => row.provider === "telegram");
|
||||
expect(bot?.health).toBe("healthy"); expect(bot?.channelId).toBe("@target");
|
||||
const id = bot?.id ?? "";
|
||||
const before = store.rows.get(id)?.credential?.ciphertext;
|
||||
expect((await api("/telegram/bot-token", "POST", { token: botToken, connectionId: id })).status).toBe(201);
|
||||
expect(store.rows.get(id)?.credential?.ciphertext.equals(before ?? Buffer.alloc(0))).toBe(false);
|
||||
expect((await listed()).filter((row) => row.provider === "telegram")).toHaveLength(1);
|
||||
expect((await api(`/${id}`, "DELETE")).status).toBe(200);
|
||||
expect(store.rows.has(id)).toBe(false);
|
||||
const providers = await (await api("/providers")).json() as Array<{ id: string; enabled: boolean; note?: string }>;
|
||||
expect(providers.find((provider) => provider.id === "peertube")?.enabled).toBe(false);
|
||||
expect((await api("/authorize", "POST", { provider: "peertube", instanceUrl: origin })).status).toBe(400);
|
||||
expect(JSON.stringify({ audit: store.audit, warn: warn.mock.calls, info: info.mock.calls })).not.toContain(botToken);
|
||||
});
|
||||
|
||||
it("runs PeerTube code flow only with an explicitly configured instance bridge", async () => {
|
||||
bridgeConfigured = true;
|
||||
expect((await finish(await begin(undefined, "peertube"))).headers.get("location")).toContain("connect=success");
|
||||
const row = (await listed()).find((connection) => connection.provider === "peertube");
|
||||
expect(row?.health).toBe("healthy");
|
||||
const id = row?.id ?? "";
|
||||
expect((await finish(await begin(id))).headers.get("location")).toContain("connect=success");
|
||||
expect((await listed()).filter((connection) => connection.id === id)).toHaveLength(1);
|
||||
const stored = store.rows.get(id)?.credential;
|
||||
if (!stored) throw new Error("PeerTube credential missing");
|
||||
const secrets = JSON.parse((await encryption.open(sessionFixture.workspaceId, id, stored)).toString()) as Record<string, string>;
|
||||
activeTokens.delete(secrets.access_token ?? "");
|
||||
expect((await listed()).find((connection) => connection.id === id)?.health).toBe("healthy");
|
||||
expect(store.audit.at(-1)?.action).toBe("credential.refresh");
|
||||
expect(store.rows.get(id)?.credential?.ciphertext.equals(stored.ciphertext)).toBe(false);
|
||||
expect((await api(`/${id}`, "DELETE")).status).toBe(200);
|
||||
bridgeConfigured = false;
|
||||
});
|
||||
|
||||
it("does not resurrect a connection removed while reconnect consent is pending", async () => {
|
||||
await finish(await begin());
|
||||
const row = (await listed())[0];
|
||||
if (!row) throw new Error("connection missing");
|
||||
const pending = await begin(row.id);
|
||||
expect((await api(`/${row.id}`, "DELETE")).status).toBe(200);
|
||||
expect((await finish(pending)).headers.get("location")).toContain("connect=failure");
|
||||
expect(store.rows.has(row.id)).toBe(false);
|
||||
});
|
||||
});
|
||||
231
apps/api/src/connections/connections.service.ts
Normal file
231
apps/api/src/connections/connections.service.ts
Normal file
|
|
@ -0,0 +1,231 @@
|
|||
import { BadRequestException, Inject, Injectable, Optional } from "@nestjs/common";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { ConnectorConnection, NormalizedIdentity } from "@tactical-media-broadcast-hub/core";
|
||||
import { AuthError, type AuthorizationRequest, type AuthorizationResult, type CredentialAccessor, type ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import { withTenant } from "../db/client.js";
|
||||
import { issueSessionInTenant } from "../auth/auth.service.js";
|
||||
import { vault, type Vault } from "../vault/vault.js";
|
||||
import { logger } from "../telemetry/logger.js";
|
||||
import { connectionRegistry } from "./connector-registry.js";
|
||||
import { PgConnectionStore, type ConnectionStore, type StoredConnection } from "./connection-store.js";
|
||||
import { AUTH_TTL_MS, browserHash, randomState, RedisPendingStore, type PendingStore } from "./pending.js";
|
||||
import { oauthFetch } from "./oauth-fetch.js";
|
||||
|
||||
export type ConnectionHealth = "healthy" | "needs_credentials" | "unavailable";
|
||||
export interface ConnectionSummary {
|
||||
id: string; provider: string; channelId: string; fields: Record<string, unknown>;
|
||||
health: ConnectionHealth; credentialRotatedAt: string | null; recentPublishes: string[]; createdAt: string;
|
||||
capabilities?: string[];
|
||||
}
|
||||
export interface ConnectionDependencies {
|
||||
store: ConnectionStore;
|
||||
pending: PendingStore;
|
||||
encryption: Vault;
|
||||
registry: (credentials: CredentialAccessor) => ConnectorRegistry;
|
||||
appUrl: string;
|
||||
callbackUrl: string;
|
||||
renewSession(session: SessionPayload): Promise<string>;
|
||||
}
|
||||
export const CONNECTION_DEPENDENCIES = Symbol("CONNECTION_DEPENDENCIES");
|
||||
|
||||
/** Each exchange has an isolated, tenant-bound working set; never writes plaintext to DB. */
|
||||
class StagedCredentials implements CredentialAccessor {
|
||||
dirty = false;
|
||||
constructor(private readonly workspaceId: string, private readonly connectionId: string, readonly secrets: Record<string, string> = {}) {}
|
||||
private assert(connection: ConnectorConnection): void {
|
||||
if (connection.workspaceId !== this.workspaceId || connection.id !== this.connectionId || connection.credentialId.id !== this.connectionId) throw new Error("Credential binding mismatch");
|
||||
}
|
||||
async getAccessToken(connection: ConnectorConnection): Promise<string> { return this.getSecret(connection, "access_token"); }
|
||||
async getSecret(connection: ConnectorConnection, key: string): Promise<string> {
|
||||
this.assert(connection);
|
||||
const value = this.secrets[key];
|
||||
if (!value) throw new Error("Credential missing");
|
||||
return value;
|
||||
}
|
||||
async put(connection: ConnectorConnection, key: string, value: string): Promise<void> {
|
||||
this.assert(connection);
|
||||
if (typeof value !== "string" || !value) throw new Error("Provider returned invalid credential");
|
||||
this.secrets[key] = value;
|
||||
this.dirty = true;
|
||||
}
|
||||
}
|
||||
function defaults(): ConnectionDependencies {
|
||||
const appUrl = process.env.APP_URL ?? "http://localhost:3000";
|
||||
return {
|
||||
renewSession: (session) => withTenant(session.workspaceId, (client) => issueSessionInTenant(client, session.workspaceId, session.userId, session.expiresAt)),
|
||||
store: new PgConnectionStore(), pending: new RedisPendingStore(), encryption: vault,
|
||||
registry: (credentials) => connectionRegistry(credentials, oauthFetch), appUrl,
|
||||
callbackUrl: process.env.CONNECTIONS_CALLBACK_URL ?? `${process.env.API_URL ?? "http://localhost:3000"}/connections/callback`,
|
||||
};
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class ConnectionsService {
|
||||
private readonly deps: ConnectionDependencies;
|
||||
constructor(@Optional() @Inject(CONNECTION_DEPENDENCIES) dependencies?: ConnectionDependencies) { this.deps = dependencies ?? defaults(); }
|
||||
|
||||
providers() {
|
||||
const registry = this.deps.registry(new StagedCredentials("", ""));
|
||||
return registry.list().map((connector) => {
|
||||
const manifest = connector.manifest();
|
||||
return {
|
||||
id: manifest.id, displayName: manifest.displayName, kind: manifest.oauth?.kind,
|
||||
enabled: manifest.oauth?.connectEnabled !== false && Boolean(connector.validateToken || (connector.buildAuthorizationUrl && connector.exchangeCode)),
|
||||
note: manifest.oauth?.connectNote, targetField: manifest.oauth?.targetField,
|
||||
instanceRequired: manifest.oauth?.kind === "per-instance-oauth2",
|
||||
capabilities: Object.entries(manifest.capabilities).filter(([, enabled]) => enabled === true).map(([key]) => key),
|
||||
};
|
||||
});
|
||||
}
|
||||
landing(status: "success" | "failure"): string {
|
||||
const url = new URL("/connections", this.deps.appUrl);
|
||||
url.searchParams.set("connect", status);
|
||||
return url.toString();
|
||||
}
|
||||
async authorize(session: SessionPayload, provider: string, instanceUrl?: string, connectionId?: string) {
|
||||
await this.deps.store.assertAuthorized(session);
|
||||
const previous = connectionId ? await this.deps.store.get(session.workspaceId, connectionId) : undefined;
|
||||
if (previous && previous.connection.provider !== provider) throw new BadRequestException("Provider mismatch");
|
||||
const id = connectionId ?? randomUUID();
|
||||
const appSecrets: Record<string, string> = {};
|
||||
if (previous?.credential) {
|
||||
const bytes = await this.deps.encryption.open(session.workspaceId, id, previous.credential);
|
||||
const secrets = JSON.parse(bytes.toString()) as Record<string, string>;
|
||||
// Reuse the registered app, but stage no previous account token.
|
||||
for (const key of ["client_id", "client_secret"]) if (secrets[key]) appSecrets[key] = secrets[key];
|
||||
}
|
||||
const staging = new StagedCredentials(session.workspaceId, id, appSecrets);
|
||||
const registry = this.deps.registry(staging);
|
||||
if (!registry.has(provider)) throw new BadRequestException("Provider is not available");
|
||||
const connector = registry.get(provider);
|
||||
if (!connector.buildAuthorizationUrl || !connector.exchangeCode || connector.manifest().oauth?.connectEnabled === false) throw new BadRequestException(connector.manifest().oauth?.connectNote ?? "Redirect linking is unavailable");
|
||||
const state = randomState();
|
||||
const binding = `${session.workspaceId}.${randomState()}`;
|
||||
const instance = previous?.connection.fields.instanceUrl ?? instanceUrl;
|
||||
const request: AuthorizationRequest = {
|
||||
state, workspaceId: session.workspaceId, connectionId: id, redirectUri: this.deps.callbackUrl,
|
||||
fields: instance ? { instanceUrl: instance } : {},
|
||||
...(connector.manifest().oauth?.pkce ? { codeVerifier: randomState() } : {}),
|
||||
};
|
||||
try {
|
||||
const redirect = await connector.buildAuthorizationUrl(request);
|
||||
request.fields = redirect.connection.fields;
|
||||
await this.deps.pending.put(state, { session, provider, request, secrets: staging.secrets, reconnect: Boolean(previous), expiresAt: Date.now() + AUTH_TTL_MS, browserHash: browserHash(binding) });
|
||||
return { redirectUrl: redirect.authorizationUrl, state, binding };
|
||||
} catch {
|
||||
logger.warn({ event: "connection.authorize_failed", provider }, "Connection authorization failed");
|
||||
throw new BadRequestException("Could not start authorization. Check the instance URL and provider configuration.");
|
||||
}
|
||||
}
|
||||
async callback(state: string, binding: string, code?: string, providerError?: string): Promise<string> {
|
||||
try {
|
||||
const pending = await this.deps.pending.consume(state, binding);
|
||||
await this.deps.store.assertAuthorized(pending.session);
|
||||
if (providerError || !code || code.length > 4096) throw new Error("Provider authorization denied");
|
||||
const id = pending.request.connectionId;
|
||||
if (!id) throw new Error("Missing connection binding");
|
||||
const staging = new StagedCredentials(pending.session.workspaceId, id, pending.secrets);
|
||||
const connector = this.deps.registry(staging).get(pending.provider);
|
||||
if (!connector.exchangeCode) throw new Error("Code exchange unavailable");
|
||||
const result = await connector.exchangeCode({ ...pending.request, code });
|
||||
try {
|
||||
await this.persist(pending.session, result, staging, pending.reconnect);
|
||||
} catch (error) {
|
||||
// A failed DB write/account check must not leave an orphan authorization.
|
||||
await connector.revokeCredentials?.(result.connection).catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
return this.landing("success");
|
||||
} catch {
|
||||
// Provider errors can echo secrets. Only a fixed event reaches logs/UI.
|
||||
logger.warn({ event: "connection.callback_failed" }, "Connection callback failed");
|
||||
return this.landing("failure");
|
||||
}
|
||||
}
|
||||
async botToken(session: SessionPayload, token: string, fields: Record<string, string>, connectionId?: string): Promise<string> {
|
||||
await this.deps.store.assertAuthorized(session);
|
||||
const previous = connectionId ? await this.deps.store.get(session.workspaceId, connectionId) : undefined;
|
||||
const id = connectionId ?? randomUUID();
|
||||
const staging = new StagedCredentials(session.workspaceId, id);
|
||||
const connector = this.deps.registry(staging).list().find((item) => item.manifest().oauth?.kind === "token" && item.validateToken);
|
||||
if (!connector?.validateToken || (previous && previous.connection.provider !== connector.manifest().id)) throw new BadRequestException("Token linking is unavailable");
|
||||
try {
|
||||
const result = await connector.validateToken({ workspaceId: session.workspaceId, connectionId: id, token, fields: { ...previous?.connection.fields, ...fields } });
|
||||
await this.persist(session, result, staging, Boolean(previous));
|
||||
return await this.deps.renewSession(session);
|
||||
} catch {
|
||||
logger.warn({ event: "connection.token_failed" }, "Bot validation failed");
|
||||
throw new BadRequestException("Bot validation failed. Check the token and try again.");
|
||||
}
|
||||
}
|
||||
async reconnect(session: SessionPayload, id: string) {
|
||||
const stored = await this.deps.store.get(session.workspaceId, id);
|
||||
return this.authorize(session, stored.connection.provider, undefined, id);
|
||||
}
|
||||
private async persist(session: SessionPayload, result: AuthorizationResult, staging: StagedCredentials, reconnect: boolean): Promise<void> {
|
||||
const { connection, identity } = result;
|
||||
if (connection.workspaceId !== session.workspaceId || identity.provider !== connection.provider || !identity.remoteId || !identity.handle) throw new Error("Provider identity invalid");
|
||||
if (reconnect) {
|
||||
const previous = await this.deps.store.get(session.workspaceId, connection.id);
|
||||
if (previous.connection.provider !== connection.provider || (previous.connection.fields.remoteId && previous.connection.fields.remoteId !== identity.remoteId)) throw new Error("Reconnect must use the same account");
|
||||
connection.channelId = previous.connection.channelId;
|
||||
} else if (!connection.fields.channelId) {
|
||||
connection.channelId = identity.remoteId;
|
||||
}
|
||||
const encrypted = await this.deps.encryption.seal(session.workspaceId, connection.id, Buffer.from(JSON.stringify(staging.secrets)));
|
||||
await this.deps.store.save(session, connection, identity, encrypted, reconnect);
|
||||
}
|
||||
private async credentials(workspaceId: string, id: string) {
|
||||
const stored = await this.deps.store.get(workspaceId, id);
|
||||
return this.storedCredentials(stored);
|
||||
}
|
||||
private async storedCredentials(stored: StoredConnection) {
|
||||
const { workspaceId, id } = stored.connection;
|
||||
if (!stored.credential) throw new AuthError("Credential missing", stored.connection.provider, false);
|
||||
const bytes = await this.deps.encryption.open(workspaceId, id, stored.credential);
|
||||
const secrets = JSON.parse(bytes.toString()) as Record<string, string>;
|
||||
const staging = new StagedCredentials(workspaceId, id, secrets);
|
||||
return { connection: stored.connection, connector: this.deps.registry(staging).get(stored.connection.provider), staging, stored };
|
||||
}
|
||||
async list(workspaceId: string): Promise<ConnectionSummary[]> {
|
||||
const rows = await this.deps.store.list(workspaceId);
|
||||
// Probe actual identity, rather than assuming a credential row means a valid token.
|
||||
for (const row of rows) {
|
||||
if (row.health === "needs_credentials") continue;
|
||||
try {
|
||||
const { connection, connector, staging, stored } = await this.credentials(workspaceId, row.id);
|
||||
let identity: NormalizedIdentity;
|
||||
try {
|
||||
identity = await connector.getIdentity(connection);
|
||||
} finally {
|
||||
// Some connectors refresh during identity reads; keep the new refresh token durable.
|
||||
if (staging.dirty && stored.credential) {
|
||||
const sealed = await this.deps.encryption.seal(workspaceId, row.id, Buffer.from(JSON.stringify(staging.secrets)));
|
||||
await this.deps.store.rotateCredential(workspaceId, row.id, stored.credential, sealed);
|
||||
}
|
||||
}
|
||||
row.health = "healthy";
|
||||
row.fields.handle = identity.handle;
|
||||
row.capabilities = Object.entries(connector.manifest().capabilities).filter(([, enabled]) => enabled === true).map(([key]) => key);
|
||||
} catch (error) {
|
||||
row.health = error instanceof AuthError ? "needs_credentials" : "unavailable";
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
async remove(session: SessionPayload, id: string): Promise<string> {
|
||||
await this.deps.store.assertAuthorized(session);
|
||||
await this.deps.store.remove(session, id, async (stored) => {
|
||||
if (!stored.credential) return;
|
||||
try {
|
||||
const { connection, connector } = await this.storedCredentials(stored);
|
||||
await connector.revokeCredentials?.(connection);
|
||||
} catch (error) {
|
||||
// Already invalid/revoked credentials are safe to remove locally.
|
||||
if (!(error instanceof AuthError)) throw new BadRequestException("Provider revocation failed. Try removing again.");
|
||||
}
|
||||
});
|
||||
return this.deps.renewSession(session);
|
||||
}
|
||||
}
|
||||
17
apps/api/src/connections/connector-registry.ts
Normal file
17
apps/api/src/connections/connector-registry.ts
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
import { ConnectorRegistry, type CredentialAccessor } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
// eslint-disable-next-line no-restricted-syntax -- Connector composition boundary.
|
||||
import { createMastodonConnector } from "@tactical-media-broadcast-hub/connector-mastodon";
|
||||
// eslint-disable-next-line no-restricted-syntax -- Connector composition boundary.
|
||||
import { createPeerTubeConnector } from "../../../../packages/connector-peertube/src/index.js";
|
||||
// eslint-disable-next-line no-restricted-syntax -- Connector composition boundary.
|
||||
import { createTelegramConnector } from "@tactical-media-broadcast-hub/connector-telegram";
|
||||
|
||||
/** Composition boundary only: services resolve manifests rather than branch on providers. */
|
||||
export function connectionRegistry(credentials: CredentialAccessor, fetchImpl: typeof fetch = fetch): ConnectorRegistry {
|
||||
const registry = new ConnectorRegistry();
|
||||
const options = { credentials, fetchImpl, env: process.env };
|
||||
registry.register(createMastodonConnector(options));
|
||||
registry.register(createPeerTubeConnector(options));
|
||||
registry.register(createTelegramConnector(options));
|
||||
return registry;
|
||||
}
|
||||
46
apps/api/src/connections/http-test-transport.ts
Normal file
46
apps/api/src/connections/http-test-transport.ts
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
import { IncomingMessage, ServerResponse } from "node:http";
|
||||
import { Duplex } from "node:stream";
|
||||
import type { Socket } from "node:net";
|
||||
|
||||
/** Runs real Node/Express HTTP handlers without binding a port (restricted CI). */
|
||||
export async function injectHttp(
|
||||
handler: (request: IncomingMessage, response: ServerResponse) => void,
|
||||
input: string | URL | Request,
|
||||
init: RequestInit = {},
|
||||
): Promise<Response> {
|
||||
const url = new URL(input instanceof Request ? input.url : String(input));
|
||||
const output: Buffer[] = [];
|
||||
const socket = new Duplex({
|
||||
read() {},
|
||||
write(chunk: Buffer, _encoding, callback) { output.push(Buffer.from(chunk)); callback(); },
|
||||
});
|
||||
// ServerResponse uses only Duplex operations; no network or unsafe dynamic values.
|
||||
const httpSocket = socket as unknown as Socket;
|
||||
const request = new IncomingMessage(httpSocket);
|
||||
request.method = init.method ?? "GET";
|
||||
request.complete = true;
|
||||
request.url = `${url.pathname}${url.search}`;
|
||||
request.headers = Object.fromEntries(new Headers(init.headers).entries());
|
||||
request.headers.host = url.host;
|
||||
const body = typeof init.body === "string" ? init.body : "";
|
||||
if (body) request.headers["content-length"] = String(Buffer.byteLength(body));
|
||||
request.push(body || null);
|
||||
if (body) request.push(null);
|
||||
const response = new ServerResponse(request);
|
||||
response.assignSocket(httpSocket);
|
||||
const completed = new Promise<Response>((resolve, reject) => {
|
||||
response.once("error", reject);
|
||||
response.once("finish", () => {
|
||||
const bytes = Buffer.concat(output);
|
||||
const bodyOffset = bytes.indexOf("\r\n\r\n") + 4;
|
||||
const headers = new Headers();
|
||||
for (const line of bytes.subarray(0, bodyOffset - 4).toString().split("\r\n").slice(1)) {
|
||||
const separator = line.indexOf(":");
|
||||
if (separator > 0) headers.append(line.slice(0, separator), line.slice(separator + 1).trim());
|
||||
}
|
||||
resolve(new Response([204, 304].includes(response.statusCode) ? null : bytes.subarray(bodyOffset), { status: response.statusCode, headers }));
|
||||
});
|
||||
});
|
||||
handler(request, response);
|
||||
return completed;
|
||||
}
|
||||
11
apps/api/src/connections/oauth-fetch.test.ts
Normal file
11
apps/api/src/connections/oauth-fetch.test.ts
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { publicAddress } from "./oauth-fetch.js";
|
||||
|
||||
describe("authorization egress policy", () => {
|
||||
it("rejects private, mapped, metadata, multicast and documentation addresses", () => {
|
||||
for (const address of ["127.0.0.1", "10.0.0.1", "172.16.0.1", "192.168.1.1", "169.254.169.254", "100.64.1.1", "0.0.0.0", "224.0.0.1", "::1", "::ffff:127.0.0.1", "fc00::1", "fe80::1", "2001:db8::1", "2002:7f00:1::"]) expect(publicAddress(address), address).toBe(false);
|
||||
expect(publicAddress("8.8.8.8")).toBe(true);
|
||||
expect(publicAddress("2606:4700:4700::1111")).toBe(true);
|
||||
expect(publicAddress("2001:4860:4860::8888")).toBe(true);
|
||||
});
|
||||
});
|
||||
1
apps/api/src/connections/oauth-fetch.ts
Normal file
1
apps/api/src/connections/oauth-fetch.ts
Normal file
|
|
@ -0,0 +1 @@
|
|||
export { publicAddress, providerFetch as oauthFetch } from "@tactical-media-broadcast-hub/connector-runtime";
|
||||
53
apps/api/src/connections/pending.test.ts
Normal file
53
apps/api/src/connections/pending.test.ts
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import { describe, expect, it, vi, afterEach } from "vitest";
|
||||
import { Vault } from "../vault/vault.js";
|
||||
import { AUTH_TTL_MS, browserHash, randomState, RedisPendingStore, type PendingAuthorization } from "./pending.js";
|
||||
import { MemoryPendingRedis } from "./testing.js";
|
||||
|
||||
function fixture() {
|
||||
const redis = new MemoryPendingRedis();
|
||||
const store = new RedisPendingStore(redis, new Vault());
|
||||
const state = randomState();
|
||||
const binding = `workspace-a.${randomState()}`;
|
||||
const pending: PendingAuthorization = {
|
||||
provider: "mastodon", request: { redirectUri: "http://localhost/callback", workspaceId: "workspace-a", state },
|
||||
session: { userId: "user-a", workspaceId: "workspace-a", role: "Owner", authzEpoch: 1, issuedAt: 0, expiresAt: Date.now() / 1000 + 600 },
|
||||
secrets: { client_secret: "registration-secret" }, reconnect: false, expiresAt: Date.now() + AUTH_TTL_MS, browserHash: browserHash(binding),
|
||||
};
|
||||
return { redis, store, state, binding, pending };
|
||||
}
|
||||
afterEach(() => vi.useRealTimers());
|
||||
describe("pending authorization binding", () => {
|
||||
it("encrypts provider secrets and consumes state exactly once", async () => {
|
||||
const { redis, store, state, binding, pending } = fixture();
|
||||
await store.put(state, pending);
|
||||
expect(JSON.stringify([...redis.entries.values()])).not.toContain("registration-secret");
|
||||
expect(await store.consume(state, binding)).toEqual(pending);
|
||||
await expect(store.consume(state, binding)).rejects.toThrow(/already used/);
|
||||
});
|
||||
it("rejects CSRF, unknown states and a different workspace without burning the valid state", async () => {
|
||||
const { store, state, binding, pending } = fixture();
|
||||
await store.put(state, pending);
|
||||
await expect(store.consume(state, "")).rejects.toThrow(/mismatch/);
|
||||
await expect(store.consume(state, binding.replace("workspace-a", "workspace-b"))).rejects.toThrow(/mismatch/);
|
||||
await expect(store.consume(randomState(), binding)).rejects.toThrow();
|
||||
await expect(store.consume("malformed", binding)).rejects.toThrow();
|
||||
expect(await store.consume(state, binding)).toEqual(pending);
|
||||
});
|
||||
it("rejects a workspace mismatch inside the encrypted request", async () => {
|
||||
const { store, state, binding, pending } = fixture();
|
||||
pending.request.workspaceId = "workspace-b";
|
||||
await store.put(state, pending);
|
||||
await expect(store.consume(state, binding)).rejects.toThrow(/workspace mismatch/);
|
||||
});
|
||||
it("expires state after ten minutes and atomically fences simultaneous callbacks", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { store, state, binding, pending } = fixture();
|
||||
await store.put(state, pending);
|
||||
const results = await Promise.allSettled([store.consume(state, binding), store.consume(state, binding)]);
|
||||
expect(results.filter((item) => item.status === "fulfilled")).toHaveLength(1);
|
||||
const next = fixture();
|
||||
await next.store.put(next.state, next.pending);
|
||||
vi.advanceTimersByTime(AUTH_TTL_MS);
|
||||
await expect(next.store.consume(next.state, next.binding)).rejects.toThrow(/expired/);
|
||||
});
|
||||
});
|
||||
57
apps/api/src/connections/pending.ts
Normal file
57
apps/api/src/connections/pending.ts
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import { createHash, randomBytes, timingSafeEqual } from "node:crypto";
|
||||
import { BadRequestException } from "@nestjs/common";
|
||||
import type { AuthorizationRequest } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import { RespControlRedis, type ControlRedis } from "../media-control/redis.js";
|
||||
import { vault, type Vault } from "../vault/vault.js";
|
||||
|
||||
export const AUTH_TTL_MS = 10 * 60_000;
|
||||
export const stateCookieName = (state: string): string => `ap_connect_${state}`;
|
||||
export const validState = (state: string): boolean => /^[A-Za-z0-9_-]{43}$/.test(state);
|
||||
export interface PendingAuthorization {
|
||||
session: SessionPayload;
|
||||
provider: string;
|
||||
request: AuthorizationRequest;
|
||||
secrets: Record<string, string>;
|
||||
reconnect: boolean;
|
||||
expiresAt: number;
|
||||
browserHash: string;
|
||||
}
|
||||
export interface PendingStore {
|
||||
put(state: string, pending: PendingAuthorization): Promise<void>;
|
||||
consume(state: string, browserBinding: string): Promise<PendingAuthorization>;
|
||||
}
|
||||
export function browserHash(binding: string): string {
|
||||
return createHash("sha256").update(binding).digest("hex");
|
||||
}
|
||||
export const randomState = (): string => randomBytes(32).toString("base64url");
|
||||
|
||||
/** GET + compare-and-delete makes callbacks one-use across API replicas. */
|
||||
export class RedisPendingStore implements PendingStore {
|
||||
constructor(
|
||||
private readonly redis: Pick<ControlRedis, "get" | "set" | "eval"> = new RespControlRedis(process.env.REDIS_URL ?? "redis://localhost:6379"),
|
||||
private readonly encryption: Vault = vault,
|
||||
) {}
|
||||
async put(state: string, pending: PendingAuthorization): Promise<void> {
|
||||
const sealed = await this.encryption.seal(pending.session.workspaceId, state, Buffer.from(JSON.stringify(pending)));
|
||||
const value = JSON.stringify({ workspaceId: pending.session.workspaceId, ciphertext: sealed.ciphertext.toString("base64") });
|
||||
if (!await this.redis.set(`oauth:pending:${state}`, value, { nx: true, pxMs: AUTH_TTL_MS })) throw new Error("state collision");
|
||||
}
|
||||
async consume(state: string, binding: string): Promise<PendingAuthorization> {
|
||||
if (!validState(state) || binding.length > 200) throw new BadRequestException("Invalid authorization state");
|
||||
const key = `oauth:pending:${state}`;
|
||||
const raw = await this.redis.get(key);
|
||||
if (!raw) throw new BadRequestException("Authorization expired or already used");
|
||||
const envelope = JSON.parse(raw) as { workspaceId: string; ciphertext: string };
|
||||
const opened = await this.encryption.open(envelope.workspaceId, state, { keyVersion: 1, ciphertext: Buffer.from(envelope.ciphertext, "base64") });
|
||||
const pending = JSON.parse(opened.toString()) as PendingAuthorization;
|
||||
const actual = Buffer.from(browserHash(binding));
|
||||
const expected = Buffer.from(pending.browserHash);
|
||||
if (pending.expiresAt <= Date.now() || pending.session.workspaceId !== envelope.workspaceId || pending.request.workspaceId !== envelope.workspaceId || !binding.startsWith(`${envelope.workspaceId}.`) || actual.length !== expected.length || !timingSafeEqual(actual, expected)) {
|
||||
throw new BadRequestException("Authorization browser or workspace mismatch");
|
||||
}
|
||||
const consumed = await this.redis.eval("if redis.call('GET', KEYS[1]) == ARGV[1] then redis.call('DEL', KEYS[1]); return 1 end; return 0", [key], [raw]);
|
||||
if (consumed !== 1) throw new BadRequestException("Authorization already used");
|
||||
return pending;
|
||||
}
|
||||
}
|
||||
23
apps/api/src/connections/testing.ts
Normal file
23
apps/api/src/connections/testing.ts
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
import type { ControlRedis } from "../media-control/redis.js";
|
||||
|
||||
/** TTL-faithful Redis substitute; compare-and-delete runs synchronously like Redis Lua. */
|
||||
export class MemoryPendingRedis implements Pick<ControlRedis, "get" | "set" | "eval"> {
|
||||
readonly entries = new Map<string, { value: string; expiresAt: number }>();
|
||||
async get(key: string): Promise<string | null> {
|
||||
const entry = this.entries.get(key);
|
||||
if (!entry || entry.expiresAt <= Date.now()) { this.entries.delete(key); return null; }
|
||||
return entry.value;
|
||||
}
|
||||
async set(key: string, value: string, options: { nx?: boolean; pxMs?: number } = {}): Promise<boolean> {
|
||||
if (options.nx && await this.get(key) !== null) return false;
|
||||
this.entries.set(key, { value, expiresAt: Date.now() + (options.pxMs ?? 1000) });
|
||||
return true;
|
||||
}
|
||||
async eval(_script: string, keys: string[], args: string[]): Promise<number> {
|
||||
const key = keys[0] ?? "";
|
||||
const entry = this.entries.get(key);
|
||||
if (!entry || entry.value !== args[0] || entry.expiresAt <= Date.now()) return 0;
|
||||
this.entries.delete(key);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
59
apps/api/src/db/client.ts
Normal file
59
apps/api/src/db/client.ts
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
import { Pool, type PoolClient } from "pg";
|
||||
import pino from "pino";
|
||||
|
||||
const logger = pino({ level: process.env.LOG_LEVEL ?? "info" });
|
||||
|
||||
export const pool = new Pool({
|
||||
connectionString: process.env.DATABASE_URL,
|
||||
max: Number(process.env.PG_POOL_MAX ?? 10),
|
||||
});
|
||||
|
||||
/**
|
||||
* Tenant-scoped transaction: sets app.workspace_id for the transaction so
|
||||
* RLS policies and SECURITY DEFINER assertions see the tenant context.
|
||||
* The setting is transaction-local (SET LOCAL) — it cannot leak.
|
||||
*/
|
||||
export async function withTenant<T>(
|
||||
workspaceId: string,
|
||||
fn: (client: PoolClient) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]);
|
||||
const result = await fn(client);
|
||||
await client.query("COMMIT");
|
||||
return result;
|
||||
} catch (err) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw err;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
/** System-level transaction (no tenant context): RLS sees NULL context and returns nothing tenant-scoped. */
|
||||
export async function withSystem<T>(fn: (client: PoolClient) => Promise<T>): Promise<T> {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
const result = await fn(client);
|
||||
await client.query("COMMIT");
|
||||
return result;
|
||||
} catch (err) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw err;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
export async function checkDatabase(): Promise<boolean> {
|
||||
try {
|
||||
await pool.query("SELECT 1");
|
||||
return true;
|
||||
} catch (err) {
|
||||
logger.warn({ err: String(err) }, "db: unreachable");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
6
apps/api/src/db/migrate.cli.ts
Normal file
6
apps/api/src/db/migrate.cli.ts
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
import { resolve } from "node:path";
|
||||
import { runMigrations } from "./migrate.js";
|
||||
const index = process.argv.indexOf("--dir");
|
||||
void runMigrations(resolve(index < 0 ? "migrations" : process.argv[index + 1] ?? "migrations"))
|
||||
.then((files) => console.log({ applied: files }))
|
||||
.catch(() => { console.error("Migration failed; schema remains unready. Inspect the phase and database records before retrying."); process.exitCode = 1; });
|
||||
12
apps/api/src/db/migrate.test.ts
Normal file
12
apps/api/src/db/migrate.test.ts
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { migrationPhases } from "./migrate.js";
|
||||
describe("migration stage grammar", () => {
|
||||
it("leaves 001–018 transactional and parses explicit stages", () => {
|
||||
expect(migrationPhases("SELECT 1;")).toEqual([{ name: "", mode: "transaction", sql: "SELECT 1;" }]);
|
||||
expect(migrationPhases("-- comment\n-- wp3:phase expand transaction\nSELECT 1;\n-- wp3:phase fill batch\nUPDATE t SET x=1;\n-- wp3:phase idx concurrent\nCREATE INDEX CONCURRENTLY t_idx ON t (x);").map((p) => [p.name, p.mode])).toEqual([["expand", "transaction"], ["fill", "batch"], ["idx", "concurrent"]]);
|
||||
});
|
||||
it("refuses duplicate phases and SQL before the first stage", () => {
|
||||
expect(() => migrationPhases("-- wp3:phase expand transaction\nSELECT 1;\n-- wp3:phase expand transaction\nSELECT 2;")).toThrow(/duplicate/);
|
||||
expect(() => migrationPhases("SELECT 0;\n-- wp3:phase expand transaction\nSELECT 1;")).toThrow(/before/);
|
||||
});
|
||||
});
|
||||
77
apps/api/src/db/migrate.ts
Normal file
77
apps/api/src/db/migrate.ts
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
import { readdir, readFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { Pool, type PoolClient } from "pg";
|
||||
|
||||
export interface MigrationPhase { name: string; mode: "transaction" | "batch" | "concurrent"; sql: string }
|
||||
/** Only explicit whole-line WP3 markers are accepted; old migrations stay transactional. */
|
||||
export function migrationPhases(sql: string): MigrationPhase[] {
|
||||
const marker = /^-- wp3:phase ([a-z][a-z0-9_]*) (transaction|batch|concurrent)\s*$/gm;
|
||||
const matches = [...sql.matchAll(marker)];
|
||||
if (matches.length === 0) return [{ name: "", mode: "transaction", sql }];
|
||||
if (sql.slice(0, matches[0]?.index).split("\n").some((line) => line.trim() && !line.trim().startsWith("--"))) throw new Error("SQL before migration phase");
|
||||
const phases = matches.map((match, index): MigrationPhase => ({ name: match[1]!, mode: match[2] as MigrationPhase["mode"], sql: sql.slice(match.index! + match[0].length, matches[index + 1]?.index).trim() }));
|
||||
if (new Set(phases.map((p) => p.name)).size !== phases.length) throw new Error("duplicate migration phase");
|
||||
return phases;
|
||||
}
|
||||
|
||||
async function concurrentIndex(client: PoolClient, sql: string): Promise<void> {
|
||||
const parsed = /^CREATE (UNIQUE )?INDEX CONCURRENTLY ([a-z][a-z0-9_]*) ON ([a-z][a-z0-9_]*) ([\s\S]+);$/.exec(sql);
|
||||
if (!parsed) throw new Error("concurrent phase must contain exactly one CREATE INDEX");
|
||||
const [, unique, name, table, tail] = parsed;
|
||||
const found = await client.query<{ indisvalid: boolean; definition: string }>("SELECT i.indisvalid, pg_get_indexdef(i.indexrelid) AS definition FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'public' AND c.relname = $1", [name]);
|
||||
const existing = found.rows[0];
|
||||
if (existing) {
|
||||
const expected = `CREATE ${unique ?? ""}INDEX ${name} ON public.${table} USING btree ${tail}`;
|
||||
const normalize = (text: string): string => text.replace(/;$/, "").replace(/\s+/g, " ").trim();
|
||||
if (normalize(existing.definition) !== normalize(expected)) throw new Error(`index ${name} definition differs`);
|
||||
if (existing.indisvalid) return;
|
||||
await client.query(`DROP INDEX CONCURRENTLY public.${name}`);
|
||||
}
|
||||
await client.query(sql);
|
||||
}
|
||||
|
||||
/** Owner-only offline runner; runtime applications never call this. */
|
||||
export async function runMigrations(dir: string): Promise<string[]> {
|
||||
if (!process.env.SUPER_DATABASE_URL) throw new Error("SUPER_DATABASE_URL is required for migrations");
|
||||
const files = (await readdir(dir)).filter((f) => f.endsWith(".sql")).sort();
|
||||
const applied: string[] = [];
|
||||
const pool = new Pool({ connectionString: process.env.SUPER_DATABASE_URL });
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("SELECT pg_advisory_lock(193674839, 3)");
|
||||
await client.query("SET lock_timeout = '5s'");
|
||||
await client.query("CREATE TABLE IF NOT EXISTS schema_migrations (name text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())");
|
||||
for (const file of files) {
|
||||
if ((await client.query("SELECT 1 FROM schema_migrations WHERE name = $1", [file])).rowCount !== 0) continue;
|
||||
for (const phase of migrationPhases(await readFile(join(dir, file), "utf8"))) {
|
||||
const key = phase.name ? `${file}:${phase.name}` : file;
|
||||
if ((await client.query("SELECT 1 FROM schema_migrations WHERE name = $1", [key])).rowCount !== 0) continue;
|
||||
try {
|
||||
if (phase.mode === "concurrent") {
|
||||
await concurrentIndex(client, phase.sql);
|
||||
await client.query("INSERT INTO schema_migrations(name) VALUES ($1)", [key]);
|
||||
} else {
|
||||
for (;;) {
|
||||
await client.query("BEGIN");
|
||||
const result = await client.query(phase.sql);
|
||||
if (phase.mode === "batch" && (result.rowCount ?? 0) > 0) { await client.query("COMMIT"); continue; }
|
||||
await client.query("INSERT INTO schema_migrations(name) VALUES ($1)", [key]);
|
||||
await client.query("COMMIT");
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw new Error(`migration ${key} failed`, { cause: error });
|
||||
}
|
||||
}
|
||||
await client.query("INSERT INTO schema_migrations(name) VALUES ($1) ON CONFLICT DO NOTHING", [file]);
|
||||
applied.push(file);
|
||||
}
|
||||
} finally {
|
||||
await client.query("SELECT pg_advisory_unlock(193674839, 3)").catch(() => undefined);
|
||||
client.release();
|
||||
await pool.end();
|
||||
}
|
||||
return applied;
|
||||
}
|
||||
326
apps/api/src/db/production-schema.integration.test.ts
Normal file
326
apps/api/src/db/production-schema.integration.test.ts
Normal file
|
|
@ -0,0 +1,326 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { join } from "node:path";
|
||||
import { Pool, type PoolClient } from "pg";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { runMigrations } from "./migrate.js";
|
||||
|
||||
/**
|
||||
* Migration 013 (streaming S1) against a live PostgreSQL: the same-channel
|
||||
* invariant fails at the DB constraint, the production CHECKs hold, and FORCE
|
||||
* RLS denies every new table across tenants. Assertions run as sp_app
|
||||
* (DATABASE_URL); setup/cleanup as owner. Skipped without DATABASE_URL.
|
||||
*/
|
||||
const dbAvailable = process.env.DATABASE_URL !== undefined;
|
||||
const ownerPool = new Pool({ connectionString: process.env.SUPER_DATABASE_URL ?? process.env.DATABASE_URL });
|
||||
const appPool = new Pool({ connectionString: process.env.DATABASE_URL });
|
||||
|
||||
const FK_VIOLATION = "23503";
|
||||
const UNIQUE_VIOLATION = "23505";
|
||||
const CHECK_VIOLATION = "23514";
|
||||
const RLS_VIOLATION = "42501";
|
||||
const ASSET = "a".repeat(64);
|
||||
|
||||
async function asTenant<T>(workspaceId: string | null, fn: (client: PoolClient) => Promise<T>): Promise<T> {
|
||||
const client = await appPool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
if (workspaceId !== null) await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]);
|
||||
const result = await fn(client);
|
||||
await client.query("COMMIT");
|
||||
return result;
|
||||
} catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function one(client: PoolClient, sql: string, params: unknown[]): Promise<string> {
|
||||
const result = await client.query(sql, params);
|
||||
return String(result.rows[0]?.id);
|
||||
}
|
||||
|
||||
interface Channel { id: string; hls: string; playlist: string; scene: string; spare: string; slot: string; overlay: string; revision: string }
|
||||
|
||||
async function seedChannel(client: PoolClient, ws: string, assetId: string): Promise<Channel> {
|
||||
const id = await one(client, "INSERT INTO virtual_channels (workspace_id, slug, name) VALUES ($1, $2, 'Channel') RETURNING id", [ws, `ch-${randomUUID().slice(0, 8)}`]);
|
||||
const hls = await one(client,
|
||||
"INSERT INTO virtual_channel_sources (workspace_id, virtual_channel_id, kind, label, remote_ref, hls_url) VALUES ($1, $2, 'hls', 'Cam 1', $3, 'https://cdn.example.com/a.m3u8') RETURNING id",
|
||||
[ws, id, `ref-${randomUUID()}`]);
|
||||
const playlist = await one(client,
|
||||
"INSERT INTO virtual_channel_sources (workspace_id, virtual_channel_id, kind, label, playlist_loop) VALUES ($1, $2, 'playlist', 'Fallback reel', true) RETURNING id",
|
||||
[ws, id]);
|
||||
await client.query("INSERT INTO playlist_items (workspace_id, virtual_channel_id, source_id, position, video_asset_id) VALUES ($1, $2, $3, 0, $4)", [ws, id, playlist, assetId]);
|
||||
const scene = await one(client, "INSERT INTO production_scenes (workspace_id, virtual_channel_id, label, layout) VALUES ($1, $2, 'Main', '2x2') RETURNING id", [ws, id]);
|
||||
const spare = await one(client, "INSERT INTO production_scenes (workspace_id, virtual_channel_id, label, layout) VALUES ($1, $2, 'Spare', 'single') RETURNING id", [ws, id]);
|
||||
const slot = await one(client,
|
||||
"INSERT INTO scene_slots (workspace_id, virtual_channel_id, scene_id, slot_index, source_id, audio_mode, gain_db) VALUES ($1, $2, $3, 0, $4, 'primary', -6) RETURNING id",
|
||||
[ws, id, scene, hls]);
|
||||
await client.query("INSERT INTO production_buses (workspace_id, virtual_channel_id, bus, scene_id) VALUES ($1, $2, 'program', $3), ($1, $2, 'preview', $3)", [ws, id, scene]);
|
||||
const overlay = await one(client, "INSERT INTO overlay_slots (workspace_id, virtual_channel_id, scene_id, kind) VALUES ($1, $2, $3, 'lower_third') RETURNING id", [ws, id, scene]);
|
||||
const revision = await one(client,
|
||||
"INSERT INTO overlay_revisions (workspace_id, virtual_channel_id, overlay_slot_id, template_id, render_state, asset_id) VALUES ($1, $2, $3, 'lower-third-v1', 'rendered', $4) RETURNING id",
|
||||
[ws, id, overlay, ASSET]);
|
||||
await client.query("UPDATE overlay_slots SET current_revision_id = $2 WHERE id = $1", [overlay, revision]);
|
||||
return { id, hls, playlist, scene, spare, slot, overlay, revision };
|
||||
}
|
||||
|
||||
describe.skipIf(!dbAvailable)("production schema 013 (live DB)", () => {
|
||||
let wsA: string;
|
||||
let wsB: string;
|
||||
let assetA: string;
|
||||
let assetB: string;
|
||||
let a1: Channel;
|
||||
let a2: Channel;
|
||||
let b1: Channel;
|
||||
|
||||
beforeAll(async () => {
|
||||
await runMigrations(process.env.MIGRATIONS_DIR ?? join(process.cwd(), "migrations"));
|
||||
const insert = "INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id";
|
||||
wsA = String((await ownerPool.query(insert, ["Prod A", `prod-a-${randomUUID()}`])).rows[0]?.id);
|
||||
wsB = String((await ownerPool.query(insert, ["Prod B", `prod-b-${randomUUID()}`])).rows[0]?.id);
|
||||
const asset = "INSERT INTO video_assets (workspace_id, storage_key, mime_type, size_bytes) VALUES ($1, 'sha256/x.mp4', 'video/mp4', 1) RETURNING id";
|
||||
assetA = await asTenant(wsA, (c) => one(c, asset, [wsA]));
|
||||
assetB = await asTenant(wsB, (c) => one(c, asset, [wsB]));
|
||||
a1 = await asTenant(wsA, (c) => seedChannel(c, wsA, assetA));
|
||||
a2 = await asTenant(wsA, (c) => seedChannel(c, wsA, assetA));
|
||||
b1 = await asTenant(wsB, (c) => seedChannel(c, wsB, assetB));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (wsA !== undefined && wsB !== undefined) await ownerPool.query("DELETE FROM workspaces WHERE id IN ($1, $2)", [wsA, wsB]);
|
||||
await appPool.end();
|
||||
await ownerPool.end();
|
||||
});
|
||||
|
||||
describe("same-channel invariant fails at the DB constraint", () => {
|
||||
it("a slot cannot reference another channel's source (same workspace)", async () => {
|
||||
await expect(asTenant(wsA, (c) => c.query(
|
||||
"INSERT INTO scene_slots (workspace_id, virtual_channel_id, scene_id, slot_index, source_id) VALUES ($1, $2, $3, 1, $4)",
|
||||
[wsA, a1.id, a1.scene, a2.hls]))).rejects.toMatchObject({ code: FK_VIOLATION });
|
||||
});
|
||||
|
||||
it("a slot cannot claim another channel to smuggle a foreign scene or source", async () => {
|
||||
await expect(asTenant(wsA, (c) => c.query(
|
||||
"INSERT INTO scene_slots (workspace_id, virtual_channel_id, scene_id, slot_index, source_id) VALUES ($1, $2, $3, 1, $4)",
|
||||
[wsA, a2.id, a1.scene, a2.hls]))).rejects.toMatchObject({ code: FK_VIOLATION });
|
||||
});
|
||||
|
||||
it("a slot cannot be re-pointed to any source in place, nor moved to another scene", async () => {
|
||||
await expect(asTenant(wsA, (c) => c.query("UPDATE scene_slots SET source_id = $2 WHERE id = $1", [a1.slot, a2.hls])))
|
||||
.rejects.toMatchObject({ code: CHECK_VIOLATION });
|
||||
await expect(asTenant(wsA, (c) => c.query("UPDATE scene_slots SET source_id = $2 WHERE id = $1", [a1.slot, a1.playlist])))
|
||||
.rejects.toThrow(/source_id is immutable/);
|
||||
await expect(asTenant(wsA, (c) => c.query("UPDATE scene_slots SET scene_id = $2 WHERE id = $1", [a1.slot, a1.spare])))
|
||||
.rejects.toThrow(/scene_id is immutable/);
|
||||
});
|
||||
|
||||
it("a bus cannot hold another channel's scene, by insert or by TAKE-style update", async () => {
|
||||
await expect(asTenant(wsA, (c) => c.query("UPDATE production_buses SET scene_id = $2 WHERE virtual_channel_id = $1 AND bus = 'program'", [a1.id, a2.scene])))
|
||||
.rejects.toMatchObject({ code: FK_VIOLATION });
|
||||
const moved = await asTenant(wsA, (c) => c.query("UPDATE production_buses SET scene_id = $2 WHERE virtual_channel_id = $1 AND bus = 'preview'", [a1.id, a1.spare]));
|
||||
expect(moved.rowCount).toBe(1);
|
||||
});
|
||||
|
||||
it("overlay slots and revisions stay inside their channel", async () => {
|
||||
await expect(asTenant(wsA, (c) => c.query(
|
||||
"INSERT INTO overlay_slots (workspace_id, virtual_channel_id, scene_id, kind) VALUES ($1, $2, $3, 'logo')",
|
||||
[wsA, a1.id, a2.scene]))).rejects.toMatchObject({ code: FK_VIOLATION });
|
||||
await expect(asTenant(wsA, (c) => c.query(
|
||||
"INSERT INTO overlay_revisions (workspace_id, virtual_channel_id, overlay_slot_id, template_id) VALUES ($1, $2, $3, 'logo')",
|
||||
[wsA, a1.id, a2.overlay]))).rejects.toMatchObject({ code: FK_VIOLATION });
|
||||
await expect(asTenant(wsA, (c) => c.query("UPDATE overlay_slots SET current_revision_id = $2 WHERE id = $1", [a1.overlay, a2.revision])))
|
||||
.rejects.toMatchObject({ code: FK_VIOLATION });
|
||||
});
|
||||
|
||||
it("a playlist item cannot name another channel's source or another tenant's asset", async () => {
|
||||
await expect(asTenant(wsA, (c) => c.query(
|
||||
"INSERT INTO playlist_items (workspace_id, virtual_channel_id, source_id, position, video_asset_id) VALUES ($1, $2, $3, 1, $4)",
|
||||
[wsA, a1.id, a2.playlist, assetA]))).rejects.toMatchObject({ code: FK_VIOLATION });
|
||||
await expect(asTenant(wsA, (c) => c.query(
|
||||
"INSERT INTO playlist_items (workspace_id, virtual_channel_id, source_id, position, video_asset_id) VALUES ($1, $2, $3, 1, $4)",
|
||||
[wsA, a1.id, a1.playlist, assetB]))).rejects.toMatchObject({ code: FK_VIOLATION });
|
||||
});
|
||||
});
|
||||
|
||||
describe("production constraints", () => {
|
||||
it("allows at most one primary audio slot per scene", async () => {
|
||||
await expect(asTenant(wsA, (c) => c.query(
|
||||
"INSERT INTO scene_slots (workspace_id, virtual_channel_id, scene_id, slot_index, source_id, audio_mode) VALUES ($1, $2, $3, 1, $4, 'primary')",
|
||||
[wsA, a1.id, a1.scene, a1.playlist]))).rejects.toMatchObject({ code: UNIQUE_VIOLATION });
|
||||
await asTenant(wsA, (c) => c.query(
|
||||
"INSERT INTO scene_slots (workspace_id, virtual_channel_id, scene_id, slot_index, source_id, audio_mode) VALUES ($1, $2, $3, 0, $4, 'primary'), ($1, $2, $5, 1, $4, 'on')",
|
||||
[wsA, a1.id, a1.spare, a1.playlist, a1.scene]));
|
||||
});
|
||||
|
||||
it("gain_db is -24..0 in 3 dB steps", async () => {
|
||||
for (const bad of [-4, 3, -27, -1]) {
|
||||
await expect(asTenant(wsA, (c) => c.query("UPDATE scene_slots SET gain_db = $2 WHERE id = $1", [a1.slot, bad])))
|
||||
.rejects.toMatchObject({ code: CHECK_VIOLATION });
|
||||
}
|
||||
for (const good of [-24, -3, 0]) await asTenant(wsA, (c) => c.query("UPDATE scene_slots SET gain_db = $2 WHERE id = $1", [a1.slot, good]));
|
||||
});
|
||||
|
||||
it("geometry is all-or-nothing, positive, and inside the frame", async () => {
|
||||
const set = (sql: string) => asTenant(wsA, (c) => c.query(`UPDATE scene_slots SET ${sql} WHERE id = $1`, [a1.slot]));
|
||||
await expect(set("x = 0, y = 0, w = 100")).rejects.toMatchObject({ code: CHECK_VIOLATION });
|
||||
await expect(set("x = 0, y = 0, w = 0, h = 100")).rejects.toMatchObject({ code: CHECK_VIOLATION });
|
||||
await expect(set("x = -1, y = 0, w = 100, h = 100")).rejects.toMatchObject({ code: CHECK_VIOLATION });
|
||||
await expect(set("x = 9000, y = 0, w = 2000, h = 100")).rejects.toMatchObject({ code: CHECK_VIOLATION });
|
||||
await set("x = 7000, y = 7000, w = 3000, h = 3000");
|
||||
});
|
||||
|
||||
it("layouts, buses and source kinds are closed enums; playlist is a source kind", async () => {
|
||||
await expect(asTenant(wsA, (c) => c.query("UPDATE production_scenes SET layout = '5x5' WHERE id = $1", [a1.spare])))
|
||||
.rejects.toMatchObject({ code: CHECK_VIOLATION });
|
||||
await expect(asTenant(wsA, (c) => c.query(
|
||||
"INSERT INTO production_buses (workspace_id, virtual_channel_id, bus) VALUES ($1, $2, 'aux')", [wsA, a1.id])))
|
||||
.rejects.toMatchObject({ code: CHECK_VIOLATION });
|
||||
await expect(asTenant(wsA, (c) => c.query(
|
||||
"INSERT INTO production_buses (workspace_id, virtual_channel_id, bus) VALUES ($1, $2, 'program')", [wsA, a1.id])))
|
||||
.rejects.toMatchObject({ code: UNIQUE_VIOLATION });
|
||||
await expect(asTenant(wsA, (c) => c.query(
|
||||
"INSERT INTO virtual_channel_sources (workspace_id, virtual_channel_id, kind, label, hls_url) VALUES ($1, $2, 'rtmp', 'X', 'https://a.example/x')",
|
||||
[wsA, a1.id]))).rejects.toMatchObject({ code: CHECK_VIOLATION });
|
||||
const kinds = await asTenant(wsA, (c) => c.query("SELECT kind FROM virtual_channel_sources WHERE virtual_channel_id = $1 ORDER BY kind", [a1.id]));
|
||||
expect(kinds.rows.map((r: { kind: string }) => r.kind)).toEqual(["hls", "playlist"]);
|
||||
});
|
||||
|
||||
it("labels with filtergraph metacharacters are rejected by the schema too", async () => {
|
||||
for (const label of ["a;b", "x[0]", "a,b", "it's", "movie=/etc/passwd"]) {
|
||||
await expect(asTenant(wsA, (c) => c.query("UPDATE production_scenes SET label = $2 WHERE id = $1", [a1.spare, label])))
|
||||
.rejects.toMatchObject({ code: CHECK_VIOLATION });
|
||||
}
|
||||
});
|
||||
|
||||
it("overlay kinds carry exactly their own parameters; rendered revisions carry a sha256 asset", async () => {
|
||||
await expect(asTenant(wsA, (c) => c.query(
|
||||
"INSERT INTO overlay_slots (workspace_id, virtual_channel_id, scene_id, kind) VALUES ($1, $2, $3, 'chyron')", [wsA, a1.id, a1.scene])))
|
||||
.rejects.toMatchObject({ code: CHECK_VIOLATION });
|
||||
await expect(asTenant(wsA, (c) => c.query(
|
||||
"INSERT INTO overlay_slots (workspace_id, virtual_channel_id, scene_id, kind, chyron_speed_px_s, chyron_strip_width_px) VALUES ($1, $2, $3, 'chyron', 401, 100)",
|
||||
[wsA, a1.id, a1.scene]))).rejects.toMatchObject({ code: CHECK_VIOLATION });
|
||||
await asTenant(wsA, (c) => c.query(
|
||||
"INSERT INTO overlay_slots (workspace_id, virtual_channel_id, scene_id, kind, chyron_speed_px_s, chyron_strip_width_px) VALUES ($1, $2, $3, 'chyron', 120, 16384)",
|
||||
[wsA, a1.id, a1.scene]));
|
||||
await expect(asTenant(wsA, (c) => c.query(
|
||||
"INSERT INTO overlay_slots (workspace_id, virtual_channel_id, scene_id, kind) VALUES ($1, $2, $3, 'clock')", [wsA, a1.id, a1.scene])))
|
||||
.rejects.toMatchObject({ code: CHECK_VIOLATION });
|
||||
await expect(asTenant(wsA, (c) => c.query(
|
||||
"INSERT INTO overlay_revisions (workspace_id, virtual_channel_id, overlay_slot_id, template_id, render_state, asset_id) VALUES ($1, $2, $3, 'lt', 'rendered', 'movie=x')",
|
||||
[wsA, a1.id, a1.overlay]))).rejects.toMatchObject({ code: CHECK_VIOLATION });
|
||||
await expect(asTenant(wsA, (c) => c.query(
|
||||
"INSERT INTO overlay_revisions (workspace_id, virtual_channel_id, overlay_slot_id, template_id, render_state) VALUES ($1, $2, $3, 'lt', 'rendered')",
|
||||
[wsA, a1.id, a1.overlay]))).rejects.toMatchObject({ code: CHECK_VIOLATION });
|
||||
});
|
||||
});
|
||||
|
||||
describe("lifecycle", () => {
|
||||
it("deleting a source empties its slots (SET NULL keeps tenant + channel); a scene on a bus cannot be deleted", async () => {
|
||||
const channel = await asTenant(wsA, (c) => seedChannel(c, wsA, assetA));
|
||||
await asTenant(wsA, (c) => c.query("DELETE FROM virtual_channel_sources WHERE id = $1", [channel.hls]));
|
||||
const slot = await asTenant(wsA, (c) => c.query("SELECT workspace_id, virtual_channel_id, source_id FROM scene_slots WHERE id = $1", [channel.slot]));
|
||||
expect(slot.rows[0]).toEqual({ workspace_id: wsA, virtual_channel_id: channel.id, source_id: null });
|
||||
await expect(asTenant(wsA, (c) => c.query("DELETE FROM production_scenes WHERE id = $1", [channel.scene])))
|
||||
.rejects.toMatchObject({ code: FK_VIOLATION });
|
||||
});
|
||||
|
||||
it("deleting a rendered revision clears the slot's current revision; deleting a channel cascades everything", async () => {
|
||||
const channel = await asTenant(wsA, (c) => seedChannel(c, wsA, assetA));
|
||||
await asTenant(wsA, (c) => c.query("DELETE FROM overlay_revisions WHERE id = $1", [channel.revision]));
|
||||
const overlay = await asTenant(wsA, (c) => c.query("SELECT current_revision_id FROM overlay_slots WHERE id = $1", [channel.overlay]));
|
||||
expect(overlay.rows[0]?.current_revision_id).toBeNull();
|
||||
await asTenant(wsA, (c) => c.query("DELETE FROM virtual_channels WHERE id = $1", [channel.id]));
|
||||
const left = await asTenant(wsA, (c) => c.query(
|
||||
`SELECT (SELECT count(*) FROM virtual_channel_sources WHERE virtual_channel_id = $1)
|
||||
+ (SELECT count(*) FROM production_scenes WHERE virtual_channel_id = $1)
|
||||
+ (SELECT count(*) FROM scene_slots WHERE virtual_channel_id = $1)
|
||||
+ (SELECT count(*) FROM production_buses WHERE virtual_channel_id = $1)
|
||||
+ (SELECT count(*) FROM overlay_slots WHERE virtual_channel_id = $1)
|
||||
+ (SELECT count(*) FROM overlay_revisions WHERE virtual_channel_id = $1)
|
||||
+ (SELECT count(*) FROM playlist_items WHERE virtual_channel_id = $1) AS n`, [channel.id]));
|
||||
expect(Number(left.rows[0]?.n)).toBe(0);
|
||||
});
|
||||
|
||||
it("a playlisted video asset cannot be deleted directly, but a workspace purge removes everything", async () => {
|
||||
await expect(asTenant(wsA, (c) => c.query("DELETE FROM video_assets WHERE id = $1", [assetA])))
|
||||
.rejects.toMatchObject({ code: FK_VIOLATION });
|
||||
const ws = String((await ownerPool.query("INSERT INTO workspaces (name, slug) VALUES ('Purge', $1) RETURNING id", [`purge-${randomUUID()}`])).rows[0]?.id);
|
||||
const asset = await asTenant(ws, (c) => one(c,
|
||||
"INSERT INTO video_assets (workspace_id, storage_key, mime_type, size_bytes) VALUES ($1, 'sha256/p.mp4', 'video/mp4', 1) RETURNING id", [ws]));
|
||||
await asTenant(ws, (c) => seedChannel(c, ws, asset));
|
||||
await ownerPool.query("DELETE FROM workspaces WHERE id = $1", [ws]);
|
||||
const left = await ownerPool.query(
|
||||
`SELECT (SELECT count(*) FROM virtual_channels WHERE workspace_id = $1)
|
||||
+ (SELECT count(*) FROM playlist_items WHERE workspace_id = $1)
|
||||
+ (SELECT count(*) FROM production_buses WHERE workspace_id = $1)
|
||||
+ (SELECT count(*) FROM overlay_revisions WHERE workspace_id = $1) AS n`, [ws]);
|
||||
expect(Number(left.rows[0]?.n)).toBe(0);
|
||||
});
|
||||
|
||||
it("workspace and channel ownership columns are immutable", async () => {
|
||||
await expect(asTenant(wsA, (c) => c.query("UPDATE production_scenes SET virtual_channel_id = $2 WHERE id = $1", [a1.spare, a2.id])))
|
||||
.rejects.toThrow(/immutable/);
|
||||
await expect(asTenant(wsA, (c) => c.query("UPDATE production_buses SET bus = 'preview' WHERE virtual_channel_id = $1 AND bus = 'program'", [a1.id])))
|
||||
.rejects.toThrow(/bus is immutable/);
|
||||
await expect(asTenant(wsA, (c) => c.query("UPDATE virtual_channels SET workspace_id = $2 WHERE id = $1", [a1.id, wsB])))
|
||||
.rejects.toThrow(/immutable|row-level security/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("RLS denial suite (FORCE, NULLIF tenant isolation)", () => {
|
||||
const tables = [
|
||||
"virtual_channels", "virtual_channel_sources", "playlist_items", "production_scenes",
|
||||
"scene_slots", "production_buses", "overlay_slots", "overlay_revisions",
|
||||
] as const;
|
||||
|
||||
it.each(tables)("%s has RLS enabled and forced", async (table) => {
|
||||
const flags = await ownerPool.query("SELECT relrowsecurity, relforcerowsecurity FROM pg_class WHERE oid = $1::regclass", [table]);
|
||||
expect(flags.rows[0]).toEqual({ relrowsecurity: true, relforcerowsecurity: true });
|
||||
});
|
||||
|
||||
it.each(tables)("%s: tenant B sees, updates and deletes none of A's rows", async (table) => {
|
||||
const seen = await asTenant(wsB, (c) => c.query(`SELECT count(*)::int AS n FROM ${table} WHERE workspace_id = $1`, [wsA]));
|
||||
expect(seen.rows[0]?.n).toBe(0);
|
||||
const updated = await asTenant(wsB, (c) => c.query(`UPDATE ${table} SET updated_at = now() WHERE workspace_id = $1`, [wsA]));
|
||||
expect(updated.rowCount).toBe(0);
|
||||
const deleted = await asTenant(wsB, (c) => c.query(`DELETE FROM ${table} WHERE workspace_id = $1`, [wsA]));
|
||||
expect(deleted.rowCount).toBe(0);
|
||||
const ownStillThere = await asTenant(wsA, (c) => c.query(`SELECT count(*)::int AS n FROM ${table}`));
|
||||
expect(ownStillThere.rows[0]?.n).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it.each(tables)("%s: no tenant context (and an empty one) sees nothing", async (table) => {
|
||||
const none = await asTenant(null, (c) => c.query(`SELECT count(*)::int AS n FROM ${table}`));
|
||||
expect(none.rows[0]?.n).toBe(0);
|
||||
const empty = await asTenant("", (c) => c.query(`SELECT count(*)::int AS n FROM ${table}`));
|
||||
expect(empty.rows[0]?.n).toBe(0);
|
||||
});
|
||||
|
||||
it("tenant B cannot write rows into A's workspace (WITH CHECK) on any table", async () => {
|
||||
const attempts: Array<[string, unknown[]]> = [
|
||||
["INSERT INTO virtual_channels (workspace_id, slug, name) VALUES ($1, 'x-rls', 'X')", [wsA]],
|
||||
["INSERT INTO virtual_channel_sources (workspace_id, virtual_channel_id, kind, label, playlist_loop) VALUES ($1, $2, 'playlist', 'X', false)", [wsA, a1.id]],
|
||||
["INSERT INTO playlist_items (workspace_id, virtual_channel_id, source_id, position, video_asset_id) VALUES ($1, $2, $3, 9, $4)", [wsA, a1.id, a1.playlist, assetA]],
|
||||
["INSERT INTO production_scenes (workspace_id, virtual_channel_id, label, layout) VALUES ($1, $2, 'X', 'single')", [wsA, a1.id]],
|
||||
["INSERT INTO scene_slots (workspace_id, virtual_channel_id, scene_id, slot_index) VALUES ($1, $2, $3, 3)", [wsA, a1.id, a1.scene]],
|
||||
["INSERT INTO production_buses (workspace_id, virtual_channel_id, bus) VALUES ($1, $2, 'program')", [wsA, a1.id]],
|
||||
["INSERT INTO overlay_slots (workspace_id, virtual_channel_id, scene_id, kind) VALUES ($1, $2, $3, 'logo')", [wsA, a1.id, a1.scene]],
|
||||
["INSERT INTO overlay_revisions (workspace_id, virtual_channel_id, overlay_slot_id, template_id) VALUES ($1, $2, $3, 'lt')", [wsA, a1.id, a1.overlay]],
|
||||
];
|
||||
for (const [sql, params] of attempts) {
|
||||
await expect(asTenant(wsB, (c) => c.query(sql, params)), sql).rejects.toMatchObject({ code: RLS_VIOLATION });
|
||||
}
|
||||
});
|
||||
|
||||
it("tenant B cannot point its own rows at A's channel, scene or source", async () => {
|
||||
await expect(asTenant(wsB, (c) => c.query(
|
||||
"INSERT INTO scene_slots (workspace_id, virtual_channel_id, scene_id, slot_index, source_id) VALUES ($1, $2, $3, 2, $4)",
|
||||
[wsB, b1.id, b1.scene, a1.hls]))).rejects.toMatchObject({ code: FK_VIOLATION });
|
||||
await expect(asTenant(wsB, (c) => c.query(
|
||||
"UPDATE production_buses SET scene_id = $2 WHERE virtual_channel_id = $1 AND bus = 'program'", [b1.id, a1.scene])))
|
||||
.rejects.toMatchObject({ code: FK_VIOLATION });
|
||||
});
|
||||
});
|
||||
});
|
||||
68
apps/api/src/db/publishing-migration.integration.test.ts
Normal file
68
apps/api/src/db/publishing-migration.integration.test.ts
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { mkdtemp, readdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { Pool } from "pg";
|
||||
import { runMigrations } from "./migrate.js";
|
||||
import { assertPublishingSchema } from "@tactical-media-broadcast-hub/publishing-execution";
|
||||
const ownerUrl = process.env.PUBLISHING_IT_SUPER_DATABASE_URL;
|
||||
const appUrl = process.env.PUBLISHING_IT_DATABASE_URL;
|
||||
describe.skipIf(!ownerUrl || !appUrl)("publishing migration upgrade and resumability (local DB)", () => {
|
||||
let admin: Pool; let owner: Pool; let app: Pool; let dir: string;
|
||||
let database: string; let previousOwner: string | undefined;
|
||||
beforeAll(async () => {
|
||||
const url = new URL(ownerUrl!); url.pathname = "/postgres";
|
||||
admin = new Pool({ connectionString: url.toString() });
|
||||
database = `wp3_migration_sol_${Date.now()}`;
|
||||
await admin.query(`CREATE DATABASE ${database}`);
|
||||
url.pathname = `/${database}`; previousOwner = process.env.SUPER_DATABASE_URL;
|
||||
process.env.SUPER_DATABASE_URL = url.toString(); owner = new Pool({ connectionString: url.toString() });
|
||||
const runtimeUrl = new URL(appUrl!); runtimeUrl.pathname = `/${database}`; app = new Pool({ connectionString: runtimeUrl.toString() });
|
||||
dir = await mkdtemp(join(tmpdir(), "wp3-migrations-sol-"));
|
||||
for (const file of (await readdir("migrations")).filter((name) => name.endsWith(".sql") && name < "018")) await writeFile(join(dir, file), await readFile(join("migrations", file)));
|
||||
await runMigrations(dir);
|
||||
}, 30_000);
|
||||
afterAll(async () => {
|
||||
if (previousOwner === undefined) delete process.env.SUPER_DATABASE_URL; else process.env.SUPER_DATABASE_URL = previousOwner;
|
||||
await owner?.end(); await app?.end();
|
||||
if (admin && database) await admin.query(`DROP DATABASE ${database}`);
|
||||
await admin?.end();
|
||||
});
|
||||
it("upgrades legacy schedules without inventing authorization and resumes 019 after failed cutover", async () => {
|
||||
const user = (await owner.query("INSERT INTO users(external_id, username, display_name) VALUES('migration-sol', 'migration-sol', 'Migration') RETURNING id")).rows[0].id as string;
|
||||
const workspace = (await owner.query("INSERT INTO workspaces(name,slug) VALUES('Migration','migration-sol') RETURNING id")).rows[0].id as string;
|
||||
const connection = (await owner.query("INSERT INTO connections(workspace_id,provider,channel_id) VALUES($1,'mastodon','channel') RETURNING id", [workspace])).rows[0].id as string;
|
||||
const keys = ["legacy-pending", "legacy-flight", "legacy-done"];
|
||||
for (const [index, key] of keys.entries()) {
|
||||
const draft = (await owner.query("INSERT INTO drafts(workspace_id,author_user_id,post,state) VALUES($1,$2,'{}','scheduled') RETURNING id", [workspace, user])).rows[0].id as string;
|
||||
await owner.query("INSERT INTO scheduled_publications(workspace_id,draft_id,connection_id,publish_at,idempotency_key,state) VALUES($1,$2,$3,now(),$4,$5)", [workspace, draft, connection, key, index === 1 ? "in_flight" : "pending"]);
|
||||
if (index === 2) await owner.query("INSERT INTO publications(workspace_id,draft_id,connection_id,idempotency_key,remote_id,published_at) VALUES($1,$2,$3,$4,'remote',now())", [workspace, draft, connection, key]);
|
||||
}
|
||||
await writeFile(join(dir, "018_publishing_execution.sql"), await readFile("migrations/018_publishing_execution.sql"));
|
||||
await runMigrations(dir);
|
||||
const upgraded = await owner.query("SELECT state, scheduled_by_user_id, execution_version FROM scheduled_publications ORDER BY idempotency_key");
|
||||
expect(upgraded.rows.map((row) => row.state)).toEqual(["done", "failed", "needs_reauthorization"]);
|
||||
expect(upgraded.rows.every((row) => row.scheduled_by_user_id === null && row.execution_version === 0)).toBe(true);
|
||||
const original = await readFile("migrations/019_publishing_integration.sql", "utf8");
|
||||
await writeFile(join(dir, "019_publishing_integration.sql"), original.replace("DROP FUNCTION publishing_due_schedules(integer);", "SELECT nonexistent_cutover_function();"));
|
||||
await expect(runMigrations(dir)).rejects.toThrow("cutover failed");
|
||||
expect((await owner.query("SELECT name FROM schema_migrations WHERE name = '019_publishing_integration.sql'")).rowCount).toBe(0);
|
||||
await expect(assertPublishingSchema(app)).rejects.toThrow();
|
||||
expect((await owner.query("SELECT name FROM schema_migrations WHERE name LIKE '019%:%'")).rowCount).toBe(3);
|
||||
await writeFile(join(dir, "019_publishing_integration.sql"), original);
|
||||
await expect(runMigrations(dir)).resolves.toEqual(["019_publishing_integration.sql"]);
|
||||
await expect(assertPublishingSchema(app)).resolves.toBeUndefined();
|
||||
await expect(runMigrations(dir)).resolves.toEqual([]);
|
||||
const role = (await app.query("SELECT current_user, rolsuper, rolbypassrls FROM pg_roles WHERE rolname = current_user")).rows[0];
|
||||
expect(role).toEqual({ current_user: "sp_app", rolsuper: false, rolbypassrls: false });
|
||||
}, 30_000);
|
||||
it("repairs an invalid concurrent index only when its definition matches", async () => {
|
||||
const file = "020_test_concurrent.sql";
|
||||
await writeFile(join(dir, file), "-- wp3:phase expand transaction\nCREATE TABLE migration_index_test(v integer); INSERT INTO migration_index_test VALUES(1),(1);\n-- wp3:phase idx concurrent\nCREATE UNIQUE INDEX CONCURRENTLY migration_index_test_idx ON migration_index_test (v);");
|
||||
await expect(runMigrations(dir)).rejects.toThrow("idx failed");
|
||||
expect((await owner.query("SELECT indisvalid FROM pg_index WHERE indexrelid = 'migration_index_test_idx'::regclass")).rows[0]?.indisvalid).toBe(false);
|
||||
await owner.query("DELETE FROM migration_index_test WHERE ctid NOT IN (SELECT min(ctid) FROM migration_index_test GROUP BY v)");
|
||||
await expect(runMigrations(dir)).resolves.toEqual([file]);
|
||||
expect((await owner.query("SELECT indisvalid FROM pg_index WHERE indexrelid = 'migration_index_test_idx'::regclass")).rows[0]?.indisvalid).toBe(true);
|
||||
}, 30_000);
|
||||
});
|
||||
176
apps/api/src/db/tenancy.integration.test.ts
Normal file
176
apps/api/src/db/tenancy.integration.test.ts
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { join } from "node:path";
|
||||
import { Pool } from "pg";
|
||||
import { pool, withTenant, withSystem } from "./client.js";
|
||||
import { runMigrations } from "./migrate.js";
|
||||
import { PublishingExecutionRepository } from "@tactical-media-broadcast-hub/publishing-execution";
|
||||
|
||||
/**
|
||||
* Audit-finding acceptance tests against a live PostgreSQL.
|
||||
* Migrations + cleanup run via the owner pool (SUPER_DATABASE_URL);
|
||||
* the assertions run as sp_app so FORCE RLS binds honestly.
|
||||
* Skipped without DATABASE_URL.
|
||||
*/
|
||||
const dbAvailable = process.env.DATABASE_URL !== undefined;
|
||||
const ownerPool = new Pool({ connectionString: process.env.SUPER_DATABASE_URL ?? process.env.DATABASE_URL });
|
||||
|
||||
async function asOwner<T>(fn: (client: import("pg").PoolClient) => Promise<T>): Promise<T> {
|
||||
const client = await ownerPool.connect();
|
||||
try {
|
||||
return await fn(client);
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
describe.skipIf(!dbAvailable)("tenancy + audit chain (live DB)", () => {
|
||||
let wsA: string;
|
||||
let wsB: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
await runMigrations(process.env.MIGRATIONS_DIR ?? join(process.cwd(), "migrations"));
|
||||
await asOwner(async (client) => {
|
||||
await client.query("DELETE FROM audit_log");
|
||||
await client.query("DELETE FROM memberships");
|
||||
await client.query("DELETE FROM connections");
|
||||
await client.query("DELETE FROM credentials");
|
||||
await client.query("DELETE FROM workspaces");
|
||||
const a = await client.query("INSERT INTO workspaces (name, slug) VALUES ('A', $1) RETURNING id", [`a-${Date.now()}`]);
|
||||
const b = await client.query("INSERT INTO workspaces (name, slug) VALUES ('B', $1) RETURNING id", [`b-${Date.now()}`]);
|
||||
wsA = a.rows[0].id as string;
|
||||
wsB = b.rows[0].id as string;
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await asOwner(async (client) => {
|
||||
await client.query("DELETE FROM workspaces WHERE id IN ($1, $2)", [wsA, wsB]);
|
||||
});
|
||||
await pool.end();
|
||||
await ownerPool.end();
|
||||
});
|
||||
|
||||
it("RLS isolates tenants: A sees only A's connections (FORCE applies to owner)", async () => {
|
||||
await withTenant(wsA, async (client) => {
|
||||
await client.query("INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-social', 'ch-a')", [wsA]);
|
||||
});
|
||||
await withTenant(wsB, async (client) => {
|
||||
await client.query("INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-social', 'ch-b')", [wsB]);
|
||||
});
|
||||
const seenFromA = await withTenant(wsA, async (client) => {
|
||||
return (await client.query("SELECT channel_id FROM connections")).rows.map((r) => r.channel_id);
|
||||
});
|
||||
expect(seenFromA).toEqual(["ch-a"]);
|
||||
const seenFromSystem = await withSystem(async (client) => {
|
||||
return (await client.query("SELECT count(*)::int AS n FROM connections")).rows[0]?.n;
|
||||
});
|
||||
expect(seenFromSystem).toBe(0);
|
||||
});
|
||||
|
||||
it("SECURITY DEFINER tenant assertion: cross-tenant call rejected (finding #3)", async () => {
|
||||
await expect(
|
||||
withTenant(wsB, async (client) => {
|
||||
await client.query("SELECT audit_append($1, NULL, 'system', 'x', 'y', 'z', NULL)", [wsA]);
|
||||
}),
|
||||
).rejects.toThrow(/tenant context mismatch/);
|
||||
});
|
||||
|
||||
it("audit branch: tenant context => tenant chain even for service callers (findings #1/#6)", async () => {
|
||||
await withTenant(wsA, async (client) => {
|
||||
const r = await client.query("SELECT audit_append($1, NULL, 'sp_scheduler', 'job.ran', 'job', 'j1', NULL) AS id", [wsA]);
|
||||
expect(r.rows[0]?.id).toBeTruthy();
|
||||
const row = await client.query("SELECT workspace_id, actor_role FROM audit_log WHERE id = $1", [r.rows[0].id]);
|
||||
expect(row.rows[0]?.workspace_id).toBe(wsA);
|
||||
expect(row.rows[0]?.actor_role).toBe("sp_scheduler");
|
||||
// system append inside tenant context must be refused
|
||||
await expect(client.query("SELECT audit_append_system(NULL, 'x', 'y', 'z', NULL)")).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
it("membership change bumps authz_epoch => sessions fail closed (finding #2)", async () => {
|
||||
const before = Number(await withSystem(async (c) => (await c.query("SELECT authz_epoch FROM workspaces WHERE id = $1", [wsA])).rows[0]?.authz_epoch));
|
||||
await asOwner(async (c) => {
|
||||
const u = await c.query("INSERT INTO users (external_id, email, display_name) VALUES ($1, 'e@x', 'E') RETURNING id", [`ext-${Date.now()}`]);
|
||||
await withTenant(wsA, async (tenantClient) => {
|
||||
await tenantClient.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, 'Publisher')", [wsA, u.rows[0].id]);
|
||||
});
|
||||
});
|
||||
const after = Number(await withSystem(async (c) => (await c.query("SELECT authz_epoch FROM workspaces WHERE id = $1", [wsA])).rows[0]?.authz_epoch));
|
||||
expect(after).toBeGreaterThan(before);
|
||||
});
|
||||
|
||||
it("hash chain detects tampering", async () => {
|
||||
await withTenant(wsA, async (client) => {
|
||||
await client.query("SELECT audit_append($1, NULL, 'system', 't.1', 't', '1', NULL)", [wsA]);
|
||||
await client.query("SELECT audit_append($1, NULL, 'system', 't.2', 't', '2', NULL)", [wsA]);
|
||||
});
|
||||
const ok = await withTenant(wsA, async (c) => (await c.query("SELECT audit_verify_chain($1) AS ok", [wsA])).rows[0]?.ok);
|
||||
expect(ok).toBe(true);
|
||||
// direct tamper (as owner, inside the tenant context, bypassing the append fn)
|
||||
await withTenant(wsA, async (c) => {
|
||||
await c.query("UPDATE audit_log SET action = 'forged' WHERE action = 't.1'");
|
||||
});
|
||||
const broken = await withTenant(wsA, async (c) => (await c.query("SELECT audit_verify_chain($1) AS ok", [wsA])).rows[0]?.ok);
|
||||
expect(broken).toBe(false);
|
||||
});
|
||||
|
||||
it("verifies a legacy v1 row followed by v2 and detects a changed v2 diff", async () => {
|
||||
await withTenant(wsB, async (client) => {
|
||||
await client.query(
|
||||
`WITH stamp AS MATERIALIZED (SELECT clock_timestamp() AS at)
|
||||
INSERT INTO audit_log (workspace_id, action, target_type, target_id, diff, at, prev_hash, hash, hash_version)
|
||||
SELECT $1, 'legacy', 'test', 'legacy', '{"value":1}'::jsonb, stamp.at, NULL,
|
||||
audit_hash(NULL, NULL, 'legacy', 'legacy', stamp.at), 1 FROM stamp`,
|
||||
[wsB],
|
||||
);
|
||||
await client.query("SELECT audit_append($1, NULL, 'system', 'modern', 'test', 'modern', $2)", [wsB, { value: 2 }]);
|
||||
const verified = await client.query("SELECT audit_verify_chain($1) AS ok", [wsB]);
|
||||
expect(verified.rows[0]?.ok).toBe(true);
|
||||
await client.query("UPDATE audit_log SET diff = $2 WHERE workspace_id = $1 AND action = 'modern'", [wsB, { value: 3 }]);
|
||||
const broken = await client.query("SELECT audit_verify_chain($1) AS ok", [wsB]);
|
||||
expect(broken.rows[0]?.ok).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
it("scheduler claim refuses a pending schedule whose draft is not approved", async () => {
|
||||
const repository = new PublishingExecutionRepository(pool);
|
||||
try {
|
||||
const user = await asOwner(async (client) => {
|
||||
const inserted = await client.query(
|
||||
"INSERT INTO users (external_id, email, display_name) VALUES ($1, 'claim@example.com', 'Claim') RETURNING id",
|
||||
[`claim-${Date.now()}`],
|
||||
);
|
||||
return inserted.rows[0]?.id as string;
|
||||
});
|
||||
const key = `claim-${Date.now()}`;
|
||||
await withTenant(wsA, async (client) => {
|
||||
const connection = await client.query(
|
||||
"INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-social', 'claim') RETURNING id",
|
||||
[wsA],
|
||||
);
|
||||
const draft = await client.query(
|
||||
"INSERT INTO drafts (workspace_id, author_user_id, post, state) VALUES ($1, $2, $3, 'in_review') RETURNING id",
|
||||
[wsA, user, { text: "unapproved", mediaAssetIds: [], visibility: "public", providerMetadata: {} }],
|
||||
);
|
||||
await client.query(
|
||||
"INSERT INTO scheduled_publications (workspace_id, draft_id, connection_id, publish_at, idempotency_key) VALUES ($1, $2, $3, now(), $4)",
|
||||
[wsA, draft.rows[0]?.id, connection.rows[0]?.id, key],
|
||||
);
|
||||
});
|
||||
const schedule = await withTenant(wsA, (client) => client.query("SELECT id, generation FROM scheduled_publications WHERE idempotency_key = $1", [key]));
|
||||
await expect(repository.claim({ workspaceId: wsA, scheduleId: String(schedule.rows[0]?.id), generation: String(schedule.rows[0]?.generation), idempotencyKey: key })).resolves.toMatchObject({ kind: "failed", code: "EXECUTION_FENCE_MISSING" });
|
||||
const state = await withTenant(wsA, (client) => client.query("SELECT state FROM scheduled_publications WHERE idempotency_key = $1", [key]));
|
||||
expect(state.rows[0]?.state).toBe("pending");
|
||||
} finally {
|
||||
// The shared repository uses the caller-owned pool, closed by suite cleanup.
|
||||
}
|
||||
});
|
||||
|
||||
it("workspace_id is immutable on tenant rows (finding #8)", async () => {
|
||||
await expect(
|
||||
withTenant(wsA, async (c) => {
|
||||
await c.query("UPDATE connections SET workspace_id = $1 WHERE channel_id = 'ch-a'", [wsB]);
|
||||
}),
|
||||
).rejects.toThrow(/workspace_id is immutable/);
|
||||
});
|
||||
});
|
||||
57
apps/api/src/demo/demo.controller.ts
Normal file
57
apps/api/src/demo/demo.controller.ts
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import { Controller, Get, Post, Body, Req, UseGuards } from "@nestjs/common";
|
||||
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import { appendTenantAudit } from "../audit/audit.service.js";
|
||||
import { withTenant } from "../db/client.js";
|
||||
import type { Request } from "express";
|
||||
|
||||
@Controller("demo")
|
||||
@UseGuards(RbacGuard)
|
||||
export class DemoController {
|
||||
@Get("health-audit")
|
||||
@RequirePermission("audit.view")
|
||||
async viewAudit(@Req() req: Request & AuthenticatedRequest): Promise<{ workspaceId: string }> {
|
||||
return { workspaceId: req.session?.workspaceId ?? "none" };
|
||||
}
|
||||
|
||||
@Post("connections")
|
||||
@RequirePermission("connection.manage")
|
||||
async addConnection(
|
||||
@Req() req: Request & AuthenticatedRequest,
|
||||
@Body() body: { provider: string; channelId: string },
|
||||
): Promise<{ id: string }> {
|
||||
const session = req.session;
|
||||
if (session === undefined) throw new Error("unreachable: guard sets session");
|
||||
const workspaceId = session.workspaceId;
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const inserted = await client.query(
|
||||
"INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, $2, $3) RETURNING id",
|
||||
[workspaceId, body.provider, body.channelId],
|
||||
);
|
||||
const id = inserted.rows[0]?.id as string;
|
||||
await client.query("SELECT audit_append($1, $2, $3, 'connection.created', 'connection', $4, $5)", [
|
||||
workspaceId,
|
||||
session.userId,
|
||||
session.role,
|
||||
id,
|
||||
JSON.stringify({ provider: body.provider }),
|
||||
]);
|
||||
return { id };
|
||||
});
|
||||
}
|
||||
|
||||
@Post("echo-audited")
|
||||
@RequirePermission("workspace.view")
|
||||
async echoAudited(@Req() req: Request & AuthenticatedRequest, @Body() body: { apiKey?: string; note?: string }): Promise<{ ok: true }> {
|
||||
const session = req.session;
|
||||
if (session === undefined) throw new Error("unreachable: guard sets session");
|
||||
await appendTenantAudit(session.workspaceId, {
|
||||
actorUserId: session.userId,
|
||||
actorRole: session.role,
|
||||
action: "demo.echo",
|
||||
targetType: "demo",
|
||||
targetId: "echo",
|
||||
diff: { apiKey: body.apiKey ?? null, note: body.note ?? null },
|
||||
});
|
||||
return { ok: true };
|
||||
}
|
||||
}
|
||||
139
apps/api/src/director/config.ts
Normal file
139
apps/api/src/director/config.ts
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
/**
|
||||
* Director channel configuration — API-side mirror of
|
||||
* apps/scheduler/src/director/config.ts.
|
||||
*
|
||||
* The API answers snapshot/fence reads from the same DIRECTOR_CHANNELS
|
||||
* document the scheduler parses, so both parsers must agree on the shape.
|
||||
* Duplicating the parser (rather than importing scheduler sources into the
|
||||
* API's dependency graph) keeps each app's build independent; the shared
|
||||
* fixture test in director.service.test.ts asserts both parse identically.
|
||||
*/
|
||||
import type { AutoDirectorRule, DirectorAction, DirectorCondition, SceneLayout, SwitchPolicy } from "@tactical-media-broadcast-hub/core";
|
||||
|
||||
export interface DirectorChannelConfig {
|
||||
readonly workspaceId: string;
|
||||
readonly channelId: string;
|
||||
readonly generation: string;
|
||||
readonly rules: readonly AutoDirectorRule[];
|
||||
readonly policy: SwitchPolicy | undefined;
|
||||
readonly fallbackSceneId: string | null;
|
||||
}
|
||||
|
||||
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
||||
const GENERATION = /^[\x21-\x7e]{1,64}$/;
|
||||
const POLICY_NAMES: readonly SwitchPolicy["name"][] = [
|
||||
"highest_priority", "oldest_live", "newest_live", "round_robin", "operator_required",
|
||||
"picture_in_picture", "two_up", "split_screen", "grid",
|
||||
];
|
||||
const LAYOUTS: readonly SceneLayout[] = ["single", "pip", "two_up", "split_horizontal", "split_vertical", "grid", "custom"];
|
||||
const CONDITION_FIELDS: readonly DirectorCondition["field"][] = ["live_source_count", "max_priority", "source_state"];
|
||||
const CONDITION_OPS: readonly DirectorCondition["op"][] = [">=", "<=", "==", ">"];
|
||||
const SELECTIONS: readonly DirectorAction["selection"][] = ["top_priority", "all_live", "oldest_live", "explicit"];
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function uuid(value: unknown, field: string): string {
|
||||
if (typeof value !== "string" || !UUID.test(value)) throw new Error(`${field} must be a UUID`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function nonNegativeInt(value: unknown, field: string): number {
|
||||
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) throw new Error(`${field} must be a non-negative integer`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseCondition(value: unknown, field: string): DirectorCondition {
|
||||
if (!isRecord(value)) throw new Error(`${field} must be an object`);
|
||||
const conditionField = value.field;
|
||||
const op = value.op;
|
||||
if (typeof conditionField !== "string" || !CONDITION_FIELDS.includes(conditionField as DirectorCondition["field"])) {
|
||||
throw new Error(`${field}.field must be one of ${CONDITION_FIELDS.join(", ")}`);
|
||||
}
|
||||
if (typeof op !== "string" || !CONDITION_OPS.includes(op as DirectorCondition["op"])) {
|
||||
throw new Error(`${field}.op must be one of ${CONDITION_OPS.join(", ")}`);
|
||||
}
|
||||
const ruleValue = value.value;
|
||||
if (typeof ruleValue !== "number" && typeof ruleValue !== "string") throw new Error(`${field}.value must be a number or string`);
|
||||
return { field: conditionField as DirectorCondition["field"], op: op as DirectorCondition["op"], value: ruleValue };
|
||||
}
|
||||
|
||||
function parseAction(value: unknown, field: string): DirectorAction {
|
||||
if (!isRecord(value)) throw new Error(`${field} must be an object`);
|
||||
const selection = value.selection;
|
||||
if (typeof selection !== "string" || !SELECTIONS.includes(selection as DirectorAction["selection"])) {
|
||||
throw new Error(`${field}.selection must be one of ${SELECTIONS.join(", ")}`);
|
||||
}
|
||||
const layout = value.layout;
|
||||
if (typeof layout !== "string" || !LAYOUTS.includes(layout as SceneLayout)) {
|
||||
throw new Error(`${field}.layout must be one of ${LAYOUTS.join(", ")}`);
|
||||
}
|
||||
let sourceIds: string[] | undefined;
|
||||
if (value.sourceIds !== undefined) {
|
||||
if (!Array.isArray(value.sourceIds)) throw new Error(`${field}.sourceIds must be an array`);
|
||||
sourceIds = value.sourceIds.map((id, i) => uuid(id, `${field}.sourceIds[${i}]`));
|
||||
if (new Set(sourceIds).size !== sourceIds.length) throw new Error(`${field}.sourceIds must be unique`);
|
||||
}
|
||||
if (selection === "explicit" && (sourceIds === undefined || sourceIds.length === 0)) {
|
||||
throw new Error(`${field}.selection explicit requires sourceIds`);
|
||||
}
|
||||
return { layout: layout as SceneLayout, selection: selection as DirectorAction["selection"], ...(sourceIds === undefined ? {} : { sourceIds }) };
|
||||
}
|
||||
|
||||
function parseRule(value: unknown, channelId: string, index: number): AutoDirectorRule {
|
||||
const field = `DIRECTOR_CHANNELS[${index}].rules[${index}]`;
|
||||
if (!isRecord(value)) throw new Error(`${field} must be an object`);
|
||||
if (value.virtualChannelId !== undefined && value.virtualChannelId !== channelId) {
|
||||
throw new Error(`${field}.virtualChannelId must match the channel it is configured on`);
|
||||
}
|
||||
return {
|
||||
id: uuid(value.id, `${field}.id`),
|
||||
virtualChannelId: channelId,
|
||||
priority: nonNegativeInt(value.priority ?? 0, `${field}.priority`),
|
||||
condition: parseCondition(value.condition, `${field}.condition`),
|
||||
action: parseAction(value.action, `${field}.action`),
|
||||
};
|
||||
}
|
||||
|
||||
function parsePolicy(value: unknown, field: string): SwitchPolicy {
|
||||
if (!isRecord(value)) throw new Error(`${field} must be an object`);
|
||||
const name = value.name;
|
||||
if (typeof name !== "string" || !POLICY_NAMES.includes(name as SwitchPolicy["name"])) {
|
||||
throw new Error(`${field}.name must be one of ${POLICY_NAMES.join(", ")}`);
|
||||
}
|
||||
return {
|
||||
name: name as SwitchPolicy["name"],
|
||||
liveConfirmations: nonNegativeInt(value.liveConfirmations ?? 2, `${field}.liveConfirmations`),
|
||||
offlineConfirmations: nonNegativeInt(value.offlineConfirmations ?? 3, `${field}.offlineConfirmations`),
|
||||
noMediaSeconds: nonNegativeInt(value.noMediaSeconds ?? 10, `${field}.noMediaSeconds`),
|
||||
minHoldMs: nonNegativeInt(value.minHoldMs ?? 2_000, `${field}.minHoldMs`),
|
||||
switchCooldownMs: nonNegativeInt(value.switchCooldownMs ?? 10_000, `${field}.switchCooldownMs`),
|
||||
};
|
||||
}
|
||||
|
||||
export function parseDirectorChannels(raw: string | undefined): DirectorChannelConfig[] {
|
||||
if (raw === undefined || raw.trim() === "") return [];
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (!Array.isArray(parsed)) throw new Error("DIRECTOR_CHANNELS must be a JSON array");
|
||||
return parsed.map((entry, index) => {
|
||||
const field = `DIRECTOR_CHANNELS[${index}]`;
|
||||
if (!isRecord(entry)) throw new Error(`${field} must be an object`);
|
||||
const channelId = uuid(entry.channelId, `${field}.channelId`);
|
||||
const generation = entry.generation;
|
||||
if (typeof generation !== "string" || !GENERATION.test(generation)) {
|
||||
throw new Error(`${field}.generation must be 1-64 printable characters`);
|
||||
}
|
||||
const rules = Array.isArray(entry.rules) ? entry.rules.map((rule, i) => parseRule(rule, channelId, i)) : [];
|
||||
const fallbackSceneId = entry.fallbackSceneId === undefined || entry.fallbackSceneId === null
|
||||
? null : uuid(entry.fallbackSceneId, `${field}.fallbackSceneId`);
|
||||
return {
|
||||
workspaceId: uuid(entry.workspaceId, `${field}.workspaceId`),
|
||||
channelId,
|
||||
generation,
|
||||
rules,
|
||||
policy: entry.policy === undefined ? undefined : parsePolicy(entry.policy, `${field}.policy`),
|
||||
fallbackSceneId,
|
||||
};
|
||||
});
|
||||
}
|
||||
84
apps/api/src/director/director.controller.ts
Normal file
84
apps/api/src/director/director.controller.ts
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
/**
|
||||
* Internal director endpoints (WP10 wiring slice).
|
||||
*
|
||||
* Process-to-process only: a shared-secret bearer (DIRECTOR_API_TOKEN) when
|
||||
* configured, plus the ref's workspace/channel UUID shape. Not registered under
|
||||
* the session RBAC guard — the scheduler worker has no browser session; its
|
||||
* authorization is the generation/version fence in DirectorService.
|
||||
*/
|
||||
import { BadRequestException, Body, Controller, Get, Headers, HttpCode, Inject, Param, Post } from "@nestjs/common";
|
||||
import { DirectorService, type DirectorApplyBody } from "./director.service.js";
|
||||
|
||||
function requireToken(token: string | undefined): void {
|
||||
const expected = process.env.DIRECTOR_API_TOKEN;
|
||||
if (expected === undefined || expected === "") return; // unset = local dev, no shared secret configured
|
||||
if (token !== `Bearer ${expected}`) throw new BadRequestException("director token is invalid");
|
||||
}
|
||||
|
||||
@Controller("internal/director/channels")
|
||||
export class DirectorController {
|
||||
constructor(@Inject(DirectorService) private readonly director: DirectorService) {}
|
||||
|
||||
@Get(":workspaceId/:channelId/state")
|
||||
async state(@Headers() headers: Record<string, string>, @Param("workspaceId") ws: string, @Param("channelId") ch: string): Promise<unknown> {
|
||||
requireToken(headers.authorization);
|
||||
return this.director.channelState(ws, ch);
|
||||
}
|
||||
|
||||
@Get(":workspaceId/:channelId/fence")
|
||||
async fence(@Headers() headers: Record<string, string>, @Param("workspaceId") ws: string, @Param("channelId") ch: string): Promise<unknown> {
|
||||
requireToken(headers.authorization);
|
||||
return this.director.fence(ws, ch);
|
||||
}
|
||||
|
||||
@Post(":workspaceId/:channelId/apply")
|
||||
@HttpCode(200)
|
||||
async apply(@Headers() headers: Record<string, string>, @Param("workspaceId") ws: string, @Param("channelId") ch: string,
|
||||
@Body() body: unknown): Promise<unknown> {
|
||||
requireToken(headers.authorization);
|
||||
return this.director.applyProgramSwitch(ws, ch, parseApply(body));
|
||||
}
|
||||
|
||||
@Post(":workspaceId/:channelId/audit")
|
||||
@HttpCode(204)
|
||||
async audit(@Headers() headers: Record<string, string>, @Param("workspaceId") ws: string, @Param("channelId") ch: string,
|
||||
@Body() body: unknown): Promise<void> {
|
||||
requireToken(headers.authorization);
|
||||
const event = body as { commandId?: unknown; outcome?: unknown; reason?: unknown };
|
||||
if (typeof event.commandId !== "string" || (event.outcome !== "SUPERSEDED" && event.outcome !== "AUTHZ_REVOKED")
|
||||
|| typeof event.reason !== "string") throw new BadRequestException("audit event is invalid");
|
||||
await this.director.audit(ws, ch, { commandId: event.commandId, outcome: event.outcome, reason: event.reason });
|
||||
}
|
||||
}
|
||||
|
||||
function parseApply(body: unknown): DirectorApplyBody {
|
||||
if (typeof body !== "object" || body === null || Array.isArray(body)) throw new BadRequestException("apply body must be an object");
|
||||
const value = body as Record<string, unknown>;
|
||||
const commandId = value.commandId;
|
||||
if (typeof commandId !== "string" || commandId === "" || commandId.length > 128) throw new BadRequestException("commandId must be 1-128 characters");
|
||||
const expectedDesiredVersion = value.expectedDesiredVersion;
|
||||
if (typeof expectedDesiredVersion !== "number" || !Number.isSafeInteger(expectedDesiredVersion) || expectedDesiredVersion < 0) {
|
||||
throw new BadRequestException("expectedDesiredVersion must be a non-negative integer");
|
||||
}
|
||||
const authzEpoch = value.authzEpoch;
|
||||
if (typeof authzEpoch !== "number" || !Number.isSafeInteger(authzEpoch) || authzEpoch < 0) {
|
||||
throw new BadRequestException("authzEpoch must be a non-negative integer");
|
||||
}
|
||||
const program = value.program;
|
||||
if (typeof program !== "object" || program === null) throw new BadRequestException("program must be an object");
|
||||
const p = program as Record<string, unknown>;
|
||||
let parsedProgram: DirectorApplyBody["program"];
|
||||
if (p.kind === "sources") {
|
||||
if (!Array.isArray(p.sourceIds) || p.sourceIds.some((id) => typeof id !== "string")) throw new BadRequestException("program.sourceIds must be strings");
|
||||
parsedProgram = { kind: "sources", sourceIds: p.sourceIds };
|
||||
} else if (p.kind === "fallback") {
|
||||
if (p.sceneId !== null && typeof p.sceneId !== "string") throw new BadRequestException("program.sceneId must be a string or null");
|
||||
parsedProgram = { kind: "fallback", sceneId: p.sceneId as string | null };
|
||||
} else throw new BadRequestException("program.kind must be sources or fallback");
|
||||
const layout = value.layout;
|
||||
if (typeof layout !== "string" || layout === "") throw new BadRequestException("layout must be a non-empty string");
|
||||
if (!Array.isArray(value.reasons) || value.reasons.some((r) => typeof r !== "string")) throw new BadRequestException("reasons must be strings");
|
||||
const releaseOverride = value.releaseOverride;
|
||||
if (releaseOverride !== undefined && typeof releaseOverride !== "string") throw new BadRequestException("releaseOverride must be a string");
|
||||
return { commandId, expectedDesiredVersion, authzEpoch, program: parsedProgram, layout, reasons: value.reasons, ...(releaseOverride === undefined ? {} : { releaseOverride }) };
|
||||
}
|
||||
31
apps/api/src/director/director.module.ts
Normal file
31
apps/api/src/director/director.module.ts
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
/**
|
||||
* Director wiring module (WP10 integration slice).
|
||||
*
|
||||
* Minimal by design: the director service reads live-monitor state and writes
|
||||
* through MediaControlService; no chatgov/feeds/virtual-channels dependency.
|
||||
* Registration is one line in app.module.ts:
|
||||
*
|
||||
* import { DirectorModule } from "./director/director.module.js";
|
||||
* imports: [..., DirectorModule]
|
||||
*/
|
||||
import { Module } from "@nestjs/common";
|
||||
import { MediaControlModule } from "../media-control/media-control.module.js";
|
||||
import { DirectorController } from "./director.controller.js";
|
||||
import { DirectorService } from "./director.service.js";
|
||||
import { MediaControlService } from "../media-control/media-control.service.js";
|
||||
import { appendTenantAudit } from "../audit/audit.service.js";
|
||||
|
||||
@Module({
|
||||
imports: [MediaControlModule],
|
||||
controllers: [DirectorController],
|
||||
providers: [
|
||||
{
|
||||
provide: DirectorService,
|
||||
inject: [MediaControlService],
|
||||
useFactory: (control: MediaControlService) =>
|
||||
new DirectorService({ control, audit: appendTenantAudit }),
|
||||
},
|
||||
],
|
||||
exports: [DirectorService],
|
||||
})
|
||||
export class DirectorModule {}
|
||||
331
apps/api/src/director/director.service.test.ts
Normal file
331
apps/api/src/director/director.service.test.ts
Normal file
|
|
@ -0,0 +1,331 @@
|
|||
import { generateKeyPairSync, randomUUID } from "node:crypto";
|
||||
import type { PoolClient } from "pg";
|
||||
import { ConflictException } from "@nestjs/common";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { decide } from "../../../scheduler/src/director/engine.js";
|
||||
import { MediaControlService } from "../media-control/media-control.service.js";
|
||||
import { MemoryRedis } from "../production/testing/memory-redis.js";
|
||||
import { Ed25519ControlSigner } from "../media-control/wire.js";
|
||||
import type { TenantTransaction } from "../production/journal.js";
|
||||
import { DirectorService, engineState, type DirectorApplyBody, type DirectorChannelConfig } from "./director.service.js";
|
||||
import { parseDirectorChannels } from "./config.js";
|
||||
|
||||
/**
|
||||
* WP10 wiring slice: the port pair against fakes.
|
||||
*
|
||||
* The database double implements exactly the three queries the service issues
|
||||
* (fence, source join, scene resolution) over a small in-memory aggregate, and
|
||||
* MediaControlService runs for real on the MemoryRedis + aggregate-transaction
|
||||
* fixture used by control-s3.test.ts. The engine itself is imported from the
|
||||
* scheduler package — the same pure decide() the evaluator workflow runs.
|
||||
*/
|
||||
|
||||
const WS = randomUUID();
|
||||
const CH = randomUUID();
|
||||
const SOURCE_A = randomUUID();
|
||||
const SOURCE_B = randomUUID();
|
||||
const SCENE_SOLO = randomUUID();
|
||||
const SCENE_FALLBACK = randomUUID();
|
||||
|
||||
interface SceneRow { sceneId: string; sources: string[] }
|
||||
|
||||
/** In-memory channel aggregate: fence + sources + scenes, in query order. */
|
||||
class ChannelDatabase {
|
||||
desiredVersion = 4;
|
||||
authzEpoch = 7;
|
||||
readonly sources: Array<{ id: string; priority: number; monitorState: string | null; version: number | null; stateEnteredAt: Date | null; lastMediaAt: string | null; lastOutcome: string | null }> = [];
|
||||
readonly scenes: SceneRow[] = [];
|
||||
failAudit = false;
|
||||
|
||||
readonly transaction: TenantTransaction = async <T>(_ws: string, fn: (client: PoolClient) => Promise<T>): Promise<T> => {
|
||||
const query = async (sql: string, values: unknown[]): Promise<{ rows: Record<string, unknown>[]; rowCount: number }> => {
|
||||
if (sql.includes("FROM virtual_channel_sources")) {
|
||||
// The SOURCE_JOIN: monitor rows keyed by (connection_id, remote_ref).
|
||||
const rows = this.sources.map((s) => ({
|
||||
id: s.id, priority: s.priority, monitor_state: s.monitorState, monitor_version: s.version,
|
||||
state_entered_at: s.stateEnteredAt,
|
||||
tier_status: s.lastMediaAt === null && s.lastOutcome === null ? null
|
||||
: { "media-health": { lastMediaAt: s.lastMediaAt, lastOutcome: s.lastOutcome } },
|
||||
}));
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (sql.includes("SELECT desired_version FROM virtual_channels")) {
|
||||
if (this.desiredVersion < 0) return { rows: [], rowCount: 0 };
|
||||
return { rows: [{ desired_version: String(this.desiredVersion) }], rowCount: 1 };
|
||||
}
|
||||
if (sql.includes("SELECT authz_epoch FROM workspaces")) {
|
||||
return { rows: [{ authz_epoch: String(this.authzEpoch) }], rowCount: 1 };
|
||||
}
|
||||
if (sql.includes("FROM production_scenes")) {
|
||||
// Scene resolution: bound slots exactly equal the selected sources.
|
||||
const selected = values.slice(2, -1).map(String);
|
||||
const match = this.scenes.find((scene) => [...scene.sources].sort().join() === [...selected].sort().join() && scene.sources.length === selected.length);
|
||||
return { rows: match === undefined ? [] : [{ scene_id: match.sceneId }], rowCount: match === undefined ? 0 : 1 };
|
||||
}
|
||||
throw new Error(`unexpected director query: ${sql.slice(0, 60)}`);
|
||||
};
|
||||
return fn({ query } as unknown as PoolClient);
|
||||
};
|
||||
}
|
||||
|
||||
/** media-control transaction double over the same aggregate (control-s3 pattern). */
|
||||
class BusDatabase {
|
||||
state = { version: 0, preview: null as string | null, program: null as string | null, receipts: [] as unknown[] };
|
||||
readonly scenes = new Set<string>();
|
||||
/** productionScenes served to the production snapshot: one slotless scene per known scene id. */
|
||||
sceneList(): Array<{ id: string; label: string; layout: string }> {
|
||||
return [...this.scenes].map((id) => ({ id, label: id === id ? "Scene" : "Scene", layout: "single" }));
|
||||
}
|
||||
readonly transaction: TenantTransaction = async <T>(_ws: string, fn: (client: PoolClient) => Promise<T>): Promise<T> => {
|
||||
const query = async (sql: string, values: unknown[] = []): Promise<{ rows: unknown[]; rowCount: number }> => {
|
||||
let rows: unknown[] = [];
|
||||
if (sql.startsWith("SELECT id FROM virtual_channels")) rows = [{ id: CH }];
|
||||
else if (sql.startsWith("SELECT desired_version")) rows = [{ desired_version: String(this.state.version), desired_status: "STOPPED" }];
|
||||
else if (sql.includes("FROM audit_log")) {
|
||||
const found = sql.includes("requestId") ? this.state.receipts.find(r => (r as { requestId?: string }).requestId === values[2]) : this.state.receipts.at(-1);
|
||||
rows = found ? [{ diff: found }] : [];
|
||||
} else if (sql.startsWith("SELECT scene_id")) rows = [{ scene_id: this.state.preview }];
|
||||
else if (sql.startsWith("INSERT INTO production_buses")) { if (sql.includes("'program'")) this.state.program = String(values[2]); else this.state.preview = String(values[2]); }
|
||||
else if (sql.startsWith("UPDATE virtual_channels")) this.state.version = Number(values[2]);
|
||||
else if (sql.startsWith("SELECT audit_append")) { this.state.receipts.push(values[6]); return { rows: [], rowCount: 1 }; }
|
||||
else if (sql.startsWith("SELECT id,label,layout")) rows = this.sceneList();
|
||||
else if (sql.startsWith("SELECT bus,scene_id")) rows = [{ bus: "preview", scene_id: this.state.preview }, { bus: "program", scene_id: this.state.program }];
|
||||
else if (sql.startsWith("SELECT id FROM production_scenes")) rows = this.scenes.has(String(values[2])) ? [{ id: values[2] }] : [];
|
||||
else if (sql.startsWith("SELECT scene_id,slot_index") || sql.includes("FROM scene_slots")) rows = [];
|
||||
else if (sql.includes("FROM virtual_channel_sources")) rows = [];
|
||||
else if (sql.includes("FROM overlay_slots")) rows = [];
|
||||
return { rows, rowCount: sql.startsWith("UPDATE") ? 1 : rows.length };
|
||||
};
|
||||
return fn({ query } as unknown as PoolClient);
|
||||
};
|
||||
}
|
||||
|
||||
function config(over: Partial<DirectorChannelConfig> = {}): DirectorChannelConfig {
|
||||
return {
|
||||
workspaceId: WS, channelId: CH, generation: "g1",
|
||||
rules: [{
|
||||
id: randomUUID(), virtualChannelId: CH, priority: 1,
|
||||
condition: { field: "live_source_count", op: ">=", value: 1 },
|
||||
action: { layout: "single", selection: "top_priority" },
|
||||
}],
|
||||
policy: { name: "highest_priority", liveConfirmations: 1, offlineConfirmations: 1, noMediaSeconds: 10, minHoldMs: 0, switchCooldownMs: 0 },
|
||||
fallbackSceneId: SCENE_FALLBACK,
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
async function fixture(channelConfig: DirectorChannelConfig): Promise<{
|
||||
db: ChannelDatabase; bus: BusDatabase; service: DirectorService; control: MediaControlService;
|
||||
}> {
|
||||
const db = new ChannelDatabase();
|
||||
const bus = new BusDatabase();
|
||||
bus.scenes.add(SCENE_SOLO);
|
||||
bus.scenes.add(SCENE_FALLBACK);
|
||||
const redis = new MemoryRedis();
|
||||
const control = new MediaControlService({
|
||||
transaction: bus.transaction, redis,
|
||||
signer: new Ed25519ControlSigner("test", generateKeyPairSync("ed25519").privateKey), assignments: [],
|
||||
});
|
||||
await control.assign(WS, CH, "media-worker-a", 1);
|
||||
const service = new DirectorService({
|
||||
transaction: db.transaction, control,
|
||||
audit: async () => randomUUID(),
|
||||
channels: () => [channelConfig],
|
||||
});
|
||||
return { db, bus, service, control };
|
||||
}
|
||||
|
||||
const apply = (over: Partial<DirectorApplyBody> = {}): DirectorApplyBody => ({
|
||||
commandId: `${CH}:9:1`, expectedDesiredVersion: 4, authzEpoch: 7,
|
||||
program: { kind: "sources", sourceIds: [SOURCE_A] }, layout: "single", reasons: ["rule"],
|
||||
...over,
|
||||
});
|
||||
|
||||
describe("engineState mapping", () => {
|
||||
it("maps monitor states and reads anything else as UNKNOWN", () => {
|
||||
for (const state of ["OFFLINE", "PROBING", "LIVE", "UNSTABLE", "ERROR"]) {
|
||||
expect(engineState(state)).toBe(state);
|
||||
}
|
||||
expect(engineState(null)).toBe("UNKNOWN");
|
||||
expect(engineState("weird")).toBe("UNKNOWN");
|
||||
});
|
||||
});
|
||||
|
||||
describe("DirectorStatePort read (channelState)", () => {
|
||||
it("joins monitor rows by source identity and renders the engine input shape", async () => {
|
||||
const entered = new Date("2026-10-09T12:00:00Z");
|
||||
const local = new ChannelDatabase();
|
||||
local.sources.push(
|
||||
{ id: SOURCE_A, priority: 50, monitorState: "LIVE", version: 3, stateEnteredAt: entered, lastMediaAt: "2026-10-09T12:00:01Z", lastOutcome: "ok" },
|
||||
{ id: SOURCE_B, priority: 10, monitorState: null, version: null, stateEnteredAt: null, lastMediaAt: null, lastOutcome: null },
|
||||
);
|
||||
const svc = new DirectorService({ transaction: local.transaction, control: { director: async () => { throw new Error("unreachable"); } }, channels: () => [config()] });
|
||||
const read = await svc.channelState(WS, CH);
|
||||
if (read.kind !== "current") throw new Error("expected current");
|
||||
expect(read.desiredVersion).toBe(4);
|
||||
expect(read.authzEpoch).toBe(7);
|
||||
expect(read.sources).toEqual([
|
||||
expect.objectContaining({ id: SOURCE_A, priority: 50, state: "LIVE", version: 3, liveSinceMs: entered.getTime(), lastMediaAtMs: Date.parse("2026-10-09T12:00:01Z") }),
|
||||
expect.objectContaining({ id: SOURCE_B, state: "UNKNOWN" }),
|
||||
]);
|
||||
// Optional engine fields are absent (never null): a null would de-confirm
|
||||
// every source as no-media inside decide().
|
||||
expect("lastMediaAtMs" in (read.sources[1] ?? {})).toBe(false);
|
||||
expect("mediaFailed" in (read.sources[0] ?? {})).toBe(false);
|
||||
expect(read.rules).toHaveLength(1);
|
||||
expect(read.policy.name).toBe("highest_priority");
|
||||
expect(read.fallbackSceneId).toBe(SCENE_FALLBACK);
|
||||
});
|
||||
|
||||
it("returns superseded when the channel is not configured for the director", async () => {
|
||||
const local = new ChannelDatabase();
|
||||
const svc = new DirectorService({ transaction: local.transaction, control: { director: async () => { throw new Error("unreachable"); } }, channels: () => [] });
|
||||
expect(await svc.channelState(WS, CH)).toEqual({ kind: "superseded", code: "SUPERSEDED" });
|
||||
expect(await svc.fence(WS, CH)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("DesiredStatePort apply (applyProgramSwitch)", () => {
|
||||
/** The evaluator loop: read -> decide, carrying memory/lastSwitch between ticks. */
|
||||
async function evaluate(service: DirectorService, db: ChannelDatabase, ticks: number): Promise<ReturnType<typeof decide>> {
|
||||
let memory: ReturnType<typeof decide>["memory"] | undefined;
|
||||
let lastSwitch: ReturnType<typeof decide>["lastSwitch"] | undefined;
|
||||
let decision: ReturnType<typeof decide> | undefined;
|
||||
for (let tick = 0; tick < ticks; tick += 1) {
|
||||
if (tick > 0) for (const source of db.sources) if (source.version !== null) source.version += 1; // a fresh monitor sample per tick
|
||||
const read = await service.channelState(WS, CH);
|
||||
if (read.kind !== "current") throw new Error("expected current");
|
||||
decision = decide({
|
||||
sources: read.sources, rules: read.rules, policy: read.policy,
|
||||
now: 100_000 + tick * 5_000, channelId: CH, fallbackSceneId: read.fallbackSceneId,
|
||||
...(memory === undefined ? {} : { memory }), ...(lastSwitch === undefined ? {} : { lastSwitch }),
|
||||
});
|
||||
memory = decision.memory; lastSwitch = decision.lastSwitch;
|
||||
}
|
||||
if (decision === undefined) throw new Error("no decision");
|
||||
return decision;
|
||||
}
|
||||
|
||||
it("applies a decision end to end: read -> decide (debounced) -> pst-select + cut through media control", async () => {
|
||||
// liveConfirmations 2: the engine must see two fresh monitor samples before
|
||||
// trusting a LIVE report — the debounce the evaluator relies on.
|
||||
const { db, bus, service } = await fixture(config({
|
||||
policy: { name: "highest_priority", liveConfirmations: 2, offlineConfirmations: 1, noMediaSeconds: 10, minHoldMs: 0, switchCooldownMs: 0 },
|
||||
}));
|
||||
db.scenes.push({ sceneId: SCENE_SOLO, sources: [SOURCE_A] });
|
||||
db.sources.push({ id: SOURCE_A, priority: 50, monitorState: "LIVE", version: 3, stateEnteredAt: new Date("2026-10-09T12:00:00Z"), lastMediaAt: null, lastOutcome: "ok" });
|
||||
|
||||
// First tick: one sample, unconfirmed — the engine holds the fallback.
|
||||
const probing = await evaluate(service, db, 1);
|
||||
expect(probing.program.kind).toBe("fallback");
|
||||
// Second tick (new monitor version): confirmed LIVE, the rule fires.
|
||||
const decision = await evaluate(service, db, 2);
|
||||
expect(decision.program).toEqual({ kind: "sources", sourceIds: [SOURCE_A] });
|
||||
expect(decision.switched).toBe(true);
|
||||
|
||||
const read = await service.channelState(WS, CH);
|
||||
if (read.kind !== "current") throw new Error("expected current");
|
||||
const outcome = await service.applyProgramSwitch(WS, CH, {
|
||||
commandId: `${CH}:0:1`, expectedDesiredVersion: read.desiredVersion, authzEpoch: read.authzEpoch,
|
||||
program: decision.program, layout: decision.layout, reasons: decision.reasons,
|
||||
});
|
||||
expect(outcome).toMatchObject({ kind: "applied" });
|
||||
expect(bus.state.preview).toBe(SCENE_SOLO);
|
||||
expect(bus.state.program).toBe(SCENE_SOLO);
|
||||
// pst + cut are two desired-version bumps
|
||||
expect(bus.state.version).toBe(2);
|
||||
});
|
||||
|
||||
it("replays the same commandId without switching twice (idempotent keys, no new receipts)", async () => {
|
||||
const { db, bus, service } = await fixture(config());
|
||||
db.scenes.push({ sceneId: SCENE_SOLO, sources: [SOURCE_A] });
|
||||
const body = apply();
|
||||
const first = await service.applyProgramSwitch(WS, CH, body);
|
||||
expect(first).toMatchObject({ kind: "applied" });
|
||||
// An activity retry arrives with the same fence: the derived idempotency
|
||||
// keys hit media-control's stored receipts, so nothing is written twice.
|
||||
const second = await service.applyProgramSwitch(WS, CH, body);
|
||||
expect(second.kind).not.toBe("conflict");
|
||||
expect(bus.state.receipts).toHaveLength(2); // pst + cut, still exactly once each
|
||||
expect(bus.state.version).toBe(2);
|
||||
// Once the fence has moved (a concurrent writer won), the same command is a conflict.
|
||||
db.desiredVersion = body.expectedDesiredVersion + 1;
|
||||
expect(await service.applyProgramSwitch(WS, CH, body)).toEqual({ kind: "conflict" });
|
||||
});
|
||||
|
||||
it("rejects a stale fence (desired version or authz epoch moved)", async () => {
|
||||
const { db, service } = await fixture(config());
|
||||
db.desiredVersion = 5;
|
||||
expect(await service.applyProgramSwitch(WS, CH, apply())).toEqual({ kind: "conflict" });
|
||||
db.desiredVersion = 4;
|
||||
db.authzEpoch = 8;
|
||||
expect(await service.applyProgramSwitch(WS, CH, apply())).toEqual({ kind: "conflict" });
|
||||
});
|
||||
|
||||
it("falls back to the configured fallback scene when no scene holds the selection", async () => {
|
||||
const { db, bus, service } = await fixture(config());
|
||||
db.scenes.push({ sceneId: randomUUID(), sources: [SOURCE_A, SOURCE_B] }); // different selection
|
||||
const outcome = await service.applyProgramSwitch(WS, CH, apply());
|
||||
expect(outcome).toMatchObject({ kind: "applied" });
|
||||
expect(bus.state.program).toBe(SCENE_FALLBACK);
|
||||
});
|
||||
|
||||
it("fails closed with a conflict when the selection has no scene and no fallback exists", async () => {
|
||||
const { service } = await fixture(config({ fallbackSceneId: null }));
|
||||
await expect(service.applyProgramSwitch(WS, CH, apply())).resolves.toEqual({ kind: "conflict" });
|
||||
});
|
||||
|
||||
it("an explicit fallback program selects the fallback scene", async () => {
|
||||
const { bus, service } = await fixture(config());
|
||||
const outcome = await service.applyProgramSwitch(WS, CH, apply({ program: { kind: "fallback", sceneId: null } }));
|
||||
expect(outcome).toMatchObject({ kind: "applied" });
|
||||
expect(bus.state.program).toBe(SCENE_FALLBACK);
|
||||
});
|
||||
|
||||
it("classifies media-control conflicts as CAS outcomes and propagates the rest", async () => {
|
||||
const failing = new DirectorService({
|
||||
transaction: new ChannelDatabase().transaction,
|
||||
control: { director: async () => { throw new ConflictException("channel is not assigned to a media worker"); } },
|
||||
channels: () => [config()],
|
||||
});
|
||||
await expect(failing.applyProgramSwitch(WS, CH, apply())).resolves.toEqual({ kind: "conflict" });
|
||||
const exploding = new DirectorService({
|
||||
transaction: new ChannelDatabase().transaction,
|
||||
control: { director: async () => { throw new Error("redis down"); } },
|
||||
channels: () => [config()],
|
||||
});
|
||||
await expect(exploding.applyProgramSwitch(WS, CH, apply())).rejects.toThrow("redis down");
|
||||
});
|
||||
|
||||
it("unwired desired-state port fails closed without touching media control", async () => {
|
||||
const svc = new DirectorService({ transaction: new ChannelDatabase().transaction, channels: () => [config()] });
|
||||
await expect(svc.applyProgramSwitch(WS, CH, apply())).resolves.toEqual({ kind: "conflict" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("config parser (API side)", () => {
|
||||
it("parses a full channel document and rejects malformed entries", () => {
|
||||
const rule = { id: randomUUID(), priority: 2, condition: { field: "live_source_count", op: ">=", value: 1 }, action: { layout: "grid", selection: "all_live" } };
|
||||
const parsed = parseDirectorChannels(JSON.stringify([{ workspaceId: WS, channelId: CH, generation: "g-1", rules: [rule], policy: { name: "grid", liveConfirmations: 1 }, fallbackSceneId: null }]));
|
||||
expect(parsed).toHaveLength(1);
|
||||
expect(parsed[0]).toMatchObject({ workspaceId: WS, channelId: CH, generation: "g-1", fallbackSceneId: null });
|
||||
expect(parsed[0]?.rules[0]).toMatchObject({ priority: 2 });
|
||||
expect(parsed[0]?.policy?.liveConfirmations).toBe(1);
|
||||
expect(parseDirectorChannels(undefined)).toEqual([]);
|
||||
expect(() => parseDirectorChannels("{}")).toThrow();
|
||||
expect(() => parseDirectorChannels(JSON.stringify([{ workspaceId: WS }]))).toThrow();
|
||||
expect(() => parseDirectorChannels(JSON.stringify([{ workspaceId: WS, channelId: "nope", generation: "g" }]))).toThrow();
|
||||
expect(() => parseDirectorChannels(JSON.stringify([{ workspaceId: WS, channelId: CH, generation: "g", rules: [{ ...rule, action: { ...rule.action, selection: "explicit" } }] }]))).toThrow(/explicit requires sourceIds/);
|
||||
});
|
||||
|
||||
it("matches the scheduler-side parser on the same document (shape compatibility)", async () => {
|
||||
const { parseDirectorChannels: schedulerParse } = await import("../../../scheduler/src/director/config.js");
|
||||
const document = JSON.stringify([{
|
||||
workspaceId: WS, channelId: CH, generation: "shared-1",
|
||||
rules: [{ id: randomUUID(), priority: 1, condition: { field: "source_state", op: "==", value: "LIVE" }, action: { layout: "pip", selection: "explicit", sourceIds: [SOURCE_A] } }],
|
||||
policy: { name: "two_up", liveConfirmations: 3, offlineConfirmations: 4, noMediaSeconds: 30, minHoldMs: 5_000, switchCooldownMs: 15_000 },
|
||||
fallbackSceneId: SCENE_FALLBACK,
|
||||
}]);
|
||||
expect(schedulerParse(document)).toEqual(parseDirectorChannels(document));
|
||||
});
|
||||
});
|
||||
322
apps/api/src/director/director.service.ts
Normal file
322
apps/api/src/director/director.service.ts
Normal file
|
|
@ -0,0 +1,322 @@
|
|||
/**
|
||||
* Director wiring service (WP10 integration slice): the API-side half of the
|
||||
* engine's two ports.
|
||||
*
|
||||
* DirectorStatePort.read <- channelState(): joins a channel's sources with
|
||||
* the live monitor (virtual_channel_sources LEFT JOIN source_monitor_states
|
||||
* on the monitor's own keyed identity (workspace_id, connection_id,
|
||||
* remote_ref)) and renders the engine's input shape.
|
||||
* DesiredStatePort <- fence()/applyProgramSwitch(): fence from
|
||||
* virtual_channels.desired_version + workspaces.authz_epoch; apply via
|
||||
* MediaControlService.director() (the transactional outbox behind
|
||||
* setDesired), with pst-select + cut under idempotency keys derived from
|
||||
* the director's commandId, so a retried activity is a duplicate, never a
|
||||
* double switch.
|
||||
*
|
||||
* The scheduler's evaluator drives these over HTTP (internal director
|
||||
* endpoints); the service is process-agnostic so tests can call it directly.
|
||||
*/
|
||||
import { BadRequestException, ConflictException, Inject, Injectable, NotFoundException, Optional } from "@nestjs/common";
|
||||
import type { PoolClient } from "pg";
|
||||
import { DEFAULT_SWITCH_POLICY } from "@tactical-media-broadcast-hub/core";
|
||||
import { withTenant } from "../db/client.js";
|
||||
import { isUuid } from "../http/validation.js";
|
||||
import type { MediaControlService } from "../media-control/media-control.service.js";
|
||||
import type { DirectorActor } from "../production/journal.js";
|
||||
import { appendTenantAudit } from "../audit/audit.service.js";
|
||||
import { parseDirectorChannels, type DirectorChannelConfig } from "./config.js";
|
||||
|
||||
export type { DirectorChannelConfig } from "./config.js";
|
||||
|
||||
/**
|
||||
* Engine-facing source row (transport twin of the scheduler's
|
||||
* DirectorChannelSource). Optional fields are ABSENT, never null: the engine
|
||||
* reads `lastMediaAtMs !== undefined` as "known", so a null would read as
|
||||
* epoch 0 and de-confirm every source as no-media (caught by the port-pair
|
||||
* test). JSON drops undefined keys, keeping wire and engine forms identical.
|
||||
*/
|
||||
export interface DirectorSourceRow {
|
||||
id: string;
|
||||
priority: number;
|
||||
enabled: boolean;
|
||||
autoSelect: boolean;
|
||||
state:
|
||||
| "OFFLINE" | "UNKNOWN" | "PROBING" | "STARTING" | "LIVE" | "UNSTABLE" | "ENDING" | "ERROR";
|
||||
version?: number;
|
||||
liveSinceMs?: number;
|
||||
lastMediaAtMs?: number;
|
||||
mediaFailed?: boolean;
|
||||
}
|
||||
|
||||
export interface DirectorChannelSnapshot {
|
||||
kind: "current";
|
||||
desiredVersion: number;
|
||||
authzEpoch: number;
|
||||
generation: string;
|
||||
sources: DirectorSourceRow[];
|
||||
rules: DirectorChannelConfig["rules"];
|
||||
policy: NonNullable<DirectorChannelConfig["policy"]>;
|
||||
fallbackSceneId: string | null;
|
||||
override: null;
|
||||
principalPermissions: Record<string, string[]>;
|
||||
}
|
||||
|
||||
export type DirectorChannelRead = DirectorChannelSnapshot | { kind: "superseded"; code: "SUPERSEDED" };
|
||||
|
||||
export interface DirectorApplyBody {
|
||||
commandId: string;
|
||||
expectedDesiredVersion: number;
|
||||
authzEpoch: number;
|
||||
program: { kind: "sources"; sourceIds: string[] } | { kind: "fallback"; sceneId: string | null };
|
||||
layout: string;
|
||||
reasons: string[];
|
||||
releaseOverride?: string;
|
||||
}
|
||||
|
||||
export type DirectorApplyOutcome =
|
||||
| { kind: "applied" | "duplicate"; desiredVersion: number }
|
||||
| { kind: "conflict" };
|
||||
|
||||
export interface DirectorFence {
|
||||
generation: string;
|
||||
desiredVersion: number;
|
||||
authzEpoch: number;
|
||||
directorAuthorized: boolean;
|
||||
}
|
||||
|
||||
export interface DirectorServiceDependencies {
|
||||
transaction: typeof withTenant;
|
||||
control: Pick<MediaControlService, "director">;
|
||||
audit?: typeof appendTenantAudit;
|
||||
/** Channel configuration; default reads DIRECTOR_CHANNELS once at construction. */
|
||||
channels?: () => DirectorChannelConfig[];
|
||||
}
|
||||
|
||||
export const DIRECTOR_DEPENDENCIES = Symbol("DIRECTOR_DEPENDENCIES");
|
||||
|
||||
/** Row-level SQL for one channel's sources joined to their monitor state. */
|
||||
const SOURCE_QUERY = `
|
||||
SELECT s.id, s.priority,
|
||||
m.state AS monitor_state, m.version AS monitor_version,
|
||||
m.state_entered_at,
|
||||
m.tier_status->'media-health'->>'lastMediaAt' AS last_media_at,
|
||||
m.tier_status->'media-health'->>'lastOutcome' AS media_last_outcome
|
||||
FROM virtual_channel_sources s
|
||||
LEFT JOIN source_monitor_states m
|
||||
ON m.workspace_id = s.workspace_id
|
||||
AND m.connection_id = s.connection_id
|
||||
AND m.remote_ref = s.remote_ref
|
||||
WHERE s.workspace_id = $1 AND s.virtual_channel_id = $2
|
||||
ORDER BY s.priority DESC, s.id`;
|
||||
|
||||
/** Monitor states the monitor schema admits (012); anything else reads UNKNOWN. */
|
||||
const MONITOR_STATES = ["OFFLINE", "PROBING", "LIVE", "UNSTABLE", "ERROR"] as const;
|
||||
|
||||
/** The engine's source state from a monitor row (never-mapped => UNKNOWN). */
|
||||
export function engineState(monitorState: string | null): DirectorSourceRow["state"] {
|
||||
return (MONITOR_STATES as readonly string[]).includes(monitorState ?? "")
|
||||
? monitorState as DirectorSourceRow["state"]
|
||||
: "UNKNOWN";
|
||||
}
|
||||
|
||||
function epochMs(value: unknown): number | null {
|
||||
if (value === null || value === undefined) return null;
|
||||
const parsed = value instanceof Date ? value.getTime() : Date.parse(String(value));
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
/** The evaluator acts as a service principal; its writes are attributed to the director itself. */
|
||||
export const DIRECTOR_ACTOR: DirectorActor = { userId: null, role: "director" };
|
||||
|
||||
@Injectable()
|
||||
export class DirectorService {
|
||||
private readonly deps: DirectorServiceDependencies;
|
||||
private readonly envChannels: DirectorChannelConfig[];
|
||||
|
||||
constructor(@Optional() @Inject(DIRECTOR_DEPENDENCIES) dependencies?: Partial<DirectorServiceDependencies>) {
|
||||
this.deps = {
|
||||
transaction: dependencies?.transaction ?? withTenant,
|
||||
control: dependencies?.control ?? {
|
||||
// No injected media-control service: this process cannot apply desired
|
||||
// state. Reads still work (the evaluator fences on them and audits
|
||||
// SUPERSEDED), but applyProgramSwitch fails closed.
|
||||
director: () => { throw new ConflictException("director desired-state port is not wired"); },
|
||||
},
|
||||
...(dependencies?.audit === undefined ? {} : { audit: dependencies.audit }),
|
||||
...(dependencies?.channels === undefined ? {} : { channels: dependencies.channels }),
|
||||
};
|
||||
this.envChannels = parseDirectorChannels(process.env.DIRECTOR_CHANNELS);
|
||||
}
|
||||
|
||||
/** Configured channel for a ref; undefined when the director is not configured for it. */
|
||||
channel(workspaceId: string, channelId: string): DirectorChannelConfig | undefined {
|
||||
const configured = this.deps.channels?.() ?? this.envChannels;
|
||||
return configured.find((c) => c.workspaceId === workspaceId && c.channelId === channelId);
|
||||
}
|
||||
|
||||
/** DirectorStatePort.read: the engine's full input for one channel. */
|
||||
async channelState(workspaceId: string, channelId: string): Promise<DirectorChannelRead> {
|
||||
assertRef(workspaceId, channelId);
|
||||
const channel = this.channel(workspaceId, channelId);
|
||||
if (channel === undefined) return { kind: "superseded", code: "SUPERSEDED" };
|
||||
return this.deps.transaction(workspaceId, async (client) => {
|
||||
const fence = await this.readFence(client, workspaceId, channelId, channel.generation);
|
||||
if (fence.generation !== channel.generation) return { kind: "superseded" as const, code: "SUPERSEDED" as const };
|
||||
const sources = await client.query(SOURCE_QUERY, [workspaceId, channelId]);
|
||||
return {
|
||||
kind: "current" as const,
|
||||
desiredVersion: fence.desiredVersion,
|
||||
authzEpoch: fence.authzEpoch,
|
||||
generation: fence.generation,
|
||||
sources: sources.rows.map((row: Record<string, unknown>) => this.sourceRow(row)),
|
||||
rules: channel.rules,
|
||||
policy: channel.policy ?? DEFAULT_SWITCH_POLICY,
|
||||
fallbackSceneId: channel.fallbackSceneId,
|
||||
override: null,
|
||||
principalPermissions: {},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
private sourceRow(row: Record<string, unknown>): DirectorSourceRow {
|
||||
const tierStatus = row.tier_status as Record<string, { lastOutcome?: string; lastMediaAt?: string }> | null | undefined;
|
||||
const mediaHealth = tierStatus?.["media-health"];
|
||||
const version = row.monitor_version === null || row.monitor_version === undefined ? undefined : Number(row.monitor_version);
|
||||
const liveSinceMs = epochMs(row.state_entered_at);
|
||||
const lastMediaAtMs = epochMs(mediaHealth?.lastMediaAt);
|
||||
const mediaFailed = mediaHealth?.lastOutcome === "fail" || mediaHealth?.lastOutcome === "error";
|
||||
return {
|
||||
id: String(row.id),
|
||||
priority: Number(row.priority ?? 0),
|
||||
enabled: true,
|
||||
autoSelect: true,
|
||||
state: engineState(row.monitor_state as string | null),
|
||||
...(version === undefined ? {} : { version }),
|
||||
...(liveSinceMs === null ? {} : { liveSinceMs }),
|
||||
...(lastMediaAtMs === null ? {} : { lastMediaAtMs }),
|
||||
...(mediaFailed ? { mediaFailed: true } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** DesiredStatePort.fence. */
|
||||
async fence(workspaceId: string, channelId: string): Promise<DirectorFence | undefined> {
|
||||
assertRef(workspaceId, channelId);
|
||||
const channel = this.channel(workspaceId, channelId);
|
||||
if (channel === undefined) return undefined;
|
||||
return this.deps.transaction(workspaceId, (client) => this.readFence(client, workspaceId, channelId, channel.generation));
|
||||
}
|
||||
|
||||
private async readFence(client: PoolClient, workspaceId: string, channelId: string, generation: string): Promise<DirectorFence> {
|
||||
const found = await client.query<{ desired_version: string | number }>(
|
||||
"SELECT desired_version FROM virtual_channels WHERE workspace_id = $1 AND id = $2", [workspaceId, channelId]);
|
||||
if (found.rowCount === 0) throw new NotFoundException(`virtual channel ${channelId} not found`);
|
||||
const desiredVersion = Number(found.rows[0]?.desired_version);
|
||||
if (!Number.isSafeInteger(desiredVersion) || desiredVersion < 0) throw new Error("desired_version is not a safe integer");
|
||||
const epoch = await client.query<{ authz_epoch: string | number }>("SELECT authz_epoch FROM workspaces WHERE id = $1", [workspaceId]);
|
||||
const authzEpoch = Number(epoch.rows[0]?.authz_epoch);
|
||||
if (!Number.isSafeInteger(authzEpoch) || authzEpoch < 0) throw new Error("authz_epoch is not a safe integer");
|
||||
return {
|
||||
generation,
|
||||
desiredVersion,
|
||||
authzEpoch,
|
||||
// Principal revocation is a later slice (it needs a persisted principal
|
||||
// per evaluator); until then the fence is generation + desiredVersion.
|
||||
directorAuthorized: true,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* DesiredStatePort.apply: a director decision as a desired-state write.
|
||||
*
|
||||
* Resolves the decision's program to a scene, then pst-select (preview) +
|
||||
* cut (preview -> program) through MediaControlService.director(), whose
|
||||
* receipt lookup makes both steps idempotent under keys derived from the
|
||||
* director's commandId: an activity retry replays the committed receipt
|
||||
* instead of switching twice.
|
||||
*/
|
||||
async applyProgramSwitch(workspaceId: string, channelId: string, body: DirectorApplyBody): Promise<DirectorApplyOutcome> {
|
||||
assertRef(workspaceId, channelId);
|
||||
if (typeof body.commandId !== "string" || body.commandId === "" || body.commandId.length > 128) {
|
||||
throw new BadRequestException("commandId must be 1-128 characters");
|
||||
}
|
||||
const channel = this.channel(workspaceId, channelId);
|
||||
if (channel === undefined) throw new ConflictException("director is not configured for this channel");
|
||||
const fence = await this.fence(workspaceId, channelId);
|
||||
if (fence === undefined) return { kind: "conflict" };
|
||||
if (fence.desiredVersion !== body.expectedDesiredVersion || fence.authzEpoch !== body.authzEpoch) return { kind: "conflict" };
|
||||
|
||||
try {
|
||||
// Resolve the decision's program to a scene the channel owns.
|
||||
const scene = await this.deps.transaction(workspaceId, (client) =>
|
||||
this.sceneForProgram(client, workspaceId, channelId, body.program, channel));
|
||||
// pst-select arms the scene on preview; cut moves preview to program.
|
||||
// Two commands, one decision: each has its own idempotency key derived
|
||||
// from the director's commandId, so replays stop at the stored receipt.
|
||||
const selected = await this.deps.control.director(workspaceId, channelId,
|
||||
{ type: "pst-select", sceneId: scene.sceneId }, `${body.commandId}:pst`, DIRECTOR_ACTOR);
|
||||
await this.deps.control.director(workspaceId, channelId,
|
||||
{ type: "cut" }, `${body.commandId}:cut`, DIRECTOR_ACTOR);
|
||||
return { kind: "applied", desiredVersion: selected.desiredVersion };
|
||||
} catch (error) {
|
||||
return classifyConflict(error);
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolve the decision's program to a scene id: an explicit scene, or a scene holding exactly those sources. */
|
||||
private async sceneForProgram(
|
||||
client: PoolClient, workspaceId: string, channelId: string,
|
||||
program: DirectorApplyBody["program"], channel: DirectorChannelConfig,
|
||||
): Promise<{ sceneId: string }> {
|
||||
if (program.kind === "fallback") {
|
||||
const sceneId = program.sceneId ?? channel.fallbackSceneId;
|
||||
if (sceneId === null) throw new ConflictException("channel has no fallback scene configured");
|
||||
return { sceneId };
|
||||
}
|
||||
if (program.sourceIds.length === 0) throw new ConflictException("program switch selected no sources");
|
||||
// A scene whose bound slots are exactly the selected sources (order and
|
||||
// multiplicity included): that scene is the director's program scene.
|
||||
const placeholders = program.sourceIds.map((_, i) => `$${i + 3}`).join(",");
|
||||
const found = await client.query<{ scene_id: string }>(
|
||||
`SELECT sc.id AS scene_id
|
||||
FROM production_scenes sc
|
||||
LEFT JOIN scene_slots sl
|
||||
ON sl.scene_id = sc.id AND sl.workspace_id = sc.workspace_id AND sl.virtual_channel_id = sc.virtual_channel_id
|
||||
WHERE sc.workspace_id = $1 AND sc.virtual_channel_id = $2
|
||||
GROUP BY sc.id
|
||||
HAVING array_length(array_agg(sl.source_id ORDER BY sl.slot_index) FILTER (WHERE sl.source_id IS NOT NULL), 1) = $${program.sourceIds.length + 3}
|
||||
AND array_agg(sl.source_id ORDER BY sl.slot_index) FILTER (WHERE sl.source_id IS NOT NULL) @> ARRAY[${placeholders}]::uuid[]`,
|
||||
[workspaceId, channelId, ...program.sourceIds, program.sourceIds.length]);
|
||||
if (found.rowCount === 0) {
|
||||
// No scene authored for this selection: switch to the channel's declared
|
||||
// fallback so the channel never stalls on an unrenderable program.
|
||||
if (channel.fallbackSceneId === null) throw new ConflictException("no scene holds the selected sources and no fallback scene is configured");
|
||||
return { sceneId: channel.fallbackSceneId };
|
||||
}
|
||||
return { sceneId: String(found.rows[0]?.scene_id) };
|
||||
}
|
||||
|
||||
/** Outcomes the evaluator audits when it cannot write (generation/authz fences). */
|
||||
async audit(workspaceId: string, channelId: string, event: { commandId: string; outcome: "SUPERSEDED" | "AUTHZ_REVOKED"; reason: string }): Promise<void> {
|
||||
assertRef(workspaceId, channelId);
|
||||
const append = this.deps.audit ?? appendTenantAudit;
|
||||
await append(workspaceId, {
|
||||
actorUserId: null,
|
||||
actorRole: "director",
|
||||
action: `director.${event.outcome.toLowerCase()}`,
|
||||
targetType: "production_command",
|
||||
targetId: channelId,
|
||||
diff: { commandId: event.commandId, outcome: event.outcome, reason: event.reason },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function assertRef(workspaceId: string, channelId: string): void {
|
||||
if (!isUuid(workspaceId)) throw new BadRequestException("workspaceId must be a UUID");
|
||||
if (!isUuid(channelId)) throw new BadRequestException("channelId must be a UUID");
|
||||
}
|
||||
|
||||
/** Media-control conflicts are CAS outcomes the evaluator re-reads on; anything else propagates. */
|
||||
function classifyConflict(error: unknown): DirectorApplyOutcome {
|
||||
if (error instanceof ConflictException || error instanceof NotFoundException) return { kind: "conflict" };
|
||||
throw error;
|
||||
}
|
||||
172
apps/api/src/director/port-pair.test.ts
Normal file
172
apps/api/src/director/port-pair.test.ts
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
import { generateKeyPairSync, randomUUID } from "node:crypto";
|
||||
import type { PoolClient } from "pg";
|
||||
import { Controller, Module, type INestApplication } from "@nestjs/common";
|
||||
import { NestFactory } from "@nestjs/core";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { DirectorController } from "./director.controller.js";
|
||||
import { DirectorService } from "./director.service.js";
|
||||
import { MediaControlService } from "../media-control/media-control.service.js";
|
||||
import { MemoryRedis } from "../production/testing/memory-redis.js";
|
||||
import { Ed25519ControlSigner } from "../media-control/wire.js";
|
||||
import type { TenantTransaction } from "../production/journal.js";
|
||||
import { decide } from "../../../scheduler/src/director/engine.js";
|
||||
import { HttpDirectorStatePort, HttpDesiredStatePort } from "../../../scheduler/src/director/adapters.js";
|
||||
import { DirectorHttpClient } from "../../../scheduler/src/director/http.client.js";
|
||||
|
||||
/**
|
||||
* Port-pair integration (WP10): the scheduler's HTTP adapters against a real
|
||||
* Nest application hosting the API's director endpoints, with media control
|
||||
* running for real on its MemoryRedis fixture. This is the exact chain the
|
||||
* production wiring uses — internal endpoints, response validation, fence,
|
||||
* engine input shape and the desired-state write — minus the database.
|
||||
*
|
||||
* The scheduler-side pieces are imported from apps/scheduler (pure modules
|
||||
* only: engine, adapters, http.client), the same cross-package pattern
|
||||
* live-monitor's poller tests use.
|
||||
*/
|
||||
const WS = "3f2b8e40-1a5c-4c7e-9d10-6f8d1a000001";
|
||||
const CH = "3f2b8e40-1a5c-4c7e-9d10-6f8d1a000002";
|
||||
const SRC_A = "3f2b8e40-1a5c-4c7e-9d10-6f8d1a000003";
|
||||
const SRC_B = "3f2b8e40-1a5c-4c7e-9d10-6f8d1a000004";
|
||||
const SCENE_SOLO = "3f2b8e40-1a5c-4c7e-9d10-6f8d1a000005";
|
||||
const SCENE_FALLBACK = "3f2b8e40-1a5c-4c7e-9d10-6f8d1a000006";
|
||||
|
||||
/** Director-side database double: fence + source join + scene resolution. */
|
||||
class DirectorDb {
|
||||
desiredVersion = 4;
|
||||
authzEpoch = 7;
|
||||
readonly sources: Array<Record<string, unknown>> = [];
|
||||
readonly scenes: Array<{ sceneId: string; sources: string[] }> = [];
|
||||
readonly transaction: TenantTransaction = async <T>(_ws: string, fn: (client: PoolClient) => Promise<T>): Promise<T> =>
|
||||
fn({ query: async (sql: string, values: unknown[]): Promise<{ rows: Record<string, unknown>[]; rowCount: number }> => {
|
||||
if (sql.includes("FROM virtual_channel_sources")) return { rows: this.sources, rowCount: this.sources.length };
|
||||
if (sql.includes("SELECT desired_version FROM virtual_channels")) return { rows: [{ desired_version: String(this.desiredVersion) }], rowCount: 1 };
|
||||
if (sql.includes("SELECT authz_epoch FROM workspaces")) return { rows: [{ authz_epoch: String(this.authzEpoch) }], rowCount: 1 };
|
||||
if (sql.includes("FROM production_scenes")) {
|
||||
const selected = values.slice(2, -1).map(String);
|
||||
const match = this.scenes.find((scene) => scene.sources.length === selected.length && [...scene.sources].sort().join() === [...selected].sort().join());
|
||||
return { rows: match === undefined ? [] : [{ scene_id: match.sceneId }], rowCount: match === undefined ? 0 : 1 };
|
||||
}
|
||||
throw new Error(`unexpected query: ${sql.slice(0, 60)}`);
|
||||
} } as unknown as PoolClient);
|
||||
}
|
||||
|
||||
/** media-control transaction double (control-s3 pattern). */
|
||||
class BusDb {
|
||||
state = { version: 0, preview: null as string | null, program: null as string | null, receipts: [] as unknown[] };
|
||||
readonly scenes = new Set<string>([SCENE_SOLO, SCENE_FALLBACK]);
|
||||
readonly transaction: TenantTransaction = async <T>(_ws: string, fn: (client: PoolClient) => Promise<T>): Promise<T> =>
|
||||
fn({ query: async (sql: string, values: unknown[] = []): Promise<{ rows: unknown[]; rowCount: number }> => {
|
||||
let rows: unknown[] = [];
|
||||
if (sql.startsWith("SELECT id FROM virtual_channels")) rows = [{ id: CH }];
|
||||
else if (sql.startsWith("SELECT desired_version")) rows = [{ desired_version: String(this.state.version), desired_status: "STOPPED" }];
|
||||
else if (sql.includes("FROM audit_log")) {
|
||||
const found = sql.includes("requestId") ? this.state.receipts.find(r => (r as { requestId?: string }).requestId === values[2]) : this.state.receipts.at(-1);
|
||||
rows = found ? [{ diff: found }] : [];
|
||||
} else if (sql.startsWith("SELECT scene_id")) rows = [{ scene_id: this.state.preview }];
|
||||
else if (sql.startsWith("INSERT INTO production_buses")) { if (sql.includes("'program'")) this.state.program = String(values[2]); else this.state.preview = String(values[2]); }
|
||||
else if (sql.startsWith("UPDATE virtual_channels")) this.state.version = Number(values[2]);
|
||||
else if (sql.startsWith("SELECT audit_append")) { this.state.receipts.push(values[6]); return { rows: [], rowCount: 1 }; }
|
||||
else if (sql.startsWith("SELECT id,label,layout")) rows = [...this.scenes].map((id) => ({ id, label: "Scene", layout: "single" }));
|
||||
else if (sql.startsWith("SELECT bus,scene_id")) rows = [{ bus: "preview", scene_id: this.state.preview }, { bus: "program", scene_id: this.state.program }];
|
||||
else if (sql.startsWith("SELECT id FROM production_scenes")) rows = this.scenes.has(String(values[2])) ? [{ id: values[2] }] : [];
|
||||
else if (sql.includes("FROM scene_slots") || sql.includes("FROM virtual_channel_sources") || sql.includes("FROM overlay_slots")) rows = [];
|
||||
return { rows, rowCount: sql.startsWith("UPDATE") ? 1 : rows.length };
|
||||
} } as unknown as PoolClient);
|
||||
}
|
||||
|
||||
const RULES = [{
|
||||
id: randomUUID(), virtualChannelId: CH, priority: 1,
|
||||
condition: { field: "live_source_count" as const, op: ">=" as const, value: 1 },
|
||||
action: { layout: "single" as const, selection: "top_priority" as const },
|
||||
}];
|
||||
const POLICY = { name: "highest_priority" as const, liveConfirmations: 1, offlineConfirmations: 1, noMediaSeconds: 600, minHoldMs: 0, switchCooldownMs: 0 };
|
||||
|
||||
/**
|
||||
* The production DirectorController, inherited verbatim: routes and handler
|
||||
* bodies are the shipped ones (Nest reads the decorators off the prototype);
|
||||
* only the service instance comes from the module below.
|
||||
*/
|
||||
@Controller("internal/director/channels")
|
||||
class BoundDirectorController extends DirectorController {}
|
||||
|
||||
async function apiStack(db: DirectorDb, bus: BusDb): Promise<INestApplication> {
|
||||
const control = new MediaControlService({
|
||||
transaction: bus.transaction, redis: new MemoryRedis(),
|
||||
signer: new Ed25519ControlSigner("test", generateKeyPairSync("ed25519").privateKey), assignments: [],
|
||||
});
|
||||
await control.assign(WS, CH, "media-worker-a", 1);
|
||||
const service = new DirectorService({
|
||||
transaction: db.transaction, control, audit: async () => randomUUID(),
|
||||
channels: () => [{ workspaceId: WS, channelId: CH, generation: "g1", rules: RULES, policy: POLICY, fallbackSceneId: SCENE_FALLBACK }],
|
||||
});
|
||||
|
||||
@Module({ controllers: [BoundDirectorController], providers: [{ provide: DirectorService, useValue: service }] })
|
||||
class PairModule {}
|
||||
const application = await NestFactory.create(PairModule, { logger: false });
|
||||
await application.init();
|
||||
return application;
|
||||
}
|
||||
|
||||
describe("director port pair over HTTP (scheduler adapters <-> API endpoints)", () => {
|
||||
it("state read -> decide -> fence -> apply, end to end", async () => {
|
||||
const db = new DirectorDb();
|
||||
const bus = new BusDb();
|
||||
db.scenes.push({ sceneId: SCENE_SOLO, sources: [SRC_A] });
|
||||
db.sources.push({ id: SRC_A, priority: 50, monitor_state: "LIVE", monitor_version: 3, state_entered_at: new Date("2026-10-09T12:00:00Z"), tier_status: null });
|
||||
db.sources.push({ id: SRC_B, priority: 10, monitor_state: null, monitor_version: null, state_entered_at: null, tier_status: null });
|
||||
const application = await apiStack(db, bus);
|
||||
try {
|
||||
await application.listen(0, "127.0.0.1");
|
||||
const port = (application.getHttpServer().address() as { port: number }).port;
|
||||
const client = new DirectorHttpClient({ baseUrl: `http://127.0.0.1:${port}` });
|
||||
const state = new HttpDirectorStatePort(client);
|
||||
const desired = new HttpDesiredStatePort(client);
|
||||
const ref = { workspaceId: WS, channelId: CH, generation: "g1" };
|
||||
|
||||
// 1. read -> decide
|
||||
const snapshot = await state.read(ref);
|
||||
if (snapshot.kind !== "current") throw new Error("expected current");
|
||||
expect(snapshot.sources).toHaveLength(2);
|
||||
expect(snapshot.sources[0]).toMatchObject({ id: SRC_A, state: "LIVE", version: 3 });
|
||||
const decision = decide({ sources: snapshot.sources, rules: snapshot.rules, policy: snapshot.policy, now: 100_000 });
|
||||
expect(decision.program).toEqual({ kind: "sources", sourceIds: [SRC_A] });
|
||||
|
||||
// 2. fence matches the snapshot the decision was based on
|
||||
const fence = await desired.fence(ref);
|
||||
expect(fence).toMatchObject({ generation: "g1", desiredVersion: snapshot.desiredVersion, authzEpoch: snapshot.authzEpoch });
|
||||
|
||||
// 3. apply lands as preview + program through media control
|
||||
const outcome = await desired.writeProgram({
|
||||
ref, commandId: `${CH}:1`, expectedDesiredVersion: fence!.desiredVersion, authzEpoch: fence!.authzEpoch,
|
||||
program: decision.program, layout: decision.layout, reasons: decision.reasons,
|
||||
});
|
||||
expect(outcome).toEqual({ kind: "applied", desiredVersion: 1 });
|
||||
expect(bus.state.preview).toBe(SCENE_SOLO);
|
||||
expect(bus.state.program).toBe(SCENE_SOLO);
|
||||
} finally {
|
||||
await application.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("a stale fence is a conflict, never a write", async () => {
|
||||
const db = new DirectorDb();
|
||||
const bus = new BusDb();
|
||||
db.scenes.push({ sceneId: SCENE_SOLO, sources: [SRC_A] });
|
||||
const application = await apiStack(db, bus);
|
||||
try {
|
||||
await application.listen(0, "127.0.0.1");
|
||||
const port = (application.getHttpServer().address() as { port: number }).port;
|
||||
const client = new DirectorHttpClient({ baseUrl: `http://127.0.0.1:${port}` });
|
||||
db.desiredVersion = 99; // a concurrent writer moved the fence
|
||||
const outcome = await new HttpDesiredStatePort(client).writeProgram({
|
||||
ref: { workspaceId: WS, channelId: CH, generation: "g1" }, commandId: "c", expectedDesiredVersion: 4, authzEpoch: 7,
|
||||
program: { kind: "sources", sourceIds: [SRC_A] }, layout: "single", reasons: [],
|
||||
});
|
||||
expect(outcome).toEqual({ kind: "conflict" });
|
||||
expect(bus.state.receipts).toHaveLength(0);
|
||||
} finally {
|
||||
await application.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
352
apps/api/src/feeds/column-scheduler.ts
Normal file
352
apps/api/src/feeds/column-scheduler.ts
Normal file
|
|
@ -0,0 +1,352 @@
|
|||
/**
|
||||
* Columns scheduler (P1 — design §3.3/§3.4/§3.5).
|
||||
*
|
||||
* Generalizes the legacy feeds poll loop into a due-SOURCES loop. The design
|
||||
* rule (§3.5): "the legacy home poll is the degenerate platform_feed source" —
|
||||
* so there is one loop, and `/feeds`'s per-connection home poll is simply the
|
||||
* `source_kind = 'home'` subset of it.
|
||||
*
|
||||
* Reused unchanged from the feeds loop:
|
||||
* - the per-key Redis lease (renewable, released in `finally`) so replicas
|
||||
* never double-poll one source;
|
||||
* - the cooldown key (`SET NX PX`) that makes repeated refresh clicks and
|
||||
* the 5-second discovery timer unable to bypass the interval;
|
||||
* - interval + 20 % jitter, SDK `RateLimitError.retryAfterMs` override.
|
||||
*
|
||||
* New per design §3.4:
|
||||
* - `column_sources` is the due list: `next_poll_at <= now`, `state <>
|
||||
* 'disabled'`, `refresh_policy.paused` false — the DB row replaces the
|
||||
* in-memory "one cycle per connection" bookkeeping;
|
||||
* - identities shared by several columns collapse to ONE poll (the fan-out
|
||||
* in `ColumnStore.storeItems` writes the item rows for every sharing
|
||||
* column), so following the same account in two columns costs one call;
|
||||
* - `consecutive_failures` drives exponential backoff (the same
|
||||
* monitor-poller curve: 2^n capped at 120 s), and the outcome is written
|
||||
* back to the source row (`state`, `last_error`, `next_poll_at`) — the
|
||||
* persisted truth the UI surfaces as `ColumnView.source.state`;
|
||||
* - per-host tokens come from `@tactical-media-broadcast-hub/egress` HostBudgets (shared
|
||||
* `host_budgets` rows — one bucket per host across every caller and host
|
||||
* class, deployment-wide), so feeds and monitor fetches to the same host
|
||||
* contend for the same tokens rather than each rate-limiting the host.
|
||||
*
|
||||
* P1 scope note: only the `home` source kind is executed against a connector
|
||||
* here (`getFeed(kind:'home')` — today's behavior); `mentions`/`search`/
|
||||
* `author` rows are planned with the P2 connector work, so they park in
|
||||
* `unsupported` rather than silently skipping. Item normalization runs through
|
||||
* the same P0c `normalizeFeedItem` funnel the legacy loop uses.
|
||||
*/
|
||||
import { Inject, Injectable, Logger, Optional, type OnApplicationBootstrap, type OnApplicationShutdown } from "@nestjs/common";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { ConnectorConnection, FeedItem } from "@tactical-media-broadcast-hub/core";
|
||||
import { RateLimitError, type ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import { DEFAULT_INTERVAL_MS } from "@tactical-media-broadcast-hub/core";
|
||||
import { pool, withTenant } from "../db/client.js";
|
||||
import { ColumnStore } from "./column-store.js";
|
||||
import { normalizeFeedItem, type NormalizedFeedItem } from "./feed-normalize.js";
|
||||
import type { ControlRedis } from "../media-control/redis.js";
|
||||
import type { ConnectionSummary } from "../connections/connections.service.js";
|
||||
|
||||
export const COLUMN_SCHEDULER_DEPENDENCIES = Symbol("COLUMN_SCHEDULER_DEPENDENCIES");
|
||||
|
||||
export interface ColumnSchedulerDependencies {
|
||||
redis: Pick<ControlRedis, "get" | "set" | "eval">;
|
||||
registry: ConnectorRegistry;
|
||||
connection(workspaceId: string, id: string): Promise<ConnectorConnection>;
|
||||
/** Legacy /feeds cache write-through, so /feeds reads stay cache-first. */
|
||||
cacheHomeFeed(workspaceId: string, connectionId: string, items: NormalizedFeedItem[]): Promise<void>;
|
||||
/** Effective host key for a provider's API host (egress hostBudgetKey). */
|
||||
hostKeyOf(provider: string, connection: ConnectorConnection): string;
|
||||
/** Draw one token from the shared per-host budget. */
|
||||
consumeBudget(hostKey: string): Promise<void>;
|
||||
random(): number;
|
||||
now(): number;
|
||||
}
|
||||
|
||||
/** Poll admission order per §3.4: due, enabled, not paused, oldest first. */
|
||||
interface DueRow {
|
||||
workspace_id: string;
|
||||
id: string;
|
||||
provider: string;
|
||||
connection_id: string | null;
|
||||
source_kind: "home" | "mentions" | "search" | "author";
|
||||
query: string | null;
|
||||
actor_ref: string | null;
|
||||
interval_ms: number;
|
||||
paused: boolean;
|
||||
consecutive_failures: number;
|
||||
}
|
||||
|
||||
const LEASE_MS = 30_000;
|
||||
const BACKOFF_CAP_MS = 120_000;
|
||||
const RENEW_LEASE = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('PEXPIRE', KEYS[1], ARGV[2]) else return 0 end";
|
||||
const RELEASE_LEASE = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) else return 0 end";
|
||||
|
||||
/** Minimum spacing between two forced refreshes of one source (cache-loss safe). */
|
||||
export const FORCE_FLOOR_MS = 15_000;
|
||||
|
||||
const DEFAULT_INTERVALS: Record<DueRow["source_kind"], number> = {
|
||||
home: DEFAULT_INTERVAL_MS.platform_feed,
|
||||
mentions: DEFAULT_INTERVAL_MS.platform_feed,
|
||||
search: DEFAULT_INTERVAL_MS.keyword,
|
||||
author: DEFAULT_INTERVAL_MS.account_list,
|
||||
};
|
||||
|
||||
/** Poll cadence: spec refresh_policy interval, else the §1.2 per-kind default. */
|
||||
export function effectiveIntervalMs(intervalMs: number | null, sourceKind: DueRow["source_kind"]): number {
|
||||
return intervalMs ?? DEFAULT_INTERVALS[sourceKind];
|
||||
}
|
||||
|
||||
/** The monitor-poller backoff curve: 2^n * interval scale, capped at 120 s. */
|
||||
export function backoffDelayMs(failures: number, intervalMs: number): number {
|
||||
return Math.min(BACKOFF_CAP_MS, intervalMs * 2 ** Math.min(7, Math.max(0, failures - 1)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Discovery cadence: sources self-schedule via next_poll_at, so the loop only
|
||||
* needs to sweep for newly due rows often enough to honor the 60 s floor.
|
||||
*/
|
||||
const SWEEP_MS = 5_000;
|
||||
|
||||
@Injectable()
|
||||
export class ColumnScheduler implements OnApplicationBootstrap, OnApplicationShutdown {
|
||||
private readonly logger = new Logger(ColumnScheduler.name);
|
||||
private running = false;
|
||||
private timer?: ReturnType<typeof setTimeout>;
|
||||
private active?: Promise<void>;
|
||||
constructor(
|
||||
private readonly store: ColumnStore,
|
||||
@Optional() @Inject(COLUMN_SCHEDULER_DEPENDENCIES) private readonly deps?: ColumnSchedulerDependencies,
|
||||
) {}
|
||||
|
||||
onApplicationBootstrap(): void {
|
||||
this.running = true;
|
||||
this.schedule(0);
|
||||
}
|
||||
|
||||
async onApplicationShutdown(): Promise<void> {
|
||||
this.running = false;
|
||||
if (this.timer) clearTimeout(this.timer);
|
||||
await this.active;
|
||||
}
|
||||
|
||||
private schedule(delay: number): void {
|
||||
if (!this.running) return;
|
||||
this.timer = setTimeout(() => {
|
||||
this.active = this.pollDue().catch(() => this.logger.warn("Column sweep unavailable")).then(() => undefined).finally(() => {
|
||||
this.active = undefined;
|
||||
this.schedule(SWEEP_MS + Math.round((this.deps?.random() ?? Math.random()) * 1_000));
|
||||
});
|
||||
}, delay);
|
||||
this.timer.unref();
|
||||
}
|
||||
|
||||
/** One pass over every workspace's due sources. Errors are per-source, never fatal. */
|
||||
async pollDue(): Promise<{ polled: number; skipped: number; failed: number }> {
|
||||
const workspaces = await this.workspaces();
|
||||
const totals = { polled: 0, skipped: 0, failed: 0 };
|
||||
for (const workspaceId of workspaces) {
|
||||
const pass = await this.pollWorkspaceDue(workspaceId);
|
||||
totals.polled += pass.polled;
|
||||
totals.skipped += pass.skipped;
|
||||
totals.failed += pass.failed;
|
||||
}
|
||||
return totals;
|
||||
}
|
||||
|
||||
/**
|
||||
* The due-source scan (§3.4). Read-only against the DB (withTenant, FORCE
|
||||
* RLS), then each source polls through its own lease/cooldown; the state
|
||||
* write-back happens in the same tenant context. Returns per-source counts
|
||||
* for the same shape the legacy loop reports.
|
||||
*/
|
||||
async pollWorkspaceDue(workspaceId: string): Promise<{ polled: number; skipped: number; failed: number }> {
|
||||
const due = await this.dueSources(workspaceId);
|
||||
const results = { polled: 0, skipped: 0, failed: 0 };
|
||||
for (const row of due) {
|
||||
const outcome = await this.pollSource(workspaceId, row);
|
||||
results[outcome] += 1;
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private async dueSources(workspaceId: string): Promise<DueRow[]> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const result = await client.query<DueRow>(
|
||||
`SELECT s.workspace_id, s.id, s.provider, s.connection_id, s.source_kind, s.query, s.actor_ref,
|
||||
(sc.refresh_policy->>'intervalMs')::int AS interval_ms,
|
||||
COALESCE((sc.refresh_policy->>'paused')::boolean, false) AS paused,
|
||||
s.consecutive_failures
|
||||
FROM column_sources s
|
||||
JOIN stream_columns sc ON sc.id = s.column_id
|
||||
WHERE s.workspace_id = $1
|
||||
AND sc.deleted_at IS NULL -- a soft-deleted column's sources retire with it
|
||||
AND s.state <> 'disabled'
|
||||
AND COALESCE((sc.refresh_policy->>'paused')::boolean, false) = false
|
||||
AND (s.next_poll_at IS NULL OR s.next_poll_at <= now())
|
||||
ORDER BY s.created_at`,
|
||||
[workspaceId],
|
||||
);
|
||||
return result.rows;
|
||||
});
|
||||
}
|
||||
|
||||
private async pollSource(workspaceId: string, row: DueRow): Promise<"polled" | "skipped" | "failed"> {
|
||||
if (this.deps === undefined) return "skipped";
|
||||
const interval = effectiveIntervalMs(row.interval_ms, row.source_kind);
|
||||
const cooldownMs = interval + Math.round(interval * 0.2 * this.deps.random());
|
||||
const key = `columns:${workspaceId}:${row.id}`;
|
||||
const leaseKey = `${key}:lease`;
|
||||
const cooldownKey = `${key}:cooldown`;
|
||||
const owner = randomUUID();
|
||||
if (!await this.deps.redis.set(leaseKey, owner, { nx: true, pxMs: LEASE_MS })) return "skipped";
|
||||
let heartbeat: ReturnType<typeof setInterval> | undefined;
|
||||
let renew: Promise<unknown> = Promise.resolve();
|
||||
let cooldown = cooldownMs;
|
||||
let started = false;
|
||||
try {
|
||||
if (!await this.deps.redis.set(cooldownKey, "idle", { nx: true, pxMs: cooldownMs })) return "skipped";
|
||||
started = true;
|
||||
heartbeat = setInterval(() => {
|
||||
renew = this.deps!.redis.eval(RENEW_LEASE, [leaseKey], [owner, String(LEASE_MS)]).catch(() => {
|
||||
this.logger.warn("Column poll lease renewal unavailable");
|
||||
});
|
||||
}, LEASE_MS / 3);
|
||||
heartbeat.unref();
|
||||
|
||||
const page = await this.fetch(workspaceId, row);
|
||||
const items = page.filter((item): item is NormalizedFeedItem => item !== null);
|
||||
await this.store.storeItems(workspaceId, row.id, items);
|
||||
await this.recordSuccess(workspaceId, row.id, interval);
|
||||
// The legacy /feeds surface stays cache-first for home sources: the
|
||||
// same items land in the hot cache the /feeds read path uses.
|
||||
if (row.source_kind === "home" && row.connection_id !== null) {
|
||||
await this.deps.cacheHomeFeed(workspaceId, row.connection_id, items);
|
||||
}
|
||||
return "polled";
|
||||
} catch (error) {
|
||||
const outcome = await this.recordFailure(workspaceId, row, error, interval);
|
||||
cooldown = outcome.cooldownMs;
|
||||
return "failed";
|
||||
} finally {
|
||||
if (heartbeat) clearInterval(heartbeat);
|
||||
await renew;
|
||||
try {
|
||||
if (started) await this.deps.redis.set(cooldownKey, "idle", { pxMs: cooldown });
|
||||
} finally {
|
||||
await this.deps.redis.eval(RELEASE_LEASE, [leaseKey], [owner]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Normalizes a page through the P0c funnel. P1 executes home only. */
|
||||
private async fetch(workspaceId: string, row: DueRow): Promise<Array<NormalizedFeedItem | null>> {
|
||||
if (this.deps === undefined) return [];
|
||||
if (row.source_kind !== "home") {
|
||||
throw new UnsupportedSourceError(row.source_kind);
|
||||
}
|
||||
if (row.connection_id === null) {
|
||||
// home is always connection-bound (a platform_feed column); a NULL
|
||||
// connection row is corrupt, not pollable.
|
||||
throw new UnsupportedSourceError("home-without-connection");
|
||||
}
|
||||
if (!this.deps.registry.has(row.provider)) throw new UnsupportedSourceError(`unknown-provider:${row.provider}`);
|
||||
const connector = this.deps.registry.get(row.provider);
|
||||
if (connector.getFeed === undefined) throw new UnsupportedSourceError(`feed-unsupported:${row.provider}`);
|
||||
const connection = await this.deps.connection(workspaceId, row.connection_id);
|
||||
const hostKey = this.deps.hostKeyOf(row.provider, connection);
|
||||
await this.deps.consumeBudget(hostKey);
|
||||
const page = await connector.getFeed(connection, { kind: "home", limit: 40 });
|
||||
return (page?.items ?? []).map((item: FeedItem) => normalizeFeedItem(item, connection));
|
||||
}
|
||||
|
||||
private async recordSuccess(workspaceId: string, sourceId: string, intervalMs: number): Promise<void> {
|
||||
await withTenant(workspaceId, async (client) => {
|
||||
await client.query(
|
||||
`UPDATE column_sources
|
||||
SET state = 'ready', last_error = NULL, last_polled_at = now(), consecutive_failures = 0,
|
||||
next_poll_at = now() + ($3::int * interval '1 millisecond'), updated_at = now()
|
||||
WHERE workspace_id = $1 AND id = $2`,
|
||||
[workspaceId, sourceId, intervalMs],
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
private async recordFailure(workspaceId: string, row: DueRow, error: unknown, intervalMs: number): Promise<{ cooldownMs: number }> {
|
||||
const rateLimited = error instanceof RateLimitError;
|
||||
const unsupported = error instanceof UnsupportedSourceError;
|
||||
const retryAfter = error instanceof RateLimitError ? error.retryAfterMs : undefined;
|
||||
const failures = row.consecutive_failures;
|
||||
const backoff = unsupported ? intervalMs : backoffDelayMs(failures + 1, intervalMs);
|
||||
const cooldownMs = rateLimited && retryAfter !== undefined ? Math.max(intervalMs, retryAfter) : backoff;
|
||||
await withTenant(workspaceId, async (client) => {
|
||||
await client.query(
|
||||
`UPDATE column_sources
|
||||
SET state = $3, last_error = $4, last_polled_at = now(),
|
||||
consecutive_failures = CASE WHEN $3 = 'unsupported' THEN consecutive_failures ELSE consecutive_failures + 1 END,
|
||||
next_poll_at = now() + ($5::bigint * interval '1 millisecond'), updated_at = now()
|
||||
WHERE workspace_id = $1 AND id = $2`,
|
||||
[workspaceId, row.id,
|
||||
unsupported ? "unsupported" : rateLimited ? "rate_limited" : "error",
|
||||
unsupported ? `unsupported source kind ${row.source_kind}` : error instanceof Error ? error.message.slice(0, 240) : "poll failed",
|
||||
cooldownMs],
|
||||
);
|
||||
}).catch(() => this.logger.warn("Column source state write-back unavailable"));
|
||||
return { cooldownMs };
|
||||
}
|
||||
|
||||
/**
|
||||
* Workspace listing spans tenants by definition, so it runs on the pool
|
||||
* with no tenant binding; every actual read/write below re-enters
|
||||
* withTenant(workspaceId) so FORCE RLS still binds per source.
|
||||
*/
|
||||
/**
|
||||
* Explicit refresh (§1.4 POST :id/refresh): mark this column's sources due
|
||||
* NOW. The per-source cooldown/lease still arbitrates the actual call, so a
|
||||
* refresh button cannot bypass the interval or double-fire mid-poll.
|
||||
*/
|
||||
/**
|
||||
* Explicit refresh (§1.4 POST :id/refresh) forces this column's sources due,
|
||||
* but not through a cooldown that is still authoritative in the DB: the
|
||||
* Redis cooldown is a cache (a flush loses it), so next_poll_at — persisted
|
||||
* — is the guard a forced refresh must still respect. FORCE_FLOOR_MS is a
|
||||
* deliberately small window: the button stays responsive for a manual
|
||||
* refresh while a double-click or a cache flush cannot double-poll a host.
|
||||
*/
|
||||
async forceColumnDue(workspaceId: string, columnId: string): Promise<number> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const result = await client.query(
|
||||
`UPDATE column_sources SET next_poll_at = now(), updated_at = now()
|
||||
WHERE workspace_id = $1 AND column_id = $2 AND state <> 'disabled'
|
||||
AND (last_polled_at IS NULL OR last_polled_at <= now() - ($3::bigint * interval '1 millisecond'))
|
||||
RETURNING id`,
|
||||
[workspaceId, columnId, FORCE_FLOOR_MS],
|
||||
);
|
||||
return result.rowCount ?? 0;
|
||||
});
|
||||
}
|
||||
|
||||
/** User-facing enable/disable of a single source (state machine §3.4). */
|
||||
async setSourceDisabled(workspaceId: string, sourceId: string, disabled: boolean): Promise<void> {
|
||||
await withTenant(workspaceId, async (client) => {
|
||||
const result = await client.query(
|
||||
`UPDATE column_sources SET state = $3, next_poll_at = CASE WHEN $3 = 'disabled' THEN NULL ELSE now() END, updated_at = now()
|
||||
WHERE workspace_id = $1 AND id = $2`,
|
||||
[workspaceId, sourceId, disabled ? "disabled" : "pending"],
|
||||
);
|
||||
if ((result.rowCount ?? 0) === 0) throw new Error(`column source ${sourceId} not found`);
|
||||
});
|
||||
}
|
||||
|
||||
private async workspaces(): Promise<string[]> {
|
||||
const result = await pool.query<{ id: string }>("SELECT id FROM workspaces ORDER BY id");
|
||||
return result.rows.map((row) => row.id);
|
||||
}
|
||||
}
|
||||
|
||||
/** Raised for source kinds P1 does not execute; parks the row in `unsupported`. */
|
||||
export class UnsupportedSourceError extends Error {
|
||||
constructor(reason: string) {
|
||||
super(`column source unsupported in P1: ${reason}`);
|
||||
this.name = "UnsupportedSourceError";
|
||||
}
|
||||
}
|
||||
652
apps/api/src/feeds/column-store.ts
Normal file
652
apps/api/src/feeds/column-store.ts
Normal file
|
|
@ -0,0 +1,652 @@
|
|||
/**
|
||||
* Columns persistence (P1 — design §1.2/§1.4).
|
||||
*
|
||||
* `stream_columns` (021) is the persisted column list; this repository owns
|
||||
* every mutation plus the resolved-source rows in `column_sources` (026) and
|
||||
* the per-source item rows in `feed_items`/`feed_item_columns` written by the
|
||||
* poll loop. Reads run under `withTenant` so FORCE RLS binds honestly.
|
||||
*
|
||||
* Everything the client can send is parsed with the P0a core validators
|
||||
* (`parseStreamColumnSpec`, `parseRefreshPolicyClamped`, `clampTitle`,
|
||||
* `clampPosition`) — the DB CHECKs are backstops, never the validation layer.
|
||||
* `StreamColumnsValidationError` is translated to 422 at the controller.
|
||||
*/
|
||||
import { BadRequestException, Inject, Injectable, NotFoundException, Optional, UnprocessableEntityException } from "@nestjs/common";
|
||||
import { createHash } from "node:crypto";
|
||||
import type { PoolClient } from "pg";
|
||||
import {
|
||||
clampPosition, clampTitle, parseRefreshPolicyClamped, parseStreamColumnSpec,
|
||||
StreamColumnsValidationError, type RefreshPolicy, type StreamColumnSpec, type StreamColumnType,
|
||||
} from "@tactical-media-broadcast-hub/core";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import { withTenant } from "../db/client.js";
|
||||
import { appendTenantAudit } from "../audit/audit.service.js";
|
||||
import type { NormalizedFeedItem } from "./feed-normalize.js";
|
||||
import type { ConnectionSummary } from "../connections/connections.service.js";
|
||||
|
||||
export type ColumnSourceKind = "home" | "mentions" | "search" | "author";
|
||||
export type ColumnSourceState =
|
||||
| "pending" | "ready" | "stale" | "error" | "unsupported" | "rate_limited" | "budget_paused" | "disabled";
|
||||
|
||||
export interface ColumnRecord {
|
||||
id: string;
|
||||
type: StreamColumnType;
|
||||
title: string;
|
||||
position: number;
|
||||
spec: StreamColumnSpec;
|
||||
refreshPolicy: RefreshPolicy;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
/** `ColumnView.source` (design §1.4): the scheduler-facing row, squashed. */
|
||||
export interface ColumnSourceView {
|
||||
id: string;
|
||||
/** Owning column — present so a bulk source query can be grouped per column. */
|
||||
columnId: string;
|
||||
provider: string;
|
||||
connectionId: string | null;
|
||||
sourceKind: ColumnSourceKind;
|
||||
query: string | null;
|
||||
actorRef: string | null;
|
||||
state: ColumnSourceState;
|
||||
lastError: string | null;
|
||||
lastPolledAt: string | null;
|
||||
nextPollAt: string | null;
|
||||
consecutiveFailures: number;
|
||||
}
|
||||
|
||||
export interface CreateColumn {
|
||||
type: StreamColumnType;
|
||||
title: string;
|
||||
position?: number;
|
||||
spec: unknown;
|
||||
refreshPolicy?: unknown;
|
||||
}
|
||||
|
||||
export interface UpdateColumn {
|
||||
title?: string;
|
||||
position?: number;
|
||||
spec?: unknown;
|
||||
refreshPolicy?: unknown;
|
||||
}
|
||||
|
||||
export interface StoredFeedItem {
|
||||
id: string;
|
||||
dedupKey: string;
|
||||
createdAtRemote: string;
|
||||
item: NormalizedFeedItem;
|
||||
}
|
||||
|
||||
/** The identity one poll resolves to — see `sourceIdentities` below. */
|
||||
interface SourceIdentity {
|
||||
provider: string;
|
||||
connectionId: string | null;
|
||||
sourceKind: ColumnSourceKind;
|
||||
query: string | null;
|
||||
actorRef: string | null;
|
||||
}
|
||||
|
||||
const identityKey = (identity: SourceIdentity): string =>
|
||||
JSON.stringify([identity.provider, identity.connectionId, identity.sourceKind, identity.query, identity.actorRef]);
|
||||
|
||||
export function mapValidationError<T>(parse: () => T): T {
|
||||
try {
|
||||
return parse();
|
||||
} catch (error) {
|
||||
if (error instanceof StreamColumnsValidationError) throw new UnprocessableEntityException(error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* sha256(workspace | provider | canonical remote id) — design §3.3. The
|
||||
* canonicalization keeps a provider that restates an item with decorations
|
||||
* (e.g. Mastodon's `scheduled:123` vs `123`) from double-storing the same
|
||||
* remote object.
|
||||
*/
|
||||
export function dedupKeyFor(workspaceId: string, provider: string, remoteId: string): string {
|
||||
const canonical = remoteId.replace(/^scheduled:/, "");
|
||||
return createHash("sha256").update(`${workspaceId}|${provider}|${canonical}`).digest("hex");
|
||||
}
|
||||
|
||||
/**
|
||||
* The polled identities a column spec resolves to (§3.3/§3.4). platform_feed
|
||||
* is the degenerate case: one connection, one kind per source row. keyword and
|
||||
* saved_search expand to one keyless search source per platform. account_list
|
||||
* expands to one author source per account entry.
|
||||
*/
|
||||
export function sourceIdentities(type: StreamColumnType, spec: StreamColumnSpec): SourceIdentity[] {
|
||||
switch (type) {
|
||||
case "platform_feed": {
|
||||
const feed = spec as import("@tactical-media-broadcast-hub/core").PlatformFeedSpec;
|
||||
const kinds = feed.kinds.length > 0 ? feed.kinds : (["home"] as const);
|
||||
return kinds.map((kind): SourceIdentity => ({
|
||||
provider: "", connectionId: feed.connectionId, sourceKind: kind, query: null, actorRef: null,
|
||||
}));
|
||||
}
|
||||
case "keyword":
|
||||
case "saved_search": {
|
||||
const keyword = spec as import("@tactical-media-broadcast-hub/core").KeywordSpec;
|
||||
return keyword.platforms.map((provider): SourceIdentity => ({
|
||||
provider, connectionId: null, sourceKind: "search", query: keyword.query, actorRef: null,
|
||||
}));
|
||||
}
|
||||
case "account_list": {
|
||||
const list = spec as import("@tactical-media-broadcast-hub/core").AccountListSpec;
|
||||
return list.accounts.map((account): SourceIdentity => ({
|
||||
provider: account.provider, connectionId: null, sourceKind: "author", query: null, actorRef: account.handle,
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Providers referenced by a spec (empty for platform_feed: the connection supplies it). */
|
||||
export function columnProviders(type: StreamColumnType, spec: StreamColumnSpec): string[] {
|
||||
switch (type) {
|
||||
case "platform_feed": return [];
|
||||
case "keyword":
|
||||
case "saved_search": return (spec as import("@tactical-media-broadcast-hub/core").KeywordSpec).platforms;
|
||||
case "account_list": return (spec as import("@tactical-media-broadcast-hub/core").AccountListSpec).accounts.map((account) => account.provider);
|
||||
}
|
||||
}
|
||||
|
||||
/** The title today's implicit column renders: "Display Name · handle" (web streams-data liveColumns). */
|
||||
export function seedTitle(row: ConnectionSummary): string {
|
||||
const displayName = String(row.fields.displayName ?? "").trim() || row.channelId;
|
||||
const handle = String(row.fields.handle ?? "").trim() || row.channelId;
|
||||
return clampTitle(`${displayName} · ${handle}`);
|
||||
}
|
||||
|
||||
interface ColumnRow {
|
||||
id: string; type: StreamColumnType; title: string; position: number;
|
||||
spec: unknown; refresh_policy: unknown; created_at: Date; updated_at: Date;
|
||||
}
|
||||
|
||||
const COLUMN_SELECT = "id, type, title, position, spec, refresh_policy, created_at, updated_at";
|
||||
|
||||
function toRecord(row: ColumnRow): ColumnRecord {
|
||||
return {
|
||||
id: row.id,
|
||||
type: row.type,
|
||||
title: row.title,
|
||||
position: row.position,
|
||||
spec: mapValidationError(() => parseStreamColumnSpec(row.type, row.spec)),
|
||||
refreshPolicy: parseRefreshPolicyClamped(row.refresh_policy ?? {}),
|
||||
createdAt: row.created_at.toISOString(),
|
||||
updatedAt: row.updated_at.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
const SOURCE_SELECT = `SELECT s.id, s.column_id, s.provider, s.connection_id, s.source_kind, s.query, s.actor_ref,
|
||||
s.state, s.last_error, s.last_polled_at, s.next_poll_at, s.consecutive_failures
|
||||
FROM column_sources s WHERE s.workspace_id = $1`;
|
||||
|
||||
function sourceFromRow(row: Record<string, unknown>): ColumnSourceView {
|
||||
return {
|
||||
id: String(row.id),
|
||||
columnId: String(row.column_id),
|
||||
provider: String(row.provider),
|
||||
connectionId: row.connection_id === null || row.connection_id === undefined ? null : String(row.connection_id),
|
||||
sourceKind: row.source_kind as ColumnSourceKind,
|
||||
query: row.query === null || row.query === undefined ? null : String(row.query),
|
||||
actorRef: row.actor_ref === null || row.actor_ref === undefined ? null : String(row.actor_ref),
|
||||
state: row.state as ColumnSourceState,
|
||||
lastError: row.last_error === null || row.last_error === undefined ? null : String(row.last_error),
|
||||
lastPolledAt: isoOrNull(row.last_polled_at),
|
||||
nextPollAt: isoOrNull(row.next_poll_at),
|
||||
consecutiveFailures: Number(row.consecutive_failures ?? 0),
|
||||
};
|
||||
}
|
||||
|
||||
function isoOrNull(value: unknown): string | null {
|
||||
return value === null || value === undefined ? null : (value as Date).toISOString();
|
||||
}
|
||||
|
||||
export interface ColumnStoreDependencies {
|
||||
/** Resolve a connection's provider for platform_feed source rows. */
|
||||
providerOf(workspaceId: string, connectionId: string): Promise<string>;
|
||||
/** Feed-capable connection summaries — the lazy seeding input. */
|
||||
connections(workspaceId: string): Promise<ConnectionSummary[]>;
|
||||
}
|
||||
|
||||
export const COLUMN_STORE_DEPENDENCIES = Symbol("COLUMN_STORE_DEPENDENCIES");
|
||||
|
||||
@Injectable()
|
||||
export class ColumnStore {
|
||||
constructor(@Optional() @Inject(COLUMN_STORE_DEPENDENCIES) private readonly deps?: ColumnStoreDependencies) {}
|
||||
|
||||
private async providerOf(workspaceId: string, connectionId: string): Promise<string> {
|
||||
if (this.deps) return this.deps.providerOf(workspaceId, connectionId);
|
||||
return withTenant(workspaceId, async (client): Promise<string> => {
|
||||
const result = await client.query<{ provider: string }>(
|
||||
"SELECT provider FROM connections WHERE id = $1 AND workspace_id = $2 AND disconnected_at IS NULL", [connectionId, workspaceId]);
|
||||
const row = result.rows[0];
|
||||
if (row === undefined) throw new NotFoundException(`connection ${connectionId} not found`);
|
||||
return row.provider;
|
||||
});
|
||||
}
|
||||
|
||||
private async connections(workspaceId: string): Promise<ConnectionSummary[]> {
|
||||
if (this.deps) return this.deps.connections(workspaceId);
|
||||
const { PgConnectionStore } = await import("../connections/connection-store.js");
|
||||
return new PgConnectionStore().list(workspaceId);
|
||||
}
|
||||
|
||||
async list(workspaceId: string): Promise<ColumnRecord[]> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const result = await client.query(
|
||||
`SELECT ${COLUMN_SELECT} FROM stream_columns WHERE deleted_at IS NULL ORDER BY position, created_at`);
|
||||
return result.rows.map((row) => toRecord(row as ColumnRow));
|
||||
});
|
||||
}
|
||||
|
||||
async get(workspaceId: string, id: string): Promise<ColumnRecord> {
|
||||
return withTenant(workspaceId, async (client) => this.read(client, workspaceId, id));
|
||||
}
|
||||
|
||||
async sources(workspaceId: string, columnId?: string): Promise<ColumnSourceView[]> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const result = columnId === undefined
|
||||
? await client.query(`${SOURCE_SELECT} ORDER BY s.created_at`, [workspaceId])
|
||||
: await client.query(`${SOURCE_SELECT} AND s.column_id = $2 ORDER BY s.created_at`, [workspaceId, columnId]);
|
||||
return result.rows.map((row) => sourceFromRow(row));
|
||||
});
|
||||
}
|
||||
|
||||
async create(session: SessionPayload, input: CreateColumn): Promise<ColumnRecord> {
|
||||
const spec = mapValidationError(() => parseStreamColumnSpec(input.type, input.spec));
|
||||
const policy = parseRefreshPolicyClamped(input.refreshPolicy ?? {});
|
||||
const title = clampTitle(input.title ?? "");
|
||||
if (title === "") throw new UnprocessableEntityException("title must be 1..80 characters after trimming");
|
||||
return withTenant(session.workspaceId, async (client) => {
|
||||
await this.assertSpecReferenceable(client, session.workspaceId, input.type, spec);
|
||||
const position = input.position === undefined
|
||||
? await this.nextPosition(client, session.workspaceId)
|
||||
: clampPosition(input.position);
|
||||
const result = await client.query(
|
||||
`INSERT INTO stream_columns (workspace_id, type, title, position, spec, refresh_policy, created_by)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7) RETURNING ${COLUMN_SELECT}`,
|
||||
[session.workspaceId, input.type, title, position, JSON.stringify(spec), JSON.stringify(policy), session.userId],
|
||||
);
|
||||
const record = toRecord(result.rows[0] as ColumnRow);
|
||||
await this.writeSources(client, session.workspaceId, record.id, input.type, spec);
|
||||
await appendTenantAudit(session.workspaceId, {
|
||||
actorUserId: session.userId, actorRole: session.role, action: "columns.created",
|
||||
targetType: "stream_column", targetId: record.id, diff: { type: input.type, title },
|
||||
});
|
||||
return record;
|
||||
});
|
||||
}
|
||||
|
||||
async update(session: SessionPayload, id: string, input: UpdateColumn): Promise<ColumnRecord> {
|
||||
if (Object.keys(input).length === 0) throw new BadRequestException("no recognized fields to update");
|
||||
return withTenant(session.workspaceId, async (client) => {
|
||||
await client.query("SELECT id FROM stream_columns WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL FOR UPDATE", [session.workspaceId, id]);
|
||||
const current = await this.read(client, session.workspaceId, id);
|
||||
const title = input.title === undefined ? current.title : clampTitle(input.title);
|
||||
if (title === "") throw new UnprocessableEntityException("title must be 1..80 characters after trimming");
|
||||
// Column type is immutable (design §1.4: PATCH carries title/position/
|
||||
// spec/refresh_policy); a different type is a new column.
|
||||
const spec = input.spec === undefined ? current.spec : mapValidationError(() => parseStreamColumnSpec(current.type, input.spec));
|
||||
const policy = input.refreshPolicy === undefined ? current.refreshPolicy : parseRefreshPolicyClamped(input.refreshPolicy);
|
||||
const position = input.position === undefined ? current.position : clampPosition(input.position);
|
||||
if (input.spec !== undefined) await this.assertSpecReferenceable(client, session.workspaceId, current.type, spec);
|
||||
const result = await client.query(
|
||||
`UPDATE stream_columns SET title = $3, position = $4, spec = $5, refresh_policy = $6, updated_at = now()
|
||||
WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL RETURNING ${COLUMN_SELECT}`,
|
||||
[session.workspaceId, id, title, position, JSON.stringify(spec), JSON.stringify(policy)],
|
||||
);
|
||||
if (result.rows[0] === undefined) throw new NotFoundException(`column ${id} not found`);
|
||||
if (input.spec !== undefined) await this.rewriteSources(client, session.workspaceId, id, current.type, spec);
|
||||
await appendTenantAudit(session.workspaceId, {
|
||||
actorUserId: session.userId, actorRole: session.role, action: "columns.updated",
|
||||
targetType: "stream_column", targetId: id,
|
||||
diff: { titleChanged: title !== current.title, positionChanged: position !== current.position, specChanged: input.spec !== undefined },
|
||||
});
|
||||
return toRecord(result.rows[0] as ColumnRow);
|
||||
});
|
||||
}
|
||||
|
||||
private async writeSources(client: PoolClient, workspaceId: string, columnId: string, type: StreamColumnType, spec: StreamColumnSpec): Promise<void> {
|
||||
for (const identity of sourceIdentities(type, spec)) {
|
||||
const provider = identity.provider !== "" ? identity.provider : await this.providerOf(workspaceId, identity.connectionId ?? "");
|
||||
await client.query(
|
||||
`INSERT INTO column_sources (workspace_id, column_id, provider, connection_id, source_kind, query, actor_ref)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7)
|
||||
ON CONFLICT (workspace_id, column_id, provider, source_kind, connection_id, query, actor_ref) DO NOTHING`,
|
||||
[workspaceId, columnId, provider, identity.connectionId, identity.sourceKind, identity.query, identity.actorRef],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Spec identity changes rewrite the source set; identities no longer referenced by this column are dropped. */
|
||||
private async rewriteSources(client: PoolClient, workspaceId: string, columnId: string, type: StreamColumnType, spec: StreamColumnSpec): Promise<void> {
|
||||
const wanted = new Set(sourceIdentities(type, spec).map(identityKey));
|
||||
const existing = await client.query<{ id: string; provider: string; connection_id: string | null; source_kind: ColumnSourceKind; query: string | null; actor_ref: string | null }>(
|
||||
"SELECT id, provider, connection_id, source_kind, query, actor_ref FROM column_sources WHERE workspace_id = $1 AND column_id = $2", [workspaceId, columnId]);
|
||||
for (const row of existing.rows) {
|
||||
const identity: SourceIdentity = { provider: row.provider, connectionId: row.connection_id, sourceKind: row.source_kind, query: row.query, actorRef: row.actor_ref };
|
||||
if (!wanted.has(identityKey(identity))) {
|
||||
await client.query("DELETE FROM column_sources WHERE workspace_id = $1 AND id = $2", [workspaceId, row.id]);
|
||||
}
|
||||
}
|
||||
await this.writeSources(client, workspaceId, columnId, type, spec);
|
||||
}
|
||||
|
||||
async remove(session: SessionPayload, id: string): Promise<void> {
|
||||
await withTenant(session.workspaceId, async (client) => {
|
||||
await client.query("SELECT id FROM stream_columns WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL FOR UPDATE", [session.workspaceId, id]);
|
||||
const result = await client.query(
|
||||
"UPDATE stream_columns SET deleted_at = now(), updated_at = now() WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL RETURNING id", [session.workspaceId, id]);
|
||||
if (result.rows[0] === undefined) throw new NotFoundException(`column ${id} not found`);
|
||||
await appendTenantAudit(session.workspaceId, {
|
||||
actorUserId: session.userId, actorRole: session.role, action: "columns.deleted",
|
||||
targetType: "stream_column", targetId: id,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Explicit order (design §1.4 POST /reorder): the client sends the full
|
||||
* ordered id list; positions are rewritten densely. Unknown, foreign or
|
||||
* missing ids are a 400 — a partial reorder would corrupt the order.
|
||||
*/
|
||||
async reorder(session: SessionPayload, ids: string[]): Promise<ColumnRecord[]> {
|
||||
if (ids.length === 0) return this.list(session.workspaceId);
|
||||
if (new Set(ids).size !== ids.length) throw new BadRequestException("reorder ids must be unique");
|
||||
return withTenant(session.workspaceId, async (client) => {
|
||||
const existing = await client.query<{ id: string }>(
|
||||
"SELECT id FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL", [session.workspaceId]);
|
||||
if (ids.length !== existing.rows.length || !existing.rows.every((row) => ids.includes(row.id))) {
|
||||
throw new BadRequestException("reorder must list every column in the workspace exactly once");
|
||||
}
|
||||
// Two-phase write: the unique (workspace_id, position) index would fire
|
||||
// mid-swap if positions were assigned in place (e.g. moving column 0 to
|
||||
// the end collides with the row already at that slot). Parking every row
|
||||
// at a high negative offset first leaves the final range untouched, so
|
||||
// the second pass can never collide. Offsets are outside the CHECK
|
||||
// (position >= 0) only transiently... so instead park at (n + i), which
|
||||
// is >= 0, unique within the transaction, and freed before phase two.
|
||||
// Phase 1 — park every row at a unique high slot (n, n+1, …) so the
|
||||
// unique (workspace_id, position) index can't fire mid-reorder.
|
||||
let park = ids.length;
|
||||
for (const id of ids) {
|
||||
await client.query("UPDATE stream_columns SET position = $3 WHERE workspace_id = $1 AND id = $2", [session.workspaceId, id, park]);
|
||||
park += 1;
|
||||
}
|
||||
// Phase 2 — assign final dense ranks in the REQUESTED order. The parked
|
||||
// range is disjoint from [0, n), so no intermediate state collides.
|
||||
const byId = new Map(ids.map((id, index) => [id, index] as const));
|
||||
const current = await client.query<ColumnRow>(
|
||||
`SELECT ${COLUMN_SELECT} FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL`, [session.workspaceId]);
|
||||
const records = current.rows
|
||||
.map((row) => toRecord(row as ColumnRow))
|
||||
.sort((a, b) => (byId.get(a.id) ?? 0) - (byId.get(b.id) ?? 0));
|
||||
let rank = 0;
|
||||
for (const record of records) {
|
||||
await client.query("UPDATE stream_columns SET position = $3, updated_at = now() WHERE workspace_id = $1 AND id = $2", [session.workspaceId, record.id, rank]);
|
||||
record.position = rank;
|
||||
rank += 1;
|
||||
}
|
||||
await appendTenantAudit(session.workspaceId, {
|
||||
actorUserId: session.userId, actorRole: session.role, action: "columns.reordered",
|
||||
targetType: "workspace", targetId: session.workspaceId, diff: { count: ids.length },
|
||||
});
|
||||
return records;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Lazy seeding (design §6 P1): on the first column-API touch for a workspace
|
||||
* with no columns at all, materialize TODAY's implicit /feeds columns — one
|
||||
* platform_feed column per feed-capable connection, titled exactly like the
|
||||
* implicit column the UI renders today ("Display · handle", kinds ["home"],
|
||||
* position by connection creation order), so the Streams page is unchanged.
|
||||
* Idempotent by construction: a second call sees a non-empty column list and
|
||||
* seeds nothing — including for connections added later.
|
||||
*/
|
||||
async ensureSeeded(workspaceId: string, connections: ConnectionSummary[], capable: (row: ConnectionSummary) => boolean): Promise<{ seeded: number }> {
|
||||
const rows = connections.filter(capable);
|
||||
if (rows.length === 0) return { seeded: 0 };
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
// Serialize concurrent first touches without taking a row lock on
|
||||
// workspaces itself: a FOR UPDATE here deadlocks with any FK insert
|
||||
// into audit_log/column_sources from a nested withTenant client,
|
||||
// because FK validation takes FOR KEY SHARE on the parent workspace
|
||||
// row. Advisory xact locks never block key-share, so seeding
|
||||
// serializes while audits and FK checks proceed.
|
||||
await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`columns-seed:${workspaceId}`]);
|
||||
const existing = await client.query<{ count: string }>(
|
||||
"SELECT count(*)::text AS count FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL", [workspaceId]);
|
||||
if (Number(existing.rows[0]?.count ?? 0) > 0) return { seeded: 0 };
|
||||
const seeder = await client.query<{ user_id: string }>(
|
||||
"SELECT m.user_id FROM memberships m JOIN users u ON u.id = m.user_id WHERE m.workspace_id = $1 ORDER BY m.created_at, u.created_at LIMIT 1", [workspaceId]);
|
||||
const createdBy = seeder.rows[0]?.user_id;
|
||||
if (createdBy === undefined) return { seeded: 0 }; // no member to attribute
|
||||
let seeded = 0;
|
||||
let position = 0;
|
||||
for (const row of rows) {
|
||||
const spec = { connectionId: row.id, kinds: ["home"] } as const;
|
||||
const inserted = await client.query<{ id: string }>(
|
||||
`INSERT INTO stream_columns (workspace_id, type, title, position, spec, refresh_policy, created_by)
|
||||
VALUES ($1,'platform_feed',$2,$3,$4,'{}',$5) RETURNING id`,
|
||||
[workspaceId, seedTitle(row), position, JSON.stringify(spec), createdBy],
|
||||
);
|
||||
const columnId = inserted.rows[0]?.id;
|
||||
if (columnId === undefined) continue;
|
||||
await client.query(
|
||||
`INSERT INTO column_sources (workspace_id, column_id, provider, connection_id, source_kind)
|
||||
VALUES ($1,$2,$3,$4,'home')
|
||||
ON CONFLICT (workspace_id, column_id, provider, source_kind, connection_id, query, actor_ref) DO NOTHING`,
|
||||
[workspaceId, columnId, row.provider, row.id],
|
||||
);
|
||||
seeded += 1;
|
||||
position += 1;
|
||||
}
|
||||
if (seeded > 0) await appendTenantAudit(workspaceId, {
|
||||
actorUserId: null, actorRole: "system", action: "columns.seeded",
|
||||
targetType: "workspace", targetId: workspaceId, diff: { seeded },
|
||||
});
|
||||
return { seeded };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* import-saved-searches (design §1.4): the localStorage saved-searches
|
||||
* contract moves server-side. Each query becomes a saved_search column over
|
||||
* the platforms the workspace can actually search, defaulting to every
|
||||
* connected feed provider when the client sends none. Idempotent per query:
|
||||
* a live saved_search column with the same query is skipped.
|
||||
*/
|
||||
async importSavedSearches(
|
||||
session: SessionPayload, queries: string[], platforms: string[],
|
||||
): Promise<{ imported: number; skipped: number }> {
|
||||
if (platforms.length === 0) return { imported: 0, skipped: queries.length };
|
||||
return withTenant(session.workspaceId, async (client) => {
|
||||
let imported = 0;
|
||||
let skipped = 0;
|
||||
for (const raw of queries) {
|
||||
const query = raw.trim();
|
||||
if (query === "") { skipped += 1; continue; }
|
||||
const duplicate = await client.query<{ id: string }>(
|
||||
`SELECT id FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL AND type = 'saved_search'
|
||||
AND spec->>'query' = $2 LIMIT 1`, [session.workspaceId, query]);
|
||||
if (duplicate.rows[0] !== undefined) { skipped += 1; continue; }
|
||||
const spec = { query, platforms, matchMode: "any" } as const;
|
||||
const title = clampTitle(query);
|
||||
if (title === "") { skipped += 1; continue; }
|
||||
const position = await this.nextPosition(client, session.workspaceId);
|
||||
const inserted = await client.query<{ id: string }>(
|
||||
`INSERT INTO stream_columns (workspace_id, type, title, position, spec, refresh_policy, created_by)
|
||||
VALUES ($1,'saved_search',$2,$3,$4,'{}',$5) RETURNING id`,
|
||||
[session.workspaceId, title, position, JSON.stringify(spec), session.userId],
|
||||
);
|
||||
const columnId = inserted.rows[0]?.id;
|
||||
if (columnId === undefined) { skipped += 1; continue; }
|
||||
await this.writeSources(client, session.workspaceId, columnId, "saved_search",
|
||||
parseStreamColumnSpec("saved_search", spec));
|
||||
imported += 1;
|
||||
}
|
||||
if (imported > 0) await appendTenantAudit(session.workspaceId, {
|
||||
actorUserId: session.userId, actorRole: session.role, action: "columns.saved_searches_imported",
|
||||
targetType: "workspace", targetId: session.workspaceId, diff: { imported, skipped },
|
||||
});
|
||||
return { imported, skipped };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Poll-time item persistence (§3.3): first writer wins per dedup key (the
|
||||
* cross-column dedup), then fan-out rows link the item to every column
|
||||
* whose source shares this source's identity — which is how "same account
|
||||
* in two columns costs one poll" still surfaces items in both.
|
||||
*/
|
||||
async storeItems(workspaceId: string, sourceId: string, items: NormalizedFeedItem[]): Promise<{ stored: number; fanned: number }> {
|
||||
if (items.length === 0) return { stored: 0, fanned: 0 };
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const found = await client.query<{ id: string; provider: string; connection_id: string | null; source_kind: ColumnSourceKind; query: string | null; actor_ref: string | null }>(
|
||||
"SELECT id, provider, connection_id, source_kind, query, actor_ref FROM column_sources WHERE workspace_id = $1 AND id = $2", [workspaceId, sourceId]);
|
||||
const source = found.rows[0];
|
||||
if (source === undefined) return { stored: 0, fanned: 0 };
|
||||
const sharing = await client.query<{ column_id: string }>(
|
||||
`SELECT column_id FROM column_sources
|
||||
WHERE workspace_id = $1 AND provider = $2 AND source_kind = $3
|
||||
AND connection_id IS NOT DISTINCT FROM $4 AND query IS NOT DISTINCT FROM $5 AND actor_ref IS NOT DISTINCT FROM $6
|
||||
AND state <> 'disabled'`,
|
||||
[workspaceId, source.provider, source.source_kind, source.connection_id, source.query, source.actor_ref]);
|
||||
const columnIds = sharing.rows.map((row) => row.column_id);
|
||||
let stored = 0;
|
||||
let fanned = 0;
|
||||
for (const item of items) {
|
||||
const dedupKey = dedupKeyFor(workspaceId, item.provider, item.remoteId);
|
||||
// First writer wins per dedup key (the workspace-wide cross-column
|
||||
// dedup, §3.3) — the returning row resolves the item id either way.
|
||||
const inserted = await client.query<{ id: string }>(
|
||||
`INSERT INTO feed_items (workspace_id, dedup_key, provider, remote_id, item_id, author_handle, author_ref, url,
|
||||
created_at_remote, fetched_at, content, plain_text, media, metrics, source_id, expires_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,now(),$10,$11,$12,$13,$14,$15)
|
||||
ON CONFLICT (workspace_id, dedup_key) DO UPDATE SET fetched_at = feed_items.fetched_at RETURNING id`,
|
||||
[workspaceId, dedupKey, item.provider, item.remoteId, item.id, item.authorHandle, item.authorRef, item.url,
|
||||
item.createdAt.toISOString(), JSON.stringify(item.content), item.plainText, JSON.stringify(item.media),
|
||||
JSON.stringify(item.metrics), sourceId, item.expiresAt === null || item.expiresAt === undefined ? null : item.expiresAt.toISOString()],
|
||||
);
|
||||
const itemId = inserted.rows[0]?.id;
|
||||
if (itemId === undefined) continue;
|
||||
stored += 1;
|
||||
// Fan out regardless of who inserted: an item ingested for one source
|
||||
// identity is visible to EVERY column sharing that identity, including
|
||||
// items an earlier poll already stored (§3.3) — otherwise the second
|
||||
// column's dedup-hit items would never get their link rows.
|
||||
for (const columnId of columnIds) {
|
||||
const linked = await client.query(
|
||||
"INSERT INTO feed_item_columns (workspace_id, item_id, column_id) VALUES ($1,$2,$3) ON CONFLICT DO NOTHING",
|
||||
[workspaceId, itemId, columnId]);
|
||||
fanned += linked.rowCount ?? 0;
|
||||
}
|
||||
}
|
||||
return { stored, fanned };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Keyset page for GET /feeds/columns/:id/items (§1.4). Cursor rule:
|
||||
* (created_at_remote DESC, dedup_key DESC) — a pure position in a total
|
||||
* order, so paging survives cache flushes and later insertions alike.
|
||||
*/
|
||||
async columnItems(
|
||||
workspaceId: string, columnId: string, options: { cursor?: { at: number; key: string }; limit: number },
|
||||
): Promise<{ items: StoredFeedItem[]; nextCursor: string | null }> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const column = await client.query("SELECT id FROM stream_columns WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL", [workspaceId, columnId]);
|
||||
if (column.rows[0] === undefined) throw new NotFoundException(`column ${columnId} not found`);
|
||||
const params: unknown[] = [workspaceId, columnId];
|
||||
const filter = options.cursor === undefined ? "" : " AND (i.created_at_remote, i.dedup_key) < ($3::timestamptz, $4)";
|
||||
if (options.cursor !== undefined) params.push(new Date(options.cursor.at).toISOString(), options.cursor.key);
|
||||
params.push(options.limit + 1);
|
||||
const result = await client.query(
|
||||
`SELECT i.id, i.dedup_key, i.created_at_remote, i.provider, i.remote_id, i.item_id, i.author_handle,
|
||||
i.author_ref, i.url, i.content, i.plain_text, i.media, i.metrics, i.expires_at
|
||||
FROM feed_items i
|
||||
JOIN feed_item_columns fic ON fic.item_id = i.id AND fic.column_id = $2
|
||||
WHERE i.workspace_id = $1${filter}
|
||||
ORDER BY i.created_at_remote DESC, i.dedup_key DESC
|
||||
LIMIT $${params.length}`,
|
||||
params,
|
||||
);
|
||||
const rows = result.rows;
|
||||
const hasMore = rows.length > options.limit;
|
||||
const items = (hasMore ? rows.slice(0, options.limit) : rows).map((row) => storedItemFromRow(row));
|
||||
const last = items[items.length - 1];
|
||||
return {
|
||||
items,
|
||||
nextCursor: hasMore && last
|
||||
? Buffer.from(JSON.stringify({ at: Date.parse(last.createdAtRemote), key: last.dedupKey })).toString("base64url")
|
||||
: null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
private async nextPosition(client: PoolClient, workspaceId: string): Promise<number> {
|
||||
const result = await client.query<{ max: number | null }>(
|
||||
"SELECT max(position) AS max FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL", [workspaceId]);
|
||||
return (result.rows[0]?.max ?? -1) + 1;
|
||||
}
|
||||
|
||||
/** A spec's references must exist in the workspace before the column can persist. */
|
||||
private async assertSpecReferenceable(client: PoolClient, workspaceId: string, type: StreamColumnType, spec: StreamColumnSpec): Promise<void> {
|
||||
if (type === "platform_feed") {
|
||||
const connectionId = (spec as import("@tactical-media-broadcast-hub/core").PlatformFeedSpec).connectionId;
|
||||
const result = await client.query(
|
||||
"SELECT 1 FROM connections WHERE workspace_id = $1 AND id = $2 AND disconnected_at IS NULL", [workspaceId, connectionId]);
|
||||
if (result.rows[0] === undefined) throw new UnprocessableEntityException("platform_feed: connectionId does not reference a connection in this workspace");
|
||||
return;
|
||||
}
|
||||
const providers = columnProviders(type, spec);
|
||||
if (providers.length === 0) return;
|
||||
const result = await client.query<{ provider: string }>(
|
||||
"SELECT DISTINCT provider FROM connections WHERE workspace_id = $1 AND disconnected_at IS NULL AND provider = ANY($2)", [workspaceId, providers]);
|
||||
const present = new Set(result.rows.map((row) => row.provider));
|
||||
const missing = [...new Set(providers)].filter((provider) => !present.has(provider));
|
||||
if (missing.length > 0) throw new UnprocessableEntityException(`no connection for provider(s): ${missing.join(", ")}`);
|
||||
}
|
||||
|
||||
private async read(client: PoolClient, workspaceId: string, id: string): Promise<ColumnRecord> {
|
||||
const result = await client.query(
|
||||
`SELECT ${COLUMN_SELECT} FROM stream_columns WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL`, [workspaceId, id]);
|
||||
const row = result.rows[0];
|
||||
if (row === undefined) throw new NotFoundException(`column ${id} not found`);
|
||||
return toRecord(row as ColumnRow);
|
||||
}
|
||||
}
|
||||
|
||||
interface FeedItemRow {
|
||||
id: string; dedup_key: string; created_at_remote: Date; provider: string; remote_id: string; item_id: string;
|
||||
author_handle: string; author_ref: string | null; url: string | null; content: unknown; plain_text: string;
|
||||
media: unknown; metrics: unknown; expires_at: Date | null;
|
||||
}
|
||||
|
||||
function storedItemFromRow(row: FeedItemRow): StoredFeedItem {
|
||||
const metrics = (typeof row.metrics === "object" && row.metrics !== null ? row.metrics : {}) as Record<string, number>;
|
||||
const item: NormalizedFeedItem = {
|
||||
id: row.item_id,
|
||||
remoteId: row.remote_id,
|
||||
provider: row.provider,
|
||||
// The store page does not carry a live connection id; the column's source
|
||||
// rows hold it, so the connectionId field is filled by the caller when it
|
||||
// matters (the items endpoint derives it from the column spec).
|
||||
connectionId: "",
|
||||
authorHandle: row.author_handle,
|
||||
authorRef: row.author_ref,
|
||||
url: row.url,
|
||||
text: row.plain_text,
|
||||
content: row.content as NormalizedFeedItem["content"],
|
||||
plainText: row.plain_text,
|
||||
createdAt: row.created_at_remote,
|
||||
mediaUrls: [],
|
||||
media: Array.isArray(row.media) ? row.media as NormalizedFeedItem["media"] : [],
|
||||
metrics,
|
||||
expiresAt: row.expires_at,
|
||||
};
|
||||
return { id: row.id, dedupKey: row.dedup_key, createdAtRemote: new Date(row.created_at_remote).toISOString(), item };
|
||||
}
|
||||
300
apps/api/src/feeds/columns.controller.ts
Normal file
300
apps/api/src/feeds/columns.controller.ts
Normal file
|
|
@ -0,0 +1,300 @@
|
|||
/**
|
||||
* Columns API (P1 — design §1.4).
|
||||
*
|
||||
* Routes (all under /feeds, the surface the Streams page already calls):
|
||||
* GET /feeds/columns list (+ lazy seeding on first touch)
|
||||
* POST /feeds/columns create
|
||||
* GET /feeds/columns/:id one column
|
||||
* PATCH /feeds/columns/:id update (title/position/spec/refreshPolicy)
|
||||
* DELETE /feeds/columns/:id soft delete
|
||||
* POST /feeds/columns/reorder explicit full-list order
|
||||
* POST /feeds/columns/import-saved-searches localStorage contract migration
|
||||
* GET /feeds/columns/:id/items keyset-cursored items
|
||||
* POST /feeds/columns/:id/refresh force a poll (cooldown-gated)
|
||||
* POST /feeds/columns/:id/sources/:sourceId/disable|enable
|
||||
*
|
||||
* Writes are gated on `streams.columns.manage` (026 seed: Owner/Admin/Editor);
|
||||
* reads stay on `draft.view` — the same permission /feeds uses, so every
|
||||
* current reader keeps reading. GET /feeds is untouched except the additive
|
||||
* `meta.columns` block.
|
||||
*/
|
||||
import { BadRequestException, Body, Controller, Delete, Get, Inject, NotFoundException, Param, Patch, Post, Query, Req, UnprocessableEntityException, UseGuards } from "@nestjs/common";
|
||||
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import { parseId, parseLimit, requireObject, requireSession, requireString } from "../http/validation.js";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import {
|
||||
ColumnStore, mapValidationError, type ColumnRecord, type ColumnSourceView, type CreateColumn, type StoredFeedItem, type UpdateColumn,
|
||||
} from "./column-store.js";
|
||||
import { ColumnScheduler } from "./column-scheduler.js";
|
||||
import { FeedsService } from "./feeds.service.js";
|
||||
import type { ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
|
||||
export interface ColumnResponse extends ColumnRecord {
|
||||
sources: ColumnSourceView[];
|
||||
}
|
||||
|
||||
export interface ColumnItemsResponse {
|
||||
items: StoredFeedItem[];
|
||||
nextCursor: string | null;
|
||||
}
|
||||
|
||||
/** DI token for the shared connector registry (single process-wide instance). */
|
||||
export const COLUMNS_REGISTRY = Symbol("COLUMNS_REGISTRY");
|
||||
|
||||
@Controller("feeds/columns")
|
||||
@UseGuards(RbacGuard)
|
||||
export class ColumnsController {
|
||||
constructor(
|
||||
@Inject(ColumnStore) private readonly store: ColumnStore,
|
||||
@Inject(ColumnScheduler) private readonly scheduler: ColumnScheduler,
|
||||
@Inject(FeedsService) private readonly feeds: FeedsService,
|
||||
@Inject(COLUMNS_REGISTRY) private readonly registry: ConnectorRegistry,
|
||||
) {}
|
||||
|
||||
@Get()
|
||||
@RequirePermission("draft.view")
|
||||
async list(@Req() request: AuthenticatedRequest): Promise<ColumnResponse[]> {
|
||||
const session = requireSession(request);
|
||||
await this.ensureSeeded(session);
|
||||
return this.views(session.workspaceId, await this.store.list(session.workspaceId));
|
||||
}
|
||||
|
||||
@Post()
|
||||
@RequirePermission("streams.columns.manage")
|
||||
async create(@Req() request: AuthenticatedRequest, @Body() body: unknown): Promise<ColumnResponse> {
|
||||
const session = requireSession(request);
|
||||
const input = parseCreate(body);
|
||||
await this.ensureSeeded(session);
|
||||
const record = await this.store.create(session, input);
|
||||
return this.view(record, await this.store.sources(session.workspaceId, record.id));
|
||||
}
|
||||
|
||||
@Get(":id")
|
||||
@RequirePermission("draft.view")
|
||||
async get(@Req() request: AuthenticatedRequest, @Param("id") id: string): Promise<ColumnResponse> {
|
||||
const session = requireSession(request);
|
||||
await this.ensureSeeded(session);
|
||||
const record = await this.store.get(session.workspaceId, parseId(id));
|
||||
return this.view(record, await this.store.sources(session.workspaceId, record.id));
|
||||
}
|
||||
|
||||
@Patch(":id")
|
||||
@RequirePermission("streams.columns.manage")
|
||||
async update(@Req() request: AuthenticatedRequest, @Param("id") id: string, @Body() body: unknown): Promise<ColumnResponse> {
|
||||
const session = requireSession(request);
|
||||
const record = await this.store.update(session, parseId(id), parseUpdate(body));
|
||||
return this.view(record, await this.store.sources(session.workspaceId, record.id));
|
||||
}
|
||||
|
||||
@Delete(":id")
|
||||
@RequirePermission("streams.columns.manage")
|
||||
async remove(@Req() request: AuthenticatedRequest, @Param("id") id: string): Promise<{ id: string; deleted: true }> {
|
||||
const session = requireSession(request);
|
||||
const parsed = parseId(id);
|
||||
await this.store.remove(session, parsed);
|
||||
return { id: parsed, deleted: true };
|
||||
}
|
||||
|
||||
@Post("reorder")
|
||||
@RequirePermission("streams.columns.manage")
|
||||
async reorder(@Req() request: AuthenticatedRequest, @Body() body: unknown): Promise<ColumnResponse[]> {
|
||||
const session = requireSession(request);
|
||||
const ids = parseIds(requireObject(body, "body").ids);
|
||||
return this.views(session.workspaceId, await this.store.reorder(session, ids));
|
||||
}
|
||||
|
||||
@Post("import-saved-searches")
|
||||
@RequirePermission("streams.columns.manage")
|
||||
async importSavedSearches(@Req() request: AuthenticatedRequest, @Body() body: unknown): Promise<{ imported: number; skipped: number }> {
|
||||
const session = requireSession(request);
|
||||
const payload = requireObject(body, "body");
|
||||
const rawQueries = payload.queries;
|
||||
if (!Array.isArray(rawQueries) || rawQueries.length === 0) {
|
||||
throw new BadRequestException("queries must be a non-empty array of saved search strings");
|
||||
}
|
||||
for (const query of rawQueries) {
|
||||
if (typeof query !== "string") throw new BadRequestException("queries must contain strings only");
|
||||
}
|
||||
if (rawQueries.length > 100) throw new BadRequestException("queries supports at most 100 entries per call");
|
||||
const rawPlatforms = payload.platforms;
|
||||
let platforms: string[];
|
||||
if (rawPlatforms === undefined || rawPlatforms === null) {
|
||||
platforms = this.searchCapableProviders(await this.connectedProviders(session));
|
||||
} else {
|
||||
if (!Array.isArray(rawPlatforms) || rawPlatforms.some((provider) => typeof provider !== "string")) {
|
||||
throw new BadRequestException("platforms must be an array of provider ids");
|
||||
}
|
||||
platforms = [...new Set(rawPlatforms as string[])];
|
||||
// Provider ids are opaque to the P0a schema by design; the registry is
|
||||
// the app-side source of truth, so an unknown id is a 422 here rather
|
||||
// than a saved_search column that can never resolve a source.
|
||||
const unknown = platforms.filter((provider) => !this.registry.has(provider));
|
||||
if (unknown.length > 0) {
|
||||
throw new UnprocessableEntityException(`unknown provider id(s): ${unknown.join(", ")}`);
|
||||
}
|
||||
}
|
||||
return this.store.importSavedSearches(session, rawQueries, platforms);
|
||||
}
|
||||
|
||||
@Get(":id/items")
|
||||
@RequirePermission("draft.view")
|
||||
async items(
|
||||
@Req() request: AuthenticatedRequest,
|
||||
@Param("id") id: string,
|
||||
@Query("cursor") cursor?: string,
|
||||
@Query("limit") limit?: string,
|
||||
): Promise<ColumnItemsResponse> {
|
||||
const session = requireSession(request);
|
||||
return this.store.columnItems(session.workspaceId, parseId(id), {
|
||||
cursor: cursor === undefined || cursor === "" ? undefined : decodeColumnCursor(cursor),
|
||||
limit: parseLimit(limit, 40, 200),
|
||||
});
|
||||
}
|
||||
|
||||
@Post(":id/refresh")
|
||||
@RequirePermission("streams.columns.manage")
|
||||
async refresh(@Req() request: AuthenticatedRequest, @Param("id") id: string): Promise<{ id: string; refreshed: number }> {
|
||||
const session = requireSession(request);
|
||||
const parsed = parseId(id);
|
||||
await this.store.get(session.workspaceId, parsed); // 404 before polling
|
||||
// Force-due this column's sources; the scheduler's cooldown/lease still
|
||||
// decides whether a call actually goes out (refresh cannot bypass them).
|
||||
const forced = await this.forceDue(session.workspaceId, parsed);
|
||||
const outcome = await this.scheduler.pollWorkspaceDue(session.workspaceId);
|
||||
return { id: parsed, refreshed: forced > 0 ? outcome.polled : 0 };
|
||||
}
|
||||
|
||||
@Post(":id/sources/:sourceId/disable")
|
||||
@RequirePermission("streams.columns.manage")
|
||||
async disableSource(@Req() request: AuthenticatedRequest, @Param("id") id: string, @Param("sourceId") sourceId: string): Promise<{ id: string; state: string }> {
|
||||
const session = requireSession(request);
|
||||
return this.setSourceState(session, parseId(id), parseId(sourceId), true);
|
||||
}
|
||||
|
||||
@Post(":id/sources/:sourceId/enable")
|
||||
@RequirePermission("streams.columns.manage")
|
||||
async enableSource(@Req() request: AuthenticatedRequest, @Param("id") id: string, @Param("sourceId") sourceId: string): Promise<{ id: string; state: string }> {
|
||||
const session = requireSession(request);
|
||||
return this.setSourceState(session, parseId(id), parseId(sourceId), false);
|
||||
}
|
||||
|
||||
// -- helpers -------------------------------------------------------------
|
||||
|
||||
private async setSourceState(session: SessionPayload, columnId: string, sourceId: string, disabled: boolean): Promise<{ id: string; state: string }> {
|
||||
const sources = await this.store.sources(session.workspaceId, columnId);
|
||||
const source = sources.find((row) => row.id === sourceId);
|
||||
if (source === undefined) throw new NotFoundException(`source ${sourceId} not found on column ${columnId}`);
|
||||
await this.scheduler.setSourceDisabled(session.workspaceId, sourceId, disabled);
|
||||
return { id: sourceId, state: disabled ? "disabled" : "pending" };
|
||||
}
|
||||
|
||||
private async ensureSeeded(session: SessionPayload): Promise<void> {
|
||||
const connections = await this.feeds.capableConnections(session.workspaceId);
|
||||
await this.store.ensureSeeded(session.workspaceId, connections, () => true);
|
||||
}
|
||||
|
||||
/** Bulk view assembly: one source query, grouped per column. */
|
||||
private async views(workspaceId: string, records: ColumnRecord[]): Promise<ColumnResponse[]> {
|
||||
if (records.length === 0) return [];
|
||||
const byColumn = new Map<string, ColumnSourceView[]>();
|
||||
for (const source of await this.store.sources(workspaceId)) {
|
||||
const list = byColumn.get(source.columnId);
|
||||
if (list === undefined) byColumn.set(source.columnId, [source]);
|
||||
else list.push(source);
|
||||
}
|
||||
return records.map((record) => ({ ...record, sources: byColumn.get(record.id) ?? [] }));
|
||||
}
|
||||
|
||||
private async view(record: ColumnRecord, sources: ColumnSourceView[]): Promise<ColumnResponse> {
|
||||
return { ...record, sources };
|
||||
}
|
||||
|
||||
private async connectedProviders(session: SessionPayload): Promise<string[]> {
|
||||
const rows = await this.feeds.capableConnections(session.workspaceId);
|
||||
return [...new Set(rows.map((row) => row.provider))];
|
||||
}
|
||||
|
||||
/** Providers whose manifest declares the P0b search capability. */
|
||||
private searchCapableProviders(providers: string[]): string[] {
|
||||
return providers.filter((provider) => {
|
||||
if (!this.registry.has(provider)) return false;
|
||||
return this.registry.get(provider).manifest().search === true;
|
||||
});
|
||||
}
|
||||
|
||||
private async forceDue(workspaceId: string, columnId: string): Promise<number> {
|
||||
return this.scheduler.forceColumnDue(workspaceId, columnId);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Request parsing (unknown body in, typed DTO out; 400 on shape errors)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function parseCreate(body: unknown): CreateColumn {
|
||||
const payload = requireObject(body, "body");
|
||||
const type = requireString(payload.type, "type");
|
||||
if (type !== "platform_feed" && type !== "keyword" && type !== "account_list" && type !== "saved_search") {
|
||||
throw new BadRequestException("type must be one of platform_feed|keyword|account_list|saved_search");
|
||||
}
|
||||
const title = requireString(payload.title, "title");
|
||||
const position = payload.position === undefined ? undefined : requirePosition(payload.position);
|
||||
return {
|
||||
type,
|
||||
title,
|
||||
position,
|
||||
spec: payload.spec === undefined ? {} : payload.spec,
|
||||
refreshPolicy: payload.refreshPolicy,
|
||||
};
|
||||
}
|
||||
|
||||
function parseUpdate(body: unknown): UpdateColumn {
|
||||
const payload = requireObject(body, "body");
|
||||
const update: UpdateColumn = {};
|
||||
if (payload.title !== undefined) update.title = requireString(payload.title, "title");
|
||||
if (payload.position !== undefined) update.position = requirePosition(payload.position);
|
||||
if (payload.spec !== undefined) update.spec = payload.spec;
|
||||
if (payload.refreshPolicy !== undefined) {
|
||||
if (typeof payload.refreshPolicy !== "object" || payload.refreshPolicy === null || Array.isArray(payload.refreshPolicy)) {
|
||||
throw new BadRequestException("refreshPolicy must be an object");
|
||||
}
|
||||
update.refreshPolicy = payload.refreshPolicy;
|
||||
}
|
||||
return update;
|
||||
}
|
||||
|
||||
function requirePosition(value: unknown): number {
|
||||
if (typeof value !== "number" || !Number.isInteger(value) || value < 0 || value > 2_147_483_647) {
|
||||
throw new BadRequestException("position must be an integer between 0 and 2147483647");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseIds(value: unknown): string[] {
|
||||
if (!Array.isArray(value) || value.length === 0) throw new BadRequestException("ids must be a non-empty array of column ids");
|
||||
const ids: string[] = [];
|
||||
for (const id of value) {
|
||||
if (typeof id !== "string") throw new BadRequestException("ids must contain strings only");
|
||||
ids.push(parseId(id));
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
/**
|
||||
* The keyset cursor: base64url JSON `{ at, key }` where `at` is the item's
|
||||
* created_at_remote epoch ms and `key` its dedup_key. Mirrors the /feeds
|
||||
* cursor contract (opaque, ≤4096 chars, position in a total order).
|
||||
*/
|
||||
export function decodeColumnCursor(cursor: string): { at: number; key: string } {
|
||||
try {
|
||||
if (cursor.length > 4096) throw new Error();
|
||||
const parsed: unknown = JSON.parse(Buffer.from(cursor, "base64url").toString());
|
||||
if (typeof parsed !== "object" || parsed === null || !("at" in parsed) || !("key" in parsed) ||
|
||||
typeof (parsed as { at: unknown }).at !== "number" || !Number.isFinite((parsed as { at: number }).at) ||
|
||||
typeof (parsed as { key: unknown }).key !== "string") throw new Error();
|
||||
return { at: (parsed as { at: number }).at, key: (parsed as { key: string }).key };
|
||||
} catch {
|
||||
throw new BadRequestException("Invalid column cursor");
|
||||
}
|
||||
}
|
||||
165
apps/api/src/feeds/columns.dependencies.ts
Normal file
165
apps/api/src/feeds/columns.dependencies.ts
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
/**
|
||||
* Columns wiring (P1): production dependency providers for the ColumnStore
|
||||
* and ColumnScheduler, mirroring feeds.dependencies.ts.
|
||||
*
|
||||
* The scheduler's per-host budget draws from `@tactical-media-broadcast-hub/egress`
|
||||
* HostBudgets — the SAME `host_budgets` rows every other egress caller uses
|
||||
* (monitor, bulk import), keyed by the egress `hostBudgetKey` rule (IDNA
|
||||
* lowercased host + effective port), so feeds polls and monitor polls to one
|
||||
* host contend for one shared bucket. That is the §3.4 requirement: budgets
|
||||
* are per-host totals, never per-process.
|
||||
*
|
||||
* The egress package is type-imported and loaded natively at runtime (the
|
||||
* same pattern as live-monitor/poller/egress.ts and virtual-channels/
|
||||
* import-egress.ts) because it is ESM-only.
|
||||
*/
|
||||
import { Inject, Injectable, Optional } from "@nestjs/common";
|
||||
import type { ConnectorConnection } from "@tactical-media-broadcast-hub/core";
|
||||
import type { CredentialAccessor, ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import { existsSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { pool } from "../db/client.js";
|
||||
import { PgConnectionStore } from "../connections/connection-store.js";
|
||||
import { RespControlRedis, type ControlRedis } from "../media-control/redis.js";
|
||||
import { vault } from "../vault/vault.js";
|
||||
import { feedRegistry } from "./feed-registry.js";
|
||||
import { FeedsService } from "./feeds.service.js";
|
||||
import {
|
||||
ColumnStore, COLUMN_STORE_DEPENDENCIES, type ColumnStoreDependencies,
|
||||
} from "./column-store.js";
|
||||
import {
|
||||
ColumnScheduler, COLUMN_SCHEDULER_DEPENDENCIES, type ColumnSchedulerDependencies,
|
||||
} from "./column-scheduler.js";
|
||||
import { COLUMNS_REGISTRY } from "./columns.controller.js";
|
||||
|
||||
type EgressPackage = typeof import("../../../../packages/egress/src/index.js");
|
||||
|
||||
const nativeImport = new Function("specifier", "return import(specifier)") as (specifier: string) => Promise<EgressPackage>;
|
||||
|
||||
function resolveEgressPackage(): string {
|
||||
for (const start of [process.cwd(), __dirname]) {
|
||||
for (let root = start; ; root = dirname(root)) {
|
||||
const directory = resolve(root, "packages/egress");
|
||||
const files = process.env.NODE_ENV === "production" ? ["dist/index.js"] : ["src/index.ts", "dist/index.js"];
|
||||
for (const file of files) {
|
||||
const candidate = resolve(directory, file);
|
||||
if (existsSync(candidate)) return candidate;
|
||||
}
|
||||
if (dirname(root) === root) break;
|
||||
}
|
||||
}
|
||||
throw new Error("Build packages/egress before starting the columns scheduler");
|
||||
}
|
||||
|
||||
/** Process-wide egress binding shared by the columns scheduler. */
|
||||
export interface ColumnsEgress {
|
||||
consume(hostKey: string): Promise<void>;
|
||||
}
|
||||
|
||||
let sharedEgress: Promise<ColumnsEgress> | undefined;
|
||||
|
||||
export function columnsEgress(): Promise<ColumnsEgress> {
|
||||
sharedEgress ??= (async () => {
|
||||
const specifier = pathToFileURL(resolveEgressPackage()).href;
|
||||
const pkg = await nativeImport(specifier);
|
||||
const budgets = new pkg.HostBudgets({ pool, hostClasses: { feeds: { capacity: 3, refillPerInterval: 10 } } });
|
||||
return { consume: (hostKey) => budgets.consume(hostKey, "feeds") };
|
||||
})();
|
||||
return sharedEgress;
|
||||
}
|
||||
|
||||
/** The API host key for a provider's feed calls, per the egress hostBudgetKey rule. */
|
||||
export function feedHostKey(provider: string, connection: ConnectorConnection): string {
|
||||
const raw = typeof connection.fields.instanceUrl === "string" && connection.fields.instanceUrl !== ""
|
||||
? connection.fields.instanceUrl
|
||||
: providerApiBaseUrl(provider);
|
||||
try {
|
||||
const url = new URL(raw);
|
||||
const host = url.hostname.replace(/^\[|\]$/g, "").replace(/\.$/, "").toLowerCase();
|
||||
const port = url.port || (url.protocol === "https:" ? "443" : "80");
|
||||
return `${host}:${port}`;
|
||||
} catch {
|
||||
return `${provider}:443`;
|
||||
}
|
||||
}
|
||||
|
||||
/** Fallback API endpoints per provider (used only when no instanceUrl exists). */
|
||||
const PROVIDER_API_HOSTS: Record<string, string> = {
|
||||
mastodon: "https://mastodon.social",
|
||||
bluesky: "https://bsky.app",
|
||||
youtube: "https://www.googleapis.com",
|
||||
reddit: "https://www.reddit.com",
|
||||
telegram: "https://api.telegram.org",
|
||||
twitter: "https://api.twitter.com",
|
||||
"mock-social": "https://mock.social",
|
||||
};
|
||||
|
||||
function providerApiBaseUrl(provider: string): string {
|
||||
return PROVIDER_API_HOSTS[provider] ?? `https://${provider}.example`;
|
||||
}
|
||||
|
||||
/** The credential accessor feedRegistry needs (shared with feeds.dependencies). */
|
||||
function credentials(): CredentialAccessor {
|
||||
const store = new PgConnectionStore();
|
||||
return {
|
||||
async getAccessToken(connection) { return credentials().getSecret(connection, "access_token"); },
|
||||
async getSecret(connection, key) {
|
||||
const stored = await store.get(connection.workspaceId, connection.id);
|
||||
if (!stored.credential) throw new Error("Credential missing");
|
||||
const bytes = await vault.open(connection.workspaceId, connection.id, stored.credential);
|
||||
const secrets = JSON.parse(bytes.toString()) as Record<string, unknown>;
|
||||
const value = secrets[key];
|
||||
if (typeof value !== "string" || !value) throw new Error("Credential missing");
|
||||
return value;
|
||||
},
|
||||
async put(connection, key, value) {
|
||||
const stored = await store.get(connection.workspaceId, connection.id);
|
||||
if (!stored.credential) throw new Error("Credential missing");
|
||||
const bytes = await vault.open(connection.workspaceId, connection.id, stored.credential);
|
||||
const secrets = JSON.parse(bytes.toString()) as Record<string, unknown>;
|
||||
secrets[key] = value;
|
||||
const sealed = await vault.seal(connection.workspaceId, connection.id, Buffer.from(JSON.stringify(secrets)));
|
||||
await store.rotateCredential(connection.workspaceId, connection.id, stored.credential, sealed);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
let sharedRegistry: ConnectorRegistry | undefined;
|
||||
|
||||
/** One registry per process, preserving per-host HTTP token buckets. */
|
||||
export function columnsRegistry(): ConnectorRegistry {
|
||||
sharedRegistry ??= feedRegistry(credentials());
|
||||
return sharedRegistry;
|
||||
}
|
||||
|
||||
/** Provider wiring for FeedsModule (see feeds.module.ts). */
|
||||
export const columnsProviders = [
|
||||
{ provide: COLUMN_STORE_DEPENDENCIES, useFactory: (): ColumnStoreDependencies => {
|
||||
const store = new PgConnectionStore();
|
||||
return {
|
||||
providerOf: async (workspaceId, connectionId) => {
|
||||
const rows = await store.list(workspaceId);
|
||||
const row = rows.find((candidate) => candidate.id === connectionId);
|
||||
if (row === undefined) throw new Error(`connection ${connectionId} not found`);
|
||||
return row.provider;
|
||||
},
|
||||
connections: (workspaceId) => store.list(workspaceId),
|
||||
};
|
||||
} },
|
||||
{ provide: COLUMNS_REGISTRY, useFactory: () => columnsRegistry() },
|
||||
{ provide: COLUMN_SCHEDULER_DEPENDENCIES, inject: [FeedsService], useFactory: (feeds: FeedsService): ColumnSchedulerDependencies => {
|
||||
const redis: Pick<ControlRedis, "get" | "set" | "eval"> = new RespControlRedis(process.env.REDIS_URL ?? "redis://127.0.0.1:6379");
|
||||
const connectionStore = new PgConnectionStore();
|
||||
return {
|
||||
redis,
|
||||
registry: columnsRegistry(),
|
||||
connection: async (workspaceId, id) => (await connectionStore.get(workspaceId, id)).connection,
|
||||
cacheHomeFeed: (workspaceId, connectionId, items) => feeds.cacheHomeFeed(workspaceId, connectionId, items),
|
||||
hostKeyOf: feedHostKey,
|
||||
consumeBudget: (hostKey) => columnsEgress().then((egress) => egress.consume(hostKey)),
|
||||
random: Math.random,
|
||||
now: Date.now,
|
||||
};
|
||||
} },
|
||||
];
|
||||
376
apps/api/src/feeds/columns.integration.test.ts
Normal file
376
apps/api/src/feeds/columns.integration.test.ts
Normal file
|
|
@ -0,0 +1,376 @@
|
|||
/**
|
||||
* Columns P1 integration tests — real Postgres (migrations ≤ 026 applied),
|
||||
* RLS-faithful via the app pool, following the video.integration.test pattern.
|
||||
*
|
||||
* Covers the DELIVER test list:
|
||||
* - CRUD + reorder, including RBAC 403 for an ungranted role
|
||||
* - seed idempotency (re-touch creates no duplicates)
|
||||
* - cooldown enforcement on the column endpoints
|
||||
* - cursor survival across a cache flush
|
||||
* - keyset items pagination (stable order, no skips/dupes across pages)
|
||||
* - dedup: same remote item from two columns sharing an identity → one row,
|
||||
* visible in both
|
||||
*/
|
||||
import "reflect-metadata";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { ForbiddenException, type CallHandler, type ExecutionContext, type INestApplication } from "@nestjs/common";
|
||||
import { NestFactory, Reflector } from "@nestjs/core";
|
||||
import { Pool } from "pg";
|
||||
import { afterAll, beforeAll, afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { InMemoryCredentials, type ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import type { ConnectorConnection, FeedItem } from "@tactical-media-broadcast-hub/core";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import { PERMISSION_KEY, RbacGuard } from "../rbac/rbac.guard.js";
|
||||
import { injectHttp } from "../connections/http-test-transport.js";
|
||||
import { feedRegistry } from "./feed-registry.js";
|
||||
import { FeedsService } from "./feeds.service.js";
|
||||
import { FeedsController } from "./feeds.controller.js";
|
||||
import { ColumnsController, COLUMNS_REGISTRY } from "./columns.controller.js";
|
||||
import { ColumnStore } from "./column-store.js";
|
||||
import { ColumnScheduler, type ColumnSchedulerDependencies } from "./column-scheduler.js";
|
||||
import { MemoryFeedRedis } from "./testing.js";
|
||||
import type { ConnectionSummary } from "../connections/connections.service.js";
|
||||
|
||||
const dbAvailable = process.env.DATABASE_URL !== undefined;
|
||||
// Setup runs as the table owner (superuser in dev) because memberships has
|
||||
// FORCE RLS and the seed path needs to insert without a tenant context.
|
||||
const ownerPool = new Pool({ connectionString: process.env.SUPER_DATABASE_URL ?? process.env.DATABASE_URL });
|
||||
|
||||
// The app pool is imported lazily where needed; ColumnStore's own fallback
|
||||
// deps use withTenant from db/client, but these tests always inject explicit
|
||||
// deps, so no module-scope side-effectful import is required.
|
||||
|
||||
const owner: SessionPayload = { userId: randomUUID(), workspaceId: randomUUID(), role: "Owner", authzEpoch: 1, issuedAt: 0, expiresAt: 9_999_999_999 };
|
||||
const viewer: SessionPayload = { ...owner, userId: randomUUID(), role: "Moderator" }; // no streams.columns.manage
|
||||
|
||||
const item = (remoteId: string, minutesAgo: number): FeedItem => ({
|
||||
id: remoteId, remoteId, provider: "mock-social", authorHandle: "alice", text: `post ${remoteId}`,
|
||||
mediaUrls: [], createdAt: new Date(Date.now() - minutesAgo * 60_000), metrics: {},
|
||||
});
|
||||
|
||||
function setupRegistry(items: FeedItem[]): ConnectorRegistry {
|
||||
const registry = feedRegistry(new InMemoryCredentials());
|
||||
const mock = registry.get("mock-social");
|
||||
const original = mock.getFeed;
|
||||
if (original === undefined) throw new Error("mock-social has no feed");
|
||||
mock.getFeed = vi.fn(async () => ({ items }));
|
||||
return registry;
|
||||
}
|
||||
|
||||
describe.skipIf(!dbAvailable)("columns P1 (live DB)", () => {
|
||||
let app: INestApplication;
|
||||
let store: ColumnStore;
|
||||
let scheduler: ColumnScheduler;
|
||||
let feeds: FeedsService;
|
||||
let redis: MemoryFeedRedis;
|
||||
let connections: Array<{ summary: ConnectionSummary; connection: ConnectorConnection }>;
|
||||
let items: FeedItem[];
|
||||
let granted = true;
|
||||
let registry: ConnectorRegistry;
|
||||
|
||||
beforeAll(async () => {
|
||||
// Users first (ids are DB-generated and captured for memberships), then
|
||||
// workspace + members, in the owner pool which bypasses RLS for setup.
|
||||
const inserted = await ownerPool.query<{ id: string }>(
|
||||
`INSERT INTO users (external_id, email, display_name, username)
|
||||
VALUES ($1,$2,'Columns P1 Owner',$3) RETURNING id`, [`u-${randomUUID()}`, `o-${randomUUID()}@example.com`, `p1o${randomUUID().slice(0, 8)}`]);
|
||||
(owner as { userId: string }).userId = String(inserted.rows[0]?.id);
|
||||
const viewerRow = await ownerPool.query<{ id: string }>(
|
||||
`INSERT INTO users (external_id, email, display_name, username)
|
||||
VALUES ($1,$2,'Columns P1 Viewer',$3) RETURNING id`, [`u-${randomUUID()}`, `v-${randomUUID()}@example.com`, `p1v${randomUUID().slice(0, 8)}`]);
|
||||
(viewer as { userId: string }).userId = String(viewerRow.rows[0]?.id);
|
||||
const workspace = await ownerPool.query(
|
||||
"INSERT INTO workspaces (name, slug) VALUES ('columns p1', $1) RETURNING id", [`cols-${randomUUID()}`]);
|
||||
const workspaceId = String(workspace.rows[0]?.id);
|
||||
// memberships is FORCE RLS: insert inside the workspace's tenant context.
|
||||
const setupClient = await ownerPool.connect();
|
||||
try {
|
||||
await setupClient.query("BEGIN");
|
||||
await setupClient.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]);
|
||||
await setupClient.query(
|
||||
"INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1,$2,'Owner'), ($1,$3,'Moderator')",
|
||||
[workspaceId, owner.userId, viewer.userId]);
|
||||
await setupClient.query("COMMIT");
|
||||
} finally {
|
||||
setupClient.release();
|
||||
}
|
||||
(owner as { workspaceId: string }).workspaceId = workspaceId;
|
||||
(viewer as { workspaceId: string }).workspaceId = workspaceId;
|
||||
|
||||
items = [item("i-1", 1), item("i-2", 5), item("i-3", 10), item("i-4", 30), item("i-5", 60)];
|
||||
registry = setupRegistry(items);
|
||||
|
||||
// Two feed-capable connections in the workspace: the seeding input.
|
||||
connections = [];
|
||||
for (let index = 0; index < 2; index += 1) {
|
||||
// The DB generates the id (connections has no id column we may set);
|
||||
// RETURNING it keeps the FK target and the summary in sync.
|
||||
const inserted = await ownerPool.query<{ id: string; created_at: Date }>(
|
||||
"INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1,'mock-social',$2) RETURNING id, created_at", [owner.workspaceId, `ch-${index}`]);
|
||||
const id = String(inserted.rows[0]?.id);
|
||||
const createdAt = inserted.rows[0]?.created_at as Date;
|
||||
const summary: ConnectionSummary = {
|
||||
id, provider: "mock-social", channelId: `ch-${index}`, fields: {}, health: "healthy",
|
||||
credentialRotatedAt: null, recentPublishes: [], createdAt: createdAt.toISOString(),
|
||||
};
|
||||
const connection: ConnectorConnection = {
|
||||
id, workspaceId: owner.workspaceId, provider: "mock-social", channelId: `ch-${index}`,
|
||||
credentialId: { kind: "credential-ref", id }, fields: {}, createdAt,
|
||||
};
|
||||
connections.push({ summary, connection });
|
||||
}
|
||||
|
||||
redis = new MemoryFeedRedis();
|
||||
store = new ColumnStore({
|
||||
providerOf: async (_ws, id) => connections.find((row) => row.summary.id === id)?.summary.provider ?? "mock-social",
|
||||
connections: async (ws) => ws === owner.workspaceId ? connections.map((row) => row.summary) : [],
|
||||
});
|
||||
feeds = new FeedsService({
|
||||
redis, registry, random: () => 0.5, requestsPerSecond: () => 0.5,
|
||||
workspaces: async () => [owner.workspaceId],
|
||||
connections: async (ws) => ws === owner.workspaceId ? connections.map((row) => row.summary) : [],
|
||||
connection: async (ws, id) => { const row = connections.find((c) => c.summary.id === id); if (!row || ws !== owner.workspaceId) throw new Error(); return row.connection; },
|
||||
visible: async (s) => s.workspaceId === owner.workspaceId ? connections.map((row) => row.summary.id) : [],
|
||||
audit: async () => undefined,
|
||||
}, store);
|
||||
scheduler = new ColumnScheduler(store, {
|
||||
redis, registry,
|
||||
connection: async (ws, id) => { const row = connections.find((c) => row_summary(c) === id); if (!row || ws !== owner.workspaceId) throw new Error(); return row.connection; },
|
||||
cacheHomeFeed: async (ws, id, normalized) => { if (ws === owner.workspaceId) await feeds.cacheHomeFeed(ws, id, normalized); },
|
||||
hostKeyOf: () => "mock.social:443",
|
||||
consumeBudget: async () => undefined,
|
||||
random: () => 0.5,
|
||||
now: () => Date.now(),
|
||||
});
|
||||
|
||||
// Both poll loops stay off: the feeds discovery timer would contend with
|
||||
// ensureSeeded's workspace row lock (this suite drives polls explicitly).
|
||||
vi.spyOn(scheduler, "onApplicationBootstrap").mockImplementation(() => undefined);
|
||||
vi.spyOn(feeds, "onApplicationBootstrap").mockImplementation(() => undefined);
|
||||
vi.spyOn(RbacGuard.prototype, "canActivate").mockImplementation(async (context: ExecutionContext) => {
|
||||
const request = context.switchToHttp().getRequest<{ headers: Record<string, string>; session?: SessionPayload }>();
|
||||
if (request.headers.authorization !== "Bearer test-session") throw new ForbiddenException("unauthenticated");
|
||||
const session = request.headers["x-test-user"] === "viewer" ? viewer : owner;
|
||||
const permission = new Reflector().get<string>(PERMISSION_KEY, context.getHandler());
|
||||
// The seeded RBAC: streams.columns.manage is Owner/Admin/Editor only.
|
||||
if (permission === "streams.columns.manage" && !granted) throw new ForbiddenException("missing permission");
|
||||
request.session = session;
|
||||
return true;
|
||||
});
|
||||
app = await NestFactory.create({
|
||||
module: class TestModule {},
|
||||
controllers: [ColumnsController, FeedsController],
|
||||
providers: [
|
||||
{ provide: FeedsService, useValue: feeds },
|
||||
{ provide: ColumnStore, useValue: store },
|
||||
{ provide: ColumnScheduler, useValue: scheduler },
|
||||
{ provide: COLUMNS_REGISTRY, useValue: registry },
|
||||
],
|
||||
}, { logger: false });
|
||||
await app.init();
|
||||
});
|
||||
|
||||
afterEach(() => { granted = true; });
|
||||
|
||||
afterAll(async () => {
|
||||
await app?.close();
|
||||
if (owner.workspaceId !== undefined) await ownerPool.query("DELETE FROM workspaces WHERE id = $1", [owner.workspaceId]);
|
||||
await ownerPool.end();
|
||||
});
|
||||
|
||||
const request = (path: string, method = "GET", body?: unknown, user: "owner" | "viewer" = "owner") =>
|
||||
injectHttp(app.getHttpAdapter().getInstance(), `http://api.test${path}`, {
|
||||
method, headers: { authorization: "Bearer test-session", "x-test-user": user, "content-type": "application/json" },
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
});
|
||||
|
||||
it("seeds one platform_feed column per feed connection, idempotently, with sources", async () => {
|
||||
const first = await request("/feeds/columns");
|
||||
expect(first.status).toBe(200);
|
||||
const seeded = await first.json() as Array<{ id: string; type: string; title: string; sources: Array<{ sourceKind: string; state: string }> }>;
|
||||
expect(seeded).toHaveLength(2);
|
||||
expect(seeded.every((column) => column.type === "platform_feed")).toBe(true);
|
||||
expect(seeded[0]?.sources).toHaveLength(1);
|
||||
expect(seeded[0]?.sources[0]?.sourceKind).toBe("home");
|
||||
// Titles reproduce the implicit column the UI renders today.
|
||||
expect(seeded[0]?.title).toContain("·");
|
||||
|
||||
// Second touch seeds nothing.
|
||||
const second = await request("/feeds/columns");
|
||||
expect(await second.json()).toHaveLength(2);
|
||||
// And the DB agrees: two columns, two source rows, one per connection.
|
||||
const rows = await ownerPool.query("SELECT count(*)::int AS c FROM stream_columns WHERE workspace_id = $1", [owner.workspaceId]);
|
||||
expect(rows.rows[0]?.c).toBe(2);
|
||||
const sources = await ownerPool.query("SELECT count(*)::int AS c FROM column_sources WHERE workspace_id = $1", [owner.workspaceId]);
|
||||
expect(sources.rows[0]?.c).toBe(2);
|
||||
});
|
||||
|
||||
it("enforces streams.columns.manage on writes while reads stay on draft.view", async () => {
|
||||
granted = false;
|
||||
expect((await request("/feeds/columns", "POST", { type: "keyword", title: "x", spec: { query: "q", platforms: ["mock-social"] } }, "viewer")).status).toBe(403);
|
||||
granted = true;
|
||||
const created = await request("/feeds/columns", "POST", { type: "keyword", title: "news", spec: { query: "anarch", platforms: ["mock-social"] } });
|
||||
expect(created.status).toBe(201);
|
||||
const column = await created.json() as { id: string; sources: Array<{ sourceKind: string }> };
|
||||
expect(column.sources.map((source) => source.sourceKind)).toEqual(["search"]);
|
||||
// Reads: viewer (no manage permission) still reads columns and items.
|
||||
expect((await request(`/feeds/columns/${column.id}`, "GET", undefined, "viewer")).status).toBe(200);
|
||||
granted = false;
|
||||
expect((await request(`/feeds/columns/${column.id}`, "PATCH", { title: "z" })).status).toBe(403);
|
||||
expect((await request(`/feeds/columns/${column.id}`, "DELETE")).status).toBe(403);
|
||||
expect((await request("/feeds/columns/reorder", "POST", { ids: [] })).status).toBe(403);
|
||||
granted = true;
|
||||
await request(`/feeds/columns/${column.id}`, "DELETE");
|
||||
});
|
||||
|
||||
it("rejects invalid specs with 422 and never stores them", async () => {
|
||||
const bad = await request("/feeds/columns", "POST", { type: "keyword", title: "bad", spec: { query: "", platforms: ["mock-social"] } });
|
||||
expect(bad.status).toBe(422);
|
||||
const badConnection = await request("/feeds/columns", "POST", { type: "platform_feed", title: "bad", spec: { connectionId: randomUUID() } });
|
||||
expect(badConnection.status).toBe(422);
|
||||
const count = await ownerPool.query(
|
||||
"SELECT count(*)::int AS c FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL", [owner.workspaceId]);
|
||||
expect(count.rows[0]?.c).toBe(2); // only the two seeded columns (the RBAC test's column was soft-deleted)
|
||||
});
|
||||
|
||||
it("polls due sources, persists items, and fans out to columns sharing an identity", async () => {
|
||||
const columns = await (await request("/feeds/columns")).json() as Array<{ id: string; sources: Array<{ id: string }> }>;
|
||||
const first = columns[0]!;
|
||||
const outcome = await scheduler.pollWorkspaceDue(owner.workspaceId);
|
||||
expect(outcome.polled).toBeGreaterThanOrEqual(2); // both seeded home sources
|
||||
|
||||
// Items landed in the store for the polled column.
|
||||
const page = await store.columnItems(owner.workspaceId, first.id, { limit: 10 });
|
||||
expect(page.items.length).toBeGreaterThanOrEqual(1);
|
||||
expect(page.items.every((stored) => stored.item.remoteId.startsWith("i-"))).toBe(true);
|
||||
// The source row moved to ready with a next poll scheduled.
|
||||
const sources = await store.sources(owner.workspaceId, first.id);
|
||||
expect(sources[0]?.state).toBe("ready");
|
||||
expect(sources[0]?.nextPollAt).not.toBeNull();
|
||||
});
|
||||
|
||||
it("keyset pagination: pages in stable order, no skips or duplicates, cursor survives cache flush", async () => {
|
||||
const columns = await (await request("/feeds/columns")).json() as Array<{ id: string }>;
|
||||
const columnId = columns[0]!.id;
|
||||
const seen: string[] = [];
|
||||
let cursor: string | null = null;
|
||||
// limit 2 over 5 items = 3 pages (2,2,1)
|
||||
for (let page = 0; page < 3; page += 1) {
|
||||
const response = await request(`/feeds/columns/${columnId}/items?limit=2${cursor === null ? "" : `&cursor=${cursor}`}`);
|
||||
expect(response.status).toBe(200);
|
||||
const body = await response.json() as { items: Array<{ item: { remoteId: string } }>; nextCursor: string | null };
|
||||
for (const stored of body.items) seen.push(stored.item.remoteId);
|
||||
cursor = body.nextCursor;
|
||||
if (cursor === null) break;
|
||||
}
|
||||
expect(seen.length).toBe(new Set(seen).size); // no duplicates
|
||||
expect(seen).toHaveLength(items.length); // no skips
|
||||
|
||||
// Cursor survival: flush every cache key, then continue from a cursor
|
||||
// minted before the flush — the store page must pick up exactly where the
|
||||
// cache left off because the cursor is a position in the stored order.
|
||||
redis.entries.clear();
|
||||
const beforeFlush = await request(`/feeds/columns/${columnId}/items?limit=1`);
|
||||
const firstCursor = ((await beforeFlush.json()) as { nextCursor: string | null }).nextCursor;
|
||||
expect(firstCursor).not.toBeNull();
|
||||
redis.entries.clear();
|
||||
const afterFlush = await request(`/feeds/columns/${columnId}/items?limit=10&cursor=${firstCursor}`);
|
||||
expect(afterFlush.status).toBe(200);
|
||||
const pageItems = (await afterFlush.json()) as { items: Array<{ item: { remoteId: string } }> };
|
||||
// Everything strictly after the first item, in keyset order.
|
||||
expect(pageItems.items.map((stored) => stored.item.remoteId)).toEqual(seen.slice(1));
|
||||
});
|
||||
|
||||
it("cooldown blocks a forced refresh from issuing a second poll", async () => {
|
||||
const columns = await (await request("/feeds/columns")).json() as Array<{ id: string }>;
|
||||
const columnId = columns[0]!.id;
|
||||
const before = registry.get("mock-social").getFeed as ReturnType<typeof vi.fn>;
|
||||
const callsBefore = before.mock.calls.length;
|
||||
const outcome = await scheduler.pollWorkspaceDue(owner.workspaceId);
|
||||
expect(outcome.polled).toBe(0); // sources just polled: next_poll_at is in the future
|
||||
// The HTTP refresh endpoint cannot bypass the persisted floor: the poll
|
||||
// happened seconds ago, so force-due refuses even though the Redis
|
||||
// cooldown keys were flushed (cursor-survival test) — next_poll_at is the
|
||||
// authoritative guard, not the cache.
|
||||
const refreshed = await request(`/feeds/columns/${columnId}/refresh`, "POST");
|
||||
expect(refreshed.status).toBe(201);
|
||||
const body = await refreshed.json() as { refreshed: number };
|
||||
expect(body.refreshed).toBe(0); // DB floor refused the forced poll
|
||||
expect(before.mock.calls.length).toBe(callsBefore);
|
||||
});
|
||||
|
||||
it("dedups the same remote item across columns sharing an identity", async () => {
|
||||
// Second column on the same connection as the first: same identity.
|
||||
const columns = await (await request("/feeds/columns")).json() as Array<{ id: string; sources: Array<{ id: string }> }>;
|
||||
const [first, second] = columns;
|
||||
expect(first && second).toBeTruthy();
|
||||
// Make both due again, then poll: one identity, one connector call.
|
||||
await ownerPool.query("UPDATE column_sources SET next_poll_at = now() WHERE workspace_id = $1", [owner.workspaceId]);
|
||||
const getFeed = registry.get("mock-social").getFeed as ReturnType<typeof vi.fn>;
|
||||
const before = getFeed.mock.calls.length;
|
||||
const outcome = await scheduler.pollWorkspaceDue(owner.workspaceId);
|
||||
expect(outcome.polled).toBe(2);
|
||||
expect(getFeed.mock.calls.length).toBe(before + 2); // two source rows, two calls (per-source semantics)
|
||||
// One feed_items row per remote item — the dedup key is workspace-wide.
|
||||
const stored = await ownerPool.query("SELECT count(*)::int AS c FROM feed_items WHERE workspace_id = $1", [owner.workspaceId]);
|
||||
expect(stored.rows[0]?.c).toBe(items.length);
|
||||
// Both columns surface the same items.
|
||||
const left = await store.columnItems(owner.workspaceId, first!.id, { limit: 10 });
|
||||
const right = await store.columnItems(owner.workspaceId, second!.id, { limit: 10 });
|
||||
expect(left.items).toHaveLength(items.length);
|
||||
expect(right.items).toHaveLength(items.length);
|
||||
// And a cursor minted on one column pages identically on the other.
|
||||
expect(left.items.map((row) => row.item.remoteId)).toEqual(right.items.map((row) => row.item.remoteId));
|
||||
});
|
||||
|
||||
it("reorder requires the full id list and rewrites positions densely", async () => {
|
||||
const columns = await (await request("/feeds/columns")).json() as Array<{ id: string; position: number }>;
|
||||
const ids = columns.map((column) => column.id);
|
||||
const partial = await request("/feeds/columns/reorder", "POST", { ids: [ids[0]!] });
|
||||
expect(partial.status).toBe(400);
|
||||
const reordered = await request("/feeds/columns/reorder", "POST", { ids: [...ids].reverse() });
|
||||
expect(reordered.status).toBe(201);
|
||||
const after = await reordered.json() as Array<{ id: string; position: number }>;
|
||||
expect(after.map((column) => column.id)).toEqual([...ids].reverse());
|
||||
expect(after.map((column) => column.position)).toEqual([0, 1]);
|
||||
});
|
||||
|
||||
it("import-saved-searches creates idempotent saved_search columns", async () => {
|
||||
// No search-capable provider is connected (mock-social declares none), so
|
||||
// the implicit-platforms path correctly skips everything rather than
|
||||
// inventing a platform the workspace cannot search.
|
||||
const skipped = await request("/feeds/columns/import-saved-searches", "POST", { queries: ["anarch"] });
|
||||
expect(skipped.status).toBe(201);
|
||||
expect(await skipped.json()).toEqual({ imported: 0, skipped: 1 });
|
||||
// Explicit platforms still import (the UI knows which it used locally).
|
||||
const imported = await request("/feeds/columns/import-saved-searches", "POST", { queries: ["anarch", "mutual aid"], platforms: ["mock-social"] });
|
||||
expect(imported.status).toBe(201);
|
||||
expect(await imported.json()).toEqual({ imported: 2, skipped: 0 });
|
||||
// Re-import: idempotent per query.
|
||||
const again = await request("/feeds/columns/import-saved-searches", "POST", { queries: ["anarch", "squatting"], platforms: ["mock-social"] });
|
||||
expect(await again.json()).toEqual({ imported: 1, skipped: 1 });
|
||||
const columns = await (await request("/feeds/columns")).json() as Array<{ type: string; title: string }>;
|
||||
const searches = columns.filter((column) => column.type === "saved_search");
|
||||
expect(searches.map((column) => column.title).sort()).toEqual(["anarch", "mutual aid", "squatting"]);
|
||||
// Unknown platform ids are rejected up front (P0a schema rules).
|
||||
const bad = await request("/feeds/columns/import-saved-searches", "POST", { queries: ["x"], platforms: ["not-a-provider"] });
|
||||
expect(bad.status).toBe(422);
|
||||
});
|
||||
|
||||
it("GET /feeds keeps its external shape and gains additive meta.columns", async () => {
|
||||
const response = await request("/feeds");
|
||||
expect(response.status).toBe(200);
|
||||
const feed = await response.json() as { items: unknown[]; nextCursor: string | null; meta: { configuredFeeds: number; columns?: Array<{ id: string }> } };
|
||||
expect(Array.isArray(feed.items)).toBe(true);
|
||||
expect(feed.meta.configuredFeeds).toBe(2);
|
||||
expect(feed.meta.columns?.length).toBeGreaterThanOrEqual(2); // seeded columns surface
|
||||
});
|
||||
|
||||
it("invalid cursor is a 400 and unknown column a 404", async () => {
|
||||
expect((await request(`/feeds/columns/${randomUUID()}/items`)).status).toBe(404);
|
||||
expect((await request("/feeds/columns/not-a-uuid/items")).status).toBe(400);
|
||||
expect((await request("/feeds/columns/00000000-0000-0000-0000-000000000000/items?cursor=garbage")).status).toBe(400);
|
||||
});
|
||||
});
|
||||
|
||||
function row_summary(row: { summary: ConnectionSummary }): string { return row.summary.id; }
|
||||
366
apps/api/src/feeds/feed-normalize.test.ts
Normal file
366
apps/api/src/feeds/feed-normalize.test.ts
Normal file
|
|
@ -0,0 +1,366 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import type { ConnectorConnection, FeedItem } from "@tactical-media-broadcast-hub/core";
|
||||
import { createMockSocialConnector, createMockSocialState } from "../../../../packages/connector-mock-social/src/index.js";
|
||||
import { DEFAULT_MEDIA_PROXY, parseProviderHtml, type CustomEmoji, type LinkPreview, type MediaProxyPolicy, type RichTextDocument } from "../../../../packages/safe-content/src/index.js";
|
||||
import { astToPlainText, feedAuthorRef, feedItemMedia, normalizeFeedItem, type MediaRef, type NormalizedFeedItem } from "./feed-normalize.js";
|
||||
import { FeedsService, feedCacheKey } from "./feeds.service.js";
|
||||
import type { FeedDependencies } from "./feeds.dependencies.js";
|
||||
import { MemoryFeedRedis } from "./testing.js";
|
||||
|
||||
/** Per-connector fixtures (design §3.2 provider table) + hostile-HTML cases. */
|
||||
interface ProviderFixture {
|
||||
provider: string;
|
||||
contentKind?: "text" | "html";
|
||||
instanceUrl?: string;
|
||||
emojis?: readonly CustomEmoji[];
|
||||
item: FeedItem;
|
||||
url?: unknown;
|
||||
expiresAt?: unknown;
|
||||
linkPreview?: unknown;
|
||||
}
|
||||
|
||||
const PROXY: MediaProxyPolicy = DEFAULT_MEDIA_PROXY;
|
||||
const connection = (fields: Record<string, string> = {}): ConnectorConnection => ({
|
||||
id: "conn-1", workspaceId: "ws-1", provider: "mastodon", channelId: "conn-1",
|
||||
credentialId: { kind: "credential-ref", id: "cred-1" }, fields, createdAt: new Date(),
|
||||
});
|
||||
|
||||
const mastodonStatus: FeedItem = {
|
||||
id: "1001", provider: "mastodon", remoteId: "1001", authorHandle: "alice@foreso.social",
|
||||
// The exact HTML shape Mastodon emits: p/br/a, mention + hashtag + invisible
|
||||
// ellipsis link spans, HTML-escaped ampersands, raw emoji <img> tags.
|
||||
text: '<p>A call for <span class="h-card"><a href="https://foreso.social/@bob" class="u-url mention">@<span>bob</span></a></span> & friends</p><p>#<a href="https://foreso.social/tags/acampada26" rel="tag">acampada26</a> kicks off <a href="https://example.org/manifesto" rel="nofollow noopener noreferrer noopener" target="_blank"><span class="invisible">https://</span><span class="ellipsis">example.org/manifes</span><span class="invisible">to</span></a> <img src="https://foreso.social/emoji/ok.png" alt=":ok:" draggable="false"><br>line two <b>bolded</b></p>',
|
||||
mediaUrls: ["https://cdn.foreso.social/media/1234.png", "https://cdn.foreso.social/media/clip.mp4"],
|
||||
createdAt: new Date("2026-10-06T12:00:00.000Z"),
|
||||
metrics: { reblogs: 2, replies: 1, favourites: 5, broken: Number.NaN, infinite: Number.POSITIVE_INFINITY },
|
||||
};
|
||||
const xssStatus: FeedItem = {
|
||||
id: "2002", provider: "mastodon", remoteId: "2002", authorHandle: "mallory@evil.example",
|
||||
text: '<p>hi <script>alert("pwned")</script><img src=x onerror="alert(1)"><iframe srcdoc="<script>alert(2)</script>"></iframe><style>body{}</style><svg><circle/></svg><a href="javascript:alert(3)">click</a><a href="https://ok.example/pass">pass</a><a href="java\tscript:alert(4)">tabbed</a></p><p>after</p>',
|
||||
mediaUrls: ["javascript:alert(5)", "data:text/html,evil", "//protocol-relative.example/x.png", "https://cdn.example/ok.png"],
|
||||
createdAt: new Date("2026-10-06T12:01:00.000Z"),
|
||||
metrics: {},
|
||||
};
|
||||
|
||||
const fixturePlainText: ProviderFixture = {
|
||||
// Plain-text provider source (Bluesky/Telegram-style): auto-linked, no HTML.
|
||||
provider: "bluesky",
|
||||
item: {
|
||||
id: "1", provider: "bluesky", remoteId: "at://did:plc:abc/app.bsky.feed.post/1",
|
||||
authorHandle: "acampada.bsky.social", mediaUrls: [], createdAt: new Date("2026-10-06T10:00:00.000Z"),
|
||||
metrics: { likes: 3 },
|
||||
text: "Line one\n\n hub https://github.com/eche/acampada26/blob/main/core/search.ts #acampada26 @alice@foreso.social",
|
||||
},
|
||||
url: "https://bsky.app/profile/acampada.bsky.social/post/1",
|
||||
};
|
||||
const fixtureMastodonHtml: ProviderFixture = {
|
||||
// Mastodon status HTML: allowlist parser, mention + ellipsis link + emoji.
|
||||
provider: "mastodon",
|
||||
contentKind: "html",
|
||||
emojis: [{ shortcode: "ok", url: "https://foreso.social/emoji/ok.png" }],
|
||||
item: mastodonStatus,
|
||||
url: "https://foreso.social/@alice/1001",
|
||||
};
|
||||
const fixtureHostileHtml: ProviderFixture = {
|
||||
// Hostile HTML from a compromised/bogus federated peer.
|
||||
provider: "mastodon",
|
||||
contentKind: "html",
|
||||
instanceUrl: "https://EVIL.example:443/",
|
||||
item: xssStatus,
|
||||
url: "javascript:alert(6)",
|
||||
linkPreview: { url: "javascript:alert(7)", title: "javascript:alert(8)", image: "javascript:alert(9)" },
|
||||
};
|
||||
const fixtureMastodonLinkPreview: ProviderFixture = {
|
||||
// Mastodon HTML + card (link-preview) payload as one source row.
|
||||
provider: "mastodon",
|
||||
contentKind: "html",
|
||||
item: { ...mastodonStatus, id: "1002", remoteId: "1002", text: '<p>with a <span class="h-card"><a href="https://foreso.social/@bob" class="u-url mention">@<span>bob</span></a></span> card</p>', mediaUrls: [] },
|
||||
url: "https://foreso.social/@alice/1002",
|
||||
linkPreview: {
|
||||
url: "https://example.org/manifesto", title: "Manifesto <scripts> & plans",
|
||||
description: "A plan", siteName: "example dot org", image: "https://cdn.example.org/cover.png",
|
||||
},
|
||||
};
|
||||
const fixtures: Record<"plainText" | "mastodonHtml" | "hostileHtml" | "mastodonLinkPreview", ProviderFixture> = {
|
||||
plainText: fixturePlainText,
|
||||
mastodonHtml: fixtureMastodonHtml,
|
||||
hostileHtml: fixtureHostileHtml,
|
||||
mastodonLinkPreview: fixtureMastodonLinkPreview,
|
||||
};
|
||||
|
||||
/** No provider CDN URL outside the media-proxy query string, ever. */
|
||||
function assertProxyOnly(media: MediaRef[]): void {
|
||||
const serialized = JSON.stringify(media);
|
||||
let index = serialized.indexOf("https://");
|
||||
while (index !== -1) {
|
||||
expect(serialized.slice(0, index)).toContain(`${PROXY.proxyPath}?url=`);
|
||||
index = serialized.indexOf("https://", index + 1);
|
||||
}
|
||||
for (const ref of media) expect(ref.proxyUrl.startsWith(`${PROXY.proxyPath}?url=`)).toBe(true);
|
||||
}
|
||||
|
||||
/** Raw-remnant scan applied to every fixture's AST, plain text and media. */
|
||||
function assertZeroRawRemnants(ast: RichTextDocument, plainText: string, media: MediaRef[]): void {
|
||||
const astJson = JSON.stringify(ast);
|
||||
const hostile: RegExp[] = [
|
||||
/<\s*(script|iframe|style|img|svg|math|template|form|object|embed|applet|frame)\b/i,
|
||||
/<\s*\/\s*/, /<!--/, /&#/, /&|<|>|"|'| /i,
|
||||
/javascript:/i, /vbscript:/i, /data:/i, /file:/i, /\/\/evil/, /\/\/protocol-relative/,
|
||||
/on(?:error|load|click|mouseover)\b|on\w+\s*=/i, /srcdoc|src\s*=|href\s*=/i, /alert\s*\(/i,
|
||||
];
|
||||
for (const haystack of [astJson, plainText, JSON.stringify(media)]) {
|
||||
for (const pattern of hostile) expect(haystack).not.toMatch(pattern);
|
||||
}
|
||||
expect(astJson).not.toContain("<p>");
|
||||
expect(astJson).not.toContain("<b>");
|
||||
assertProxyOnly(media);
|
||||
}
|
||||
|
||||
function normalizeFixture(fixture: ProviderFixture) {
|
||||
return normalizeFeedItem(fixture.item, { ...connection(fixture.instanceUrl ? { instanceUrl: fixture.instanceUrl } : {}), provider: fixture.provider }, {
|
||||
contentKind: fixture.contentKind,
|
||||
emojis: fixture.emojis,
|
||||
url: fixture.url,
|
||||
expiresAt: fixture.expiresAt,
|
||||
linkPreview: fixture.linkPreview,
|
||||
});
|
||||
}
|
||||
|
||||
const flat = (ast: RichTextDocument): string[] =>
|
||||
ast.children.map((paragraph) => paragraph.children.map((node) => node.type).join(","));
|
||||
|
||||
describe("feed normalization v2 (columns P0c)", () => {
|
||||
it("keeps legacy /feeds fields byte-identical while adding the v2 fields additively", () => {
|
||||
const normalized = normalizeFeedItem(mastodonStatus, connection(), { url: "https://foreso.social/@alice/1001" });
|
||||
expect(normalized).not.toBeNull();
|
||||
expect(normalized).toMatchObject({
|
||||
id: "1001", remoteId: "1001", provider: "mastodon", connectionId: "conn-1",
|
||||
authorHandle: "alice@foreso.social", text: mastodonStatus.text,
|
||||
mediaUrls: mastodonStatus.mediaUrls,
|
||||
metrics: { reblogs: 2, replies: 1, favourites: 5 }, // NaN/Infinity dropped
|
||||
createdAt: new Date("2026-10-06T12:00:00.000Z"),
|
||||
});
|
||||
expect(Object.keys(normalized?.metrics ?? {})).toEqual(["reblogs", "replies", "favourites"]);
|
||||
expect(normalized?.authorRef).toBe("mastodon:alice@foreso.social");
|
||||
expect(normalized?.url).toBe("https://foreso.social/@alice/1001");
|
||||
expect(normalized?.content?.type).toBe("doc");
|
||||
expect(typeof normalized?.plainText).toBe("string");
|
||||
expect(Array.isArray(normalized?.media)).toBe(true);
|
||||
expect(normalized?.expiresAt).toBeNull();
|
||||
});
|
||||
|
||||
it("normalizes the plain-text fixture with auto-linked http(s) URLs only", () => {
|
||||
const normalized = normalizeFixture(fixturePlainText);
|
||||
expect(normalized).not.toBeNull();
|
||||
assertZeroRawRemnants(normalized?.content as RichTextDocument, normalized?.plainText as string, normalized?.media as MediaRef[]);
|
||||
expect(flat(normalized?.content as RichTextDocument)).toEqual([
|
||||
"text",
|
||||
"text,link,text,hashtag,text,mention",
|
||||
]);
|
||||
const link = normalized?.content?.children[1]?.children.find((node) => node.type === "link");
|
||||
expect(link?.type === "link" && link.href).toBe("https://github.com/eche/acampada26/blob/main/core/search.ts");
|
||||
expect(link?.type === "link" && link.host).toBe("github.com");
|
||||
expect(normalized?.plainText).toBe("Line one\n\n hub github.com/eche/acampada26/blob/main/core/search.ts #acampada26 @alice@foreso.social");
|
||||
expect(normalized?.authorRef).toBe("bluesky:acampada.bsky.social");
|
||||
expect(normalized?.url).toBe("https://bsky.app/profile/acampada.bsky.social/post/1");
|
||||
expect(normalized?.media).toEqual([]);
|
||||
expect(normalized?.expiresAt).toBeNull();
|
||||
});
|
||||
|
||||
it("normalizes the Mastodon HTML fixture (mention, ellipsis link, emoji, breaks) with zero raw remnants", () => {
|
||||
const normalized = normalizeFixture(fixtureMastodonHtml);
|
||||
expect(normalized).not.toBeNull();
|
||||
const ast = normalized?.content as RichTextDocument;
|
||||
assertZeroRawRemnants(ast, normalized?.plainText as string, normalized?.media as MediaRef[]);
|
||||
expect(ast.truncated).toBe(false);
|
||||
// Invisible spans drop their URL scheme, the ellipsis span becomes "…",
|
||||
// the mention qualifies its handle from the href host, the emoji <img>
|
||||
// alt becomes a proxied emoji node, <b> unwraps to plain text.
|
||||
expect(normalized?.plainText).toBe("A call for @bob@foreso.social & friends\n\n#acampada26 kicks off example.org/manifes… :ok:\nline two bolded");
|
||||
expect(flat(ast)).toEqual([
|
||||
"text,mention,text",
|
||||
"text,link,text,link,text,emoji,linebreak,text",
|
||||
]);
|
||||
const mention = ast.children[0]?.children[1];
|
||||
expect(mention).toEqual({ type: "mention", handle: "bob@foreso.social", href: "https://foreso.social/@bob" });
|
||||
const link = ast.children[1]?.children.find((node) => node.type === "link" && node.href === "https://example.org/manifesto");
|
||||
expect(link?.type === "link" && link.host).toBe("example.org");
|
||||
expect(link?.type === "link" && link.children.map((child) => child.type === "text" ? child.text : null).join("")).toBe("example.org/manifes…");
|
||||
const emoji = ast.children[1]?.children.find((node) => node.type === "emoji");
|
||||
expect(emoji?.type === "emoji" && emoji.src).toBe(`${PROXY.proxyPath}?url=${encodeURIComponent("https://foreso.social/emoji/ok.png")}`);
|
||||
// Attachments become proxy media refs; kind is sniffed from the extension.
|
||||
expect(normalized?.media).toEqual([
|
||||
{ kind: "image", proxyUrl: `${PROXY.proxyPath}?url=${encodeURIComponent("https://cdn.foreso.social/media/1234.png")}` },
|
||||
{ kind: "video", proxyUrl: `${PROXY.proxyPath}?url=${encodeURIComponent("https://cdn.foreso.social/media/clip.mp4")}` },
|
||||
]);
|
||||
assertProxyOnly(normalized?.media as MediaRef[]);
|
||||
});
|
||||
|
||||
it("normalizes the hostile-HTML fixture: scripts, iframes, handlers and unsafe schemes leave no remnants", () => {
|
||||
const normalized = normalizeFixture(fixtureHostileHtml);
|
||||
expect(normalized).not.toBeNull();
|
||||
const ast = normalized?.content as RichTextDocument;
|
||||
assertZeroRawRemnants(ast, normalized?.plainText as string, normalized?.media as MediaRef[]);
|
||||
// Script/style/iframe/svg contents vanish entirely; img alt is empty; the
|
||||
// javascript: link degrades to its inert label; the tab-split "java\tscript:"
|
||||
// scheme is refused; only the one safe link survives as a real link node.
|
||||
expect(flat(ast)).toEqual(["text,link,text", "text"]);
|
||||
const onlyLink = ast.children[0]?.children[1];
|
||||
expect(onlyLink).toEqual({
|
||||
type: "link", href: "https://ok.example/pass", host: "ok.example",
|
||||
children: [{ type: "text", text: "pass" }],
|
||||
});
|
||||
expect(normalized?.plainText).toBe("hi clickpasstabbed\n\nafter");
|
||||
expect(normalized?.plainText).not.toMatch(/alert|pwned|onerror|javascript|srcdoc/i);
|
||||
// Federated host appends to the author ref via instanceUrl (punycoded, port stripped).
|
||||
expect(normalized?.authorRef).toBe("mastodon:mallory@evil.example");
|
||||
// Permalink with an unsafe scheme is refused, not carried.
|
||||
expect(normalized?.url).toBeNull();
|
||||
// Hostile media URLs are dropped wholesale; only the safe https URL is proxied.
|
||||
expect(normalized?.mediaUrls).toHaveLength(4); // legacy field stays as the source delivered it
|
||||
expect(normalized?.media).toEqual([{ kind: "image", proxyUrl: `${PROXY.proxyPath}?url=${encodeURIComponent("https://cdn.example/ok.png")}` }]);
|
||||
assertProxyOnly(normalized?.media as MediaRef[]);
|
||||
// Hostile card: unsafe URL/title/image all rejected → no preview survives.
|
||||
expect(normalized?.linkPreview).toBeUndefined();
|
||||
});
|
||||
|
||||
it("sanitizes the Mastodon link-preview card payload (text fields, proxied image, safe href)", () => {
|
||||
const normalized = normalizeFixture(fixtureMastodonLinkPreview);
|
||||
expect(normalized).not.toBeNull();
|
||||
const preview = normalized?.linkPreview as LinkPreview;
|
||||
expect(preview).not.toBeNull();
|
||||
expect(preview).toEqual({
|
||||
type: "linkPreview", href: "https://example.org/manifesto", host: "example.org",
|
||||
title: "Manifesto <scripts> & plans", description: "A plan", siteName: "example dot org",
|
||||
image: `${PROXY.proxyPath}?url=${encodeURIComponent("https://cdn.example.org/cover.png")}`,
|
||||
});
|
||||
// OpenGraph fields are text: the entity-decoded title carries no live markup.
|
||||
expect(preview.title).toContain("<scripts>");
|
||||
expect(preview.title).not.toMatch(/<|&/);
|
||||
assertZeroRawRemnants(normalized?.content as RichTextDocument, normalized?.plainText as string, normalized?.media as MediaRef[]);
|
||||
});
|
||||
|
||||
it("round-trips every fixture AST through JSON (storage shape) with a stable plainText", () => {
|
||||
for (const [name, fixture] of Object.entries(fixtures)) {
|
||||
const normalized = normalizeFixture(fixture);
|
||||
expect(normalized, name).not.toBeNull();
|
||||
const ast = normalized?.content as RichTextDocument;
|
||||
assertZeroRawRemnants(ast, normalized?.plainText as string, normalized?.media as MediaRef[]);
|
||||
const roundTripped = JSON.parse(JSON.stringify(ast)) as RichTextDocument;
|
||||
expect(roundTripped, name).toEqual(ast);
|
||||
expect(astToPlainText(roundTripped), name).toBe(normalized?.plainText);
|
||||
}
|
||||
});
|
||||
|
||||
it("matches the reference parseProviderHtml derivation for the Mastodon fixture", () => {
|
||||
const fixture = fixtureMastodonHtml;
|
||||
const reference = parseProviderHtml(fixture.item.text, { emojis: fixture.emojis });
|
||||
const normalized = normalizeFixture(fixture);
|
||||
expect(normalized?.content).toEqual(reference);
|
||||
expect(astToPlainText(reference)).toBe(normalized?.plainText);
|
||||
});
|
||||
|
||||
it("builds canonical author refs: handle host wins, instanceUrl fills federated hosts, IDN punycodes", () => {
|
||||
expect(feedAuthorRef("mastodon", "alice@foreso.social")).toBe("mastodon:alice@foreso.social");
|
||||
expect(feedAuthorRef("mastodon", "alice", "https://fedoraproject.org/@alice")).toBe("mastodon:alice@fedoraproject.org");
|
||||
expect(feedAuthorRef("mastodon", "alice", "https://bücher.example:8443/path")).toBe("mastodon:alice@xn--bcher-kva.example");
|
||||
expect(feedAuthorRef("mastodon", "alice", "gopher://odd.example")).toBe("mastodon:alice");
|
||||
expect(feedAuthorRef("mastodon", "@bob@", "https://ok.example")).toBeNull();
|
||||
expect(feedAuthorRef("mastodon", "has space")).toBeNull();
|
||||
expect(feedAuthorRef("mastodon", "")).toBeNull();
|
||||
});
|
||||
|
||||
it("expires items per ToS: explicit Date, date-like string, and unparseable → null", () => {
|
||||
const base = { ...mastodonStatus, mediaUrls: [] };
|
||||
const at = normalizeFeedItem(base, connection(), { expiresAt: new Date("2026-11-05T00:00:00.000Z") });
|
||||
expect(at?.expiresAt).toEqual(new Date("2026-11-05T00:00:00.000Z"));
|
||||
const fromString = normalizeFeedItem(base, connection(), { expiresAt: "2026-11-05T00:00:00.000Z" });
|
||||
expect(fromString?.expiresAt).toEqual(new Date("2026-11-05T00:00:00.000Z"));
|
||||
expect(normalizeFeedItem(base, connection(), { expiresAt: "not-a-date" })?.expiresAt).toBeNull();
|
||||
expect(normalizeFeedItem(base, connection())?.expiresAt).toBeNull();
|
||||
});
|
||||
|
||||
it("drops items failing the legacy validation", () => {
|
||||
expect(normalizeFeedItem({ ...mastodonStatus, remoteId: 42 as unknown as string }, connection())).toBeNull();
|
||||
expect(normalizeFeedItem({ ...mastodonStatus, createdAt: new Date(Number.NaN) }, connection())).toBeNull();
|
||||
expect(normalizeFeedItem({ ...mastodonStatus, authorHandle: 7 as unknown as string }, connection())).toBeNull();
|
||||
expect(normalizeFeedItem(mastodonStatus, connection())).not.toBeNull();
|
||||
});
|
||||
|
||||
it("proxies only safe media: scheme tricks, protocol-relative and oversized URLs are dropped", () => {
|
||||
const media = feedItemMedia([
|
||||
"https://cdn.example/a.png", "javascript:alert(1)", "data:text/html,x", "//rel.example/b.png",
|
||||
"https://user:pass@cred.example/c.jpg", "not a url", "https://cdn.example/d.webm?v=2", 42,
|
||||
]);
|
||||
// Credential-bearing URLs are refused by the same safe-URL gate.
|
||||
expect(media.map((ref) => ref.kind)).toEqual(["image", "video"]); // extension sniff
|
||||
expect(media[1]?.proxyUrl).toBe(`${PROXY.proxyPath}?url=${encodeURIComponent("https://cdn.example/d.webm?v=2")}`);
|
||||
assertProxyOnly(media);
|
||||
expect(feedItemMedia("not-an-array")).toEqual([]);
|
||||
});
|
||||
|
||||
it("serves legacy cache payloads (no AST) and v2 payloads through the unchanged /feeds shape", async () => {
|
||||
const state = createMockSocialState();
|
||||
state.feed = [];
|
||||
const mock = createMockSocialConnector(state);
|
||||
const registry = new ConnectorRegistry(); registry.register(mock);
|
||||
const redis = new MemoryFeedRedis();
|
||||
const connectionId = randomUUID();
|
||||
const deps: FeedDependencies = {
|
||||
redis, registry, workspaces: async () => ["ws-1"],
|
||||
connections: async () => [{ id: connectionId, provider: "mock-social", channelId: connectionId, fields: {}, health: "healthy", credentialRotatedAt: null, recentPublishes: [], createdAt: new Date().toISOString() }],
|
||||
connection: async (_ws, id) => ({ id, workspaceId: "ws-1", provider: "mock-social", channelId: id, credentialId: { kind: "credential-ref", id }, fields: {}, createdAt: new Date() }),
|
||||
visible: async () => [connectionId], audit: async () => undefined, requestsPerSecond: () => 0.5, random: () => 0.5,
|
||||
};
|
||||
const service = new FeedsService(deps);
|
||||
const session = { workspaceId: "ws-1", userId: randomUUID(), role: "Owner", authzEpoch: 1, issuedAt: 0, expiresAt: 9_999_999_999 };
|
||||
const key = feedCacheKey("ws-1", connectionId);
|
||||
const legacy = { id: "legacy", remoteId: "legacy", provider: "mock-social", connectionId, authorHandle: "old@social", text: "legacy item", mediaUrls: ["https://cdn.old.example/a.png"], createdAt: new Date("2026-10-01T00:00:00.000Z"), metrics: { likes: 1 } };
|
||||
await redis.set(key, JSON.stringify({ items: [legacy], fetchedAt: new Date().toISOString(), handle: "old@social", displayName: "Old" }), { pxMs: 99_000 });
|
||||
const served = await service.read(session);
|
||||
expect(served.items).toHaveLength(1);
|
||||
expect(served.items[0]).toMatchObject({ remoteId: "legacy", text: "legacy item", metrics: { likes: 1 } });
|
||||
expect(served.items[0]?.createdAt).toBeInstanceOf(Date);
|
||||
expect("content" in (served.items[0] as object)).toBe(false); // additive: absent, not null
|
||||
expect("expiresAt" in (served.items[0] as object)).toBe(false);
|
||||
// A v2 payload keeps the legacy fields and adds the new ones through the same response.
|
||||
const normalized = normalizeFeedItem({ ...legacy, id: "v2", remoteId: "v2" }, { ...connection(), provider: "mock-social" }, { url: "https://old.example/legacy/2", expiresAt: new Date("2026-11-01T00:00:00.000Z") });
|
||||
await redis.set(key, JSON.stringify({ items: [normalized], fetchedAt: new Date().toISOString(), handle: "old@social", displayName: "Old" }), { pxMs: 99_000 });
|
||||
const v2Served = await service.read(session);
|
||||
expect(v2Served.items[0]).toMatchObject({ remoteId: "v2", text: "legacy item", authorRef: "mock-social:old@social", url: "https://old.example/legacy/2" });
|
||||
expect((v2Served.items[0] as { content?: { type?: string } }).content?.type).toBe("doc");
|
||||
expect(v2Served.items[0]?.createdAt).toBeInstanceOf(Date);
|
||||
expect((v2Served.items[0] as unknown as { expiresAt?: Date }).expiresAt).toEqual(new Date("2026-11-01T00:00:00.000Z"));
|
||||
});
|
||||
|
||||
it("still polls and serves end-to-end: connector feed → v2 cache → /feeds response", async () => {
|
||||
const state = createMockSocialState();
|
||||
state.feed = [{
|
||||
id: "e2e", remoteId: "e2e", provider: "mock-social", authorHandle: "carol@social",
|
||||
text: "End to end #acampada", mediaUrls: ["https://cdn.social/e2e.png"],
|
||||
createdAt: new Date("2026-10-06T09:00:00.000Z"), metrics: { likes: 2 },
|
||||
}];
|
||||
const mock = createMockSocialConnector(state);
|
||||
const registry = new ConnectorRegistry(); registry.register(mock);
|
||||
const deps: FeedDependencies = {
|
||||
redis: new MemoryFeedRedis(), registry, workspaces: async () => ["ws-1"],
|
||||
connections: async () => [{ id: "conn-e2e", provider: "mock-social", channelId: "conn-e2e", fields: {}, health: "healthy", credentialRotatedAt: null, recentPublishes: [], createdAt: new Date().toISOString() }],
|
||||
connection: async (_ws, id) => ({ id, workspaceId: "ws-1", provider: "mock-social", channelId: id, credentialId: { kind: "credential-ref", id }, fields: {}, createdAt: new Date() }),
|
||||
visible: async () => ["conn-e2e"], audit: async () => undefined, requestsPerSecond: () => 0.5, random: () => 0.5,
|
||||
};
|
||||
const service = new FeedsService(deps);
|
||||
expect(await service.pollWorkspace("ws-1")).toEqual({ polled: 1, skipped: 0, failed: 0 });
|
||||
const response = await service.read({ workspaceId: "ws-1", userId: randomUUID(), role: "Owner", authzEpoch: 1, issuedAt: 0, expiresAt: 9_999_999_999 });
|
||||
expect(response.items).toHaveLength(1);
|
||||
const item = response.items[0] as NormalizedFeedItem | undefined;
|
||||
expect(item).toBeDefined();
|
||||
expect(item).toMatchObject({ remoteId: "e2e", text: "End to end #acampada", authorRef: "mock-social:carol@social", metrics: { likes: 2 } });
|
||||
expect(item?.content?.type).toBe("doc");
|
||||
expect(item?.content?.children[0]?.children.map((node) => node.type)).toEqual(["text", "hashtag"]);
|
||||
expect(item?.plainText).toBe("End to end #acampada");
|
||||
expect(item?.media?.[0]?.proxyUrl).toBe(`${PROXY.proxyPath}?url=${encodeURIComponent("https://cdn.social/e2e.png")}`);
|
||||
assertZeroRawRemnants(item?.content as RichTextDocument, item?.plainText as string, item?.media as MediaRef[]);
|
||||
});
|
||||
});
|
||||
211
apps/api/src/feeds/feed-normalize.ts
Normal file
211
apps/api/src/feeds/feed-normalize.ts
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
/**
|
||||
* Feed-item normalization v2 (columns P0c — design §3.2 "Normalization" + §6).
|
||||
*
|
||||
* Every source type funnels into ONE `normalizeFeedItem()`. The legacy `/feeds`
|
||||
* response fields (`text`, `mediaUrls`, `metrics`, dates, ids) are unchanged;
|
||||
* the columns fields are strictly additive:
|
||||
*
|
||||
* - `authorRef` canonical `{provider}:{handle@host}`; the host is appended
|
||||
* only when resolvable (from the handle itself, else the connection's
|
||||
* `instanceUrl` field, IDN→punycode + lowercased by `URL`). No host →
|
||||
* `{provider}:{handle}`; invalid handle → null.
|
||||
* - `url` provider permalink candidate — only absolute, credential-free
|
||||
* http(s) URLs survive (`parseSafeHttpUrl`); anything else → null.
|
||||
* - `content` safe-content rich-text AST. The ONLY render source: raw HTML
|
||||
* never survives as markup (`contentKind: "html"` feeds Mastodon-style
|
||||
* sanitized HTML through the allowlist parser; plain text is auto-linked
|
||||
* http(s)-only). Scripts, iframes, styles, event handlers and unsafe
|
||||
* schemes are dropped or reduced to inert text by the parser.
|
||||
* - `plainText` derived from the AST at ingest (stable function of the AST)
|
||||
* for saved-search client filtering and previews.
|
||||
* - `media` proxy-minted media refs; raw provider CDN URLs are only ever
|
||||
* carried inside the proxy query (`/api/media-proxy?url=…`), never as the
|
||||
* browser-facing URL.
|
||||
* - `expiresAt` nullable ToS expiry (YouTube Data-API items: fetched+30d).
|
||||
* - `linkPreview` sanitized Mastodon `card`/OpenGraph payload or null.
|
||||
*
|
||||
* Legacy validation semantics are retained verbatim: finite dates, string
|
||||
* remoteId/handle/text, finite metrics; invalid items are dropped (null).
|
||||
*/
|
||||
import type { ConnectorConnection, FeedItem } from "@tactical-media-broadcast-hub/core";
|
||||
import {
|
||||
DEFAULT_MEDIA_PROXY,
|
||||
neutralizeText,
|
||||
parseLinkPreview,
|
||||
parsePlainText,
|
||||
parseProviderHtml,
|
||||
parseSafeHttpUrl,
|
||||
toProxiedMediaUrl,
|
||||
type CustomEmoji,
|
||||
type LinkPreview,
|
||||
type MediaProxyPolicy,
|
||||
type ParseOptions,
|
||||
type RichTextDocument,
|
||||
} from "../../../../packages/safe-content/src/index.js";
|
||||
import type { StreamFeedItem } from "./feeds.service.js";
|
||||
|
||||
/** Media reference (design §1.2 `feed_items.media`). */
|
||||
export interface MediaRef {
|
||||
kind: "image" | "video";
|
||||
/** Media-proxy URL — never the raw provider CDN URL. */
|
||||
proxyUrl: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
alt?: string;
|
||||
}
|
||||
|
||||
/** Fully normalized v2 item — everything the persisted `feed_items` row needs. */
|
||||
export type NormalizedFeedItem = StreamFeedItem & {
|
||||
authorRef: string | null;
|
||||
url: string | null;
|
||||
content: RichTextDocument;
|
||||
plainText: string;
|
||||
media: MediaRef[];
|
||||
expiresAt: Date | null;
|
||||
linkPreview?: LinkPreview;
|
||||
};
|
||||
|
||||
export type FeedContentKind = "text" | "html";
|
||||
|
||||
export interface NormalizeFeedItemOptions {
|
||||
/** "html" = Mastodon-style sanitized HTML → allowlist AST; default plain text. */
|
||||
contentKind?: FeedContentKind;
|
||||
/** Provider permalink candidate; relative/unsafe schemes → null. */
|
||||
url?: unknown;
|
||||
/** ToS expiry (YouTube Data API: fetched_at + 30d). Unparseable → null. */
|
||||
expiresAt?: unknown;
|
||||
/** Mastodon `card` / OpenGraph payload → sanitized preview; hostile → null. */
|
||||
linkPreview?: unknown;
|
||||
mediaProxy?: MediaProxyPolicy;
|
||||
emojis?: readonly CustomEmoji[];
|
||||
}
|
||||
|
||||
/** Upper bound on media refs per item; Mastodon ≤4, YouTube 1 — generous cap. */
|
||||
const MAX_MEDIA = 16;
|
||||
/** Extension sniff for kind when the source only offers bare URLs. */
|
||||
const VIDEO_EXT = /\.(?:mp4|webm|mov|m4v|ogv)(?:[?#]|$)/i;
|
||||
/** Same shape safe-content enforces for handles (`user` | `user@host`). */
|
||||
const HANDLE = /^[\p{L}\p{N}_.-]{1,64}(?:@[\p{L}\p{N}.-]{1,253})?$/u;
|
||||
|
||||
/**
|
||||
* Canonical author ref `{provider}:{handle@host}` (design §3.1/§3.2). The host
|
||||
* comes from the handle itself, else from a federated `instanceUrl` connection
|
||||
* field; `URL` lowercases and punycodes it, and the port never participates.
|
||||
*/
|
||||
export function feedAuthorRef(provider: string, authorHandle: string, instanceUrl?: unknown): string | null {
|
||||
const handle = neutralizeText(authorHandle).trim().replace(/^@+/, "");
|
||||
if (!HANDLE.test(handle)) return null;
|
||||
const host = handle.includes("@") ? null : instanceHost(instanceUrl);
|
||||
return `${provider}:${handle}${host ? `@${host}` : ""}`;
|
||||
}
|
||||
|
||||
function instanceHost(raw: unknown): string | null {
|
||||
if (typeof raw !== "string" || raw.trim() === "") return null;
|
||||
try {
|
||||
const url = new URL(raw.trim());
|
||||
return url.protocol === "http:" || url.protocol === "https:" ? url.hostname : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Derives proxy-minted media refs; unsafe origin URLs are dropped entirely. */
|
||||
export function feedItemMedia(urls: unknown, policy: MediaProxyPolicy = DEFAULT_MEDIA_PROXY): MediaRef[] {
|
||||
if (!Array.isArray(urls)) return [];
|
||||
const out: MediaRef[] = [];
|
||||
for (const raw of urls.slice(0, MAX_MEDIA)) {
|
||||
const proxyUrl = toProxiedMediaUrl(raw, policy);
|
||||
if (proxyUrl === null) continue;
|
||||
out.push({ kind: typeof raw === "string" && VIDEO_EXT.test(raw) ? "video" : "image", proxyUrl });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Flat text derived from a rich-text document. Pure and total over the closed
|
||||
* AST: same document in, same string out — stable across storage round-trips.
|
||||
* Links contribute their visible label (for parsed plain text that is the URL
|
||||
* without the scheme, so host/path terms stay filterable).
|
||||
*/
|
||||
export function astToPlainText(doc: RichTextDocument): string {
|
||||
const lines: string[] = [];
|
||||
for (const paragraph of doc.children) {
|
||||
let line = "";
|
||||
for (const node of paragraph.children) {
|
||||
switch (node.type) {
|
||||
case "text":
|
||||
line += node.text;
|
||||
break;
|
||||
case "emoji":
|
||||
line += `:${node.shortcode}:`;
|
||||
break;
|
||||
case "linebreak":
|
||||
line += "\n";
|
||||
break;
|
||||
case "link":
|
||||
for (const child of node.children) line += child.type === "text" ? child.text : `:${child.shortcode}:`;
|
||||
break;
|
||||
case "mention":
|
||||
line += `@${node.handle}`;
|
||||
break;
|
||||
case "hashtag":
|
||||
line += `#${node.tag}`;
|
||||
break;
|
||||
}
|
||||
}
|
||||
lines.push(line);
|
||||
}
|
||||
return lines.join("\n\n");
|
||||
}
|
||||
|
||||
function normalizeExpiresAt(raw: unknown): Date | null {
|
||||
if (raw instanceof Date) return Number.isFinite(raw.getTime()) ? raw : null;
|
||||
if (typeof raw === "string" || typeof raw === "number") {
|
||||
const date = new Date(raw);
|
||||
return Number.isFinite(date.getTime()) ? date : null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The single funnel (design §3.2). Returns null for items failing the legacy
|
||||
* validation, otherwise the legacy shape plus the additive v2 fields. Content
|
||||
* is parsed here — callers never pass an AST, so raw provider HTML cannot
|
||||
* bypass the allowlist parser.
|
||||
*/
|
||||
export function normalizeFeedItem(
|
||||
item: FeedItem,
|
||||
connection: ConnectorConnection,
|
||||
options: NormalizeFeedItemOptions = {},
|
||||
): NormalizedFeedItem | null {
|
||||
const createdAt = new Date(item.createdAt);
|
||||
if (!Number.isFinite(createdAt.getTime()) || typeof item.remoteId !== "string" || typeof item.text !== "string" || typeof item.authorHandle !== "string") {
|
||||
return null;
|
||||
}
|
||||
const mediaProxy = options.mediaProxy ?? DEFAULT_MEDIA_PROXY;
|
||||
const parseOptions: ParseOptions = { mediaProxy, ...(options.emojis ? { emojis: options.emojis } : {}) };
|
||||
const content = options.contentKind === "html"
|
||||
? parseProviderHtml(item.text, parseOptions)
|
||||
: parsePlainText(item.text, parseOptions);
|
||||
const safeUrl = parseSafeHttpUrl(options.url);
|
||||
const preview = parseLinkPreview(options.linkPreview, mediaProxy);
|
||||
const mediaUrls = Array.isArray(item.mediaUrls) ? item.mediaUrls.filter((url): url is string => typeof url === "string") : [];
|
||||
return {
|
||||
id: typeof item.id === "string" ? item.id : item.remoteId,
|
||||
remoteId: item.remoteId,
|
||||
provider: connection.provider,
|
||||
connectionId: connection.id,
|
||||
authorHandle: item.authorHandle,
|
||||
authorRef: feedAuthorRef(connection.provider, item.authorHandle, connection.fields.instanceUrl),
|
||||
url: safeUrl === null ? null : safeUrl.href,
|
||||
text: item.text,
|
||||
content,
|
||||
plainText: astToPlainText(content),
|
||||
createdAt,
|
||||
mediaUrls,
|
||||
media: feedItemMedia(item.mediaUrls, mediaProxy),
|
||||
metrics: Object.fromEntries(Object.entries(item.metrics ?? {}).filter(([, value]) => typeof value === "number" && Number.isFinite(value))),
|
||||
expiresAt: normalizeExpiresAt(options.expiresAt),
|
||||
...(preview === null ? {} : { linkPreview: preview }),
|
||||
};
|
||||
}
|
||||
18
apps/api/src/feeds/feed-registry.ts
Normal file
18
apps/api/src/feeds/feed-registry.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
import type { CredentialAccessor } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import { connectionRegistry } from "../connections/connector-registry.js";
|
||||
import { oauthFetch } from "../connections/oauth-fetch.js";
|
||||
import { createMockSocialConnector, createMockSocialState } from "../../../../packages/connector-mock-social/src/index.js";
|
||||
|
||||
/** Composition boundary: the demo connector uses deterministic public data, no OAuth. */
|
||||
export function feedRegistry(credentials: CredentialAccessor) {
|
||||
const registry = connectionRegistry(credentials, oauthFetch);
|
||||
const state = createMockSocialState();
|
||||
const now = Date.now();
|
||||
state.feed = Array.from({ length: 12 }, (_, index) => ({
|
||||
id: `demo-feed-${index}`, remoteId: `demo-feed-${index}`, provider: "mock-social",
|
||||
authorHandle: "mock@social", text: `Mock Social feed item ${index + 1}: ready for Streams.`,
|
||||
mediaUrls: [], createdAt: new Date(now - index * 60_000), metrics: { likes: index },
|
||||
}));
|
||||
registry.register(createMockSocialConnector(state));
|
||||
return registry;
|
||||
}
|
||||
23
apps/api/src/feeds/feeds.controller.ts
Normal file
23
apps/api/src/feeds/feeds.controller.ts
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
import { Controller, Get, Inject, Param, Post, Query, Req, UseGuards } from "@nestjs/common";
|
||||
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import { parseId, parseLimit, requireSession } from "../http/validation.js";
|
||||
import { FeedsService } from "./feeds.service.js";
|
||||
|
||||
@Controller("feeds")
|
||||
@UseGuards(RbacGuard)
|
||||
export class FeedsController {
|
||||
constructor(@Inject(FeedsService) private readonly feeds: FeedsService) {}
|
||||
@Get()
|
||||
@RequirePermission("draft.view")
|
||||
list(@Req() request: AuthenticatedRequest, @Query("cursor") cursor?: string, @Query("limit") limit?: string) {
|
||||
return this.feeds.read(requireSession(request), { cursor, limit: parseLimit(limit, 40, 200) });
|
||||
}
|
||||
@Post("refresh")
|
||||
@RequirePermission("draft.view")
|
||||
refresh(@Req() request: AuthenticatedRequest) { return this.feeds.refresh(requireSession(request)); }
|
||||
@Get(":connectionId")
|
||||
@RequirePermission("draft.view")
|
||||
connection(@Req() request: AuthenticatedRequest, @Param("connectionId") connectionId: string, @Query("cursor") cursor?: string, @Query("limit") limit?: string) {
|
||||
return this.feeds.read(requireSession(request), { connectionId: parseId(connectionId), cursor, limit: parseLimit(limit, 40, 200) });
|
||||
}
|
||||
}
|
||||
72
apps/api/src/feeds/feeds.dependencies.ts
Normal file
72
apps/api/src/feeds/feeds.dependencies.ts
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
import type { ConnectorConnection } from "@tactical-media-broadcast-hub/core";
|
||||
import type { ConnectorRegistry, CredentialAccessor } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import { appendTenantAudit } from "../audit/audit.service.js";
|
||||
import { PgConnectionStore } from "../connections/connection-store.js";
|
||||
import type { ConnectionSummary } from "../connections/connections.service.js";
|
||||
import { pool } from "../db/client.js";
|
||||
import { GroupsService, hasWorkspaceWideAccess } from "../groups/groups.service.js";
|
||||
import { RespControlRedis, type ControlRedis } from "../media-control/redis.js";
|
||||
import { vault } from "../vault/vault.js";
|
||||
import { feedRegistry } from "./feed-registry.js";
|
||||
|
||||
export interface FeedDependencies {
|
||||
redis: Pick<ControlRedis, "get" | "set" | "eval" | "close">;
|
||||
registry: ConnectorRegistry;
|
||||
workspaces(): Promise<string[]>;
|
||||
connections(workspaceId: string): Promise<ConnectionSummary[]>;
|
||||
connection(workspaceId: string, id: string): Promise<ConnectorConnection>;
|
||||
visible(session: SessionPayload): Promise<string[]>;
|
||||
audit(session: SessionPayload): Promise<void>;
|
||||
/** Conservative share of the connector's SDK budget; remaining capacity serves other operations. */
|
||||
requestsPerSecond(provider: string): number;
|
||||
random(): number;
|
||||
}
|
||||
|
||||
export function feedDefaults(): FeedDependencies {
|
||||
const store = new PgConnectionStore();
|
||||
// A single registry preserves the connector's per-host HTTP token buckets.
|
||||
const credentials: CredentialAccessor = {
|
||||
async getAccessToken(connection) { return credentials.getSecret(connection, "access_token"); },
|
||||
async getSecret(connection, key) {
|
||||
const stored = await store.get(connection.workspaceId, connection.id);
|
||||
if (!stored.credential) throw new Error("Credential missing");
|
||||
const bytes = await vault.open(connection.workspaceId, connection.id, stored.credential);
|
||||
const secrets = JSON.parse(bytes.toString()) as Record<string, unknown>;
|
||||
const value = secrets[key];
|
||||
if (typeof value !== "string" || !value) throw new Error("Credential missing");
|
||||
return value;
|
||||
},
|
||||
async put(connection, key, value) {
|
||||
const stored = await store.get(connection.workspaceId, connection.id);
|
||||
if (!stored.credential) throw new Error("Credential missing");
|
||||
const bytes = await vault.open(connection.workspaceId, connection.id, stored.credential);
|
||||
const secrets = JSON.parse(bytes.toString()) as Record<string, unknown>;
|
||||
secrets[key] = value;
|
||||
const sealed = await vault.seal(connection.workspaceId, connection.id, Buffer.from(JSON.stringify(secrets)));
|
||||
await store.rotateCredential(connection.workspaceId, connection.id, stored.credential, sealed);
|
||||
},
|
||||
};
|
||||
return {
|
||||
redis: new RespControlRedis(process.env.REDIS_URL ?? "redis://127.0.0.1:6379"),
|
||||
registry: feedRegistry(credentials),
|
||||
async workspaces() {
|
||||
const result = await pool.query<{ id: string }>("SELECT id FROM workspaces ORDER BY id");
|
||||
return result.rows.map((row) => row.id);
|
||||
},
|
||||
connections: (workspaceId) => store.list(workspaceId),
|
||||
connection: async (workspaceId, id) => (await store.get(workspaceId, id)).connection,
|
||||
visible: async (session) => (await new GroupsService().getVisibleConnections(session.workspaceId, session.userId, hasWorkspaceWideAccess(session.role))).connectionIds,
|
||||
audit: async (session) => { await appendTenantAudit(session.workspaceId, {
|
||||
actorUserId: session.userId, actorRole: session.role, action: "feeds.refresh",
|
||||
targetType: "workspace", targetId: session.workspaceId,
|
||||
}); },
|
||||
// The smallest current feed connector SDK budget is 0.5 RPS. Never raise it
|
||||
// via an env override; lowering the SDK policy also lowers poll frequency.
|
||||
requestsPerSecond: (provider) => {
|
||||
const configured = Number(process.env[`CONNECTOR_RATE_${provider.toUpperCase().replace(/-/g, "_")}_RPS`]);
|
||||
return Number.isFinite(configured) && configured > 0 ? Math.min(0.5, configured) : 0.5;
|
||||
},
|
||||
random: Math.random,
|
||||
};
|
||||
}
|
||||
102
apps/api/src/feeds/feeds.integration.test.ts
Normal file
102
apps/api/src/feeds/feeds.integration.test.ts
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
import "reflect-metadata";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { Module, UnauthorizedException, ForbiddenException, type ExecutionContext, type INestApplication } from "@nestjs/common";
|
||||
import { NestFactory, Reflector } from "@nestjs/core";
|
||||
import { InMemoryCredentials } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { DemoController } from "../demo/demo.controller.js";
|
||||
import { PERMISSION_KEY, RbacGuard } from "../rbac/rbac.guard.js";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import { injectHttp } from "../connections/http-test-transport.js";
|
||||
import type { ConnectorConnection } from "@tactical-media-broadcast-hub/core";
|
||||
import { feedRegistry } from "./feed-registry.js";
|
||||
import { FeedsService, type FeedResponse } from "./feeds.service.js";
|
||||
import { FeedsController } from "./feeds.controller.js";
|
||||
import { MemoryFeedRedis } from "./testing.js";
|
||||
|
||||
const memory = vi.hoisted(() => ({ rows: new Map<string, { id: string; workspaceId: string; provider: string; channelId: string }>(), audits: [] as string[] }));
|
||||
vi.mock("../db/client.js", () => ({
|
||||
pool: {},
|
||||
withTenant: async (workspaceId: string, callback: (client: { query(sql: string, values: unknown[]): Promise<{ rows: Array<{ id: string }>; rowCount: number }> }) => Promise<unknown>) => callback({
|
||||
async query(sql, values) {
|
||||
if (sql.startsWith("INSERT INTO connections")) {
|
||||
const id = randomUUID();
|
||||
memory.rows.set(id, { id, workspaceId, provider: String(values[1]), channelId: String(values[2]) });
|
||||
return { rows: [{ id }], rowCount: 1 };
|
||||
}
|
||||
if (sql.includes("audit_append")) memory.audits.push(sql);
|
||||
return { rows: [], rowCount: 1 };
|
||||
},
|
||||
}),
|
||||
}));
|
||||
const session: SessionPayload = { userId: randomUUID(), workspaceId: randomUUID(), role: "Owner", authzEpoch: 1, issuedAt: 0, expiresAt: 9_999_999_999 };
|
||||
@Module({})
|
||||
class TestModule {}
|
||||
|
||||
describe("HTTP demo connection → connector poll → GET feeds", () => {
|
||||
let app: INestApplication;
|
||||
let canView = true;
|
||||
const registry = feedRegistry(new InMemoryCredentials());
|
||||
const service = new FeedsService({
|
||||
redis: new MemoryFeedRedis(), registry, random: () => 0, requestsPerSecond: () => 0.5,
|
||||
workspaces: async () => [session.workspaceId],
|
||||
connections: async (ws) => [...memory.rows.values()].filter((row) => row.workspaceId === ws).map((row) => ({ ...row, fields: {}, health: "needs_credentials", recentPublishes: [], credentialRotatedAt: null, createdAt: new Date().toISOString() })),
|
||||
connection: async (ws, id): Promise<ConnectorConnection> => {
|
||||
const row = memory.rows.get(id); if (!row || row.workspaceId !== ws) throw new Error("Missing row");
|
||||
return { ...row, fields: {}, credentialId: { kind: "credential-ref", id }, createdAt: new Date() };
|
||||
},
|
||||
visible: async (s) => [...memory.rows.values()].filter((row) => row.workspaceId === s.workspaceId).map((row) => row.id),
|
||||
audit: async () => { memory.audits.push("feeds.refresh"); },
|
||||
});
|
||||
beforeAll(async () => {
|
||||
vi.spyOn(service, "onApplicationBootstrap").mockImplementation(() => undefined);
|
||||
vi.spyOn(RbacGuard.prototype, "canActivate").mockImplementation(async (context: ExecutionContext) => {
|
||||
const req = context.switchToHttp().getRequest<{ headers: Record<string, string>; session?: SessionPayload }>();
|
||||
if (req.headers.authorization !== "Bearer test-session") throw new UnauthorizedException();
|
||||
const permission = new Reflector().get<string>(PERMISSION_KEY, context.getHandler());
|
||||
if (permission === "draft.view" && !canView) throw new ForbiddenException();
|
||||
req.session = session; return true;
|
||||
});
|
||||
app = await NestFactory.create({ module: TestModule, controllers: [DemoController, FeedsController], providers: [{ provide: FeedsService, useValue: service }] }, { logger: false });
|
||||
await app.init();
|
||||
});
|
||||
afterAll(async () => { await app?.close(); vi.restoreAllMocks(); });
|
||||
const request = (path: string, method = "GET", body?: unknown, auth = true) => injectHttp(app.getHttpAdapter().getInstance(), `http://api.test${path}`, {
|
||||
method, headers: { ...(auth ? { authorization: "Bearer test-session" } : {}), "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body),
|
||||
});
|
||||
it("seeds through the real demo route without OAuth, then returns only feed-capable items", async () => {
|
||||
const created = await request("/demo/connections", "POST", { provider: "mock-social", channelId: "demo-channel" });
|
||||
expect(created.status).toBe(201);
|
||||
const mockId = (await created.json() as { id: string }).id;
|
||||
const other = await request("/demo/connections", "POST", { provider: "peertube", channelId: "video-channel" });
|
||||
const otherId = (await other.json() as { id: string }).id;
|
||||
const refreshed = await request("/feeds/refresh", "POST");
|
||||
expect(refreshed.status).toBe(201);
|
||||
expect(await refreshed.json()).toEqual({ polled: 1, skipped: 0, failed: 0 });
|
||||
const response = await request("/feeds");
|
||||
expect(response.status).toBe(200);
|
||||
const feed = await response.json() as FeedResponse;
|
||||
expect(feed.items).toHaveLength(12);
|
||||
expect(feed.items.every((item) => item.connectionId === mockId && item.provider === "mock-social")).toBe(true);
|
||||
expect(feed.meta.excludedConnections).toEqual([{ id: otherId, provider: "peertube", reason: "feed_not_supported" }]);
|
||||
expect(feed.meta.connections[0]?.handle).toBe("mock@social");
|
||||
expect(memory.audits.filter((entry) => entry === "feeds.refresh")).toHaveLength(1);
|
||||
await service.pollWorkspace(session.workspaceId);
|
||||
expect(memory.audits.filter((entry) => entry === "feeds.refresh")).toHaveLength(1);
|
||||
const direct = await request(`/feeds/${mockId}?limit=2`);
|
||||
expect((await direct.json() as FeedResponse).items).toHaveLength(2);
|
||||
const absent = await request(`/feeds/${otherId}`);
|
||||
expect((await absent.json() as FeedResponse).items).toEqual([]);
|
||||
});
|
||||
it("requires draft.view on all feed routes and rejects invalid cursor and ID", async () => {
|
||||
expect((await request("/feeds", "GET", undefined, false)).status).toBe(401);
|
||||
canView = false;
|
||||
expect((await request("/feeds")).status).toBe(403);
|
||||
expect((await request("/feeds/refresh", "POST")).status).toBe(403);
|
||||
expect((await request(`/feeds/${[...memory.rows.keys()][0]}`)).status).toBe(403);
|
||||
canView = true;
|
||||
expect((await request("/feeds/not-a-uuid")).status).toBe(400);
|
||||
expect((await request("/feeds?cursor=garbage")).status).toBe(400);
|
||||
expect((await request("/feeds?limit=-1")).status).toBe(400);
|
||||
});
|
||||
});
|
||||
24
apps/api/src/feeds/feeds.module.ts
Normal file
24
apps/api/src/feeds/feeds.module.ts
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
import { Module } from "@nestjs/common";
|
||||
import { FeedsController } from "./feeds.controller.js";
|
||||
import { FeedsService } from "./feeds.service.js";
|
||||
import { ColumnsController } from "./columns.controller.js";
|
||||
import { ColumnStore } from "./column-store.js";
|
||||
import { ColumnScheduler } from "./column-scheduler.js";
|
||||
import { columnsProviders } from "./columns.dependencies.js";
|
||||
|
||||
/**
|
||||
* Feeds + columns (P1): the columns controller/store/scheduler join the feeds
|
||||
* module — one surface, one connector registry, one hot-cache dependency set.
|
||||
* ColumnStore is injected into FeedsService for the additive meta.columns.
|
||||
*/
|
||||
@Module({
|
||||
// Controllers are ordered deliberately: ColumnsController's static paths
|
||||
// (/feeds/columns, /feeds/columns/reorder, …) must register BEFORE
|
||||
// FeedsController's parametric /feeds/:connectionId, or that route would
|
||||
// capture "columns" as a connection id (Express matches in declaration
|
||||
// order in Express/Nest's router).
|
||||
controllers: [ColumnsController, FeedsController],
|
||||
providers: [FeedsService, ColumnStore, ColumnScheduler, ...columnsProviders],
|
||||
exports: [FeedsService, ColumnStore, ColumnScheduler],
|
||||
})
|
||||
export class FeedsModule {}
|
||||
257
apps/api/src/feeds/feeds.service.ts
Normal file
257
apps/api/src/feeds/feeds.service.ts
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
import { BadRequestException, Inject, Injectable, Logger, NotFoundException, Optional, type OnApplicationBootstrap, type OnApplicationShutdown } from "@nestjs/common";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { ConnectorConnection, FeedItem } from "@tactical-media-broadcast-hub/core";
|
||||
import { RateLimitError } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import type { ConnectionSummary } from "../connections/connections.service.js";
|
||||
import { normalizeFeedItem, type NormalizedFeedItem } from "./feed-normalize.js";
|
||||
import { feedDefaults, type FeedDependencies } from "./feeds.dependencies.js";
|
||||
import { ColumnStore, type ColumnRecord } from "./column-store.js";
|
||||
|
||||
export const FEED_DEPENDENCIES = Symbol("FEED_DEPENDENCIES");
|
||||
export const FEED_CONCURRENCY = 4;
|
||||
const DISCOVERY_MS = 5_000;
|
||||
const CACHE_SLACK_MS = 30_000;
|
||||
const LEASE_MS = 30_000;
|
||||
const RENEW_LEASE = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('PEXPIRE', KEYS[1], ARGV[2]) else return 0 end";
|
||||
const RELEASE_LEASE = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) else return 0 end";
|
||||
export const feedCacheKey = (workspaceId: string, id: string): string => `feeds:${workspaceId}:${id}:home`;
|
||||
export function feedIntervalMs(rps: number): number {
|
||||
if (!Number.isFinite(rps) || rps <= 0) throw new Error("Feed rate must be positive");
|
||||
// Identity + feed, allocating at most half the declared budget to polling.
|
||||
return Math.max(60_000, Math.ceil(4_000 / rps));
|
||||
}
|
||||
export const feedCacheTtlMs = (intervalMs: number): number => Math.ceil(intervalMs * 1.2) + CACHE_SLACK_MS;
|
||||
export interface StreamFeedItem extends FeedItem { connectionId: string }
|
||||
interface CachedFeed { items: NormalizedFeedItem[]; fetchedAt: string; handle: string; displayName: string }
|
||||
export interface FeedResponse {
|
||||
items: StreamFeedItem[];
|
||||
nextCursor: string | null;
|
||||
meta: {
|
||||
configuredFeeds: number;
|
||||
connections: Array<{ id: string; provider: string; handle: string; displayName: string; capabilities: string[]; fetchedAt: string | null; status: "ready" | "pending" }>;
|
||||
excludedConnections: Array<{ id: string; provider: string; reason: "feed_not_supported" }>;
|
||||
/**
|
||||
* Additive (P1): the persisted column list, when the workspace has any.
|
||||
* Absent (not null) for workspaces that never touched the columns API, so
|
||||
* the legacy shape is byte-identical for existing clients.
|
||||
*/
|
||||
columns?: Array<{ id: string; type: string; title: string; position: number }>;
|
||||
};
|
||||
}
|
||||
interface Position { at: number; key: string }
|
||||
const itemKey = (item: StreamFeedItem): string => JSON.stringify([item.connectionId, item.id]);
|
||||
function position(item: StreamFeedItem): Position { return { at: item.createdAt.getTime(), key: itemKey(item) }; }
|
||||
function compare(a: Position, b: Position): number { return b.at - a.at || a.key.localeCompare(b.key); }
|
||||
function decodeCursor(cursor: string): Position {
|
||||
try {
|
||||
if (cursor.length > 4096) throw new Error();
|
||||
const parsed: unknown = JSON.parse(Buffer.from(cursor, "base64url").toString());
|
||||
if (typeof parsed !== "object" || parsed === null || !("at" in parsed) || !("key" in parsed) ||
|
||||
typeof parsed.at !== "number" || !Number.isFinite(parsed.at) || typeof parsed.key !== "string") throw new Error();
|
||||
return { at: parsed.at, key: parsed.key };
|
||||
} catch { throw new BadRequestException("Invalid feed cursor"); }
|
||||
}
|
||||
function normalize(item: FeedItem, connection: ConnectorConnection): NormalizedFeedItem | null {
|
||||
return normalizeFeedItem(item, connection);
|
||||
}
|
||||
export { normalize };
|
||||
|
||||
@Injectable()
|
||||
export class FeedsService implements OnApplicationBootstrap, OnApplicationShutdown {
|
||||
private readonly deps: FeedDependencies;
|
||||
private readonly logger = new Logger(FeedsService.name);
|
||||
private timer?: ReturnType<typeof setTimeout>;
|
||||
private running = false;
|
||||
private active?: Promise<void>;
|
||||
private readonly cycles = new Map<string, Promise<{ polled: number; skipped: number; failed: number }>>();
|
||||
private inFlight = 0;
|
||||
private readonly waiters: Array<() => void> = [];
|
||||
private readonly columns?: ColumnStore;
|
||||
constructor(@Optional() @Inject(FEED_DEPENDENCIES) dependencies?: FeedDependencies, @Optional() @Inject(ColumnStore) columns?: ColumnStore) {
|
||||
this.deps = dependencies ?? feedDefaults();
|
||||
this.columns = columns;
|
||||
}
|
||||
|
||||
/** Persisted columns for meta.columns (additive; absent when never created). */
|
||||
async columnSummaries(workspaceId: string): Promise<Array<{ id: string; type: string; title: string; position: number }>> {
|
||||
if (this.columns === undefined) return [];
|
||||
return columnSummariesOf(await this.columns.list(workspaceId));
|
||||
}
|
||||
|
||||
onApplicationBootstrap(): void {
|
||||
this.running = true;
|
||||
this.schedule(0);
|
||||
}
|
||||
async onApplicationShutdown(): Promise<void> {
|
||||
this.running = false;
|
||||
if (this.timer) clearTimeout(this.timer);
|
||||
await this.active;
|
||||
await Promise.allSettled(this.cycles.values());
|
||||
this.deps.redis.close();
|
||||
}
|
||||
private schedule(delay: number): void {
|
||||
if (!this.running) return;
|
||||
this.timer = setTimeout(() => {
|
||||
this.active = this.pollAll().catch(() => this.logger.warn("Feed discovery unavailable")).finally(() => {
|
||||
this.active = undefined;
|
||||
this.schedule(DISCOVERY_MS + Math.round(this.deps.random() * 1_000));
|
||||
});
|
||||
}, delay);
|
||||
this.timer.unref();
|
||||
}
|
||||
async pollAll(): Promise<void> {
|
||||
for (const workspaceId of await this.deps.workspaces()) await this.pollWorkspace(workspaceId);
|
||||
}
|
||||
/** Feed-capable connection summaries — the lazy-seeding input for columns. */
|
||||
async capableConnections(workspaceId: string): Promise<ConnectionSummary[]> {
|
||||
return (await this.deps.connections(workspaceId)).filter((row) => this.capable(row));
|
||||
}
|
||||
|
||||
private capable(row: ConnectionSummary): boolean {
|
||||
if (!this.deps.registry.has(row.provider)) return false;
|
||||
const connector = this.deps.registry.get(row.provider);
|
||||
return connector.manifest().capabilities.feed === true && connector.getFeed !== undefined;
|
||||
}
|
||||
/** Refresh shares the scheduler's cooldowns and concurrency limit; repeated clicks cannot bypass them. */
|
||||
async refresh(session: SessionPayload) {
|
||||
await this.deps.audit(session);
|
||||
return this.pollWorkspace(session.workspaceId, await this.deps.visible(session));
|
||||
}
|
||||
pollWorkspace(workspaceId: string, visibleIds?: string[]): Promise<{ polled: number; skipped: number; failed: number }> {
|
||||
const key = JSON.stringify([workspaceId, visibleIds?.slice().sort() ?? null]);
|
||||
const current = this.cycles.get(key);
|
||||
if (current) return current;
|
||||
const cycle = this.cycle(workspaceId, visibleIds).finally(() => this.cycles.delete(key));
|
||||
this.cycles.set(key, cycle);
|
||||
return cycle;
|
||||
}
|
||||
private async cycle(workspaceId: string, visibleIds?: string[]) {
|
||||
const rows = (await this.deps.connections(workspaceId)).filter((row) => this.capable(row) && (!visibleIds || visibleIds.includes(row.id)));
|
||||
const results = { polled: 0, skipped: 0, failed: 0 };
|
||||
let index = 0;
|
||||
await Promise.all(Array.from({ length: Math.min(FEED_CONCURRENCY, rows.length) }, async () => {
|
||||
for (let row = rows[index++]; row; row = rows[index++]) {
|
||||
if (this.inFlight >= FEED_CONCURRENCY) await new Promise<void>((resolve) => this.waiters.push(resolve));
|
||||
else this.inFlight += 1;
|
||||
try { results[await this.poll(workspaceId, row)] += 1; }
|
||||
catch { results.failed += 1; this.logger.warn("Feed poll unavailable"); }
|
||||
finally {
|
||||
const waiter = this.waiters.shift();
|
||||
if (waiter) waiter(); else this.inFlight -= 1;
|
||||
}
|
||||
}
|
||||
}));
|
||||
return results;
|
||||
}
|
||||
private async poll(workspaceId: string, row: ConnectionSummary): Promise<"polled" | "skipped" | "failed"> {
|
||||
const interval = feedIntervalMs(this.deps.requestsPerSecond(row.provider));
|
||||
const delay = interval + Math.round(interval * 0.2 * this.deps.random());
|
||||
const key = feedCacheKey(workspaceId, row.id);
|
||||
const leaseKey = `${key}:lease`;
|
||||
const owner = randomUUID();
|
||||
if (!await this.deps.redis.set(leaseKey, owner, { nx: true, pxMs: LEASE_MS })) return "skipped";
|
||||
let heartbeat: ReturnType<typeof setInterval> | undefined;
|
||||
let renew: Promise<unknown> = Promise.resolve();
|
||||
let cooldown = delay;
|
||||
let started = false;
|
||||
try {
|
||||
if (!await this.deps.redis.set(`${key}:cooldown`, "idle", { nx: true, pxMs: delay })) return "skipped";
|
||||
started = true;
|
||||
heartbeat = setInterval(() => {
|
||||
renew = this.deps.redis.eval(RENEW_LEASE, [leaseKey], [owner, String(LEASE_MS)]).catch(() => {
|
||||
this.logger.warn("Feed lease renewal unavailable");
|
||||
});
|
||||
}, LEASE_MS / 3);
|
||||
heartbeat.unref();
|
||||
const connection = await this.deps.connection(workspaceId, row.id);
|
||||
const connector = this.deps.registry.get(connection.provider);
|
||||
const identity = await connector.getIdentity(connection);
|
||||
const page = await connector.getFeed?.(connection, { kind: "home", limit: 40 });
|
||||
const items = [...new Map((page?.items ?? []).map((item) => normalize(item, connection)).filter((item): item is NormalizedFeedItem => item !== null).map((item) => [itemKey(item), item])).values()];
|
||||
await this.deps.redis.set(key, JSON.stringify({ items, fetchedAt: new Date().toISOString(), handle: identity.handle, displayName: identity.displayName ?? identity.handle } satisfies CachedFeed), { pxMs: feedCacheTtlMs(interval) });
|
||||
return "polled";
|
||||
} catch (error) {
|
||||
await this.deps.redis.set(key, "null", { pxMs: feedCacheTtlMs(interval) });
|
||||
if (error instanceof RateLimitError && error.retryAfterMs) cooldown = Math.max(delay, error.retryAfterMs);
|
||||
return "failed";
|
||||
} finally {
|
||||
if (heartbeat) clearInterval(heartbeat);
|
||||
await renew;
|
||||
// Completing a long SDK call starts a fresh cooldown. The renewable
|
||||
// lease prevents another replica from entering while that call runs.
|
||||
try {
|
||||
if (started) await this.deps.redis.set(`${key}:cooldown`, "idle", { pxMs: cooldown });
|
||||
} finally { await this.deps.redis.eval(RELEASE_LEASE, [leaseKey], [owner]); }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Hot-cache write-through for the legacy /feeds surface (design §3.4): the
|
||||
* column scheduler's home-source polls push their normalized items here so
|
||||
* /feeds stays cache-first with no external polling of its own.
|
||||
*/
|
||||
async cacheHomeFeed(workspaceId: string, connectionId: string, items: NormalizedFeedItem[]): Promise<void> {
|
||||
const rows = await this.deps.connections(workspaceId);
|
||||
const row = rows.find((candidate) => candidate.id === connectionId);
|
||||
if (row === undefined) return;
|
||||
const interval = feedIntervalMs(this.deps.requestsPerSecond(row.provider));
|
||||
let handle = String(row.fields.handle ?? row.channelId);
|
||||
let displayName = String(row.fields.displayName ?? row.channelId);
|
||||
try {
|
||||
const connection = await this.deps.connection(workspaceId, connectionId);
|
||||
const identity = await this.deps.registry.get(connection.provider).getIdentity(connection);
|
||||
handle = identity.handle;
|
||||
displayName = identity.displayName ?? identity.handle;
|
||||
} catch { /* Identity is best-effort here; the cached text still serves. */ }
|
||||
await this.deps.redis.set(feedCacheKey(workspaceId, connectionId), JSON.stringify({
|
||||
items, fetchedAt: new Date().toISOString(), handle, displayName,
|
||||
} satisfies CachedFeed), { pxMs: feedCacheTtlMs(interval) });
|
||||
}
|
||||
|
||||
async read(session: SessionPayload, options: { connectionId?: string; cursor?: string; limit?: number } = {}): Promise<FeedResponse> {
|
||||
const limit = options.limit ?? 40;
|
||||
if (!Number.isInteger(limit) || limit < 1 || limit > 200) throw new BadRequestException("Invalid feed limit");
|
||||
const after = options.cursor ? decodeCursor(options.cursor) : null;
|
||||
const visible = new Set(await this.deps.visible(session));
|
||||
const rows = (await this.deps.connections(session.workspaceId)).filter((row) => visible.has(row.id) && (!options.connectionId || row.id === options.connectionId));
|
||||
if (options.connectionId && rows.length === 0) throw new NotFoundException("Connection not found");
|
||||
const meta: FeedResponse["meta"] = { configuredFeeds: 0, connections: [], excludedConnections: [] };
|
||||
const merged: StreamFeedItem[] = [];
|
||||
for (const row of rows) {
|
||||
if (!this.capable(row)) { meta.excludedConnections.push({ id: row.id, provider: row.provider, reason: "feed_not_supported" }); continue; }
|
||||
meta.configuredFeeds += 1;
|
||||
const raw = await this.deps.redis.get(feedCacheKey(session.workspaceId, row.id));
|
||||
let cached: CachedFeed | null = null;
|
||||
try { cached = raw ? JSON.parse(raw) as CachedFeed | null : null; } catch { /* An invalid cache is pending until the next poll. */ }
|
||||
meta.connections.push({
|
||||
id: row.id, provider: row.provider, handle: cached?.handle ?? String(row.fields.handle ?? row.channelId),
|
||||
displayName: cached?.displayName ?? String(row.fields.displayName ?? row.channelId),
|
||||
capabilities: Object.entries(this.deps.registry.get(row.provider).manifest().capabilities).filter(([, enabled]) => enabled === true).map(([capability]) => capability),
|
||||
fetchedAt: cached?.fetchedAt ?? null, status: cached ? "ready" : "pending",
|
||||
});
|
||||
for (const cachedItem of cached?.items ?? []) {
|
||||
merged.push({
|
||||
...cachedItem,
|
||||
connectionId: row.id, provider: row.provider,
|
||||
createdAt: new Date(cachedItem.createdAt),
|
||||
...(cachedItem.expiresAt === undefined ? {} : { expiresAt: cachedItem.expiresAt === null ? null : new Date(cachedItem.expiresAt) }),
|
||||
} satisfies StreamFeedItem);
|
||||
}
|
||||
}
|
||||
const sorted = merged.sort((a, b) => compare(position(a), position(b))).filter((item) => !after || compare(position(item), after) > 0);
|
||||
const items = sorted.slice(0, limit);
|
||||
const last = items[items.length - 1];
|
||||
// Additive meta.columns: only when the workspace has persisted columns —
|
||||
// a workspace that never touched the columns API gets the legacy shape.
|
||||
if (this.columns !== undefined) {
|
||||
const summaries = await columnSummariesOf(await this.columns.list(session.workspaceId));
|
||||
if (summaries.length > 0) meta.columns = summaries;
|
||||
}
|
||||
return { items, nextCursor: sorted.length > limit && last ? Buffer.from(JSON.stringify(position(last))).toString("base64url") : null, meta };
|
||||
}
|
||||
}
|
||||
|
||||
function columnSummariesOf(records: ColumnRecord[]): Array<{ id: string; type: string; title: string; position: number }> {
|
||||
return records.map((record) => ({ id: record.id, type: record.type, title: record.title, position: record.position }));
|
||||
}
|
||||
195
apps/api/src/feeds/feeds.test.ts
Normal file
195
apps/api/src/feeds/feeds.test.ts
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
import "reflect-metadata";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { ConnectorRegistry, InMemoryCredentials, RateLimitError, type PlatformConnector } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import type { ConnectorConnection, FeedItem } from "@tactical-media-broadcast-hub/core";
|
||||
import { createMockSocialConnector, createMockSocialState } from "../../../../packages/connector-mock-social/src/index.js";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import type { ConnectionSummary } from "../connections/connections.service.js";
|
||||
import { FeedsService, feedCacheKey, feedCacheTtlMs, feedIntervalMs } from "./feeds.service.js";
|
||||
import type { FeedDependencies } from "./feeds.dependencies.js";
|
||||
import { connectionRegistry } from "../connections/connector-registry.js";
|
||||
import { MemoryFeedRedis } from "./testing.js";
|
||||
|
||||
const session: SessionPayload = { workspaceId: randomUUID(), userId: randomUUID(), role: "Owner", authzEpoch: 1, issuedAt: 0, expiresAt: 9_999_999_999 };
|
||||
const item = (remoteId: string, createdAt = new Date()): FeedItem => ({ id: remoteId, remoteId, provider: "mock-social", authorHandle: "alice", text: remoteId, mediaUrls: [], createdAt, metrics: {} });
|
||||
function setup(count = 1, rps = 0.5) {
|
||||
const state = createMockSocialState();
|
||||
state.feed = [item("first")];
|
||||
const mock = createMockSocialConnector(state);
|
||||
const implementation = mock.getFeed;
|
||||
if (!implementation) throw new Error("Mock feed missing");
|
||||
const getFeed = vi.fn(implementation);
|
||||
mock.getFeed = getFeed;
|
||||
const registry = new ConnectorRegistry(); registry.register(mock);
|
||||
const connections: ConnectorConnection[] = Array.from({ length: count }, () => {
|
||||
const id = randomUUID();
|
||||
return { id, workspaceId: session.workspaceId, provider: "mock-social", channelId: id, credentialId: { kind: "credential-ref", id }, fields: {}, createdAt: new Date() };
|
||||
});
|
||||
const redis = new MemoryFeedRedis();
|
||||
const summaries = (): ConnectionSummary[] => connections.map((row) => ({ id: row.id, provider: row.provider, channelId: row.channelId, fields: row.fields, health: "healthy", credentialRotatedAt: null, recentPublishes: [], createdAt: row.createdAt.toISOString() }));
|
||||
const deps: FeedDependencies = {
|
||||
redis, registry, workspaces: async () => [session.workspaceId], connections: async (ws) => ws === session.workspaceId ? summaries() : [],
|
||||
connection: async (_ws, id) => { const row = connections.find((row) => row.id === id); if (!row) throw new Error(); return row; },
|
||||
visible: async (s) => s.workspaceId === session.workspaceId ? connections.map((row) => row.id) : [],
|
||||
audit: vi.fn(async () => undefined), requestsPerSecond: () => rps, random: () => 0.5,
|
||||
};
|
||||
return { service: new FeedsService(deps), deps, connections, mock, state, getFeed, redis };
|
||||
}
|
||||
afterEach(() => vi.useRealTimers());
|
||||
describe("Feeds pipeline", () => {
|
||||
it("caches normalized JSON with interval + jitter + slack TTL and expires it", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { service, redis, connections } = setup();
|
||||
await service.pollWorkspace(session.workspaceId);
|
||||
const key = feedCacheKey(session.workspaceId, connections[0]?.id ?? "");
|
||||
const entry = redis.entries.get(key);
|
||||
expect(entry?.expiresAt).toBe(Date.now() + feedCacheTtlMs(60_000));
|
||||
const response = await service.read(session);
|
||||
expect(response.items[0]?.createdAt).toBeInstanceOf(Date);
|
||||
expect(response.items[0]?.connectionId).toBe(connections[0]?.id);
|
||||
await vi.advanceTimersByTimeAsync(feedCacheTtlMs(60_000));
|
||||
expect(await redis.get(key)).toBeNull();
|
||||
expect((await service.read(session)).meta.connections[0]?.status).toBe("pending");
|
||||
});
|
||||
it("merges newest first with a stable tie-break cursor, even when new items arrive", async () => {
|
||||
const { service, connections, redis } = setup(2);
|
||||
const date = new Date();
|
||||
for (const row of connections) await redis.set(feedCacheKey(session.workspaceId, row.id), JSON.stringify({ fetchedAt: date.toISOString(), handle: "alice", displayName: "Alice", items: [item("same", date), item("older", new Date(date.getTime() - 1000))] }), { pxMs: 99_000 });
|
||||
const first = await service.read(session, { limit: 1 });
|
||||
expect(first.nextCursor).not.toBeNull();
|
||||
const added = connections[0];
|
||||
if (added) {
|
||||
const key = feedCacheKey(session.workspaceId, added.id);
|
||||
const raw = await redis.get(key);
|
||||
const cached = JSON.parse(raw ?? "{}") as { items: FeedItem[] };
|
||||
cached.items.unshift(item("new-arrival", new Date(date.getTime() + 1000)));
|
||||
await redis.set(key, JSON.stringify(cached), { pxMs: 99_000 });
|
||||
}
|
||||
const next = await service.read(session, { limit: 1, cursor: first.nextCursor ?? undefined });
|
||||
expect(next.items[0]?.connectionId).not.toBe(first.items[0]?.connectionId);
|
||||
expect(next.items[0]?.remoteId).toBe("same");
|
||||
const third = await service.read(session, { limit: 2, cursor: next.nextCursor ?? undefined });
|
||||
expect(third.items.map((row) => row.remoteId)).toEqual(["older", "older"]);
|
||||
expect(third.nextCursor).toBeNull();
|
||||
await expect(service.read(session, { cursor: "bad" })).rejects.toThrow("Invalid feed cursor");
|
||||
});
|
||||
it("filters capabilities and missing methods, exposes exclusions without polling them", async () => {
|
||||
const { service, deps, connections, mock } = setup(3);
|
||||
const noFeed: PlatformConnector = { ...mock, manifest: () => ({ ...mock.manifest(), id: "no-feed", capabilities: { publishText: true, feed: false } }), getFeed: vi.fn() };
|
||||
const missing: PlatformConnector = { ...mock, manifest: () => ({ ...mock.manifest(), id: "missing-feed" }), getFeed: undefined };
|
||||
deps.registry.register(noFeed); deps.registry.register(missing);
|
||||
if (connections[1]) connections[1].provider = "no-feed";
|
||||
if (connections[2]) connections[2].provider = "missing-feed";
|
||||
expect(await service.pollWorkspace(session.workspaceId)).toEqual({ polled: 1, skipped: 0, failed: 0 });
|
||||
const response = await service.read(session);
|
||||
expect(response.meta.configuredFeeds).toBe(1);
|
||||
expect(response.meta.excludedConnections).toHaveLength(2);
|
||||
expect(response.items).toHaveLength(1);
|
||||
expect(noFeed.getFeed).not.toHaveBeenCalled();
|
||||
expect((await service.read(session, { connectionId: connections[1]?.id })).items).toEqual([]);
|
||||
});
|
||||
it("applies group visibility to merged, per-account and refresh operations", async () => {
|
||||
const { service, deps, connections } = setup(2);
|
||||
deps.visible = async () => [connections[0]?.id ?? ""];
|
||||
await service.refresh(session);
|
||||
const response = await service.read(session);
|
||||
expect(response.meta.connections).toHaveLength(1);
|
||||
await expect(service.read(session, { connectionId: connections[1]?.id })).rejects.toThrow("Connection not found");
|
||||
expect((await service.read({ ...session, workspaceId: randomUUID() })).items).toEqual([]);
|
||||
});
|
||||
it("paces low-budget scheduling with positive jitter; refresh and other replicas respect cooldown", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { service, deps, getFeed } = setup(1, 0.01);
|
||||
expect(feedIntervalMs(0.01)).toBe(400_000);
|
||||
service.onApplicationBootstrap();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(getFeed).toHaveBeenCalledTimes(1);
|
||||
const replica = new FeedsService(deps);
|
||||
expect((await replica.refresh(session)).skipped).toBe(1);
|
||||
await vi.advanceTimersByTimeAsync(439_999);
|
||||
expect(getFeed).toHaveBeenCalledTimes(1);
|
||||
await vi.advanceTimersByTimeAsync(6_000);
|
||||
expect(getFeed).toHaveBeenCalledTimes(2);
|
||||
expect(deps.audit).toHaveBeenCalledTimes(1);
|
||||
await service.onApplicationShutdown();
|
||||
await vi.advanceTimersByTimeAsync(900_000);
|
||||
expect(getFeed).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
it("renews a lease during slow polls and never overlaps refresh from another replica", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { service, deps, getFeed } = setup();
|
||||
let resolveFeed: (() => void) | undefined;
|
||||
getFeed.mockImplementationOnce(() => new Promise((resolve) => { resolveFeed = () => resolve({ items: [item("slow")] }); }));
|
||||
const active = service.pollWorkspace(session.workspaceId);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
await vi.advanceTimersByTimeAsync(150_000);
|
||||
expect((await new FeedsService(deps).pollWorkspace(session.workspaceId)).skipped).toBe(1);
|
||||
resolveFeed?.(); await active;
|
||||
expect(getFeed).toHaveBeenCalledTimes(1);
|
||||
expect((await service.pollWorkspace(session.workspaceId)).skipped).toBe(1);
|
||||
});
|
||||
it("uses stored credentials and the real connector HTTP limiter across accounts on one host", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { deps, connections } = setup(2);
|
||||
const credentials = new InMemoryCredentials();
|
||||
const requests: Array<{ at: number; token: string; path: string }> = [];
|
||||
const start = Date.now();
|
||||
const fetchImpl: typeof fetch = async (input, init) => {
|
||||
const url = new URL(String(input));
|
||||
const token = new Headers(init?.headers).get("authorization") ?? "";
|
||||
requests.push({ at: Date.now() - start, token, path: url.pathname });
|
||||
const account = { id: "alice", username: "alice", acct: "alice", display_name: "Alice", avatar: "", url: "https://social.example/@alice" };
|
||||
const data = url.pathname.endsWith("verify_credentials") ? account : [{
|
||||
id: "status-1", created_at: new Date().toISOString(), content: "<p>A real timeline post</p>",
|
||||
account, media_attachments: [], reblogs_count: 2, replies_count: 1, favourites_count: 3, reblog: null,
|
||||
}];
|
||||
return new Response(JSON.stringify(data), { headers: { "content-type": "application/json" } });
|
||||
};
|
||||
deps.registry = connectionRegistry(credentials, fetchImpl);
|
||||
for (const row of connections) {
|
||||
row.provider = "mastodon"; row.fields.instanceUrl = "https://social.example";
|
||||
await credentials.put(row, "access_token", `secret-${row.id}`);
|
||||
}
|
||||
const service = new FeedsService(deps);
|
||||
const polling = service.pollWorkspace(session.workspaceId);
|
||||
await vi.advanceTimersByTimeAsync(12_000);
|
||||
expect((await polling).polled).toBe(2);
|
||||
expect(requests).toHaveLength(4);
|
||||
expect(requests.every((request) => request.token.startsWith("Bearer secret-"))).toBe(true);
|
||||
for (let i = 1; i < requests.length; i += 1) expect((requests[i]?.at ?? 0) - (requests[i - 1]?.at ?? 0)).toBeGreaterThanOrEqual(2000);
|
||||
const response = await service.read(session);
|
||||
expect(response.items).toHaveLength(2);
|
||||
expect(response.items.every((post) => post.text === "A real timeline post" && post.metrics.favourites === 3)).toBe(true);
|
||||
});
|
||||
it("bounds concurrency across simultaneous workspace cycles", async () => {
|
||||
const { service, deps, getFeed, connections } = setup(10);
|
||||
let running = 0; let max = 0;
|
||||
const releases: Array<() => void> = [];
|
||||
getFeed.mockImplementation(() => new Promise((resolve) => {
|
||||
running += 1; max = Math.max(max, running);
|
||||
releases.push(() => { running -= 1; resolve({ items: [] }); });
|
||||
}));
|
||||
const polls = [service.pollWorkspace(session.workspaceId, connections.slice(0, 5).map((row) => row.id)), service.pollWorkspace(session.workspaceId, connections.slice(5).map((row) => row.id))];
|
||||
for (let i = 0; i < 50; i += 1) { await new Promise<void>((resolve) => setImmediate(resolve)); for (const release of releases.splice(0)) release(); }
|
||||
await Promise.all(polls);
|
||||
expect(max).toBe(4);
|
||||
expect(deps.audit).not.toHaveBeenCalled();
|
||||
});
|
||||
it("fails connections independently, clears stale data and honors Retry-After", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { service, getFeed, mock, connections } = setup();
|
||||
await service.pollWorkspace(session.workspaceId);
|
||||
await vi.advanceTimersByTimeAsync(66_000);
|
||||
getFeed.mockRejectedValueOnce(new RateLimitError("mock-social", 200_000));
|
||||
expect((await service.pollWorkspace(session.workspaceId)).failed).toBe(1);
|
||||
expect((await service.read(session)).items).toEqual([]);
|
||||
await vi.advanceTimersByTimeAsync(100_000);
|
||||
expect((await service.pollWorkspace(session.workspaceId)).skipped).toBe(1);
|
||||
await vi.advanceTimersByTimeAsync(100_000);
|
||||
vi.spyOn(mock, "getIdentity").mockRejectedValueOnce(new Error("Token revoked"));
|
||||
expect((await service.pollWorkspace(session.workspaceId)).failed).toBe(1);
|
||||
expect((await service.read(session, { connectionId: connections[0]?.id })).items).toEqual([]);
|
||||
expect(getFeed).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
28
apps/api/src/feeds/testing.ts
Normal file
28
apps/api/src/feeds/testing.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
import type { FeedDependencies } from "./feeds.dependencies.js";
|
||||
|
||||
/** TTL-faithful Redis surface, including ownership checks for renewable leases. */
|
||||
type FeedRedis = FeedDependencies["redis"];
|
||||
export class MemoryFeedRedis implements FeedRedis {
|
||||
readonly entries = new Map<string, { value: string; expiresAt: number }>();
|
||||
async get(key: string): Promise<string | null> {
|
||||
const entry = this.entries.get(key);
|
||||
if (!entry || entry.expiresAt <= Date.now()) { this.entries.delete(key); return null; }
|
||||
return entry.value;
|
||||
}
|
||||
async set(key: string, value: string, options: { nx?: boolean; pxMs?: number } = {}): Promise<boolean> {
|
||||
// Atomic NX check: do not yield between reading and writing.
|
||||
const entry = this.entries.get(key);
|
||||
if (options.nx && entry && entry.expiresAt > Date.now()) return false;
|
||||
this.entries.set(key, { value, expiresAt: Date.now() + (options.pxMs ?? 1000) });
|
||||
return true;
|
||||
}
|
||||
async eval(script: string, keys: string[], args: string[]): Promise<number> {
|
||||
const key = keys[0] ?? "";
|
||||
const entry = this.entries.get(key);
|
||||
if (!entry || entry.value !== args[0] || entry.expiresAt <= Date.now()) return 0;
|
||||
if (script.includes("PEXPIRE")) entry.expiresAt = Date.now() + Number(args[1]);
|
||||
else this.entries.delete(key);
|
||||
return 1;
|
||||
}
|
||||
close(): void {}
|
||||
}
|
||||
161
apps/api/src/groups/groups.controller.ts
Normal file
161
apps/api/src/groups/groups.controller.ts
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
import { BadRequestException, Body, Controller, Delete, Get, Inject, Param, Patch, Post, Req, UseGuards } from "@nestjs/common";
|
||||
import type { Request } from "express";
|
||||
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import { optionalString, parseId, requireObject, requireSession, requireString } from "../http/validation.js";
|
||||
import {
|
||||
GroupsService,
|
||||
hasWorkspaceWideAccess,
|
||||
type GroupActor,
|
||||
type GroupDetail,
|
||||
type GroupMembershipResult,
|
||||
type GroupSummary,
|
||||
type VisibleConnections,
|
||||
} from "./groups.service.js";
|
||||
|
||||
type AuthedRequest = Request & AuthenticatedRequest;
|
||||
|
||||
function actorOf(request: AuthedRequest): GroupActor {
|
||||
const session = requireSession(request);
|
||||
return { userId: session.userId, role: session.role };
|
||||
}
|
||||
|
||||
function optionalDescription(value: unknown): string | undefined {
|
||||
if (value === undefined || value === null) return undefined;
|
||||
if (typeof value !== "string") throw new BadRequestException("description must be a string");
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Member add/remove and group delete bump the workspace authz epoch, which
|
||||
* invalidates every live session; those responses carry `token` — the caller's
|
||||
* replacement session (same contract as /team).
|
||||
*/
|
||||
@Controller("groups")
|
||||
@UseGuards(RbacGuard)
|
||||
export class GroupsController {
|
||||
constructor(@Inject(GroupsService) private readonly groups: GroupsService) {}
|
||||
|
||||
/**
|
||||
* The connections the caller may publish through (Compose calls this).
|
||||
* Declared before `:id` so the literal path wins route matching.
|
||||
* session.role is current: role changes bump the epoch and the guard rejects stale sessions.
|
||||
*/
|
||||
@Get("visible-connections")
|
||||
@RequirePermission("workspace.view")
|
||||
async visibleConnections(@Req() request: AuthedRequest): Promise<VisibleConnections> {
|
||||
const session = requireSession(request);
|
||||
return this.groups.getVisibleConnections(session.workspaceId, session.userId, hasWorkspaceWideAccess(session.role));
|
||||
}
|
||||
|
||||
@Get("mine")
|
||||
@RequirePermission("workspace.view")
|
||||
async mine(@Req() request: AuthedRequest): Promise<GroupSummary[]> {
|
||||
const session = requireSession(request);
|
||||
return this.groups.getUserGroups(session.workspaceId, session.userId);
|
||||
}
|
||||
|
||||
@Post()
|
||||
@RequirePermission("workspace.members.manage")
|
||||
async create(@Req() request: AuthedRequest, @Body() body: unknown): Promise<GroupSummary> {
|
||||
const dto = requireObject(body, "body");
|
||||
return this.groups.createGroup(
|
||||
requireSession(request).workspaceId,
|
||||
requireString(dto.name, "name"),
|
||||
optionalDescription(dto.description) ?? "",
|
||||
actorOf(request),
|
||||
);
|
||||
}
|
||||
|
||||
@Get()
|
||||
@RequirePermission("workspace.view")
|
||||
async list(@Req() request: AuthedRequest): Promise<GroupSummary[]> {
|
||||
return this.groups.listGroups(requireSession(request).workspaceId);
|
||||
}
|
||||
|
||||
@Get(":id")
|
||||
@RequirePermission("workspace.view")
|
||||
async detail(@Req() request: AuthedRequest, @Param("id") id: string): Promise<GroupDetail> {
|
||||
return this.groups.getGroupDetail(requireSession(request).workspaceId, parseId(id));
|
||||
}
|
||||
|
||||
@Patch(":id")
|
||||
@RequirePermission("workspace.members.manage")
|
||||
async rename(@Req() request: AuthedRequest, @Param("id") id: string, @Body() body: unknown): Promise<GroupSummary> {
|
||||
const dto = requireObject(body, "body");
|
||||
return this.groups.renameGroup(
|
||||
requireSession(request).workspaceId,
|
||||
parseId(id),
|
||||
{ name: optionalString(dto.name, "name"), description: optionalDescription(dto.description) },
|
||||
actorOf(request),
|
||||
);
|
||||
}
|
||||
|
||||
@Delete(":id")
|
||||
@RequirePermission("workspace.members.manage")
|
||||
async remove(@Req() request: AuthedRequest, @Param("id") id: string): Promise<GroupMembershipResult> {
|
||||
return this.groups.deleteGroup(requireSession(request).workspaceId, parseId(id), actorOf(request));
|
||||
}
|
||||
|
||||
@Post(":id/members")
|
||||
@RequirePermission("workspace.members.manage")
|
||||
async addMember(@Req() request: AuthedRequest, @Param("id") id: string, @Body() body: unknown): Promise<GroupMembershipResult> {
|
||||
const dto = requireObject(body, "body");
|
||||
return this.groups.addMember(
|
||||
requireSession(request).workspaceId,
|
||||
parseId(id),
|
||||
parseId(requireString(dto.userId, "userId"), "userId"),
|
||||
actorOf(request),
|
||||
);
|
||||
}
|
||||
|
||||
@Delete(":id/members/:userId")
|
||||
@RequirePermission("workspace.members.manage")
|
||||
async removeMember(
|
||||
@Req() request: AuthedRequest,
|
||||
@Param("id") id: string,
|
||||
@Param("userId") userId: string,
|
||||
): Promise<GroupMembershipResult> {
|
||||
return this.groups.removeMember(requireSession(request).workspaceId, parseId(id), parseId(userId, "userId"), actorOf(request));
|
||||
}
|
||||
|
||||
@Post(":id/connections")
|
||||
@RequirePermission("connection.manage")
|
||||
async assignConnection(@Req() request: AuthedRequest, @Param("id") id: string, @Body() body: unknown): Promise<GroupSummary & GroupMembershipResult> {
|
||||
const dto = requireObject(body, "body");
|
||||
return this.groups.assignConnection(
|
||||
requireSession(request).workspaceId,
|
||||
parseId(id),
|
||||
parseId(requireString(dto.connectionId, "connectionId"), "connectionId"),
|
||||
actorOf(request),
|
||||
);
|
||||
}
|
||||
|
||||
@Delete(":id/connections/:connectionId")
|
||||
@RequirePermission("connection.manage")
|
||||
async unassignConnection(
|
||||
@Req() request: AuthedRequest,
|
||||
@Param("id") id: string,
|
||||
@Param("connectionId") connectionId: string,
|
||||
): Promise<GroupSummary & GroupMembershipResult> {
|
||||
return this.groups.unassignConnection(
|
||||
requireSession(request).workspaceId,
|
||||
parseId(id),
|
||||
parseId(connectionId, "connectionId"),
|
||||
actorOf(request),
|
||||
);
|
||||
}
|
||||
|
||||
@Post(":id/assets")
|
||||
@RequirePermission("media.manage")
|
||||
async assignAsset(@Req() request: AuthedRequest, @Param("id") id: string, @Body() body: unknown): Promise<GroupSummary> {
|
||||
const dto = requireObject(body, "body");
|
||||
return this.groups.assignAsset(requireSession(request).workspaceId, parseId(id), requireString(dto.assetId, "assetId"), actorOf(request));
|
||||
}
|
||||
|
||||
/** Asset ids are storage keys and may contain `/`: pass them URL-encoded. */
|
||||
@Delete(":id/assets/:assetId")
|
||||
@RequirePermission("media.manage")
|
||||
async unassignAsset(@Req() request: AuthedRequest, @Param("id") id: string, @Param("assetId") assetId: string): Promise<GroupSummary> {
|
||||
return this.groups.unassignAsset(requireSession(request).workspaceId, parseId(id), assetId, actorOf(request));
|
||||
}
|
||||
}
|
||||
251
apps/api/src/groups/groups.integration.test.ts
Normal file
251
apps/api/src/groups/groups.integration.test.ts
Normal file
|
|
@ -0,0 +1,251 @@
|
|||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { join } from "node:path";
|
||||
import type { ExecutionContext } from "@nestjs/common";
|
||||
import { Reflector } from "@nestjs/core";
|
||||
import { Pool, type PoolClient } from "pg";
|
||||
|
||||
/**
|
||||
* Groups against a live PostgreSQL, as sp_app (FORCE RLS binds).
|
||||
* Uses its OWN database (GROUPS_IT_DATABASE_URL, plus GROUPS_IT_SUPER_DATABASE_URL
|
||||
* for migrations/seeding) for the same reason as auth.integration.test.ts:
|
||||
* tenancy.integration.test.ts wipes every workspace in the shared DB and vitest
|
||||
* runs files in parallel. Skipped when unset.
|
||||
*/
|
||||
const appUrl = process.env.GROUPS_IT_DATABASE_URL;
|
||||
const ownerUrl = process.env.GROUPS_IT_SUPER_DATABASE_URL;
|
||||
|
||||
type Modules = {
|
||||
groups: typeof import("./groups.service.js");
|
||||
guard: typeof import("../rbac/rbac.guard.js");
|
||||
session: typeof import("../auth/session.js");
|
||||
db: typeof import("../db/client.js");
|
||||
};
|
||||
|
||||
describe.skipIf(appUrl === undefined || ownerUrl === undefined)("groups (live DB)", () => {
|
||||
let m: Modules;
|
||||
let ownerPool: Pool;
|
||||
const suffix = `${Date.now()}${Math.floor(Math.random() * 1e6)}`;
|
||||
let wsA: string;
|
||||
let wsB: string;
|
||||
let owner: string;
|
||||
let admin: string;
|
||||
let member: string;
|
||||
let outsider: string;
|
||||
let connShared: string;
|
||||
let connPrivate: string;
|
||||
let connB: string;
|
||||
|
||||
async function asOwner<T>(fn: (client: PoolClient) => Promise<T>): Promise<T> {
|
||||
const client = await ownerPool.connect();
|
||||
try {
|
||||
return await fn(client);
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function epoch(ws: string): Promise<number> {
|
||||
return asOwner(async (c) => Number((await c.query("SELECT authz_epoch FROM workspaces WHERE id = $1", [ws])).rows[0]?.authz_epoch));
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
// The pool reads DATABASE_URL at import time: point it at the isolated DB first.
|
||||
process.env.DATABASE_URL = appUrl;
|
||||
process.env.SUPER_DATABASE_URL = ownerUrl;
|
||||
const { runMigrations } = await import("../db/migrate.js");
|
||||
await runMigrations(join(process.cwd(), "migrations"));
|
||||
m = {
|
||||
groups: await import("./groups.service.js"),
|
||||
guard: await import("../rbac/rbac.guard.js"),
|
||||
session: await import("../auth/session.js"),
|
||||
db: await import("../db/client.js"),
|
||||
};
|
||||
ownerPool = new Pool({ connectionString: ownerUrl });
|
||||
|
||||
// Seed via the owner pool (FORCE RLS still applies to the owner, so tenant
|
||||
// rows are written inside a tenant context).
|
||||
await asOwner(async (c) => {
|
||||
const ws = async (name: string): Promise<string> =>
|
||||
(await c.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id", [name, `${name}-${suffix}`])).rows[0].id as string;
|
||||
wsA = await ws("grp-a");
|
||||
wsB = await ws("grp-b");
|
||||
const user = async (tag: string): Promise<string> =>
|
||||
(
|
||||
await c.query("INSERT INTO users (external_id, username, email, display_name) VALUES ($1, $2, $3, $4) RETURNING id", [
|
||||
`grp:${tag}:${suffix}`,
|
||||
`test-${tag}-${suffix}`,
|
||||
`${tag}-${suffix}@example.com`,
|
||||
tag,
|
||||
])
|
||||
).rows[0].id as string;
|
||||
owner = await user("owner");
|
||||
admin = await user("admin");
|
||||
member = await user("member");
|
||||
outsider = await user("outsider");
|
||||
|
||||
const inTenant = async (wsId: string, fn: () => Promise<void>): Promise<void> => {
|
||||
await c.query("BEGIN");
|
||||
await c.query("SELECT set_config('app.workspace_id', $1, true)", [wsId]);
|
||||
await fn();
|
||||
await c.query("COMMIT");
|
||||
};
|
||||
await inTenant(wsA, async () => {
|
||||
for (const [u, role] of [
|
||||
[owner, "Owner"],
|
||||
[admin, "Admin"],
|
||||
[member, "Publisher"],
|
||||
[outsider, "Publisher"],
|
||||
] as const) {
|
||||
await c.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, $3)", [wsA, u, role]);
|
||||
}
|
||||
const conn = async (ch: string): Promise<string> =>
|
||||
(await c.query("INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-social', $2) RETURNING id", [wsA, ch]))
|
||||
.rows[0].id as string;
|
||||
connShared = await conn("shared");
|
||||
connPrivate = await conn("private");
|
||||
});
|
||||
await inTenant(wsB, async () => {
|
||||
await c.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, 'Owner')", [wsB, owner]);
|
||||
connB = (await c.query("INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-social', 'b') RETURNING id", [wsB]))
|
||||
.rows[0].id as string;
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (ownerPool !== undefined) {
|
||||
await asOwner(async (c) => {
|
||||
await c.query("DELETE FROM workspaces WHERE id = ANY($1::uuid[])", [[wsA, wsB]]);
|
||||
await c.query("DELETE FROM users WHERE external_id LIKE $1", [`grp:%:${suffix}`]);
|
||||
});
|
||||
await ownerPool.end();
|
||||
}
|
||||
await m?.db.pool.end();
|
||||
});
|
||||
|
||||
let groupId: string;
|
||||
|
||||
it("create group -> assign connection -> add member: the member sees exactly that connection", async () => {
|
||||
const svc = new m.groups.GroupsService();
|
||||
const actor = { userId: admin, role: "Admin" };
|
||||
const group = await svc.createGroup(wsA, "Newsroom", "front desk", actor);
|
||||
groupId = group.id;
|
||||
await expect(svc.createGroup(wsA, "Newsroom", "", actor)).rejects.toMatchObject({ status: 409 });
|
||||
|
||||
await svc.assignConnection(wsA, groupId, connShared, actor);
|
||||
const before = await epoch(wsA);
|
||||
const added = await svc.addMember(wsA, groupId, member, actor);
|
||||
expect(await epoch(wsA)).toBeGreaterThan(before);
|
||||
// The replacement token carries the post-bump epoch.
|
||||
expect(m.session.verifySession(added.token)?.authzEpoch).toBe(await epoch(wsA));
|
||||
|
||||
expect(await svc.getVisibleConnections(wsA, member, false)).toEqual({ scope: "groups", connectionIds: [connShared] });
|
||||
|
||||
const detail = await svc.getGroupDetail(wsA, groupId);
|
||||
expect(detail.members.map((x) => x.userId)).toEqual([member]);
|
||||
expect(detail.connections.map((x) => x.connectionId)).toEqual([connShared]);
|
||||
expect((await svc.listGroups(wsA)).find((g) => g.id === groupId)).toMatchObject({ memberCount: 1, connectionCount: 1 });
|
||||
expect((await svc.getUserGroups(wsA, member)).map((g) => g.id)).toEqual([groupId]);
|
||||
|
||||
const actions = await asOwner(async (c) => {
|
||||
await c.query("BEGIN");
|
||||
await c.query("SELECT set_config('app.workspace_id', $1, true)", [wsA]);
|
||||
const rows = (await c.query("SELECT action FROM audit_log WHERE target_id = $1 ORDER BY seq", [groupId])).rows as Array<{ action: string }>;
|
||||
await c.query("COMMIT");
|
||||
return rows.map((r) => r.action);
|
||||
});
|
||||
expect(actions).toEqual(["group.created", "group.connection_assigned", "group.member_added"]);
|
||||
});
|
||||
|
||||
it("a non-member does NOT see the connection", async () => {
|
||||
const svc = new m.groups.GroupsService();
|
||||
expect(await svc.getVisibleConnections(wsA, outsider, false)).toEqual({ scope: "groups", connectionIds: [] });
|
||||
});
|
||||
|
||||
it("Owner/Admin see ALL workspace connections regardless of group membership", async () => {
|
||||
const svc = new m.groups.GroupsService();
|
||||
for (const u of [owner, admin]) {
|
||||
const visible = await svc.getVisibleConnections(wsA, u, true);
|
||||
expect(visible.scope).toBe("workspace");
|
||||
expect(new Set(visible.connectionIds)).toEqual(new Set([connShared, connPrivate]));
|
||||
}
|
||||
});
|
||||
|
||||
it("@RequireGroupAccess: member passes for the shared connection, outsider and private connection are forbidden, Owner passes", async () => {
|
||||
const epochNow = await epoch(wsA);
|
||||
const tokenFor = (userId: string, role: string): string =>
|
||||
m.session.issueSession({ userId, workspaceId: wsA, role, authzEpoch: epochNow });
|
||||
const check = async (token: string, connectionId: string): Promise<boolean | string> => {
|
||||
const handler = (): void => undefined;
|
||||
Reflect.defineMetadata(m.guard.PERMISSION_KEY, "publication.publish", handler);
|
||||
Reflect.defineMetadata(m.guard.GROUP_ACCESS_KEY, true, handler);
|
||||
const req = { headers: { authorization: `Bearer ${token}` }, params: { connectionId } };
|
||||
const ctx = { getHandler: () => handler, switchToHttp: () => ({ getRequest: () => req }) } as unknown as ExecutionContext;
|
||||
try {
|
||||
return await new m.guard.RbacGuard(new Reflector()).canActivate(ctx);
|
||||
} catch (err) {
|
||||
return (err as Error).constructor.name;
|
||||
}
|
||||
};
|
||||
expect(await check(tokenFor(member, "Publisher"), connShared)).toBe(true);
|
||||
expect(await check(tokenFor(member, "Publisher"), connPrivate)).toBe("ForbiddenException");
|
||||
expect(await check(tokenFor(outsider, "Publisher"), connShared)).toBe("ForbiddenException");
|
||||
expect(await check(tokenFor(owner, "Owner"), connPrivate)).toBe(true);
|
||||
});
|
||||
|
||||
it("cross-tenant scoping: a connection or user from another workspace cannot be attached", async () => {
|
||||
const svc = new m.groups.GroupsService();
|
||||
const actor = { userId: admin, role: "Admin" };
|
||||
await expect(svc.assignConnection(wsA, groupId, connB, actor)).rejects.toMatchObject({ status: 404 });
|
||||
// owner IS a member of B, but this checks a user with no wsA membership is refused:
|
||||
const stranger = await asOwner(
|
||||
async (c) =>
|
||||
(await c.query(
|
||||
"INSERT INTO users (external_id, username, email, display_name) VALUES ($1, $2, $3, 'S') RETURNING id",
|
||||
[`grp:stranger:${suffix}`, `test-stranger-${suffix}`, `s-${suffix}@x.io`],
|
||||
))
|
||||
.rows[0].id as string,
|
||||
);
|
||||
await expect(svc.addMember(wsA, groupId, stranger, actor)).rejects.toMatchObject({ status: 400 });
|
||||
});
|
||||
|
||||
it("remove member -> authz_epoch bumps -> connection no longer visible", async () => {
|
||||
const svc = new m.groups.GroupsService();
|
||||
const before = await epoch(wsA);
|
||||
await svc.removeMember(wsA, groupId, member, { userId: admin, role: "Admin" });
|
||||
expect(await epoch(wsA)).toBeGreaterThan(before);
|
||||
expect(await svc.getVisibleConnections(wsA, member, false)).toEqual({ scope: "groups", connectionIds: [] });
|
||||
await expect(svc.removeMember(wsA, groupId, member, { userId: admin, role: "Admin" })).rejects.toMatchObject({ status: 404 });
|
||||
});
|
||||
|
||||
it("deleting a group with members bumps the epoch via the cascade", async () => {
|
||||
const svc = new m.groups.GroupsService();
|
||||
const actor = { userId: admin, role: "Admin" };
|
||||
const g = await svc.createGroup(wsA, "Ephemeral", "", actor);
|
||||
await svc.addMember(wsA, g.id, member, actor);
|
||||
const before = await epoch(wsA);
|
||||
await svc.deleteGroup(wsA, g.id, actor);
|
||||
expect(await epoch(wsA)).toBeGreaterThan(before);
|
||||
await expect(svc.getGroupDetail(wsA, g.id)).rejects.toMatchObject({ status: 404 });
|
||||
});
|
||||
|
||||
it("RLS: workspace A's group rows are invisible from workspace B's context and from no context", async () => {
|
||||
const { withTenant, withSystem } = m.db;
|
||||
const count = async (client: PoolClient, table: string): Promise<number> =>
|
||||
Number((await client.query(`SELECT count(*)::int AS n FROM ${table}`)).rows[0]?.n);
|
||||
await new m.groups.GroupsService().assignAsset(wsA, groupId, "media/abc.png", { userId: admin, role: "Admin" });
|
||||
|
||||
const fromA = await withTenant(wsA, async (c) => ({ groups: await count(c, "groups"), assets: await count(c, "group_assets") }));
|
||||
expect(fromA.groups).toBeGreaterThan(0);
|
||||
expect(fromA.assets).toBe(1);
|
||||
for (const table of ["groups", "group_members", "group_connections", "group_assets"]) {
|
||||
expect(await withTenant(wsB, (c) => count(c, table))).toBe(0);
|
||||
expect(await withSystem((c) => count(c, table))).toBe(0);
|
||||
}
|
||||
await expect(new m.groups.GroupsService().getGroupDetail(wsB, groupId)).rejects.toMatchObject({ status: 404 });
|
||||
// WITH CHECK: a B-context write cannot target A.
|
||||
await expect(
|
||||
withTenant(wsB, (c) => c.query("INSERT INTO groups (workspace_id, name) VALUES ($1, 'smuggled')", [wsA])),
|
||||
).rejects.toThrow(/row-level security/);
|
||||
});
|
||||
});
|
||||
472
apps/api/src/groups/groups.service.ts
Normal file
472
apps/api/src/groups/groups.service.ts
Normal file
|
|
@ -0,0 +1,472 @@
|
|||
import { BadRequestException, ConflictException, Injectable, NotFoundException } from "@nestjs/common";
|
||||
import type { PoolClient } from "pg";
|
||||
import { scrub, type AuditEvent } from "../audit/audit.service.js";
|
||||
import { issueSessionInTenant } from "../auth/auth.service.js";
|
||||
import { withTenant } from "../db/client.js";
|
||||
|
||||
/**
|
||||
* Groups: sub-workspace sharing scopes. Owner/Admin publish through every
|
||||
* connection; everyone else publishes only through connections assigned to a
|
||||
* group they belong to.
|
||||
*
|
||||
* Every mutation writes its audit row on the SAME transaction (audit_append on
|
||||
* the tenant client), so a change can never land without its audit entry.
|
||||
*
|
||||
* Group-membership changes bump the workspace authz epoch (008 trigger), which
|
||||
* invalidates all live sessions — including the caller's. Those mutations return
|
||||
* a fresh token for the caller, minted inside the same transaction (same
|
||||
* contract as TeamService).
|
||||
*/
|
||||
|
||||
export const WORKSPACE_WIDE_ROLES: readonly string[] = ["Owner", "Admin"];
|
||||
|
||||
export function hasWorkspaceWideAccess(role: string): boolean {
|
||||
return WORKSPACE_WIDE_ROLES.includes(role);
|
||||
}
|
||||
|
||||
export interface GroupActor {
|
||||
userId: string;
|
||||
role: string;
|
||||
}
|
||||
|
||||
export interface GroupSummary {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
memberCount: number;
|
||||
connectionCount: number;
|
||||
assetCount: number;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface GroupMember {
|
||||
userId: string;
|
||||
email: string;
|
||||
displayName: string;
|
||||
addedBy: string | null;
|
||||
addedAt: string;
|
||||
}
|
||||
|
||||
export interface GroupConnection {
|
||||
connectionId: string;
|
||||
provider: string;
|
||||
channelId: string;
|
||||
assignedBy: string | null;
|
||||
assignedAt: string;
|
||||
}
|
||||
|
||||
export interface GroupAsset {
|
||||
assetId: string;
|
||||
assignedBy: string | null;
|
||||
assignedAt: string;
|
||||
}
|
||||
|
||||
export interface GroupDetail extends GroupSummary {
|
||||
members: GroupMember[];
|
||||
connections: GroupConnection[];
|
||||
assets: GroupAsset[];
|
||||
}
|
||||
|
||||
/** Result of a mutation that bumps the authz epoch: the caller's replacement session. */
|
||||
export interface GroupMembershipResult {
|
||||
token: string;
|
||||
}
|
||||
|
||||
export interface VisibleConnections {
|
||||
/** "workspace": Owner/Admin — every connection. "groups": union of the user's groups. */
|
||||
scope: "workspace" | "groups";
|
||||
connectionIds: string[];
|
||||
}
|
||||
|
||||
interface GroupRow {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
created_at: Date;
|
||||
member_count: string | number;
|
||||
connection_count: string | number;
|
||||
asset_count: string | number;
|
||||
}
|
||||
|
||||
const GROUP_SELECT = `SELECT g.id, g.name, g.description, g.created_at,
|
||||
(SELECT count(*) FROM group_members gm WHERE gm.group_id = g.id) AS member_count,
|
||||
(SELECT count(*) FROM group_connections gc WHERE gc.group_id = g.id) AS connection_count,
|
||||
(SELECT count(*) FROM group_assets ga WHERE ga.group_id = g.id) AS asset_count
|
||||
FROM groups g`;
|
||||
|
||||
function summaryFromRow(row: GroupRow): GroupSummary {
|
||||
return {
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
description: row.description,
|
||||
memberCount: Number(row.member_count),
|
||||
connectionCount: Number(row.connection_count),
|
||||
assetCount: Number(row.asset_count),
|
||||
createdAt: new Date(row.created_at).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
function iso(value: Date): string {
|
||||
return new Date(value).toISOString();
|
||||
}
|
||||
|
||||
/** node-postgres surfaces SQLSTATE on `code`. */
|
||||
function pgCode(err: unknown): string | undefined {
|
||||
if (typeof err !== "object" || err === null || !("code" in err)) return undefined;
|
||||
const code = (err as { code: unknown }).code;
|
||||
return typeof code === "string" ? code : undefined;
|
||||
}
|
||||
|
||||
const UNIQUE_VIOLATION = "23505";
|
||||
const FK_VIOLATION = "23503";
|
||||
|
||||
async function audit(client: PoolClient, workspaceId: string, event: AuditEvent): Promise<void> {
|
||||
await client.query("SELECT audit_append($1, $2, $3, $4, $5, $6, $7)", [
|
||||
workspaceId,
|
||||
event.actorUserId,
|
||||
event.actorRole,
|
||||
event.action,
|
||||
event.targetType,
|
||||
event.targetId,
|
||||
scrub(event.diff ?? null),
|
||||
]);
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class GroupsService {
|
||||
async createGroup(workspaceId: string, name: string, description: string, actor: GroupActor): Promise<GroupSummary> {
|
||||
const cleanName = name.trim();
|
||||
if (cleanName === "") throw new BadRequestException("name must be a non-empty string");
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
let id: string;
|
||||
try {
|
||||
const res = await client.query(
|
||||
"INSERT INTO groups (workspace_id, name, description) VALUES ($1, $2, $3) RETURNING id",
|
||||
[workspaceId, cleanName, description],
|
||||
);
|
||||
id = res.rows[0]?.id as string;
|
||||
} catch (err) {
|
||||
if (pgCode(err) === UNIQUE_VIOLATION) throw new ConflictException(`a group named "${cleanName}" already exists`);
|
||||
throw err;
|
||||
}
|
||||
await audit(client, workspaceId, {
|
||||
actorUserId: actor.userId,
|
||||
actorRole: actor.role,
|
||||
action: "group.created",
|
||||
targetType: "group",
|
||||
targetId: id,
|
||||
diff: { name: cleanName, description },
|
||||
});
|
||||
return this.summary(client, id);
|
||||
});
|
||||
}
|
||||
|
||||
async renameGroup(
|
||||
workspaceId: string,
|
||||
groupId: string,
|
||||
changes: { name?: string; description?: string },
|
||||
actor: GroupActor,
|
||||
): Promise<GroupSummary> {
|
||||
const name = changes.name?.trim();
|
||||
if (name === "") throw new BadRequestException("name must be a non-empty string");
|
||||
if (name === undefined && changes.description === undefined) {
|
||||
throw new BadRequestException("nothing to update: pass name and/or description");
|
||||
}
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const before = await this.summary(client, groupId, true);
|
||||
try {
|
||||
await client.query(
|
||||
"UPDATE groups SET name = COALESCE($2, name), description = COALESCE($3, description) WHERE id = $1",
|
||||
[groupId, name ?? null, changes.description ?? null],
|
||||
);
|
||||
} catch (err) {
|
||||
if (pgCode(err) === UNIQUE_VIOLATION) throw new ConflictException(`a group named "${name ?? ""}" already exists`);
|
||||
throw err;
|
||||
}
|
||||
const after = await this.summary(client, groupId);
|
||||
const diff: Record<string, unknown> = {};
|
||||
if (after.name !== before.name) diff.name = [before.name, after.name];
|
||||
if (after.description !== before.description) diff.description = [before.description, after.description];
|
||||
if (Object.keys(diff).length > 0) {
|
||||
await audit(client, workspaceId, {
|
||||
actorUserId: actor.userId,
|
||||
actorRole: actor.role,
|
||||
action: "group.renamed",
|
||||
targetType: "group",
|
||||
targetId: groupId,
|
||||
diff,
|
||||
});
|
||||
}
|
||||
return after;
|
||||
});
|
||||
}
|
||||
|
||||
/** Cascades to members/connections/assets; member rows going away bump the epoch. */
|
||||
async deleteGroup(workspaceId: string, groupId: string, actor: GroupActor): Promise<GroupMembershipResult> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]);
|
||||
const before = await this.summary(client, groupId, true);
|
||||
await client.query("DELETE FROM groups WHERE id = $1", [groupId]);
|
||||
await audit(client, workspaceId, {
|
||||
actorUserId: actor.userId,
|
||||
actorRole: actor.role,
|
||||
action: "group.deleted",
|
||||
targetType: "group",
|
||||
targetId: groupId,
|
||||
diff: {
|
||||
name: before.name,
|
||||
memberCount: before.memberCount,
|
||||
connectionCount: before.connectionCount,
|
||||
assetCount: before.assetCount,
|
||||
},
|
||||
});
|
||||
return { token: await issueSessionInTenant(client, workspaceId, actor.userId) };
|
||||
});
|
||||
}
|
||||
|
||||
async addMember(workspaceId: string, groupId: string, userId: string, actor: GroupActor): Promise<GroupMembershipResult> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]);
|
||||
await this.summary(client, groupId);
|
||||
let inserted: number;
|
||||
try {
|
||||
const res = await client.query(
|
||||
`INSERT INTO group_members (workspace_id, group_id, user_id, added_by) VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (group_id, user_id) DO NOTHING`,
|
||||
[workspaceId, groupId, userId, actor.userId],
|
||||
);
|
||||
inserted = res.rowCount ?? 0;
|
||||
} catch (err) {
|
||||
// Composite FK to memberships: the user is not a member of this workspace.
|
||||
if (pgCode(err) === FK_VIOLATION) throw new BadRequestException("user is not a member of this workspace");
|
||||
throw err;
|
||||
}
|
||||
if (inserted === 0) throw new ConflictException("user is already a member of this group");
|
||||
await audit(client, workspaceId, {
|
||||
actorUserId: actor.userId,
|
||||
actorRole: actor.role,
|
||||
action: "group.member_added",
|
||||
targetType: "group",
|
||||
targetId: groupId,
|
||||
diff: { userId },
|
||||
});
|
||||
return { token: await issueSessionInTenant(client, workspaceId, actor.userId) };
|
||||
});
|
||||
}
|
||||
|
||||
async removeMember(workspaceId: string, groupId: string, userId: string, actor: GroupActor): Promise<GroupMembershipResult> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]);
|
||||
const res = await client.query("DELETE FROM group_members WHERE group_id = $1 AND user_id = $2", [groupId, userId]);
|
||||
if ((res.rowCount ?? 0) === 0) throw new NotFoundException("user is not a member of this group");
|
||||
await audit(client, workspaceId, {
|
||||
actorUserId: actor.userId,
|
||||
actorRole: actor.role,
|
||||
action: "group.member_removed",
|
||||
targetType: "group",
|
||||
targetId: groupId,
|
||||
diff: { userId },
|
||||
});
|
||||
return { token: await issueSessionInTenant(client, workspaceId, actor.userId) };
|
||||
});
|
||||
}
|
||||
|
||||
async assignConnection(workspaceId: string, groupId: string, connectionId: string, actor: GroupActor): Promise<GroupSummary & GroupMembershipResult> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]);
|
||||
await this.summary(client, groupId);
|
||||
let inserted: number;
|
||||
try {
|
||||
const res = await client.query(
|
||||
`INSERT INTO group_connections (workspace_id, group_id, connection_id, assigned_by) VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (group_id, connection_id) DO NOTHING`,
|
||||
[workspaceId, groupId, connectionId, actor.userId],
|
||||
);
|
||||
inserted = res.rowCount ?? 0;
|
||||
} catch (err) {
|
||||
if (pgCode(err) === FK_VIOLATION) throw new NotFoundException(`connection ${connectionId} not found`);
|
||||
throw err;
|
||||
}
|
||||
if (inserted === 0) throw new ConflictException("connection is already assigned to this group");
|
||||
await audit(client, workspaceId, {
|
||||
actorUserId: actor.userId,
|
||||
actorRole: actor.role,
|
||||
action: "group.connection_assigned",
|
||||
targetType: "group",
|
||||
targetId: groupId,
|
||||
diff: { connectionId },
|
||||
});
|
||||
return { ...await this.summary(client, groupId), token: await issueSessionInTenant(client, workspaceId, actor.userId) };
|
||||
});
|
||||
}
|
||||
|
||||
async unassignConnection(workspaceId: string, groupId: string, connectionId: string, actor: GroupActor): Promise<GroupSummary & GroupMembershipResult> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]);
|
||||
const res = await client.query("DELETE FROM group_connections WHERE group_id = $1 AND connection_id = $2", [
|
||||
groupId,
|
||||
connectionId,
|
||||
]);
|
||||
if ((res.rowCount ?? 0) === 0) throw new NotFoundException("connection is not assigned to this group");
|
||||
await audit(client, workspaceId, {
|
||||
actorUserId: actor.userId,
|
||||
actorRole: actor.role,
|
||||
action: "group.connection_unassigned",
|
||||
targetType: "group",
|
||||
targetId: groupId,
|
||||
diff: { connectionId },
|
||||
});
|
||||
return { ...await this.summary(client, groupId), token: await issueSessionInTenant(client, workspaceId, actor.userId) };
|
||||
});
|
||||
}
|
||||
|
||||
async assignAsset(workspaceId: string, groupId: string, assetId: string, actor: GroupActor): Promise<GroupSummary> {
|
||||
const cleanAssetId = assetId.trim();
|
||||
if (cleanAssetId === "") throw new BadRequestException("assetId must be a non-empty string");
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
await this.summary(client, groupId);
|
||||
const res = await client.query(
|
||||
`INSERT INTO group_assets (workspace_id, group_id, asset_id, assigned_by) VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (group_id, asset_id) DO NOTHING`,
|
||||
[workspaceId, groupId, cleanAssetId, actor.userId],
|
||||
);
|
||||
if ((res.rowCount ?? 0) === 0) throw new ConflictException("asset is already assigned to this group");
|
||||
await audit(client, workspaceId, {
|
||||
actorUserId: actor.userId,
|
||||
actorRole: actor.role,
|
||||
action: "group.asset_assigned",
|
||||
targetType: "group",
|
||||
targetId: groupId,
|
||||
diff: { assetId: cleanAssetId },
|
||||
});
|
||||
return this.summary(client, groupId);
|
||||
});
|
||||
}
|
||||
|
||||
async unassignAsset(workspaceId: string, groupId: string, assetId: string, actor: GroupActor): Promise<GroupSummary> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const res = await client.query("DELETE FROM group_assets WHERE group_id = $1 AND asset_id = $2", [groupId, assetId]);
|
||||
if ((res.rowCount ?? 0) === 0) throw new NotFoundException("asset is not assigned to this group");
|
||||
await audit(client, workspaceId, {
|
||||
actorUserId: actor.userId,
|
||||
actorRole: actor.role,
|
||||
action: "group.asset_unassigned",
|
||||
targetType: "group",
|
||||
targetId: groupId,
|
||||
diff: { assetId },
|
||||
});
|
||||
return this.summary(client, groupId);
|
||||
});
|
||||
}
|
||||
|
||||
async listGroups(workspaceId: string): Promise<GroupSummary[]> {
|
||||
const res = await withTenant(workspaceId, (client) => client.query(`${GROUP_SELECT} ORDER BY g.name ASC`));
|
||||
return (res.rows as GroupRow[]).map(summaryFromRow);
|
||||
}
|
||||
|
||||
async getGroupDetail(workspaceId: string, groupId: string): Promise<GroupDetail> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const summary = await this.summary(client, groupId);
|
||||
const members = await client.query(
|
||||
`SELECT gm.user_id, u.email, u.display_name, gm.added_by, gm.created_at
|
||||
FROM group_members gm JOIN users u ON u.id = gm.user_id
|
||||
WHERE gm.group_id = $1 ORDER BY gm.created_at ASC`,
|
||||
[groupId],
|
||||
);
|
||||
const connections = await client.query(
|
||||
`SELECT gc.connection_id, c.provider, c.channel_id, gc.assigned_by, gc.created_at
|
||||
FROM group_connections gc JOIN connections c ON c.id = gc.connection_id
|
||||
WHERE gc.group_id = $1 ORDER BY gc.created_at ASC`,
|
||||
[groupId],
|
||||
);
|
||||
const assets = await client.query(
|
||||
"SELECT asset_id, assigned_by, created_at FROM group_assets WHERE group_id = $1 ORDER BY created_at ASC",
|
||||
[groupId],
|
||||
);
|
||||
return {
|
||||
...summary,
|
||||
members: (
|
||||
members.rows as Array<{ user_id: string; email: string; display_name: string; added_by: string | null; created_at: Date }>
|
||||
).map((r) => ({ userId: r.user_id, email: r.email, displayName: r.display_name, addedBy: r.added_by, addedAt: iso(r.created_at) })),
|
||||
connections: (
|
||||
connections.rows as Array<{ connection_id: string; provider: string; channel_id: string; assigned_by: string | null; created_at: Date }>
|
||||
).map((r) => ({
|
||||
connectionId: r.connection_id,
|
||||
provider: r.provider,
|
||||
channelId: r.channel_id,
|
||||
assignedBy: r.assigned_by,
|
||||
assignedAt: iso(r.created_at),
|
||||
})),
|
||||
assets: (assets.rows as Array<{ asset_id: string; assigned_by: string | null; created_at: Date }>).map((r) => ({
|
||||
assetId: r.asset_id,
|
||||
assignedBy: r.assigned_by,
|
||||
assignedAt: iso(r.created_at),
|
||||
})),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async getUserGroups(workspaceId: string, userId: string): Promise<GroupSummary[]> {
|
||||
const res = await withTenant(workspaceId, (client) =>
|
||||
client.query(
|
||||
`${GROUP_SELECT} WHERE EXISTS (SELECT 1 FROM group_members gm WHERE gm.group_id = g.id AND gm.user_id = $1)
|
||||
ORDER BY g.name ASC`,
|
||||
[userId],
|
||||
),
|
||||
);
|
||||
return (res.rows as GroupRow[]).map(summaryFromRow);
|
||||
}
|
||||
|
||||
async getVisibleConnections(workspaceId: string, userId: string, isOwnerOrAdmin: boolean): Promise<VisibleConnections> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
if (isOwnerOrAdmin) {
|
||||
const res = await client.query("SELECT id FROM connections ORDER BY created_at ASC");
|
||||
return { scope: "workspace", connectionIds: (res.rows as Array<{ id: string }>).map((r) => r.id) };
|
||||
}
|
||||
const res = await client.query(
|
||||
`SELECT c.id FROM connections c
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM group_connections gc
|
||||
JOIN group_members gm ON gm.group_id = gc.group_id
|
||||
WHERE gc.connection_id = c.id AND gm.user_id = $1
|
||||
)
|
||||
ORDER BY c.created_at ASC`,
|
||||
[userId],
|
||||
);
|
||||
return { scope: "groups", connectionIds: (res.rows as Array<{ id: string }>).map((r) => r.id) };
|
||||
});
|
||||
}
|
||||
|
||||
private async summary(client: PoolClient, groupId: string, lock = false): Promise<GroupSummary> {
|
||||
const res = await client.query(`${GROUP_SELECT} WHERE g.id = $1${lock ? " FOR UPDATE OF g" : ""}`, [groupId]);
|
||||
const row = res.rows[0] as GroupRow | undefined;
|
||||
if (row === undefined) throw new NotFoundException(`group ${groupId} not found`);
|
||||
return summaryFromRow(row);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Guard-side check (same tenant client): can this user publish through this
|
||||
* connection? Role is read from the DB, not the token, so it is the live value.
|
||||
*/
|
||||
export async function userCanUseConnection(
|
||||
client: PoolClient,
|
||||
workspaceId: string,
|
||||
userId: string,
|
||||
connectionId: string,
|
||||
): Promise<boolean> {
|
||||
const res = await client.query(
|
||||
`SELECT 1 FROM memberships m
|
||||
WHERE m.workspace_id = $1 AND m.user_id = $2 AND m.status = 'active'
|
||||
AND (
|
||||
m.role = ANY($4::text[])
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM group_connections gc
|
||||
JOIN group_members gm ON gm.group_id = gc.group_id
|
||||
WHERE gc.connection_id = $3 AND gm.user_id = m.user_id
|
||||
)
|
||||
)`,
|
||||
[workspaceId, userId, connectionId, WORKSPACE_WIDE_ROLES],
|
||||
);
|
||||
return (res.rowCount ?? 0) > 0;
|
||||
}
|
||||
336
apps/api/src/groups/groups.test.ts
Normal file
336
apps/api/src/groups/groups.test.ts
Normal file
|
|
@ -0,0 +1,336 @@
|
|||
import { BadRequestException, ConflictException, ForbiddenException, NotFoundException, type ExecutionContext } from "@nestjs/common";
|
||||
import { Reflector } from "@nestjs/core";
|
||||
import type { QueryResult } from "pg";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
// vi.mock calls below are hoisted above these imports.
|
||||
import { issueSession } from "../auth/session.js";
|
||||
import { GROUP_ACCESS_KEY, PERMISSION_KEY, RbacGuard } from "../rbac/rbac.guard.js";
|
||||
import { GroupsService, hasWorkspaceWideAccess } from "./groups.service.js";
|
||||
|
||||
/**
|
||||
* Unit tests: the tenant client is faked so we can assert on the exact SQL
|
||||
* parameters and the in-transaction audit_append call. Live-DB behaviour
|
||||
* (RLS, epoch trigger, FK scoping) is in groups.integration.test.ts — that
|
||||
* file cannot share this one because vi.mock is file-wide.
|
||||
*/
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
interface Call {
|
||||
sql: string;
|
||||
values: readonly unknown[];
|
||||
}
|
||||
|
||||
const fake = vi.hoisted(() => ({
|
||||
calls: [] as Array<{ sql: string; values: readonly unknown[] }>,
|
||||
responders: [] as Array<(sql: string, values: readonly unknown[]) => Array<Record<string, unknown>> | Error | undefined>,
|
||||
tenants: [] as string[],
|
||||
epoch: 7,
|
||||
}));
|
||||
|
||||
function result(rows: Row[]): QueryResult<Row> {
|
||||
return { command: "", rowCount: rows.length, oid: 0, fields: [], rows };
|
||||
}
|
||||
|
||||
const client = {
|
||||
async query(sql: string, values: readonly unknown[] = []): Promise<QueryResult<Row>> {
|
||||
fake.calls.push({ sql, values });
|
||||
for (const respond of fake.responders) {
|
||||
const out = respond(sql, values);
|
||||
if (out instanceof Error) throw out;
|
||||
if (out !== undefined) return result(out);
|
||||
}
|
||||
return result([]);
|
||||
},
|
||||
};
|
||||
|
||||
vi.mock("../db/client.js", () => ({
|
||||
pool: {
|
||||
query: async (sql: string, values: readonly unknown[] = []) => client.query(sql, values),
|
||||
},
|
||||
withTenant: async <T>(workspaceId: string, fn: (c: typeof client) => Promise<T>): Promise<T> => {
|
||||
fake.tenants.push(workspaceId);
|
||||
return fn(client);
|
||||
},
|
||||
withSystem: async <T>(fn: (c: typeof client) => Promise<T>): Promise<T> => fn(client),
|
||||
}));
|
||||
|
||||
vi.mock("../auth/auth.service.js", () => ({
|
||||
issueSessionInTenant: async (_c: unknown, workspaceId: string, userId: string) => `token:${workspaceId}:${userId}:${fake.epoch}`,
|
||||
}));
|
||||
|
||||
|
||||
const ws = "10000000-0000-4000-8000-000000000001";
|
||||
const groupId = "20000000-0000-4000-8000-000000000002";
|
||||
const userId = "30000000-0000-4000-8000-000000000003";
|
||||
const connectionId = "40000000-0000-4000-8000-000000000004";
|
||||
const actor = { userId: "50000000-0000-4000-8000-000000000005", role: "Admin" };
|
||||
|
||||
const groupRow = (over: Row = {}): Row => ({
|
||||
id: groupId,
|
||||
name: "Newsroom",
|
||||
description: "",
|
||||
created_at: new Date("2026-10-06T10:00:00.000Z"),
|
||||
member_count: "1",
|
||||
connection_count: "2",
|
||||
asset_count: "0",
|
||||
...over,
|
||||
});
|
||||
|
||||
function pgError(code: string): Error {
|
||||
return Object.assign(new Error(`pg ${code}`), { code });
|
||||
}
|
||||
|
||||
function respond(match: RegExp, out: Row[] | Error | ((values: readonly unknown[]) => Row[] | Error)): void {
|
||||
fake.responders.push((sql, values) => (match.test(sql) ? (typeof out === "function" ? out(values) : out) : undefined));
|
||||
}
|
||||
|
||||
function auditCalls(): Call[] {
|
||||
return fake.calls.filter((c) => c.sql.includes("audit_append"));
|
||||
}
|
||||
|
||||
function insertCall(table: string): Call {
|
||||
const call = fake.calls.find((c) => c.sql.includes(`INSERT INTO ${table}`));
|
||||
if (call === undefined) throw new Error(`no INSERT INTO ${table}`);
|
||||
return call;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
fake.calls.length = 0;
|
||||
fake.responders.length = 0;
|
||||
fake.tenants.length = 0;
|
||||
});
|
||||
|
||||
describe("GroupsService (unit)", () => {
|
||||
const service = new GroupsService();
|
||||
|
||||
it("createGroup inserts under the tenant, trims the name, audits group.created on the same client", async () => {
|
||||
respond(/INSERT INTO groups/, [{ id: groupId }]);
|
||||
respond(/FROM groups g WHERE g.id/, [groupRow({ member_count: 0, connection_count: 0 })]);
|
||||
const created = await service.createGroup(ws, " Newsroom ", "desk", actor);
|
||||
|
||||
expect(created).toMatchObject({ id: groupId, name: "Newsroom", memberCount: 0, connectionCount: 0 });
|
||||
expect(fake.tenants).toEqual([ws]);
|
||||
expect(insertCall("groups").values).toEqual([ws, "Newsroom", "desk"]);
|
||||
expect(auditCalls()).toHaveLength(1);
|
||||
expect(auditCalls()[0]?.values).toEqual([ws, actor.userId, "Admin", "group.created", "group", groupId, { name: "Newsroom", description: "desk" }]);
|
||||
});
|
||||
|
||||
it("createGroup maps a duplicate name to 409 and writes no audit", async () => {
|
||||
respond(/INSERT INTO groups/, pgError("23505"));
|
||||
await expect(service.createGroup(ws, "Newsroom", "", actor)).rejects.toBeInstanceOf(ConflictException);
|
||||
expect(auditCalls()).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("createGroup rejects a blank name before touching the DB", async () => {
|
||||
await expect(service.createGroup(ws, " ", "", actor)).rejects.toBeInstanceOf(BadRequestException);
|
||||
expect(fake.calls).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("renameGroup audits only the fields that changed", async () => {
|
||||
let renamed = false;
|
||||
respond(/UPDATE groups SET/, () => {
|
||||
renamed = true;
|
||||
return [];
|
||||
});
|
||||
respond(/FROM groups g WHERE g.id/, () => [groupRow(renamed ? { name: "Desk" } : {})]);
|
||||
const after = await service.renameGroup(ws, groupId, { name: "Desk" }, actor);
|
||||
|
||||
expect(after.name).toBe("Desk");
|
||||
const update = fake.calls.find((c) => c.sql.includes("UPDATE groups SET"));
|
||||
expect(update?.values).toEqual([groupId, "Desk", null]);
|
||||
expect(auditCalls()[0]?.values.slice(3)).toEqual(["group.renamed", "group", groupId, { name: ["Newsroom", "Desk"] }]);
|
||||
});
|
||||
|
||||
it("renameGroup with nothing to change is a 400", async () => {
|
||||
await expect(service.renameGroup(ws, groupId, {}, actor)).rejects.toBeInstanceOf(BadRequestException);
|
||||
});
|
||||
|
||||
it("deleteGroup audits the cascade counts and returns a replacement token", async () => {
|
||||
respond(/FROM groups g WHERE g.id/, [groupRow()]);
|
||||
const out = await service.deleteGroup(ws, groupId, actor);
|
||||
expect(out.token).toBe(`token:${ws}:${actor.userId}:7`);
|
||||
expect(fake.calls.some((c) => c.sql.includes("FOR UPDATE OF g"))).toBe(true);
|
||||
expect(fake.calls.find((c) => c.sql.startsWith("DELETE FROM groups"))?.values).toEqual([groupId]);
|
||||
expect(auditCalls()[0]?.values.slice(3)).toEqual([
|
||||
"group.deleted",
|
||||
"group",
|
||||
groupId,
|
||||
{ name: "Newsroom", memberCount: 1, connectionCount: 2, assetCount: 0 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("deleteGroup on an unknown/foreign group is a 404 (RLS hides it)", async () => {
|
||||
await expect(service.deleteGroup(ws, groupId, actor)).rejects.toBeInstanceOf(NotFoundException);
|
||||
expect(auditCalls()).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("addMember inserts with workspace + actor, audits group.member_added, returns a fresh token", async () => {
|
||||
respond(/FROM groups g WHERE g.id/, [groupRow()]);
|
||||
respond(/INSERT INTO group_members/, [{}]);
|
||||
const out = await service.addMember(ws, groupId, userId, actor);
|
||||
|
||||
expect(insertCall("group_members").values).toEqual([ws, groupId, userId, actor.userId]);
|
||||
expect(auditCalls()[0]?.values).toEqual([ws, actor.userId, "Admin", "group.member_added", "group", groupId, { userId }]);
|
||||
expect(out.token).toContain(actor.userId);
|
||||
});
|
||||
|
||||
it("addMember: non-workspace user (composite FK) is a 400; duplicate is a 409", async () => {
|
||||
respond(/FROM groups g WHERE g.id/, [groupRow()]);
|
||||
respond(/INSERT INTO group_members/, pgError("23503"));
|
||||
await expect(service.addMember(ws, groupId, userId, actor)).rejects.toBeInstanceOf(BadRequestException);
|
||||
|
||||
fake.responders.length = 0;
|
||||
respond(/FROM groups g WHERE g.id/, [groupRow()]);
|
||||
respond(/INSERT INTO group_members/, []);
|
||||
await expect(service.addMember(ws, groupId, userId, actor)).rejects.toBeInstanceOf(ConflictException);
|
||||
expect(auditCalls()).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("removeMember deletes by (group, user) and audits group.member_removed", async () => {
|
||||
respond(/DELETE FROM group_members/, [{}]);
|
||||
const out = await service.removeMember(ws, groupId, userId, actor);
|
||||
expect(fake.calls.find((c) => c.sql.includes("DELETE FROM group_members"))?.values).toEqual([groupId, userId]);
|
||||
expect(auditCalls()[0]?.values.slice(3)).toEqual(["group.member_removed", "group", groupId, { userId }]);
|
||||
expect(out.token).toBeTruthy();
|
||||
});
|
||||
|
||||
it("removeMember of a non-member is a 404 with no audit", async () => {
|
||||
await expect(service.removeMember(ws, groupId, userId, actor)).rejects.toBeInstanceOf(NotFoundException);
|
||||
expect(auditCalls()).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("assignConnection / unassignConnection pass the right params and audit", async () => {
|
||||
respond(/FROM groups g WHERE g.id/, [groupRow()]);
|
||||
respond(/INSERT INTO group_connections/, [{}]);
|
||||
respond(/DELETE FROM group_connections/, [{}]);
|
||||
await service.assignConnection(ws, groupId, connectionId, actor);
|
||||
await service.unassignConnection(ws, groupId, connectionId, actor);
|
||||
|
||||
expect(insertCall("group_connections").values).toEqual([ws, groupId, connectionId, actor.userId]);
|
||||
expect(fake.calls.find((c) => c.sql.includes("DELETE FROM group_connections"))?.values).toEqual([groupId, connectionId]);
|
||||
expect(auditCalls().map((c) => c.values[3])).toEqual(["group.connection_assigned", "group.connection_unassigned"]);
|
||||
expect(auditCalls().map((c) => c.values[6])).toEqual([{ connectionId }, { connectionId }]);
|
||||
});
|
||||
|
||||
it("assignConnection of a connection outside the workspace (composite FK) is a 404", async () => {
|
||||
respond(/FROM groups g WHERE g.id/, [groupRow()]);
|
||||
respond(/INSERT INTO group_connections/, pgError("23503"));
|
||||
await expect(service.assignConnection(ws, groupId, connectionId, actor)).rejects.toBeInstanceOf(NotFoundException);
|
||||
});
|
||||
|
||||
it("assignAsset / unassignAsset audit with the asset id", async () => {
|
||||
respond(/FROM groups g WHERE g.id/, [groupRow()]);
|
||||
respond(/INSERT INTO group_assets/, [{}]);
|
||||
respond(/DELETE FROM group_assets/, [{}]);
|
||||
await service.assignAsset(ws, groupId, " media/ab12.png ", actor);
|
||||
await service.unassignAsset(ws, groupId, "media/ab12.png", actor);
|
||||
|
||||
expect(insertCall("group_assets").values).toEqual([ws, groupId, "media/ab12.png", actor.userId]);
|
||||
expect(auditCalls().map((c) => [c.values[3], c.values[6]])).toEqual([
|
||||
["group.asset_assigned", { assetId: "media/ab12.png" }],
|
||||
["group.asset_unassigned", { assetId: "media/ab12.png" }],
|
||||
]);
|
||||
});
|
||||
|
||||
it("listGroups maps counts to numbers", async () => {
|
||||
respond(/FROM groups g ORDER BY/, [groupRow(), groupRow({ id: "x", name: "Ops", member_count: "0" })]);
|
||||
const list = await service.listGroups(ws);
|
||||
expect(list.map((g) => [g.name, g.memberCount, g.connectionCount])).toEqual([
|
||||
["Newsroom", 1, 2],
|
||||
["Ops", 0, 2],
|
||||
]);
|
||||
});
|
||||
|
||||
it("getUserGroups filters by the user's membership", async () => {
|
||||
respond(/FROM groups g WHERE EXISTS/, [groupRow()]);
|
||||
const mine = await service.getUserGroups(ws, userId);
|
||||
expect(mine).toHaveLength(1);
|
||||
expect(fake.calls[0]?.values).toEqual([userId]);
|
||||
});
|
||||
|
||||
it("getGroupDetail returns members, connections and assets", async () => {
|
||||
const at = new Date("2026-10-06T11:00:00.000Z");
|
||||
respond(/FROM groups g WHERE g.id/, [groupRow()]);
|
||||
respond(/FROM group_members gm JOIN users/, [{ user_id: userId, email: "a@x.io", display_name: "A", added_by: actor.userId, created_at: at }]);
|
||||
respond(/FROM group_connections gc JOIN connections/, [
|
||||
{ connection_id: connectionId, provider: "mock-social", channel_id: "ch", assigned_by: null, created_at: at },
|
||||
]);
|
||||
respond(/FROM group_assets/, [{ asset_id: "media/1", assigned_by: null, created_at: at }]);
|
||||
const detail = await service.getGroupDetail(ws, groupId);
|
||||
expect(detail.members).toEqual([{ userId, email: "a@x.io", displayName: "A", addedBy: actor.userId, addedAt: at.toISOString() }]);
|
||||
expect(detail.connections[0]).toMatchObject({ connectionId, provider: "mock-social", channelId: "ch" });
|
||||
expect(detail.assets[0]?.assetId).toBe("media/1");
|
||||
});
|
||||
|
||||
it("getVisibleConnections: Owner/Admin get the whole workspace; others get the group-scoped query", async () => {
|
||||
respond(/SELECT id FROM connections/, [{ id: "c1" }, { id: "c2" }]);
|
||||
respond(/JOIN group_members gm/, [{ id: "c1" }]);
|
||||
|
||||
expect(await service.getVisibleConnections(ws, userId, true)).toEqual({ scope: "workspace", connectionIds: ["c1", "c2"] });
|
||||
expect(await service.getVisibleConnections(ws, userId, false)).toEqual({ scope: "groups", connectionIds: ["c1"] });
|
||||
expect(fake.calls[1]?.values).toEqual([userId]);
|
||||
});
|
||||
|
||||
it("hasWorkspaceWideAccess is Owner/Admin only", () => {
|
||||
expect(["Owner", "Admin", "Publisher", "Editor"].map(hasWorkspaceWideAccess)).toEqual([true, true, false, false]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("RbacGuard @RequireGroupAccess (unit)", () => {
|
||||
const token = issueSession({ userId, workspaceId: ws, role: "Publisher", authzEpoch: 3 });
|
||||
|
||||
function context(opts: { permission?: string; groupAccess?: boolean; params?: Record<string, string> }): {
|
||||
ctx: ExecutionContext;
|
||||
reflector: Reflector;
|
||||
} {
|
||||
const handler = (): void => undefined;
|
||||
if (opts.permission !== undefined) Reflect.defineMetadata(PERMISSION_KEY, opts.permission, handler);
|
||||
if (opts.groupAccess === true) Reflect.defineMetadata(GROUP_ACCESS_KEY, true, handler);
|
||||
const req = { headers: { authorization: `Bearer ${token}` }, params: opts.params ?? {} };
|
||||
const ctx = { getHandler: () => handler, switchToHttp: () => ({ getRequest: () => req }) } as unknown as ExecutionContext;
|
||||
return { ctx, reflector: new Reflector() };
|
||||
}
|
||||
|
||||
async function run(opts: Parameters<typeof context>[0]): Promise<true | string> {
|
||||
const { ctx, reflector } = context(opts);
|
||||
try {
|
||||
return (await new RbacGuard(reflector).canActivate(ctx)) as true;
|
||||
} catch (err) {
|
||||
return (err as Error).constructor.name;
|
||||
}
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
respond(/SELECT authz_epoch FROM workspaces/, [{ authz_epoch: "3" }]);
|
||||
respond(/FROM role_permissions rp/, [{ "?column?": 1 }]);
|
||||
});
|
||||
|
||||
it("routes without the decorator are unchanged: no group query runs", async () => {
|
||||
expect(await run({ permission: "publication.publish", params: { connectionId } })).toBe(true);
|
||||
expect(fake.calls.some((c) => c.sql.includes("group_connections"))).toBe(false);
|
||||
});
|
||||
|
||||
it("passes when the membership/group query matches, with (workspace, user, connection, roles) params", async () => {
|
||||
respond(/group_connections gc/, [{ "?column?": 1 }]);
|
||||
expect(await run({ permission: "publication.publish", groupAccess: true, params: { connectionId } })).toBe(true);
|
||||
const check = fake.calls.find((c) => c.sql.includes("group_connections gc"));
|
||||
expect(check?.values).toEqual([ws, userId, connectionId, ["Owner", "Admin"]]);
|
||||
});
|
||||
|
||||
it("forbids when the user is neither Owner/Admin nor in a group holding the connection", async () => {
|
||||
expect(await run({ permission: "publication.publish", groupAccess: true, params: { connectionId } })).toBe(ForbiddenException.name);
|
||||
});
|
||||
|
||||
it("fails closed when the route has no :connectionId, and 400s a non-UUID", async () => {
|
||||
expect(await run({ groupAccess: true })).toBe(ForbiddenException.name);
|
||||
expect(await run({ groupAccess: true, params: { connectionId: "nope" } })).toBe(BadRequestException.name);
|
||||
});
|
||||
|
||||
it("still enforces the permission before the group check", async () => {
|
||||
fake.responders.length = 0;
|
||||
respond(/SELECT authz_epoch FROM workspaces/, [{ authz_epoch: "3" }]);
|
||||
respond(/group_connections gc/, [{ "?column?": 1 }]);
|
||||
expect(await run({ permission: "publication.publish", groupAccess: true, params: { connectionId } })).toBe(ForbiddenException.name);
|
||||
expect(fake.calls.some((c) => c.sql.includes("group_connections"))).toBe(false);
|
||||
});
|
||||
});
|
||||
26
apps/api/src/health/health.controller.ts
Normal file
26
apps/api/src/health/health.controller.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import { Controller, Get, Res, Inject } from "@nestjs/common";
|
||||
import type { Response } from "express";
|
||||
import { HealthService, type ComponentHealth } from "./health.service";
|
||||
|
||||
interface ReadinessBody {
|
||||
status: "ready" | "degraded";
|
||||
components: Record<string, ComponentHealth>;
|
||||
}
|
||||
|
||||
@Controller()
|
||||
export class HealthController {
|
||||
constructor(@Inject(HealthService) private readonly health: HealthService) {}
|
||||
|
||||
@Get("healthz")
|
||||
liveness(@Res() res: Response): void {
|
||||
res.status(200).json({ status: "alive" });
|
||||
}
|
||||
|
||||
@Get("readyz")
|
||||
async readiness(@Res() res: Response): Promise<void> {
|
||||
const components = await this.health.components();
|
||||
const ready = Object.values(components).every((c) => c.status === "up");
|
||||
const body: ReadinessBody = { status: ready ? "ready" : "degraded", components };
|
||||
res.status(ready ? 200 : 503).json(body);
|
||||
}
|
||||
}
|
||||
2
apps/api/src/health/health.deps.ts
Normal file
2
apps/api/src/health/health.deps.ts
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
export { Injectable } from "@nestjs/common";
|
||||
export { logger } from "../telemetry/logger";
|
||||
10
apps/api/src/health/health.module.ts
Normal file
10
apps/api/src/health/health.module.ts
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
import { Module } from "@nestjs/common";
|
||||
import { HealthController } from "./health.controller";
|
||||
import { HealthService } from "./health.service";
|
||||
|
||||
@Module({
|
||||
providers: [HealthService],
|
||||
controllers: [HealthController],
|
||||
exports: [HealthService],
|
||||
})
|
||||
export class HealthModule {}
|
||||
50
apps/api/src/health/health.service.ts
Normal file
50
apps/api/src/health/health.service.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
import { assertPublishingSchema } from "@tactical-media-broadcast-hub/publishing-execution";
|
||||
import { pool } from "../db/client.js";
|
||||
import * as net from "net";
|
||||
import { Injectable } from "@nestjs/common";
|
||||
|
||||
export type ComponentStatus = "up" | "down";
|
||||
|
||||
export interface ComponentHealth {
|
||||
status: ComponentStatus;
|
||||
}
|
||||
|
||||
interface ServiceEndpoint {
|
||||
host: string;
|
||||
port: number;
|
||||
}
|
||||
|
||||
const serviceEnv: Record<string, ServiceEndpoint> = {
|
||||
postgres: { host: process.env.POSTGRES_HOST ?? "localhost", port: Number(process.env.POSTGRES_PORT ?? 5433) },
|
||||
redis: { host: process.env.REDIS_HOST ?? "localhost", port: Number(process.env.REDIS_PORT ?? 6379) },
|
||||
temporal: { host: process.env.TEMPORAL_HOST ?? "localhost", port: Number(process.env.TEMPORAL_PORT ?? 7233) },
|
||||
};
|
||||
|
||||
function probeTcp(host: string, port: number, timeoutMs: number): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
const socket = new net.Socket();
|
||||
const done = (ok: boolean): void => {
|
||||
socket.destroy();
|
||||
resolve(ok);
|
||||
};
|
||||
socket.setTimeout(timeoutMs);
|
||||
socket.once("connect", () => done(true));
|
||||
socket.once("timeout", () => done(false));
|
||||
socket.once("error", () => done(false));
|
||||
socket.connect(port, host);
|
||||
});
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class HealthService {
|
||||
async components(): Promise<Record<string, ComponentHealth>> {
|
||||
const entries = await Promise.all(
|
||||
Object.entries(serviceEnv).map(async ([name, { host, port }]) => {
|
||||
const up = await probeTcp(host, port, 1500);
|
||||
return [name, { status: up ? "up" : "down" } as ComponentHealth] as const;
|
||||
}),
|
||||
);
|
||||
const schema = await assertPublishingSchema(pool).then(() => "up" as const).catch(() => "down" as const);
|
||||
return { ...Object.fromEntries(entries), publishing_schema: { status: schema } };
|
||||
}
|
||||
}
|
||||
79
apps/api/src/http/validation.ts
Normal file
79
apps/api/src/http/validation.ts
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
import { BadRequestException } from "@nestjs/common";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import type { AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
|
||||
/** Shared request-shape guards for controllers that take `unknown` bodies. */
|
||||
|
||||
export function requireSession(request: AuthenticatedRequest): SessionPayload {
|
||||
if (request.session === undefined) throw new Error("unreachable: RBAC guard did not attach a session");
|
||||
return request.session;
|
||||
}
|
||||
|
||||
export function requireObject(value: unknown, field: string): Record<string, unknown> {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
||||
throw new BadRequestException(`${field} must be an object`);
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
export function requireString(value: unknown, field: string): string {
|
||||
if (typeof value !== "string" || value.trim() === "") {
|
||||
throw new BadRequestException(`${field} must be a non-empty string`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function optionalString(value: unknown, field: string): string | undefined {
|
||||
if (value === undefined || value === null || value === "") return undefined;
|
||||
return requireString(value, field);
|
||||
}
|
||||
|
||||
const EMAIL = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
|
||||
|
||||
export function requireEmail(value: unknown, field = "email"): string {
|
||||
const email = requireString(value, field).trim().toLowerCase();
|
||||
if (!EMAIL.test(email) || email.length > 254) throw new BadRequestException(`${field} must be an email address`);
|
||||
return email;
|
||||
}
|
||||
|
||||
export function optionalEmail(value: unknown, field = "email"): string | undefined {
|
||||
const email = optionalString(value, field);
|
||||
return email === undefined ? undefined : requireEmail(email, field);
|
||||
}
|
||||
|
||||
export const USERNAME_PATTERN = /^[a-z0-9][a-z0-9_.-]{1,31}$/;
|
||||
|
||||
export function requireUsername(value: unknown, field = "username"): string {
|
||||
const username = requireString(value, field).trim().toLowerCase();
|
||||
if (!USERNAME_PATTERN.test(username)) {
|
||||
throw new BadRequestException(`${field} must be 2-32 characters: lowercase letters, digits, dot, underscore or hyphen`);
|
||||
}
|
||||
return username;
|
||||
}
|
||||
|
||||
/** Positive integer within an inclusive cap (e.g. expiresInHours 1..720, maxUses 1..10000). */
|
||||
export function requirePositiveInt(value: unknown, field: string, max: number): number {
|
||||
const n = typeof value === "number" ? value : NaN;
|
||||
if (!Number.isInteger(n) || n < 1 || n > max) {
|
||||
throw new BadRequestException(`${field} must be an integer between 1 and ${max}`);
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
|
||||
export function isUuid(value: string): boolean {
|
||||
return UUID.test(value);
|
||||
}
|
||||
|
||||
export function parseId(value: string, field = "id"): string {
|
||||
if (!isUuid(value)) throw new BadRequestException(`${field} must be a UUID`);
|
||||
return value;
|
||||
}
|
||||
|
||||
export function parseLimit(value: unknown, fallback: number, max: number): number {
|
||||
if (value === undefined || value === "") return fallback;
|
||||
const n = typeof value === "string" ? Number(value) : NaN;
|
||||
if (!Number.isInteger(n) || n < 1) throw new BadRequestException("limit must be a positive integer");
|
||||
return Math.min(n, max);
|
||||
}
|
||||
114
apps/api/src/intake/adapters/adapter.ts
Normal file
114
apps/api/src/intake/adapters/adapter.ts
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
/**
|
||||
* Intake adapter contracts.
|
||||
*
|
||||
* One shape in, one shape out:
|
||||
*
|
||||
* - `IntakeAdapter` consumes a source row (kind + params + carried adapter
|
||||
* state) and emits `RawIntakeItem`s — provider-shaped, pre-normalize.
|
||||
* - `RawIntakeItem` is deliberately the `FeedItem` shape from
|
||||
* @tactical-media-broadcast-hub/core plus the P0c options, because the task mandate is
|
||||
* that intake items flow through feeds/feed-normalize.ts unchanged: the
|
||||
* adapter builds a `ConnectorConnection` stand-in for its (provider, host)
|
||||
* pair, `normalizeFeedItem` parses the text/html through the safe-content
|
||||
* allowlist, mints the authorRef and proxied media refs, and the service
|
||||
* derives `text` (the FTS corpus) from `astToPlainText(content)`.
|
||||
*
|
||||
* Fetch surface: `IntakeFetch` is the injectable egress client seam. Tests
|
||||
* install a fake that records (url, hostClass) draws so budget accounting is
|
||||
* asserted; production binds `intakeEgress()`.
|
||||
*/
|
||||
import type { ConnectorConnection, FeedItem } from "@tactical-media-broadcast-hub/core";
|
||||
import type { NormalizeFeedItemOptions } from "../../feeds/feed-normalize.js";
|
||||
import { HOST_CLASS_BY_KIND, PROVIDER_BY_KIND, type IntakeKind } from "../intake-kinds.js";
|
||||
import type { IntakeParams } from "../intake-params.js";
|
||||
import { IntakeHttpError } from "../intake-egress.js";
|
||||
|
||||
/** The egress fetch surface an adapter draws on. */
|
||||
export interface IntakeFetch {
|
||||
/**
|
||||
* One budgeted outbound GET. `hostClass` selects the budget row class;
|
||||
* non-2xx resolves to a thrown `IntakeHttpError` carrying status/headers,
|
||||
* except 304 which the caller classifies (free revalidation, not an error
|
||||
* path for the scheduler).
|
||||
*/
|
||||
fetch(url: string | URL, options: { headers?: Record<string, string>; hostClass: string }): Promise<Response>;
|
||||
}
|
||||
|
||||
/** Per-source adapter state carried in intake_sources.state (etag etc.). */
|
||||
export type AdapterState = Record<string, unknown>;
|
||||
|
||||
/** Provider-shaped item an adapter emits, pre-normalization. */
|
||||
export interface RawIntakeItem extends FeedItem {
|
||||
/** P0c options: content kind (html for Mastodon/RSS summaries) + permalink. */
|
||||
normalize: NormalizeFeedItemOptions;
|
||||
/** Platform-native id (guid, status id, yt:video:…, url#hash). Defaults to remoteId. */
|
||||
remoteRef?: string;
|
||||
/** attribution jsonb written to intake_items (author, url, title, extras). */
|
||||
attribution: Record<string, unknown>;
|
||||
/** Harvestable media: direct file URLs, fetched+stored when policy allows. */
|
||||
media?: Array<{ url: string; kind: "image" | "video" }>;
|
||||
}
|
||||
|
||||
export interface AdapterPollResult {
|
||||
items: RawIntakeItem[];
|
||||
/** Next carried state (etag/lastModified/lastContentHash/lastRemoteRef). */
|
||||
state: AdapterState;
|
||||
/** True when upstream returned 304/unchanged — a clean, cheap poll. */
|
||||
unchanged?: boolean;
|
||||
}
|
||||
|
||||
export interface AdapterContext {
|
||||
workspaceId: string;
|
||||
sourceId: string;
|
||||
params: IntakeParams["params"];
|
||||
kind: IntakeKind;
|
||||
state: AdapterState;
|
||||
fetch: IntakeFetch;
|
||||
}
|
||||
|
||||
export interface IntakeAdapter {
|
||||
readonly kind: IntakeKind;
|
||||
poll(context: AdapterContext): Promise<AdapterPollResult>;
|
||||
}
|
||||
|
||||
/**
|
||||
* The ConnectorConnection stand-in handed to normalizeFeedItem: provider is
|
||||
* the per-kind platform id (so authorRef renders `{provider}:{handle@host}`)
|
||||
* and `fields.instanceUrl` carries the federated instance origin when the
|
||||
* kind has one — exactly the field feed-normalize already reads.
|
||||
*/
|
||||
export function connectionFor(kind: IntakeKind, params: IntakeParams["params"], connectionId: string, workspaceId: string): ConnectorConnection {
|
||||
const fields: Record<string, string> = {};
|
||||
if ("instance" in params && typeof params.instance === "string") fields.instanceUrl = params.instance;
|
||||
if ("url" in params && typeof params.url === "string") fields.instanceUrl = new URL(params.url).origin;
|
||||
return {
|
||||
id: connectionId,
|
||||
workspaceId,
|
||||
provider: PROVIDER_BY_KIND[kind],
|
||||
channelId: connectionId,
|
||||
credentialId: { kind: "credential-ref", id: connectionId },
|
||||
fields,
|
||||
createdAt: new Date(0),
|
||||
};
|
||||
}
|
||||
|
||||
/** The host class an adapter's own feed fetch draws on. */
|
||||
export function hostClassFor(kind: IntakeKind): string {
|
||||
return HOST_CLASS_BY_KIND[kind];
|
||||
}
|
||||
|
||||
/** Budgeted GET with JSON parsing; throws IntakeHttpError on bad status. */
|
||||
export async function fetchJson<T>(fetcher: IntakeFetch, url: string | URL, hostClass: string, headers?: Record<string, string>): Promise<{ body: T; response: Response }> {
|
||||
const response = await fetcher.fetch(url, { headers: { accept: "application/json", ...headers }, hostClass });
|
||||
if (!response.ok) throw new IntakeHttpError(response.status, response.headers, String(url));
|
||||
const body = JSON.parse(await response.text()) as T;
|
||||
return { body, response };
|
||||
}
|
||||
|
||||
/** Budgeted GET with text body; 304 resolves as `{ unchanged: true }`. */
|
||||
export async function fetchText(fetcher: IntakeFetch, url: string | URL, hostClass: string, headers?: Record<string, string>): Promise<{ text: string; response: Response; unchanged: boolean }> {
|
||||
const response = await fetcher.fetch(url, { headers: { accept: "application/xml, text/xml, text/html;q=0.9, */*;q=0.8", ...headers }, hostClass });
|
||||
if (response.status === 304) return { text: "", response, unchanged: true };
|
||||
if (!response.ok) throw new IntakeHttpError(response.status, response.headers, String(url));
|
||||
return { text: await response.text(), response, unchanged: false };
|
||||
}
|
||||
415
apps/api/src/intake/adapters/adapters.test.ts
Normal file
415
apps/api/src/intake/adapters/adapters.test.ts
Normal file
|
|
@ -0,0 +1,415 @@
|
|||
/**
|
||||
* Intake adapter unit tests — fixtures per source kind (task requirement 7).
|
||||
* No DB, no network: every adapter runs against a recording fake egress
|
||||
* client so per-request budget draws are asserted, and the RSS/Atom/Mastodon
|
||||
* fixtures are the payloads the research appendix verified live.
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createHash } from "node:crypto";
|
||||
import { parseFeed } from "./rss-parse.js";
|
||||
import { feedToItems, RssAdapter } from "./rss.adapter.js";
|
||||
import { MastodonAccountAdapter, MastodonTagAdapter, carryState, freshItems, statusesToItems } from "./mastodon.adapter.js";
|
||||
import { PeerTubeChannelAdapter, videosToItems } from "./peertube.adapter.js";
|
||||
import { YouTubeChannelAdapter } from "./youtube.adapter.js";
|
||||
import { WebWatchAdapter, normalizePageText, pageHash } from "./web-watch.adapter.js";
|
||||
import { BlueskyKeywordAdapter, TelegramChannelAdapter, matchesKeywords } from "./push-seams.js";
|
||||
import { adapterFor } from "./index.js";
|
||||
import type { AdapterContext, IntakeFetch } from "./adapter.js";
|
||||
import { itemContentHash, collapseText } from "../intake-ingest.js";
|
||||
import { parseIntakeParams, paramsHash, canonicalParams, egressOrigin } from "../intake-params.js";
|
||||
import { isIntakeKind, KIND_TELEGRAM_CHANNEL, INTAKE_KINDS } from "../intake-kinds.js";
|
||||
import { DEFAULT_POLL_INTERVAL_SEC, HOST_CLASS_BY_KIND, PROVIDER_BY_KIND } from "../intake-kinds.js";
|
||||
import { backoffDelayMs, jitteredIntervalMs } from "../intake.service.js";
|
||||
import { intakeRetryAfterMs, IntakeHttpError } from "../intake-egress.js";
|
||||
|
||||
/** Recording fake: every draw + its host class, fixture-served. */
|
||||
class FakeEgress implements IntakeFetch {
|
||||
readonly draws: Array<{ url: string; hostClass: string }> = [];
|
||||
constructor(private readonly routes: Record<string, () => Response | { status: number; headers: Record<string, string>; body?: string }>) {}
|
||||
|
||||
async fetch(url: string | URL, options: { headers?: Record<string, string>; hostClass: string }): Promise<Response> {
|
||||
const key = String(url);
|
||||
this.draws.push({ url: key, hostClass: options.hostClass });
|
||||
const route = Object.entries(this.routes).find(([prefix]) => key.startsWith(prefix));
|
||||
if (route === undefined) return new Response("not found", { status: 404 });
|
||||
const result = route[1]();
|
||||
if (result instanceof Response) return result;
|
||||
if (result.status === 204 || result.status === 304) return new Response(null, { status: 304, headers: result.headers });
|
||||
return new Response(result.body ?? "", { status: result.status, headers: result.headers });
|
||||
}
|
||||
}
|
||||
|
||||
function context(kind: string, params: Record<string, unknown>, state: Record<string, unknown> = {}, fetcher: IntakeFetch): AdapterContext {
|
||||
return {
|
||||
workspaceId: "ws", sourceId: "src",
|
||||
kind: kind as AdapterContext["kind"],
|
||||
params: params as unknown as AdapterContext["params"],
|
||||
state, fetch: fetcher,
|
||||
};
|
||||
}
|
||||
|
||||
const RSS_FIXTURE = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:media="http://search.yahoo.com/mrss/">
|
||||
<channel>
|
||||
<title>Water Working Group</title>
|
||||
<link>https://example.org/water</link>
|
||||
<item>
|
||||
<title>Water rights are a community responsibility</title>
|
||||
<link>https://example.org/water/2026-10-09</link>
|
||||
<guid isPermaLink="true">https://example.org/water/2026-10-09</guid>
|
||||
<pubDate>Fri, 09 Oct 2026 12:00:00 GMT</pubDate>
|
||||
<dc:creator xmlns:dc="http://purl.org/dc/elements/1.1/">Water WG</dc:creator>
|
||||
<description><![CDATA[<p>The hearing is postponed to February.</p>]]></description>
|
||||
<media:thumbnail url="https://example.org/water.jpg" />
|
||||
</item>
|
||||
<item>
|
||||
<title>Repair café this Saturday</title>
|
||||
<link>https://example.org/water/repair</link>
|
||||
<guid>repair-2</guid>
|
||||
<pubDate>Thu, 08 Oct 2026 09:00:00 GMT</pubDate>
|
||||
<description>Bring your bikes and sewing machines.</description>
|
||||
<enclosure url="https://example.org/repair.jpg" type="image/jpeg" length="1000" />
|
||||
</item>
|
||||
</channel>
|
||||
</rss>`;
|
||||
|
||||
const MASTODON_TAG_FIXTURE = JSON.stringify([
|
||||
{
|
||||
id: "110001", uri: "https://social.example/users/alice/statuses/110001", url: "https://social.example/@alice/110001",
|
||||
created_at: "2026-10-09T11:00:00.000Z",
|
||||
content: "<p>Solidarity with the <a href=\"https://social.example/tags/water\" class=\"mention hashtag\" rel=\"tag\">#<span>water</span></a> strikers & their families</p>",
|
||||
account: { id: "1", acct: "alice", display_name: "Alice", username: "alice" },
|
||||
media_attachments: [{ id: "9", url: "https://files.social.example/media/9.png", preview_url: "https://files.social.example/media/9_small.png", type: "image", mime_type: "image/png" }],
|
||||
visibility: "public",
|
||||
},
|
||||
{ id: "110002", created_at: "2026-10-09T10:00:00.000Z", content: "<p>unrelated post</p>", account: { acct: "bob@example.social", username: "bob" }, visibility: "public" },
|
||||
]);
|
||||
|
||||
const PEERTUBE_FIXTURE = JSON.stringify({
|
||||
total: 2,
|
||||
data: [
|
||||
{ uuid: "9c1de6de-8df0-4e3a-97a6-7e3a1d6e9a11", name: "General assembly recording", publishedAt: "2026-10-08T18:00:00.000Z", channel: { displayName: "Collective Media", name: "collective" }, thumbnailPath: "/static/thumbnails/a11.jpg", duration: 5400 },
|
||||
{ uuid: "0b2ef7ef-9e01-4f4b-88b7-8f4b2e7f9b22", name: "Street medics training", publishedAt: "2026-10-07T18:00:00.000Z", channel: { displayName: "Collective Media" }, isLive: false },
|
||||
],
|
||||
});
|
||||
|
||||
const YOUTUBE_FIXTURE = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feed xmlns:yt="http://www.youtube.com/xml/schemas/2015" xmlns="http://www.w3.org/2005/Atom">
|
||||
<link rel="self" href="https://www.youtube.com/feeds/videos.xml?channel_id=UCtest0000000000000000"/>
|
||||
<title>Collective Channel</title>
|
||||
<entry>
|
||||
<id>yt:video:AAAAAAAAAAA</id>
|
||||
<yt:videoId>AAAAAAAAAAA</yt:videoId>
|
||||
<yt:channelId>UCtest0000000000000000</yt:channelId>
|
||||
<title>Community assembly livestream</title>
|
||||
<link rel="alternate" href="https://www.youtube.com/watch?v=AAAAAAAAAAA"/>
|
||||
<published>2026-10-09T08:00:00.000Z</published>
|
||||
<updated>2026-10-09T08:00:00.000Z</updated>
|
||||
<author><name>Collective Channel</name></author>
|
||||
<media:group xmlns:media="http://search.yahoo.com/mrss/">
|
||||
<media:thumbnail url="https://i.ytimg.com/vi/AAAAAAAAAAA/hqdefault.jpg"/>
|
||||
</media:group>
|
||||
</entry>
|
||||
</feed>`;
|
||||
|
||||
const WEB_PAGE_A = `<!doctype html><html><head><style>.x{color:red}</style></head>
|
||||
<body><div id="main"><p>Next hearing scheduled for March 2027 at the courthouse.</p>
|
||||
<span data-token="9f8e7d6c5b4a3f2e1d0c9b8a7f6e5d4c"></span></div></body></html>`;
|
||||
const WEB_PAGE_B = `<!doctype html><html><body><div id="main"><p>Next hearing scheduled for April 2027 at the courthouse.</p>
|
||||
<span data-token="1a2b3c4d5e6f708192a3b4c5d6e7f809"></span></div></body></html>`;
|
||||
|
||||
describe("intake params validation + canonicalization", () => {
|
||||
it("validates and canonicalizes per kind", () => {
|
||||
const a = parseIntakeParams("rss", { url: "https://example.org/feed.xml" });
|
||||
const b = parseIntakeParams("rss", { url: "https://example.org/feed.xml#" });
|
||||
expect(paramsHash(a)).toBe(paramsHash(b)); // hash-stripped
|
||||
const c = parseIntakeParams("mastodon_tag", { instance: "Mastodon.Social", tag: "Water" });
|
||||
expect(c.params).toEqual({ instance: "https://mastodon.social", tag: "water" });
|
||||
const d = parseIntakeParams("mastodon_account", { instance: "https://mastodon.social", accountId: "000123", handle: "@Alice@mastodon.social" });
|
||||
expect(d.params).toEqual({ instance: "https://mastodon.social", accountId: "000123", handle: "alice@mastodon.social" });
|
||||
expect(() => parseIntakeParams("mastodon_account", { instance: "https://mastodon.social", accountId: "abc" })).toThrow();
|
||||
expect(() => parseIntakeParams("rss", { url: "file:///etc/passwd" })).toThrow();
|
||||
expect(() => parseIntakeParams("rss", { url: "https://user:pass@example.org/f" })).toThrow();
|
||||
expect(() => parseIntakeParams("youtube_channel", { channelId: "not-a-channel" })).toThrow();
|
||||
expect(() => parseIntakeParams("nope", {})).toThrow();
|
||||
expect(() => parseIntakeParams("telegram_channel", { channelId: "@durov" })).toThrow();
|
||||
const e = parseIntakeParams("peertube_channel", { instance: "https://video.example", handle: "@collective@video.example" });
|
||||
expect(e.params).toEqual({ instance: "https://video.example", handle: "collective@video.example" });
|
||||
const w = parseIntakeParams("web_watch", { url: "https://example.org/page", selector: "#main" });
|
||||
expect(w.params).toEqual({ url: "https://example.org/page", selector: "#main" });
|
||||
});
|
||||
|
||||
it("canonical form is key-order independent", () => {
|
||||
const left = parseIntakeParams("mastodon_tag", { tag: "water", instance: "https://mastodon.social" });
|
||||
const right = parseIntakeParams("mastodon_tag", { instance: "https://mastodon.social", tag: "water" });
|
||||
expect(canonicalParams(left)).toBe(canonicalParams(right));
|
||||
});
|
||||
|
||||
it("maps kinds to poll defaults, host classes, and providers", () => {
|
||||
expect(DEFAULT_POLL_INTERVAL_SEC.telegram_channel).toBe(0);
|
||||
expect(DEFAULT_POLL_INTERVAL_SEC.bluesky_keyword).toBe(0);
|
||||
expect(DEFAULT_POLL_INTERVAL_SEC.rss).toBeGreaterThan(0);
|
||||
expect(HOST_CLASS_BY_KIND.mastodon_tag).toBe("intake-social");
|
||||
expect(PROVIDER_BY_KIND.mastodon_tag).toBe("mastodon");
|
||||
expect(PROVIDER_BY_KIND.web_watch).toBe("web");
|
||||
for (const kind of INTAKE_KINDS) expect(isIntakeKind(kind)).toBe(true);
|
||||
expect(egressOrigin(parseIntakeParams("youtube_channel", { channelId: "UCtest0000000000000000" })))
|
||||
.toBe("https://www.youtube.com");
|
||||
expect(egressOrigin(parseIntakeParams("rss", { url: "https://example.org/f.xml" }))).toBe("https://example.org");
|
||||
});
|
||||
});
|
||||
|
||||
describe("rss parse + adapter", () => {
|
||||
it("parses RSS 2.0 with CDATA, entities, dc:creator, media:thumbnail", () => {
|
||||
const feed = parseFeed(RSS_FIXTURE);
|
||||
expect(feed.type).toBe("rss");
|
||||
expect(feed.channelTitle).toBe("Water Working Group");
|
||||
expect(feed.entries).toHaveLength(2);
|
||||
const first = feed.entries[0]!;
|
||||
expect(first.title).toBe("Water rights are a community responsibility");
|
||||
expect(first.link).toBe("https://example.org/water/2026-10-09");
|
||||
expect(first.publishedAt?.toISOString()).toBe("2026-10-09T12:00:00.000Z");
|
||||
expect(first.author).toBe("Water WG");
|
||||
expect(first.contentHtml).toContain("postponed to February");
|
||||
expect(first.mediaThumbnails).toEqual(["https://example.org/water.jpg"]);
|
||||
expect(feed.entries[1]!.title).toBe("Repair café this Saturday");
|
||||
expect(feed.entries[1]!.enclosures[0]?.type).toBe("image/jpeg");
|
||||
});
|
||||
|
||||
it("parses Atom (YouTube shape) with yt:videoId", () => {
|
||||
const feed = parseFeed(YOUTUBE_FIXTURE);
|
||||
expect(feed.type).toBe("atom");
|
||||
expect(feed.channelTitle).toBe("Collective Channel");
|
||||
expect(feed.entries[0]!.videoId).toBe("AAAAAAAAAAA");
|
||||
expect(feed.entries[0]!.link).toBe("https://www.youtube.com/watch?v=AAAAAAAAAAA");
|
||||
});
|
||||
|
||||
it("classifies enclosure + thumbnail media kinds", () => {
|
||||
const items = feedToItems(parseFeed(RSS_FIXTURE), new Date("2026-10-09T12:00:00Z"));
|
||||
expect(items[0]?.media).toEqual([{ url: "https://example.org/water.jpg", kind: "image" }]);
|
||||
expect(items[1]?.media?.some((entry) => entry.url === "https://example.org/repair.jpg")).toBe(true);
|
||||
expect(items[0]?.remoteId).toBe("https://example.org/water/2026-10-09");
|
||||
expect(items).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("conditional GET: 304 is free and emits nothing; etag carries", async () => {
|
||||
let etag = "\"v1\"";
|
||||
const egress = new FakeEgress({
|
||||
"https://example.org/feed.xml": () => ({ status: 200, headers: { etag, "content-type": "application/xml" }, body: RSS_FIXTURE }),
|
||||
});
|
||||
const adapter = new RssAdapter();
|
||||
const first = await adapter.poll(context("rss", { url: "https://example.org/feed.xml" }, {}, egress));
|
||||
expect(first.items).toHaveLength(2);
|
||||
expect(first.state.etag).toBe(etag);
|
||||
expect(egress.draws[0]!.hostClass).toBe("intake-rss");
|
||||
|
||||
etag = "\"v2\"";
|
||||
const second = await adapter.poll(context("rss", { url: "https://example.org/feed.xml" }, { etag: "\"v1\"" },
|
||||
new FakeEgress({ "https://example.org/feed.xml": () => ({ status: 304, headers: {} }) })));
|
||||
expect(second.unchanged).toBe(true);
|
||||
expect(second.items).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("is total over malformed XML", () => {
|
||||
const feed = parseFeed("<rss><channel><item><title>only");
|
||||
expect(feed.entries).toHaveLength(1);
|
||||
expect(parseFeed("").entries).toHaveLength(0);
|
||||
expect(parseFeed("not xml at all").entries).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("mastodon adapters", () => {
|
||||
it("tag timeline: html status → items with attachments, budget class intake-social", async () => {
|
||||
const egress = new FakeEgress({
|
||||
"https://mastodon.social/api/v1/timelines/tag/water": () => new Response(MASTODON_TAG_FIXTURE, { headers: { "content-type": "application/json", "x-ratelimit-limit": "300" } }),
|
||||
});
|
||||
const adapter = new MastodonTagAdapter();
|
||||
const result = await adapter.poll(context("mastodon_tag", { instance: "https://mastodon.social", tag: "water" }, {}, egress));
|
||||
expect(result.items).toHaveLength(2);
|
||||
expect(egress.draws[0]!.hostClass).toBe("intake-social");
|
||||
expect(egress.draws[0]!.url).toContain("/api/v1/timelines/tag/water?limit=40");
|
||||
const first = result.items[0]!;
|
||||
expect(first.remoteId).toBe("110001");
|
||||
expect(first.media![0]).toEqual({ url: "https://files.social.example/media/9.png", kind: "image" });
|
||||
expect(first.attribution.authorHandle).toBe("alice");
|
||||
// remote federation: acct carries @host
|
||||
expect(result.items[1]!.attribution.authorHandle).toBe("bob@example.social");
|
||||
const state = carryState(result.items, {});
|
||||
expect(state.lastRemoteRef).toBe("110001");
|
||||
});
|
||||
|
||||
it("account timeline: exclude_replies + freshness window from carried state", async () => {
|
||||
const egress = new FakeEgress({
|
||||
"https://mastodon.social/api/v1/accounts/000123/statuses": () => new Response(MASTODON_TAG_FIXTURE, { headers: { "content-type": "application/json" } }),
|
||||
});
|
||||
const adapter = new MastodonAccountAdapter();
|
||||
const first = await adapter.poll(context("mastodon_account", { instance: "https://mastodon.social", accountId: "000123" }, {}, egress));
|
||||
expect(first.items).toHaveLength(2);
|
||||
expect(egress.draws[0]!.url).toContain("exclude_replies=true");
|
||||
// Second delivery of the same window: the high-water mark (110001, the
|
||||
// newest) is present, so nothing newer exists and 110002 was already
|
||||
// delivered in the first poll — zero new items.
|
||||
const second = await adapter.poll(context("mastodon_account", { instance: "https://mastodon.social", accountId: "000123" },
|
||||
{ lastRemoteRef: "110001", lastSeenIds: ["110001", "110002"] }, egress));
|
||||
expect(second.items).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("freshItems cuts at the high-water mark and dedups seen ids", () => {
|
||||
const items = statusesToItems([
|
||||
{ id: "3", created_at: "2026-10-09T11:00:00Z", content: "c" },
|
||||
{ id: "2", created_at: "2026-10-09T10:00:00Z", content: "b" },
|
||||
{ id: "1", created_at: "2026-10-09T09:00:00Z", content: "a" },
|
||||
], new Date());
|
||||
expect(freshItems(items, "2", ["1"]).map((i) => i.remoteId)).toEqual(["3"]);
|
||||
expect(freshItems(items, undefined, ["1", "2"]).map((i) => i.remoteId)).toEqual(["3"]);
|
||||
});
|
||||
|
||||
it("drops private/direct statuses", () => {
|
||||
const items = statusesToItems([
|
||||
{ id: "1", visibility: "private", content: "x" },
|
||||
{ id: "2", visibility: "direct", content: "x" },
|
||||
{ id: "3", visibility: "public", content: "x" },
|
||||
], new Date());
|
||||
expect(items).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("peertube adapter", () => {
|
||||
it("channel videos via public REST, thumbnail media, duration metric", async () => {
|
||||
const egress = new FakeEgress({
|
||||
"https://video.example/api/v1/video-channels/collective%40video.example/videos": () => new Response(PEERTUBE_FIXTURE, { headers: { "content-type": "application/json" } }),
|
||||
});
|
||||
const adapter = new PeerTubeChannelAdapter();
|
||||
const result = await adapter.poll(context("peertube_channel", { instance: "https://video.example", handle: "collective@video.example" }, {}, egress));
|
||||
expect(result.items).toHaveLength(2);
|
||||
expect(egress.draws[0]!.hostClass).toBe("intake-peertube");
|
||||
expect(result.items[0]!.remoteId).toBe("9c1de6de-8df0-4e3a-97a6-7e3a1d6e9a11");
|
||||
expect(result.items[0]!.media![0]!.url).toBe("https://video.example/static/thumbnails/a11.jpg");
|
||||
expect(result.items[0]!.metrics.durationSec).toBe(5400);
|
||||
const second = await adapter.poll(context("peertube_channel", { instance: "https://video.example", handle: "collective@video.example" }, result.state, egress));
|
||||
expect(second.items).toHaveLength(0); // seen-set window
|
||||
});
|
||||
|
||||
it("videosToItems tolerates missing uuid/dates", () => {
|
||||
expect(videosToItems([{ name: "no id" }], "https://video.example", new Date())).toHaveLength(0);
|
||||
const one = videosToItems([{ uuid: "x", name: "n" }], "https://video.example", new Date("2026-10-09T00:00:00Z"));
|
||||
expect(one[0]!.createdAt).toEqual(new Date("2026-10-09T00:00:00Z"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("youtube adapter", () => {
|
||||
it("channel Atom: videoId remote ref, watch URL, thumbnail; seen window suppresses re-emission", async () => {
|
||||
const egress = new FakeEgress({
|
||||
"https://www.youtube.com/feeds/videos.xml": () => new Response(YOUTUBE_FIXTURE, { headers: { "content-type": "application/atom+xml", etag: "\"yt1\"" } }),
|
||||
});
|
||||
const adapter = new YouTubeChannelAdapter();
|
||||
const result = await adapter.poll(context("youtube_channel", { channelId: "UCtest0000000000000000" }, {}, egress));
|
||||
expect(result.items).toHaveLength(1);
|
||||
const item = result.items[0]!;
|
||||
expect(item.remoteRef).toBe("yt:video:AAAAAAAAAAA");
|
||||
expect(item.attribution.url).toBe("https://www.youtube.com/watch?v=AAAAAAAAAAA");
|
||||
expect(item.media![0]!.kind).toBe("image");
|
||||
expect(result.state.etag).toBe("\"yt1\"");
|
||||
const second = await adapter.poll(context("youtube_channel", { channelId: "UCtest0000000000000000" }, result.state, egress));
|
||||
expect(second.items).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("web watch adapter", () => {
|
||||
it("diff normalization strips churn and hash only moves on real change", () => {
|
||||
const a = normalizePageText(WEB_PAGE_A);
|
||||
const b = normalizePageText(WEB_PAGE_B);
|
||||
const churn = normalizePageText(WEB_PAGE_A.replace("9f8e7d6c5b4a3f2e1d0c9b8a7f6e5d4c", "ffffffffffffffffffffffffffffffff").replace("March", "March"));
|
||||
expect(pageHash(a)).toBe(pageHash(churn)); // token + identical text → same hash
|
||||
expect(pageHash(a)).not.toBe(pageHash(b)); // March → April
|
||||
expect(a).toContain("courthouse");
|
||||
expect(a).not.toContain("csrf");
|
||||
expect(a).not.toContain("color:red");
|
||||
});
|
||||
|
||||
it("first fetch seeds the baseline without emitting; change emits one item", async () => {
|
||||
let body = WEB_PAGE_A;
|
||||
const egress = new FakeEgress({ "https://example.org/hearing": () => new Response(body, { headers: { "content-type": "text/html" } }) });
|
||||
const adapter = new WebWatchAdapter();
|
||||
const params = { url: "https://example.org/hearing", selector: "#main" };
|
||||
const seed = await adapter.poll(context("web_watch", params, {}, egress));
|
||||
expect(seed.items).toHaveLength(0);
|
||||
expect(typeof seed.state.contentHash).toBe("string");
|
||||
body = WEB_PAGE_B;
|
||||
const changed = await adapter.poll(context("web_watch", params, seed.state, egress));
|
||||
expect(changed.items).toHaveLength(1);
|
||||
expect(changed.items[0]!.attribution.diffHash).toBe(changed.state.contentHash);
|
||||
const stable = await adapter.poll(context("web_watch", params, changed.state, egress));
|
||||
expect(stable.unchanged).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("push-only seams", () => {
|
||||
it("bluesky keyword gate is exact-substring, multi-term AND", () => {
|
||||
expect(matchesKeywords("Water rights now", "water rights")).toBe(true);
|
||||
expect(matchesKeywords("Water rights now", "water luxury")).toBe(false);
|
||||
expect(matchesKeywords("WATER RIGHTS", "water")).toBe(true);
|
||||
expect(matchesKeywords("anything", "")).toBe(false);
|
||||
const adapter = new BlueskyKeywordAdapter();
|
||||
expect(adapter.gate("solidarity with water strikers", { keywords: ["water strikers"] })).toBe(true);
|
||||
expect(adapter.gate("unrelated", { keywords: ["water strikers"] })).toBe(false);
|
||||
});
|
||||
|
||||
it("telegram + bluesky poll() are clean no-ops", async () => {
|
||||
for (const Adapter of [TelegramChannelAdapter, BlueskyKeywordAdapter]) {
|
||||
const egress = new FakeEgress({});
|
||||
const result = await new Adapter().poll(context("telegram_channel", { channelId: "-100123" }, {}, egress));
|
||||
expect(result.items).toHaveLength(0);
|
||||
expect(result.unchanged).toBe(true);
|
||||
expect(egress.draws).toHaveLength(0); // never fetches
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("adapter registry", () => {
|
||||
it("is total over kinds", () => {
|
||||
for (const kind of INTAKE_KINDS) {
|
||||
const adapter = adapterFor(kind);
|
||||
expect(adapter.kind).toBe(kind);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("content hash + text collapse", () => {
|
||||
it("collapses whitespace and NFKC before hashing; media identity is sorted+unique", () => {
|
||||
expect(collapseText(" a\u00a0 b ")).toBe("a b");
|
||||
const h1 = itemContentHash("same text", ["https://b.example/x", "https://a.example/y"]);
|
||||
const h2 = itemContentHash("same text", ["https://a.example/y", "https://b.example/x", "https://a.example/y"]);
|
||||
expect(h1).toBe(h2); // whitespace + order + duplicates do not matter
|
||||
expect(h1).not.toBe(itemContentHash("same text", []));
|
||||
expect(h1).toMatch(/^[0-9a-f]{64}$/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("scheduler math + error classification helpers", () => {
|
||||
it("backoff curve: 2^n capped at 15 min", () => {
|
||||
expect(backoffDelayMs(1, 60_000)).toBe(60_000);
|
||||
expect(backoffDelayMs(2, 60_000)).toBe(120_000);
|
||||
expect(backoffDelayMs(7, 60_000)).toBe(900_000); // 60s·2^6 = 64 min → capped at 15 min
|
||||
expect(backoffDelayMs(1, 60_000)).toBeLessThanOrEqual(15 * 60_000);
|
||||
});
|
||||
|
||||
it("jitter stays within ±20 % and validates inputs", () => {
|
||||
expect(jitteredIntervalMs(100, 0)).toBe(80);
|
||||
expect(jitteredIntervalMs(100, 1)).toBe(120);
|
||||
expect(jitteredIntervalMs(100, 0.5)).toBe(100);
|
||||
expect(() => jitteredIntervalMs(0, 0.5)).toThrow();
|
||||
expect(() => jitteredIntervalMs(100, 1.5)).toThrow();
|
||||
});
|
||||
|
||||
it("retry-after parses seconds and HTTP-date", () => {
|
||||
const seconds = new Headers({ "retry-after": "30" });
|
||||
expect(intakeRetryAfterMs(seconds, 0)).toBe(30_000);
|
||||
const past = new Headers({ "retry-after": new Date(0).toUTCString() });
|
||||
expect(intakeRetryAfterMs(past, 10_000)).toBe(0);
|
||||
expect(intakeRetryAfterMs(new Headers(), 0)).toBe(0);
|
||||
const err = new IntakeHttpError(429, new Headers({ "retry-after": "5" }), "https://x");
|
||||
expect(err.rateLimited && err.transient).toBe(true);
|
||||
expect(new IntakeHttpError(410, new Headers(), "https://x").gone).toBe(true);
|
||||
});
|
||||
});
|
||||
29
apps/api/src/intake/adapters/index.ts
Normal file
29
apps/api/src/intake/adapters/index.ts
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
/**
|
||||
* The per-kind adapter registry — one entry per intake kind, mirroring the
|
||||
* feed-registry composition boundary. Adding a source kind = one adapter +
|
||||
* one registry line; the scheduler and service never branch on kind.
|
||||
*/
|
||||
import type { IntakeAdapter } from "./adapter.js";
|
||||
import { RssAdapter } from "./rss.adapter.js";
|
||||
import { MastodonAccountAdapter, MastodonTagAdapter } from "./mastodon.adapter.js";
|
||||
import { PeerTubeChannelAdapter } from "./peertube.adapter.js";
|
||||
import { YouTubeChannelAdapter } from "./youtube.adapter.js";
|
||||
import { WebWatchAdapter } from "./web-watch.adapter.js";
|
||||
import { BlueskyKeywordAdapter, TelegramChannelAdapter } from "./push-seams.js";
|
||||
import type { IntakeKind } from "../intake-kinds.js";
|
||||
|
||||
export const INTAKE_ADAPTERS: Readonly<Record<IntakeKind, () => IntakeAdapter>> = {
|
||||
rss: () => new RssAdapter(),
|
||||
mastodon_tag: () => new MastodonTagAdapter(),
|
||||
mastodon_account: () => new MastodonAccountAdapter(),
|
||||
bluesky_keyword: () => new BlueskyKeywordAdapter(),
|
||||
telegram_channel: () => new TelegramChannelAdapter(),
|
||||
peertube_channel: () => new PeerTubeChannelAdapter(),
|
||||
youtube_channel: () => new YouTubeChannelAdapter(),
|
||||
web_watch: () => new WebWatchAdapter(),
|
||||
};
|
||||
|
||||
/** A fresh adapter per source poll (adapters are stateless; context carries state). */
|
||||
export function adapterFor(kind: IntakeKind): IntakeAdapter {
|
||||
return INTAKE_ADAPTERS[kind]();
|
||||
}
|
||||
141
apps/api/src/intake/adapters/mastodon.adapter.ts
Normal file
141
apps/api/src/intake/adapters/mastodon.adapter.ts
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
/**
|
||||
* Mastodon adapters — public anon tag + account timelines.
|
||||
*
|
||||
* Verified facts (research §1.3, 2026-10-09): `/api/v1/timelines/tag/{tag}`
|
||||
* and `/api/v1/accounts/{id}/statuses` answer 200 anonymously; the federated
|
||||
* public/local timelines are 422 for anon users, so this adapter never builds
|
||||
* on those. Budget: `x-ratelimit-limit: 300` per 5 min per IP — modeled by the
|
||||
* intake-social host class (1 req/s ceiling), and `X-RateLimit-Remaining` /
|
||||
* `Retry-After` headers are already honored by the shared egress bucket's
|
||||
* observe() path, which parks the host on 429/0-remaining.
|
||||
*
|
||||
* Status HTML runs through the P0c allowlist parser (`contentKind: "html"`);
|
||||
* `media_attachments[].url` (directly fetchable, no auth) become harvest
|
||||
* candidates alongside the proxied preview refs.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import { fetchJson, hostClassFor, type AdapterContext, type AdapterPollResult, type IntakeAdapter, type RawIntakeItem } from "./adapter.js";
|
||||
import { KIND_MASTODON_ACCOUNT, KIND_MASTODON_TAG } from "../intake-kinds.js";
|
||||
import type { MastodonAccountParams, MastodonTagParams } from "../intake-params.js";
|
||||
|
||||
export const MASTODON_PAGE_LIMIT = 40;
|
||||
|
||||
interface MastodonStatus {
|
||||
id: string;
|
||||
uri?: string;
|
||||
url?: string | null;
|
||||
created_at?: string;
|
||||
content?: string;
|
||||
account?: { id?: string; acct?: string; display_name?: string; url?: string; username?: string };
|
||||
media_attachments?: Array<{ id?: string; url?: string | null; preview_url?: string | null; type?: string; mime_type?: string }>;
|
||||
reblog?: MastodonStatus | null;
|
||||
visibility?: string;
|
||||
}
|
||||
|
||||
const isStatus = (value: unknown): value is MastodonStatus =>
|
||||
typeof value === "object" && value !== null && typeof (value as MastodonStatus).id === "string";
|
||||
|
||||
const MEDIA_KIND: Record<string, "image" | "video"> = { image: "image", gifv: "image", video: "video", audio: "video" };
|
||||
|
||||
export function statusesToItems(statuses: MastodonStatus[], now: Date): RawIntakeItem[] {
|
||||
const out: RawIntakeItem[] = [];
|
||||
for (const status of statuses) {
|
||||
if (!isStatus(status) || status.visibility === "private" || status.visibility === "direct") continue;
|
||||
const author = status.account?.acct ?? status.account?.username ?? "unknown";
|
||||
const host = author.includes("@") ? author.split("@").at(-1) : null;
|
||||
const media: Array<{ url: string; kind: "image" | "video" }> = [];
|
||||
for (const attachment of status.media_attachments ?? []) {
|
||||
const url = typeof attachment.url === "string" ? attachment.url : attachment.preview_url;
|
||||
if (typeof url === "string" && url !== "") {
|
||||
media.push({ url, kind: MEDIA_KIND[attachment.type ?? ""] ?? (/^video\//i.test(attachment.mime_type ?? "") ? "video" : "image") });
|
||||
}
|
||||
}
|
||||
const url = status.url ?? status.uri ?? null;
|
||||
out.push({
|
||||
id: status.id,
|
||||
remoteId: status.id,
|
||||
remoteRef: status.id,
|
||||
provider: "mastodon",
|
||||
authorHandle: author,
|
||||
text: status.content ?? "",
|
||||
mediaUrls: media.map((entry) => entry.url),
|
||||
createdAt: parseDate(status.created_at) ?? now,
|
||||
metrics: {},
|
||||
normalize: { contentKind: "html", url: url ?? undefined },
|
||||
attribution: {
|
||||
authorHandle: author,
|
||||
authorRef: host ? `mastodon:${author}` : `mastodon:${author}`,
|
||||
url,
|
||||
provider: "mastodon",
|
||||
statusId: status.id,
|
||||
},
|
||||
media,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function parseDate(raw: string | undefined): Date | null {
|
||||
if (typeof raw !== "string") return null;
|
||||
const date = new Date(raw);
|
||||
return Number.isFinite(date.getTime()) ? date : null;
|
||||
}
|
||||
|
||||
/** Only statuses newer than the carried high-water mark are emitted. */
|
||||
export function freshItems(items: RawIntakeItem[], lastRemoteRef: string | undefined, lastSeenIds: string[] | undefined): RawIntakeItem[] {
|
||||
const seen = new Set(lastSeenIds ?? []);
|
||||
const refOf = (item: RawIntakeItem): string => item.remoteRef ?? item.remoteId;
|
||||
if (lastRemoteRef !== undefined && lastRemoteRef !== "") {
|
||||
const cut = items.findIndex((item) => refOf(item) === lastRemoteRef);
|
||||
if (cut >= 0) return items.slice(0, cut).filter((item) => !seen.has(refOf(item)));
|
||||
}
|
||||
return items.filter((item) => !seen.has(refOf(item)));
|
||||
}
|
||||
|
||||
export function carryState(items: RawIntakeItem[], previous: Record<string, unknown>): Record<string, unknown> {
|
||||
const priorIds = Array.isArray(previous.lastSeenIds) ? previous.lastSeenIds.filter((id): id is string => typeof id === "string") : [];
|
||||
const priorRef = typeof previous.lastRemoteRef === "string" ? previous.lastRemoteRef : undefined;
|
||||
const ids = [...priorIds, ...items.map((item) => item.remoteRef ?? item.remoteId)].slice(-80);
|
||||
return { lastRemoteRef: items[0] !== undefined ? (items[0].remoteRef ?? items[0].remoteId) : priorRef, lastSeenIds: ids };
|
||||
}
|
||||
|
||||
export class MastodonTagAdapter implements IntakeAdapter {
|
||||
readonly kind = KIND_MASTODON_TAG;
|
||||
|
||||
async poll(context: AdapterContext): Promise<AdapterPollResult> {
|
||||
const params = context.params as MastodonTagParams;
|
||||
const { body } = await fetchJson<unknown>(
|
||||
context.fetch,
|
||||
`${params.instance}/api/v1/timelines/tag/${encodeURIComponent(params.tag)}?limit=${MASTODON_PAGE_LIMIT}&local=false`,
|
||||
hostClassFor(KIND_MASTODON_TAG),
|
||||
);
|
||||
const statuses = Array.isArray(body) ? body.filter(isStatus) : [];
|
||||
const items = freshItems(statusesToItems(statuses, new Date()),
|
||||
typeof context.state.lastRemoteRef === "string" ? context.state.lastRemoteRef : undefined,
|
||||
Array.isArray(context.state.lastSeenIds) ? context.state.lastSeenIds.filter((id): id is string => typeof id === "string") : undefined);
|
||||
return { items, state: carryState(items, context.state) };
|
||||
}
|
||||
}
|
||||
|
||||
export class MastodonAccountAdapter implements IntakeAdapter {
|
||||
readonly kind = KIND_MASTODON_ACCOUNT;
|
||||
|
||||
async poll(context: AdapterContext): Promise<AdapterPollResult> {
|
||||
const params = context.params as MastodonAccountParams;
|
||||
const { body } = await fetchJson<unknown>(
|
||||
context.fetch,
|
||||
`${params.instance}/api/v1/accounts/${encodeURIComponent(params.accountId)}/statuses?limit=${MASTODON_PAGE_LIMIT}&exclude_replies=true`,
|
||||
hostClassFor(KIND_MASTODON_ACCOUNT),
|
||||
);
|
||||
const statuses = Array.isArray(body) ? body.filter(isStatus) : [];
|
||||
const items = freshItems(statusesToItems(statuses, new Date()),
|
||||
typeof context.state.lastRemoteRef === "string" ? context.state.lastRemoteRef : undefined,
|
||||
Array.isArray(context.state.lastSeenIds) ? context.state.lastSeenIds.filter((id): id is string => typeof id === "string") : undefined);
|
||||
return { items, state: carryState(items, context.state) };
|
||||
}
|
||||
}
|
||||
|
||||
/** Deterministic per-status scratch id (tests + dedup helpers). */
|
||||
export function statusScratchId(status: MastodonStatus): string {
|
||||
return createHash("sha256").update(status.id).digest("hex").slice(0, 16);
|
||||
}
|
||||
96
apps/api/src/intake/adapters/peertube.adapter.ts
Normal file
96
apps/api/src/intake/adapters/peertube.adapter.ts
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
/**
|
||||
* PeerTube channel adapter — public channel videos over the same REST the
|
||||
* connector and the live-monitor use: `GET /api/v1/video-channels/{handle}/videos`
|
||||
* (anonymous, per research §1.5; instance default 50 req/10 s per IP, our
|
||||
* intake-peertube host class runs stricter at 5/10 s). No webhooks exist
|
||||
* upstream (verified), so polling is the only path.
|
||||
*
|
||||
* Media policy is thumbnail-first (research §2.5): the thumbnail is a harvest
|
||||
* candidate; the full webseed file is only a candidate when the source opted
|
||||
* into full-harvest, which the media policy gate in the service enforces.
|
||||
*/
|
||||
import { fetchJson, hostClassFor, type AdapterContext, type AdapterPollResult, type IntakeAdapter, type RawIntakeItem } from "./adapter.js";
|
||||
import { KIND_PEERTUBE_CHANNEL } from "../intake-kinds.js";
|
||||
import type { PeerTubeChannelParams } from "../intake-params.js";
|
||||
|
||||
interface PtVideo {
|
||||
uuid?: string;
|
||||
shortUUID?: string;
|
||||
name?: string;
|
||||
createdAt?: string;
|
||||
publishedAt?: string;
|
||||
account?: { name?: string; displayName?: string };
|
||||
channel?: { displayName?: string; name?: string };
|
||||
thumbnailPath?: string;
|
||||
previewPath?: string;
|
||||
isLive?: boolean;
|
||||
duration?: number;
|
||||
}
|
||||
|
||||
interface PtList { total?: number; data?: PtVideo[] }
|
||||
|
||||
function dateOf(...values: Array<string | undefined>): Date | null {
|
||||
for (const value of values) {
|
||||
if (typeof value !== "string") continue;
|
||||
const date = new Date(value);
|
||||
if (Number.isFinite(date.getTime())) return date;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function videosToItems(videos: PtVideo[], instance: string, now: Date): RawIntakeItem[] {
|
||||
const out: RawIntakeItem[] = [];
|
||||
for (const video of videos) {
|
||||
const uuid = video.uuid ?? video.shortUUID;
|
||||
if (typeof uuid !== "string" || uuid === "") continue;
|
||||
const url = `${instance}/videos/watch/${uuid}`;
|
||||
const thumb = typeof video.thumbnailPath === "string" && video.thumbnailPath !== ""
|
||||
? `${instance}${video.thumbnailPath}` : typeof video.previewPath === "string" && video.previewPath !== ""
|
||||
? `${instance}${video.previewPath}` : null;
|
||||
const handle = video.channel?.displayName ?? video.channel?.name ?? video.account?.name ?? "channel";
|
||||
out.push({
|
||||
id: uuid,
|
||||
remoteId: uuid,
|
||||
remoteRef: uuid,
|
||||
provider: "peertube",
|
||||
authorHandle: handle,
|
||||
text: video.name ?? "",
|
||||
mediaUrls: thumb ? [thumb] : [],
|
||||
createdAt: dateOf(video.publishedAt, video.createdAt) ?? now,
|
||||
metrics: typeof video.duration === "number" ? { durationSec: video.duration } : {},
|
||||
normalize: { contentKind: "text", url },
|
||||
attribution: {
|
||||
title: video.name ?? null,
|
||||
url,
|
||||
channelTitle: video.channel?.displayName ?? null,
|
||||
videoId: uuid,
|
||||
provider: "peertube",
|
||||
...(video.isLive === true ? { live: true } : {}),
|
||||
},
|
||||
media: thumb ? [{ url: thumb, kind: "image" }] : [],
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export class PeerTubeChannelAdapter implements IntakeAdapter {
|
||||
readonly kind = KIND_PEERTUBE_CHANNEL;
|
||||
|
||||
async poll(context: AdapterContext): Promise<AdapterPollResult> {
|
||||
const params = context.params as PeerTubeChannelParams;
|
||||
const { body } = await fetchJson<PtList>(
|
||||
context.fetch,
|
||||
`${params.instance}/api/v1/video-channels/${encodeURIComponent(params.handle)}/videos?count=25&sort=-publishedAt`,
|
||||
hostClassFor(KIND_PEERTUBE_CHANNEL),
|
||||
);
|
||||
const videos = Array.isArray(body.data) ? body.data : [];
|
||||
const seen = new Set(Array.isArray(context.state.seenVideoIds)
|
||||
? context.state.seenVideoIds.filter((id): id is string => typeof id === "string") : []);
|
||||
const items = videosToItems(videos, params.instance, new Date()).filter((item) => !seen.has(item.remoteRef ?? item.remoteId));
|
||||
for (const item of items) seen.add(item.remoteRef ?? item.remoteId);
|
||||
return {
|
||||
items,
|
||||
state: { ...context.state, seenVideoIds: [...seen].slice(-50) },
|
||||
};
|
||||
}
|
||||
}
|
||||
69
apps/api/src/intake/adapters/push-seams.ts
Normal file
69
apps/api/src/intake/adapters/push-seams.ts
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
/**
|
||||
* Push-only seams (research §1.2/§1.4): these kinds are never poll-due
|
||||
* (DEFAULT_POLL_INTERVAL_SEC = 0). The adapters exist so the scheduler has a
|
||||
* complete registry and so the ingest halves are testable now:
|
||||
*
|
||||
* - bluesky_keyword: the keyword gate consumed by the Jetstream WS worker.
|
||||
* `matchesKeywords(text, keywords)` is the exact predicate the future
|
||||
* consumer applies to every firehose event (~52 posts/s measured); it is
|
||||
* exported and unit-tested so the seam is honest, not a stub that always
|
||||
* returns true.
|
||||
*
|
||||
* - telegram_channel: intake rows for the EXISTING archive webhook path.
|
||||
* The webhook already normalizes posts, harvests media and upserts into
|
||||
* archived_feed_items (apps/api/src/archive/normalizer.ts + service);
|
||||
* this adapter's poll() is unreachable by design, and the service's
|
||||
* ingest path bridges archive items into intake rows (see
|
||||
* IntakeService.ingestArchiveItem).
|
||||
*/
|
||||
import type { AdapterContext, AdapterPollResult, IntakeAdapter } from "./adapter.js";
|
||||
import { KIND_BLUESKY_KEYWORD, KIND_TELEGRAM_CHANNEL } from "../intake-kinds.js";
|
||||
import type { BlueskyKeywordParams } from "../intake-params.js";
|
||||
|
||||
/**
|
||||
* Case/diacritic-folded substring gate. Multi-term keywords require BOTH
|
||||
* terms (AND) — the research noise mitigation: one common word alone must
|
||||
* not pull a trending meme into intake.
|
||||
*/
|
||||
export function matchesKeywords(text: string, keyword: string): boolean {
|
||||
const needle = keyword.trim().toLowerCase();
|
||||
if (needle === "") return false;
|
||||
const terms = needle.split(/\s+/).filter((term) => term !== "");
|
||||
if (terms.length === 0) return false;
|
||||
const haystack = text.toLowerCase();
|
||||
return terms.every((term) => haystack.includes(term));
|
||||
}
|
||||
|
||||
export function filterByKeywords<T>(items: T[], textOf: (item: T) => string, keywords: readonly string[]): T[] {
|
||||
return items.filter((item) => {
|
||||
const text = textOf(item);
|
||||
return keywords.some((keyword) => matchesKeywords(text, keyword));
|
||||
});
|
||||
}
|
||||
|
||||
export class BlueskyKeywordAdapter implements IntakeAdapter {
|
||||
readonly kind = KIND_BLUESKY_KEYWORD;
|
||||
|
||||
async poll(_context: AdapterContext): Promise<AdapterPollResult> {
|
||||
// Push-only: the Jetstream WS consumer owns this kind's write path; poll()
|
||||
// exists so the registry is total over kinds and always answers a
|
||||
// misconfigured cadence with a clean no-op instead of a fetch.
|
||||
return { items: [], state: _context.state, unchanged: true };
|
||||
}
|
||||
|
||||
/** The gate the WS worker applies per firehose event. */
|
||||
gate(text: string, params: BlueskyKeywordParams): boolean {
|
||||
return params.keywords.some((keyword) => matchesKeywords(text, keyword));
|
||||
}
|
||||
}
|
||||
|
||||
export class TelegramChannelAdapter implements IntakeAdapter {
|
||||
readonly kind = KIND_TELEGRAM_CHANNEL;
|
||||
|
||||
async poll(_context: AdapterContext): Promise<AdapterPollResult> {
|
||||
// Push-only: telegram rows arrive through the existing archive webhook
|
||||
// (bot-admin channels) and the future MTProto sidecar; see the service's
|
||||
// ingestArchiveItem bridge.
|
||||
return { items: [], state: _context.state, unchanged: true };
|
||||
}
|
||||
}
|
||||
276
apps/api/src/intake/adapters/rss-parse.ts
Normal file
276
apps/api/src/intake/adapters/rss-parse.ts
Normal file
|
|
@ -0,0 +1,276 @@
|
|||
/**
|
||||
* Minimal RSS 2.0 / Atom / RSS-in-HTML-entity parser for the intake adapters.
|
||||
*
|
||||
* In-process by the research verdict ("rss-parser in-process is ~100 lines;
|
||||
* Miniflux only if feed edge-cases bite"). No external dependency: feeds are
|
||||
* a closed, well-known XML subset, and the extractor below is total over
|
||||
* arbitrary input — unknown elements are ignored, missing dates fall back to
|
||||
* fetch time, and the caller (feed-normalize) owns all safety-critical
|
||||
* parsing of text/URLs.
|
||||
*
|
||||
* XML entity decoding covers the five predefined entities plus numeric
|
||||
* references; unknown entities pass through verbatim (a literal "&foo;" in a
|
||||
* title is not a decoding failure).
|
||||
*/
|
||||
const ENTITY: Record<string, string> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'",
|
||||
nbsp: "\u00a0", eacute: "é", egrave: "è", agrave: "à", ccedil: "ç", auml: "ä", ouml: "ö", uuml: "ü",
|
||||
laquo: "«", raquo: "»", ldquo: "\u201c", rdquo: "\u201d", lsquo: "\u2018", rsquo: "\u2019",
|
||||
ndash: "–", mdash: "—", hellip: "…", rsquo2: "’" };
|
||||
|
||||
export function decodeEntities(text: string): string {
|
||||
return text.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*);/g, (whole, entity: string) => {
|
||||
if (entity.startsWith("#x") || entity.startsWith("#X")) {
|
||||
const code = Number.parseInt(entity.slice(2), 16);
|
||||
return Number.isInteger(code) && code > 0 && code <= 0x10ffff ? safeCodePoint(code) : whole;
|
||||
}
|
||||
if (entity.startsWith("#")) {
|
||||
const code = Number.parseInt(entity.slice(1), 10);
|
||||
return Number.isInteger(code) && code > 0 && code <= 0x10ffff ? safeCodePoint(code) : whole;
|
||||
}
|
||||
return ENTITY[entity] ?? whole;
|
||||
});
|
||||
}
|
||||
|
||||
function safeCodePoint(code: number): string {
|
||||
try { return String.fromCodePoint(code); } catch { return "\uFFFD"; }
|
||||
}
|
||||
|
||||
interface XmlNode {
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
children: XmlNode[];
|
||||
text: string;
|
||||
}
|
||||
|
||||
const TAG_TOKEN = /<!--[\s\S]*?-->|<!\[CDATA\[([\s\S]*?)\]\]>|<\/( [a-zA-Z][a-zA-Z0-9:_-]* )\s*>|<([a-zA-Z][a-zA-Z0-9:_-]*)((?:\s+[^<>]*?)?)(\/?)>/g;
|
||||
const NAME_RE = /^([a-zA-Z][a-zA-Z0-9:_-]*)\s*$/;
|
||||
|
||||
/** Strip any namespace prefix: "dc:creator" -> "creator". */
|
||||
function localName(tag: string): string {
|
||||
const at = tag.indexOf(":");
|
||||
return at > 0 ? tag.slice(at + 1) : tag;
|
||||
}
|
||||
|
||||
function parseAttrs(raw: string | undefined): Record<string, string> {
|
||||
const attrs: Record<string, string> = {};
|
||||
if (!raw) return attrs;
|
||||
for (const match of raw.matchAll(/([a-zA-Z_:][a-zA-Z0-9_:.-]*)\s*=\s*(?:"([^"]*)"|'([^']*)')/g)) {
|
||||
attrs[match[1]!.toLowerCase()] = decodeEntities(match[2] ?? match[3] ?? "");
|
||||
}
|
||||
return attrs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tolerant single-pass scan into a flat tree: one regex walks every tag token
|
||||
* in document order, a stack tracks open elements, text between tags
|
||||
* accumulates on the current node. Malformed markup (unterminated tags,
|
||||
* mismatched closes) degrades to whatever structure is recoverable — never
|
||||
* an exception, because feeds in the wild are routinely broken.
|
||||
*/
|
||||
export function parseXml(input: string): XmlNode {
|
||||
const root: XmlNode = { tag: "#root", attrs: {}, children: [], text: "" };
|
||||
const stack: XmlNode[] = [root];
|
||||
let cursor = 0;
|
||||
let textChunk = "";
|
||||
const flush = (): void => {
|
||||
if (textChunk !== "") {
|
||||
stack[stack.length - 1]!.text += decodeEntities(textChunk);
|
||||
textChunk = "";
|
||||
}
|
||||
};
|
||||
const token = /<!--[\s\S]*?-->|<!\[CDATA\[([\s\S]*?)\]\]>|<\/([a-zA-Z][a-zA-Z0-9:_-]*)\s*>|<([a-zA-Z][a-zA-Z0-9:_-]*)((?:\s+[^<>]*?)?)(\/?)>/g;
|
||||
for (const match of input.matchAll(token)) {
|
||||
textChunk += input.slice(cursor, match.index);
|
||||
cursor = match.index! + match[0].length;
|
||||
if (match[1] !== undefined) { textChunk += match[1]; continue; } // CDATA: raw text
|
||||
if (match[0].startsWith("<!")) continue; // comment / doctype
|
||||
if (match[2] !== undefined) { // closing tag
|
||||
flush();
|
||||
const close = localName(match[2].toLowerCase());
|
||||
const at = stack.map((node) => node.tag).lastIndexOf(close);
|
||||
if (at > 0) stack.length = at; // pop to the matching open; stray closes are ignored
|
||||
continue;
|
||||
}
|
||||
// Local-name only (dc:creator -> creator, yt:videoId -> videoid,
|
||||
// media:thumbnail -> thumbnail): feed namespaces are decorative for the
|
||||
// fields this extractor reads, and matching on the local name keeps every
|
||||
// lookup below prefix-agnostic.
|
||||
const tag = localName(match[3]!.toLowerCase());
|
||||
flush();
|
||||
const node: XmlNode = { tag, attrs: parseAttrs(match[4]), children: [], text: "" };
|
||||
stack[stack.length - 1]!.children.push(node);
|
||||
if (match[5] !== "/") stack.push(node); // not self-closing: descend
|
||||
}
|
||||
textChunk += input.slice(cursor);
|
||||
flush();
|
||||
return root;
|
||||
}
|
||||
|
||||
function findFirst(node: XmlNode, tag: string): XmlNode | undefined {
|
||||
for (const child of node.children) {
|
||||
if (child.tag === tag) return child;
|
||||
const nested = findFirst(child, tag);
|
||||
if (nested) return nested;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function findAll(node: XmlNode, tag: string): XmlNode[] {
|
||||
const out: XmlNode[] = [];
|
||||
const walk = (current: XmlNode): void => {
|
||||
for (const child of current.children) {
|
||||
if (child.tag === tag) out.push(child);
|
||||
walk(child);
|
||||
}
|
||||
};
|
||||
walk(node);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** text of a child tag by name, "" when absent — keeps author lookups flat. */
|
||||
function childText(node: XmlNode, tag: string): string {
|
||||
return textOf(childTag(node, tag));
|
||||
}
|
||||
|
||||
/** Raw inner text (entities decoded, whitespace preserved) — CDATA HTML. */
|
||||
function rawText(node: XmlNode): string {
|
||||
return decodeEntities(node.text);
|
||||
}
|
||||
|
||||
/** Inner text of a node across nested markup, whitespace-collapsed. */
|
||||
function textOf(node: XmlNode | undefined): string {
|
||||
if (!node) return "";
|
||||
let out = node.text;
|
||||
for (const child of node.children) out += " " + textOf(child);
|
||||
return out.replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
export interface FeedEntry {
|
||||
id: string;
|
||||
title: string;
|
||||
link: string | null;
|
||||
contentHtml: string | null;
|
||||
contentText: string | null;
|
||||
publishedAt: Date | null;
|
||||
updatedAt: Date | null;
|
||||
author: string | null;
|
||||
enclosures: Array<{ url: string; type: string }>;
|
||||
mediaThumbnails: string[];
|
||||
/** youtube atom: yt:videoId */
|
||||
videoId: string | null;
|
||||
}
|
||||
|
||||
export interface ParsedFeed {
|
||||
type: "rss" | "atom";
|
||||
channelTitle: string | null;
|
||||
channelLink: string | null;
|
||||
entries: FeedEntry[];
|
||||
}
|
||||
|
||||
function childTag(node: XmlNode, tag: string): XmlNode | undefined {
|
||||
return node.children.find((child) => child.tag === tag);
|
||||
}
|
||||
|
||||
/** RSS <author><name>…</name></author> / Atom author/name text. */
|
||||
function authorNameOf(node: XmlNode): string {
|
||||
const author = childTag(node, "author");
|
||||
return author === undefined ? "" : textOf(childTag(author, "name") ?? author);
|
||||
}
|
||||
|
||||
function linkOf(item: XmlNode, channelLink: string | null): string | null {
|
||||
// Atom: <link rel="alternate" href="…"/> wins; rel absent is alternate too.
|
||||
// Self/service links (rel="self") are skipped. RSS: <link>text</link>.
|
||||
const links = item.children.filter((child) => child.tag === "link");
|
||||
for (const link of links) {
|
||||
const rel = link.attrs.rel;
|
||||
const href = link.attrs.href;
|
||||
if (href !== undefined && (rel === undefined || rel === "alternate")) return href;
|
||||
}
|
||||
const plain = links.map((link) => textOf(link)).find((text) => text !== "");
|
||||
if (plain) return plain;
|
||||
const guid = childTag(item, "guid");
|
||||
if (guid && /^https?:\/\//i.test(guid.text.trim())) return guid.text.trim();
|
||||
return channelLink;
|
||||
}
|
||||
|
||||
function dateOf(...candidates: Array<XmlNode | undefined>): Date | null {
|
||||
for (const node of candidates) {
|
||||
const raw = textOf(node);
|
||||
if (raw === "") continue;
|
||||
const parsed = new Date(raw);
|
||||
if (Number.isFinite(parsed.getTime())) return parsed;
|
||||
// RFC822 without timezone ("Thu, 09 Oct 2026 12:00:00") is common in the wild
|
||||
const zoned = new Date(`${raw} GMT`);
|
||||
if (Number.isFinite(zoned.getTime())) return zoned;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function parseFeed(xml: string): ParsedFeed {
|
||||
const root = parseXml(xml);
|
||||
const rssChannel = findFirst(root, "channel");
|
||||
const atomFeed = root.children.find((child) => child.tag === "feed");
|
||||
if (rssChannel) {
|
||||
const channelTitle = textOf(childTag(rssChannel, "title")) || null;
|
||||
const channelLink = textOf(childTag(rssChannel, "link")) || null;
|
||||
const entries = findAll(rssChannel, "item").map((item): FeedEntry => {
|
||||
const encoded = childTag(item, "encoded"); // content:encoded (local name)
|
||||
const description = childTag(item, "description");
|
||||
// A description carrying markup IS provider HTML (CDATA-wrapped in
|
||||
// practice); plain descriptions stay text so P0c parses them as text.
|
||||
const descriptionHtml = encoded !== undefined ? textOf(encoded)
|
||||
: description !== undefined && /<\s*[a-zA-Z]/.test(rawText(description)) ? rawText(description) : null;
|
||||
const enclosures = item.children.filter((child) => child.tag === "enclosure")
|
||||
.map((enclosure) => ({ url: enclosure.attrs.url ?? "", type: enclosure.attrs.type ?? "" }))
|
||||
.filter((entry) => entry.url !== "");
|
||||
const group = findFirst(item, "group"); // media:group
|
||||
const thumbnails = [...findAll(item, "thumbnail"), ...(group ? findAll(group, "thumbnail") : [])]
|
||||
.map((thumb) => thumb.attrs.url ?? "").filter((url) => url !== "");
|
||||
return {
|
||||
id: textOf(childTag(item, "guid")) || textOf(childTag(item, "link")) || "",
|
||||
title: textOf(childTag(item, "title")),
|
||||
link: linkOf(item, channelLink),
|
||||
contentHtml: descriptionHtml,
|
||||
contentText: descriptionHtml === null && description !== undefined ? textOf(description) : null,
|
||||
publishedAt: dateOf(childTag(item, "pubdate"), childTag(item, "date"), childTag(item, "published")),
|
||||
updatedAt: dateOf(childTag(item, "updated")),
|
||||
author: textOf(childTag(item, "creator")) || childText(item, "author") || authorNameOf(item) || null,
|
||||
enclosures,
|
||||
mediaThumbnails: thumbnails,
|
||||
videoId: null,
|
||||
};
|
||||
});
|
||||
return { type: "rss", channelTitle, channelLink, entries };
|
||||
}
|
||||
if (atomFeed) {
|
||||
const channelTitle = textOf(childTag(atomFeed, "title")) || null;
|
||||
const channelLink = linkOf(atomFeed, null);
|
||||
const entries = findAll(atomFeed, "entry").map((entry): FeedEntry => {
|
||||
const contentHtml = textOf(childTag(entry, "content"));
|
||||
const summary = textOf(childTag(entry, "summary"));
|
||||
const enclosures = entry.children.filter((child) => child.tag === "link" && child.attrs.rel === "enclosure")
|
||||
.map((link) => ({ url: link.attrs.href ?? "", type: link.attrs.type ?? "" }))
|
||||
.filter((item) => item.url !== "");
|
||||
const videoIdNode = entry.children.find((child) => child.tag === "videoid");
|
||||
// yt:videoId is a leaf: raw text only (no nested aggregation).
|
||||
const group = findFirst(entry, "group");
|
||||
const thumbnails = [...findAll(entry, "thumbnail"), ...(group ? findAll(group, "thumbnail") : [])]
|
||||
.map((thumb) => thumb.attrs.url ?? "").filter((url) => url !== "");
|
||||
return {
|
||||
id: textOf(childTag(entry, "id")) || "",
|
||||
title: textOf(childTag(entry, "title")),
|
||||
link: linkOf(entry, channelLink),
|
||||
contentHtml: contentHtml !== "" ? contentHtml : null,
|
||||
contentText: contentHtml === "" && summary !== "" ? summary : null,
|
||||
publishedAt: dateOf(childTag(entry, "published"), childTag(entry, "updated")),
|
||||
updatedAt: dateOf(childTag(entry, "updated")),
|
||||
author: authorNameOf(entry) || childText(entry, "name") || null,
|
||||
enclosures,
|
||||
mediaThumbnails: thumbnails,
|
||||
videoId: videoIdNode ? videoIdNode.text.trim() || null : null,
|
||||
};
|
||||
});
|
||||
return { type: "atom", channelTitle, channelLink, entries };
|
||||
}
|
||||
return { type: "rss", channelTitle: null, channelLink: null, entries: [] };
|
||||
}
|
||||
78
apps/api/src/intake/adapters/rss.adapter.ts
Normal file
78
apps/api/src/intake/adapters/rss.adapter.ts
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
/**
|
||||
* RSS/Atom adapter — conditional GET via ETag / Last-Modified.
|
||||
*
|
||||
* The feed is fetched once per poll through the egress client (one budget
|
||||
* draw); when upstream answers 304 the poll is free and emits nothing, and
|
||||
* the carried ETag/Last-Modified state rides in intake_sources.state. Text
|
||||
* lands as sanitized HTML through the P0c allowlist parser (feed summaries
|
||||
* are HTML in practice), enclosures/thumbnails become harvest candidates.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import { fetchText, type AdapterContext, type AdapterPollResult, type IntakeAdapter, type RawIntakeItem } from "./adapter.js";
|
||||
import { parseFeed, type ParsedFeed } from "./rss-parse.js";
|
||||
import { hostClassFor } from "./adapter.js";
|
||||
import { KIND_RSS } from "../intake-kinds.js";
|
||||
import type { RssParams } from "../intake-params.js";
|
||||
|
||||
export const MAX_FEED_ITEMS = 100;
|
||||
|
||||
function stateHeaders(state: Record<string, unknown>): Record<string, string> {
|
||||
const headers: Record<string, string> = {};
|
||||
if (typeof state.etag === "string" && state.etag !== "") headers["if-none-match"] = state.etag;
|
||||
if (typeof state.lastModified === "string" && state.lastModified !== "") headers["if-modified-since"] = state.lastModified;
|
||||
return headers;
|
||||
}
|
||||
|
||||
export function feedToItems(feed: ParsedFeed, publishedFallback: Date): RawIntakeItem[] {
|
||||
return feed.entries.slice(0, MAX_FEED_ITEMS).map((entry) => {
|
||||
const media: Array<{ url: string; kind: "image" | "video" }> = [];
|
||||
for (const enclosure of entry.enclosures) {
|
||||
if (/^video\//i.test(enclosure.type)) media.push({ url: enclosure.url, kind: "video" });
|
||||
else if (/^image\//i.test(enclosure.type) || /\.(?:png|jpe?g|webp|gif)(?:[?#]|$)/i.test(enclosure.url)) media.push({ url: enclosure.url, kind: "image" });
|
||||
else if (/^audio\//i.test(enclosure.type)) media.push({ url: enclosure.url, kind: "video" });
|
||||
}
|
||||
for (const thumb of entry.mediaThumbnails.slice(0, 4)) media.push({ url: thumb, kind: "image" });
|
||||
const text = entry.contentHtml ?? entry.contentText ?? entry.title;
|
||||
const scratch = createHash("sha256").update(text).digest("hex").slice(0, 32);
|
||||
const ref = entry.id || entry.link || `sha256:${scratch}`;
|
||||
return {
|
||||
id: entry.id || entry.link || scratch,
|
||||
remoteId: ref,
|
||||
remoteRef: ref,
|
||||
provider: "rss",
|
||||
authorHandle: entry.author ?? feed.channelTitle ?? "feed",
|
||||
text,
|
||||
mediaUrls: media.map((entry_media) => entry_media.url),
|
||||
createdAt: entry.publishedAt ?? entry.updatedAt ?? publishedFallback,
|
||||
metrics: {},
|
||||
normalize: { contentKind: entry.contentHtml !== null ? "html" : "text", url: entry.link ?? undefined },
|
||||
attribution: {
|
||||
title: entry.title || null,
|
||||
url: entry.link ?? null,
|
||||
author: entry.author ?? null,
|
||||
channelTitle: feed.channelTitle,
|
||||
provider: "rss",
|
||||
...(entry.publishedAt ? { publishedAt: entry.publishedAt.toISOString() } : {}),
|
||||
},
|
||||
media,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export class RssAdapter implements IntakeAdapter {
|
||||
readonly kind = KIND_RSS;
|
||||
|
||||
async poll(context: AdapterContext): Promise<AdapterPollResult> {
|
||||
const params = context.params as RssParams;
|
||||
const hostClass = hostClassFor(KIND_RSS);
|
||||
const { text, response, unchanged } = await fetchText(context.fetch, params.url, hostClass, stateHeaders(context.state));
|
||||
const state: Record<string, unknown> = { ...context.state };
|
||||
const etag = response.headers.get("etag");
|
||||
const lastModified = response.headers.get("last-modified");
|
||||
if (etag !== null) state.etag = etag;
|
||||
if (lastModified !== null) state.lastModified = lastModified;
|
||||
if (unchanged) return { items: [], state, unchanged: true };
|
||||
const feed = parseFeed(text);
|
||||
return { items: feedToItems(feed, new Date()), state };
|
||||
}
|
||||
}
|
||||
100
apps/api/src/intake/adapters/web-watch.adapter.ts
Normal file
100
apps/api/src/intake/adapters/web-watch.adapter.ts
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
/**
|
||||
* Page-watch adapter — the changedetection.io pattern in ~100 lines
|
||||
* (research §1.8): fetch the page through the egress budget, normalize the
|
||||
* text (strip scripts/styles, collapse whitespace, strip rotating
|
||||
* timestamps/CSRF nonces), hash, and emit an item only when the hash moves.
|
||||
*
|
||||
* Plain fetch only — no headless browser (per the task's "no headless"
|
||||
* bound); a page that needs JS renders as its static DOM, which is the right
|
||||
* default for press pages and gazettes.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import { fetchText, hostClassFor, type AdapterContext, type AdapterPollResult, type IntakeAdapter, type RawIntakeItem } from "./adapter.js";
|
||||
import { KIND_WEB_WATCH } from "../intake-kinds.js";
|
||||
import type { WebWatchParams } from "../intake-params.js";
|
||||
|
||||
/**
|
||||
* Diff normalization: what changes on a page without changing its content —
|
||||
* numbers (counts, timestamps), long hex/base64 tokens (CSRF, cache busters).
|
||||
* Hashing the normalized text is what keeps phantom diffs out of the archive.
|
||||
*/
|
||||
export function normalizePageText(html: string, selector?: string): string {
|
||||
let work = html
|
||||
.replace(/<script[\s\S]*?<\/script>/gi, " ")
|
||||
.replace(/<style[\s\S]*?<\/style>/gi, " ")
|
||||
.replace(/<noscript[\s\S]*?<\/noscript>/gi, " ")
|
||||
.replace(/<!--[\s\S]*?-->/g, " ")
|
||||
.replace(/<[^>]+>/g, " ")
|
||||
.replace(/ /gi, " ")
|
||||
.replace(/&/gi, "&")
|
||||
.replace(/</gi, "<")
|
||||
.replace(/>/gi, ">")
|
||||
.replace(/"/gi, '"')
|
||||
.replace(/�?39;/g, "'");
|
||||
if (selector !== undefined && selector.trim() !== "") {
|
||||
// Cheap id/class containment: keep only lines containing the selector's
|
||||
// leaf token when the selector looks like #id or .class (a full CSS
|
||||
// engine is out of scope for phase 1; a wrong selector degrades to the
|
||||
// whole page's diff, which is still correct — just noisier).
|
||||
const leaf = selector.trim().split(/[ >]+/).at(-1) ?? "";
|
||||
const match = /^([#.])([\w-]+)$/.exec(leaf);
|
||||
if (match) {
|
||||
// Extract the region around the marker token from the raw HTML instead.
|
||||
const marker = match[1] === "#" ? `id="${match[2]}"` : `class="${match[2]}"`;
|
||||
const at = html.toLowerCase().indexOf(marker.toLowerCase());
|
||||
if (at >= 0) {
|
||||
const from = html.lastIndexOf("<", at);
|
||||
const to = html.indexOf("</body>", at);
|
||||
work = normalizePageText(html.slice(from > 0 ? from : 0, to > 0 ? to : undefined));
|
||||
}
|
||||
}
|
||||
}
|
||||
return work
|
||||
.replace(/\b[0-9]{2,}\b/g, " ") // counts, epochs, timestamps
|
||||
.replace(/\b[0-9a-f]{16,}\b/gi, " ") // CSRF tokens, digests
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function pageHash(normalized: string): string {
|
||||
return createHash("sha256").update(normalized).digest("hex");
|
||||
}
|
||||
|
||||
export class WebWatchAdapter implements IntakeAdapter {
|
||||
readonly kind = KIND_WEB_WATCH;
|
||||
|
||||
async poll(context: AdapterContext): Promise<AdapterPollResult> {
|
||||
const params = context.params as WebWatchParams;
|
||||
const { text, unchanged } = await fetchText(context.fetch, params.url, hostClassFor(KIND_WEB_WATCH), {
|
||||
"cache-control": "no-cache",
|
||||
});
|
||||
if (unchanged) return { items: [], state: context.state, unchanged: true };
|
||||
const normalized = normalizePageText(text, params.selector);
|
||||
const hash = pageHash(normalized);
|
||||
if (typeof context.state.contentHash === "string" && context.state.contentHash === hash) {
|
||||
return { items: [], state: context.state, unchanged: true };
|
||||
}
|
||||
const now = new Date();
|
||||
const items: RawIntakeItem[] = [];
|
||||
if (typeof context.state.contentHash === "string") {
|
||||
// First fetch seeds the baseline; only a CHANGED page (second fetch
|
||||
// onward) emits — a new subscription never manufactures an item for a
|
||||
// page that was already in that state.
|
||||
items.push({
|
||||
id: hash.slice(0, 24),
|
||||
remoteId: hash.slice(0, 24),
|
||||
remoteRef: `${params.url}#${hash.slice(0, 16)}`,
|
||||
provider: "web",
|
||||
authorHandle: new URL(params.url).hostname,
|
||||
text: normalized.slice(0, 8_000),
|
||||
mediaUrls: [],
|
||||
createdAt: now,
|
||||
metrics: {},
|
||||
normalize: { contentKind: "text", url: params.url },
|
||||
attribution: { url: params.url, provider: "web", changedAt: now.toISOString(), diffHash: hash },
|
||||
media: [],
|
||||
});
|
||||
}
|
||||
return { items, state: { ...context.state, contentHash: hash, lastChangedAt: now.toISOString() } };
|
||||
}
|
||||
}
|
||||
59
apps/api/src/intake/adapters/youtube.adapter.ts
Normal file
59
apps/api/src/intake/adapters/youtube.adapter.ts
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
/**
|
||||
* YouTube channel adapter — the channel Atom feed, no key, no quota
|
||||
* (verified live, research §1.1/§1.6: `feeds/videos.xml?channel_id=…`
|
||||
* returns Atom with yt:videoId, media thumbnails; 15-entry window, ~15 min
|
||||
* freshness → poll ≤ every 10 min). The Data API is deliberately NOT used
|
||||
* here: search.list costs 100 units against a 10k/day project budget while
|
||||
* the feed is free.
|
||||
*/
|
||||
import { fetchText, hostClassFor, type AdapterContext, type AdapterPollResult, type IntakeAdapter, type RawIntakeItem } from "./adapter.js";
|
||||
import { parseFeed } from "./rss-parse.js";
|
||||
import { KIND_YOUTUBE_CHANNEL, YOUTUBE_FEED_URL } from "../intake-kinds.js";
|
||||
import type { YouTubeChannelParams } from "../intake-params.js";
|
||||
|
||||
export class YouTubeChannelAdapter implements IntakeAdapter {
|
||||
readonly kind = KIND_YOUTUBE_CHANNEL;
|
||||
|
||||
async poll(context: AdapterContext): Promise<AdapterPollResult> {
|
||||
const params = context.params as YouTubeChannelParams;
|
||||
const { text, response, unchanged } = await fetchText(context.fetch, YOUTUBE_FEED_URL(params.channelId), hostClassFor(KIND_YOUTUBE_CHANNEL));
|
||||
const state: Record<string, unknown> = { ...context.state };
|
||||
const etag = response.headers.get("etag");
|
||||
if (etag !== null) state.etag = etag;
|
||||
if (unchanged) return { items: [], state, unchanged: true };
|
||||
const feed = parseFeed(text);
|
||||
const seen = new Set(Array.isArray(state.seenVideoIds)
|
||||
? state.seenVideoIds.filter((id): id is string => typeof id === "string") : []);
|
||||
const items: RawIntakeItem[] = [];
|
||||
for (const entry of feed.entries) {
|
||||
const videoId = entry.videoId ?? (entry.id.startsWith("yt:video:") ? entry.id.slice("yt:video:".length) : null);
|
||||
if (videoId === null || seen.has(videoId)) continue;
|
||||
const watchUrl = `https://www.youtube.com/watch?v=${videoId}`;
|
||||
items.push({
|
||||
id: videoId,
|
||||
remoteId: `yt:video:${videoId}`,
|
||||
remoteRef: `yt:video:${videoId}`,
|
||||
provider: "youtube",
|
||||
authorHandle: entry.author ?? feed.channelTitle ?? params.channelId,
|
||||
text: entry.title,
|
||||
mediaUrls: entry.mediaThumbnails.slice(0, 1),
|
||||
createdAt: entry.publishedAt ?? entry.updatedAt ?? new Date(),
|
||||
metrics: {},
|
||||
normalize: { contentKind: "text", url: watchUrl },
|
||||
attribution: {
|
||||
title: entry.title || null,
|
||||
url: watchUrl,
|
||||
channelTitle: feed.channelTitle,
|
||||
channelId: params.channelId,
|
||||
videoId,
|
||||
provider: "youtube",
|
||||
},
|
||||
// metadata-only default: the thumbnail is harvested, never the video.
|
||||
media: entry.mediaThumbnails.slice(0, 1).map((url) => ({ url, kind: "image" as const })),
|
||||
});
|
||||
seen.add(videoId);
|
||||
}
|
||||
state.seenVideoIds = [...seen].slice(-30);
|
||||
return { items, state };
|
||||
}
|
||||
}
|
||||
108
apps/api/src/intake/intake-egress.ts
Normal file
108
apps/api/src/intake/intake-egress.ts
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
/**
|
||||
* Intake egress binding — THE single outbound surface for every intake
|
||||
* adapter (task requirement 2: all outbound through packages/egress per-host
|
||||
* budgets).
|
||||
*
|
||||
* Same shape as live-monitor/poller/egress.ts and virtual-channels/
|
||||
* import-egress.ts: the egress package is ESM-only while the API emits
|
||||
* CommonJS, so it is type-imported (erased at build) and loaded natively at
|
||||
* runtime. Every request drawn by an adapter goes through `egressFetch`,
|
||||
* which means SSRF policy (private-address denial, scheme/port policy), the
|
||||
* global timeout, the response-size cap and — the point — the shared
|
||||
* `host_budgets` rows all apply. Two replicas polling mastodon.social contend
|
||||
* for one bucket; a 429 parks that host for every caller.
|
||||
*
|
||||
* `IntakeHttpError` carries the status + headers so adapters can classify
|
||||
* 304-not-modified (free), 429/Retry-After (budget/park) and 5xx (backoff)
|
||||
* the way the monitor poller's BackoffPollDriver does.
|
||||
*/
|
||||
import { existsSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { pool } from "../db/client.js";
|
||||
import { INTAKE_HOST_CLASSES } from "./intake-kinds.js";
|
||||
// Type-only (erased in the CommonJS build); loaded natively below.
|
||||
import type { createEgressClient, EgressError, EgressPolicy } from "../../../../packages/egress/src/index.js";
|
||||
|
||||
type EgressPackage = typeof import("../../../../packages/egress/src/index.js");
|
||||
|
||||
export interface IntakeEgress {
|
||||
egressFetch: ReturnType<typeof createEgressClient>["egressFetch"];
|
||||
policy: EgressPolicy;
|
||||
isEgressError(error: unknown): error is EgressError;
|
||||
}
|
||||
|
||||
/** Response body cap for feed/page/JSON fetches (not media downloads). */
|
||||
export const INTAKE_MAX_BYTES = 8 * 1024 * 1024;
|
||||
export const INTAKE_TIMEOUT_MS = 20_000;
|
||||
/** Media harvest ceiling per file (research §2.5 default image cap). */
|
||||
export const INTAKE_MEDIA_MAX_BYTES = 50 * 1024 * 1024;
|
||||
|
||||
const nativeImport = new Function("specifier", "return import(specifier)") as (specifier: string) => Promise<EgressPackage>;
|
||||
let shared: Promise<IntakeEgress> | undefined;
|
||||
|
||||
function resolveEgressPackage(): string {
|
||||
for (const start of [process.cwd(), __dirname]) {
|
||||
for (let root = start; ; root = dirname(root)) {
|
||||
const directory = resolve(root, "packages/egress");
|
||||
const files = process.env.NODE_ENV === "production" ? ["dist/index.js"] : ["src/index.ts", "dist/index.js"];
|
||||
for (const file of files) {
|
||||
const candidate = resolve(directory, file);
|
||||
if (existsSync(candidate)) return candidate;
|
||||
}
|
||||
if (dirname(root) === root) break;
|
||||
}
|
||||
}
|
||||
throw new Error("Build packages/egress before starting intake");
|
||||
}
|
||||
|
||||
/** Process-wide egress client shared by every intake adapter. */
|
||||
export function intakeEgress(): Promise<IntakeEgress> {
|
||||
shared ??= (async () => {
|
||||
const pkg = await nativeImport(pathToFileURL(resolveEgressPackage()).href);
|
||||
const client = pkg.createEgressClient({
|
||||
pool,
|
||||
timeoutMs: INTAKE_TIMEOUT_MS,
|
||||
maxResponseBytes: INTAKE_MAX_BYTES,
|
||||
hostClasses: INTAKE_HOST_CLASSES,
|
||||
});
|
||||
return { ...client, isEgressError: (error: unknown): error is EgressError => error instanceof pkg.EgressError };
|
||||
})();
|
||||
return shared;
|
||||
}
|
||||
|
||||
/** Non-2xx/3xx egress response surfaced with its headers (Retry-After, ETag…). */
|
||||
export class IntakeHttpError extends Error {
|
||||
constructor(readonly status: number, readonly headers: Headers, readonly url: string) {
|
||||
super(`intake HTTP ${status}`);
|
||||
this.name = "IntakeHttpError";
|
||||
}
|
||||
get notModified(): boolean { return this.status === 304; }
|
||||
get gone(): boolean { return this.status === 404 || this.status === 410; }
|
||||
get rateLimited(): boolean { return this.status === 429; }
|
||||
get transient(): boolean { return this.status === 429 || this.status >= 500; }
|
||||
}
|
||||
|
||||
/** Retry-After as a lower-bound delay in ms, including HTTP-date form. */
|
||||
export function intakeRetryAfterMs(headers: Headers, now: number): number {
|
||||
const raw = headers.get("retry-after");
|
||||
if (raw === null) return 0;
|
||||
if (/^\d+(?:\.\d+)?$/.test(raw.trim())) {
|
||||
const delay = Number(raw) * 1000;
|
||||
return Number.isFinite(delay) ? delay : 0;
|
||||
}
|
||||
const deadline = Date.parse(raw);
|
||||
return Number.isFinite(deadline) ? Math.max(0, deadline - now) : 0;
|
||||
}
|
||||
|
||||
/** Reads a response body to a string under the shared size cap. */
|
||||
export async function readBodyText(response: Response): Promise<string> {
|
||||
return response.text();
|
||||
}
|
||||
|
||||
/** Reads a response body to a Buffer under the media cap, before S3 put. */
|
||||
export async function readBodyBytes(response: Response): Promise<Buffer> {
|
||||
const bytes = Buffer.from(await response.arrayBuffer());
|
||||
if (bytes.byteLength > INTAKE_MEDIA_MAX_BYTES) throw new Error("intake media exceeds size cap");
|
||||
return bytes;
|
||||
}
|
||||
122
apps/api/src/intake/intake-ingest.ts
Normal file
122
apps/api/src/intake/intake-ingest.ts
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
/**
|
||||
* The intake ingest funnel: raw adapter items → P0c normalize → content-hash
|
||||
* → idempotent upsert into intake_items (027).
|
||||
*
|
||||
* Every item passes `normalizeFeedItem` (task requirement 3), so `content`
|
||||
* lands as the closed safe-content AST — never raw provider HTML — and media
|
||||
* as proxy-minted refs. `text` (the FTS corpus) is derived from the AST via
|
||||
* `astToPlainText`, never the raw provider string.
|
||||
*
|
||||
* `content_hash` is the schema's dedup key — UNIQUE (workspace_id,
|
||||
* content_hash): sha256 of the NFKC + whitespace-collapsed AST plain text
|
||||
* plus the sorted media origin identity. The same statement arriving via RSS
|
||||
* and a Mastodon boost (or the same URL watched twice) hashes equal and
|
||||
* stores once; first writer wins, later deliveries no-op.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import type { PoolClient } from "pg";
|
||||
import { normalizeFeedItem, astToPlainText, type MediaRef, type NormalizedFeedItem } from "../feeds/feed-normalize.js";
|
||||
import { connectionFor, type RawIntakeItem } from "./adapters/adapter.js";
|
||||
import { PROVIDER_BY_KIND, type IntakeKind } from "./intake-kinds.js";
|
||||
import type { IntakeParams } from "./intake-params.js";
|
||||
import type { HarvestedMedia } from "./media-harvest.js";
|
||||
|
||||
export type IntakeMediaRef = MediaRef & {
|
||||
/** S3 content-addressed key when the harvest stored the origin file. */
|
||||
storageKey?: string;
|
||||
hash?: string;
|
||||
sizeBytes?: number;
|
||||
contentType?: string;
|
||||
harvestStatus?: "stored" | "skipped" | "too_large" | "failed";
|
||||
originUrl?: string;
|
||||
};
|
||||
|
||||
export interface NormalizeOutcome {
|
||||
normalized: NormalizedFeedItem | null;
|
||||
plainText: string;
|
||||
contentHash: string;
|
||||
}
|
||||
|
||||
/** NFKC + whitespace-collapse, per the 027 content_hash note. */
|
||||
export function collapseText(plainText: string): string {
|
||||
return plainText.normalize("NFKC").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
/** sha256(collapsed plain text + sorted unique media origin URLs). */
|
||||
export function itemContentHash(plainText: string, mediaUrls: readonly string[]): string {
|
||||
const origins = [...new Set(mediaUrls.filter((url) => typeof url === "string" && url !== ""))].sort();
|
||||
return createHash("sha256")
|
||||
.update(JSON.stringify([collapseText(plainText), origins]))
|
||||
.digest("hex");
|
||||
}
|
||||
|
||||
/** Normalize one raw item; normalized = null when the funnel's validation drops it. */
|
||||
export function normalizeRawItem(raw: RawIntakeItem, kind: IntakeKind, params: IntakeParams["params"],
|
||||
workspaceId: string, sourceId: string): NormalizeOutcome {
|
||||
const connection = connectionFor(kind, params, sourceId, workspaceId);
|
||||
const normalized = normalizeFeedItem(raw, connection, raw.normalize);
|
||||
if (normalized === null) return { normalized: null, plainText: "", contentHash: "" };
|
||||
const plainText = astToPlainText(normalized.content);
|
||||
return { normalized, plainText, contentHash: itemContentHash(plainText, raw.mediaUrls) };
|
||||
}
|
||||
|
||||
/** Merge the P0c proxied media refs with the harvest outcome per origin URL. */
|
||||
export function mergeMediaRefs(normalized: NormalizedFeedItem, harvested: HarvestedMedia[]): IntakeMediaRef[] {
|
||||
const byUrl = new Map(harvested.map((entry) => [entry.url, entry]));
|
||||
return normalized.media.map((ref): IntakeMediaRef => {
|
||||
const origin = originOfProxyUrl(ref.proxyUrl);
|
||||
const hit = origin === null ? undefined : byUrl.get(origin);
|
||||
if (!hit) return ref;
|
||||
return {
|
||||
...ref,
|
||||
originUrl: hit.url,
|
||||
...(hit.status === "stored" ? {
|
||||
storageKey: hit.storageKey,
|
||||
hash: hit.hash,
|
||||
sizeBytes: hit.sizeBytes,
|
||||
contentType: hit.contentType,
|
||||
harvestStatus: "stored" as const,
|
||||
} : { harvestStatus: hit.status }),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** Extracts the origin URL back out of a media-proxy URL. */
|
||||
export function originOfProxyUrl(proxyUrl: string): string | null {
|
||||
const marker = "url=";
|
||||
const at = proxyUrl.indexOf(marker);
|
||||
if (at < 0) return null;
|
||||
try { return decodeURIComponent(proxyUrl.slice(at + marker.length)); } catch { return null; }
|
||||
}
|
||||
|
||||
export interface UpsertResult {
|
||||
inserted: number;
|
||||
unchanged: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Idempotent upsert inside the caller's tenant transaction. The conflict
|
||||
* target is the content-addressed key — (workspace_id, content_hash) — so a
|
||||
* re-poll that yields the same content writes nothing, and cross-source
|
||||
* duplicates (RSS + Mastodon boost of the same statement) collapse too.
|
||||
*/
|
||||
export async function upsertItems(client: PoolClient, workspaceId: string, sourceId: string, kind: IntakeKind,
|
||||
rows: Array<{ raw: RawIntakeItem; outcome: NormalizeOutcome; harvested: HarvestedMedia[] }>): Promise<UpsertResult> {
|
||||
const result: UpsertResult = { inserted: 0, unchanged: 0 };
|
||||
for (const entry of rows) {
|
||||
const { normalized, plainText, contentHash } = entry.outcome;
|
||||
if (normalized === null) continue;
|
||||
const mediaRefs = mergeMediaRefs(normalized, entry.harvested);
|
||||
const attribution: Record<string, unknown> = { ...entry.raw.attribution, provider: PROVIDER_BY_KIND[kind] };
|
||||
const inserted = await client.query(
|
||||
`INSERT INTO intake_items (workspace_id, source_id, remote_ref, content_hash, published_at,
|
||||
text, content, media_refs, attribution)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
|
||||
ON CONFLICT (workspace_id, content_hash) DO NOTHING`,
|
||||
[workspaceId, sourceId, entry.raw.remoteRef, contentHash, normalized.createdAt.toISOString(),
|
||||
plainText, JSON.stringify(normalized.content), JSON.stringify(mediaRefs), JSON.stringify(attribution)]);
|
||||
if ((inserted.rowCount ?? 0) > 0) result.inserted += 1;
|
||||
else result.unchanged += 1;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
128
apps/api/src/intake/intake-kinds.ts
Normal file
128
apps/api/src/intake/intake-kinds.ts
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
/**
|
||||
* Intake source kinds (migration 027 CHECK) and the per-kind maps the rest of
|
||||
* the module keys off.
|
||||
*
|
||||
* This file is the ONE place provider-named string literals live outside the
|
||||
* connector packages — the same standing as archive's ARCHIVE_PROVIDER
|
||||
* boundary: the intake adapters ARE the provider integration surface for
|
||||
* keyless public sources, so naming the platform they adapt is the boundary,
|
||||
* not provider branching. Everything else in apps/api/src/intake references
|
||||
* these constants; nothing else declares the literals.
|
||||
*/
|
||||
/* eslint-disable no-restricted-syntax -- intake kinds name the platform they adapt; this file is the provider boundary (archive ARCHIVE_PROVIDER pattern). */
|
||||
|
||||
export const INTAKE_KINDS = [
|
||||
"rss", "mastodon_tag", "mastodon_account", "bluesky_keyword",
|
||||
"telegram_channel", "peertube_channel", "youtube_channel", "web_watch",
|
||||
] as const;
|
||||
|
||||
export type IntakeKind = (typeof INTAKE_KINDS)[number];
|
||||
|
||||
export function isIntakeKind(value: unknown): value is IntakeKind {
|
||||
return typeof value === "string" && (INTAKE_KINDS as readonly string[]).includes(value);
|
||||
}
|
||||
|
||||
export const KIND_RSS: IntakeKind = "rss";
|
||||
export const KIND_MASTODON_TAG: IntakeKind = "mastodon_tag";
|
||||
export const KIND_MASTODON_ACCOUNT: IntakeKind = "mastodon_account";
|
||||
export const KIND_BLUESKY_KEYWORD: IntakeKind = "bluesky_keyword";
|
||||
export const KIND_TELEGRAM_CHANNEL: IntakeKind = "telegram_channel";
|
||||
export const KIND_PEERTUBE_CHANNEL: IntakeKind = "peertube_channel";
|
||||
export const KIND_YOUTUBE_CHANNEL: IntakeKind = "youtube_channel";
|
||||
export const KIND_WEB_WATCH: IntakeKind = "web_watch";
|
||||
|
||||
/** Kinds whose default posture is push-only (poll_interval_sec = 0, never due). */
|
||||
export const PUSH_ONLY_KINDS: readonly IntakeKind[] = [KIND_BLUESKY_KEYWORD, KIND_TELEGRAM_CHANNEL];
|
||||
|
||||
/**
|
||||
* The provider id handed to the P0c `normalizeFeedItem` funnel, so intake
|
||||
* items carry the same canonical `{provider}:{handle@host}` authorRef shape
|
||||
* every other feed item does.
|
||||
*/
|
||||
export const PROVIDER_BY_KIND: Record<IntakeKind, string> = {
|
||||
rss: "rss",
|
||||
mastodon_tag: "mastodon",
|
||||
mastodon_account: "mastodon",
|
||||
bluesky_keyword: "bluesky",
|
||||
telegram_channel: "telegram",
|
||||
peertube_channel: "peertube",
|
||||
youtube_channel: "youtube",
|
||||
web_watch: "web",
|
||||
};
|
||||
|
||||
/**
|
||||
* Default poll cadence per kind (seconds), from the firehose research source
|
||||
* inventory: RSS 5–15 min, YouTube channel RSS ≤10 min (15-entry window),
|
||||
* Mastodon 300/5min budget comfortably supports 60 s hashtags, PeerTube 10 min,
|
||||
* page-watch hourly. 0 = push-only (never due; the telegram webhook bridge and
|
||||
* the Jetstream seam park there).
|
||||
*/
|
||||
export const DEFAULT_POLL_INTERVAL_SEC: Record<IntakeKind, number> = {
|
||||
rss: 900,
|
||||
mastodon_tag: 300,
|
||||
mastodon_account: 300,
|
||||
bluesky_keyword: 0,
|
||||
telegram_channel: 0,
|
||||
peertube_channel: 600,
|
||||
youtube_channel: 600,
|
||||
web_watch: 3_600,
|
||||
};
|
||||
|
||||
/**
|
||||
* Egress host class per kind — the budget row class every outbound request of
|
||||
* that adapter draws from (packages/egress HostBudgets, shared host_budgets
|
||||
* rows deployment-wide). intake-social runs at capacity 3 / refill 10 per 10 s
|
||||
* = 1 req/s ceiling = exactly the verified Mastodon 300/5min-per-IP budget.
|
||||
*/
|
||||
export const HOST_CLASS_BY_KIND: Record<IntakeKind, string> = {
|
||||
rss: "intake-rss",
|
||||
mastodon_tag: "intake-social",
|
||||
mastodon_account: "intake-social",
|
||||
bluesky_keyword: "intake-jetstream",
|
||||
telegram_channel: "intake-telegram",
|
||||
peertube_channel: "intake-peertube",
|
||||
youtube_channel: "intake-rss",
|
||||
web_watch: "intake-web",
|
||||
};
|
||||
|
||||
/** Host-class budget shapes handed to createEgressClient. */
|
||||
export const INTAKE_HOST_CLASSES = {
|
||||
"intake-rss": { capacity: 3, refillPerInterval: 10 },
|
||||
"intake-social": { capacity: 3, refillPerInterval: 10 },
|
||||
"intake-peertube": { capacity: 3, refillPerInterval: 5 },
|
||||
"intake-web": { capacity: 2, refillPerInterval: 2 },
|
||||
"intake-telegram": { capacity: 3, refillPerInterval: 10 },
|
||||
"intake-jetstream": { capacity: 1, refillPerInterval: 1 },
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Media policy defaults per kind (research §2.5): images are cheap and safe to
|
||||
* harvest; full video files are opt-in per source because YouTube-class full
|
||||
* harvest is the storage wildcard.
|
||||
*/
|
||||
export type MediaPolicy = "metadata" | "images" | "full";
|
||||
|
||||
export const DEFAULT_MEDIA_POLICY: Record<IntakeKind, MediaPolicy> = {
|
||||
rss: "images",
|
||||
mastodon_tag: "images",
|
||||
mastodon_account: "images",
|
||||
bluesky_keyword: "metadata",
|
||||
telegram_channel: "metadata",
|
||||
peertube_channel: "images",
|
||||
youtube_channel: "metadata",
|
||||
web_watch: "images",
|
||||
};
|
||||
|
||||
/**
|
||||
* Public endpoints the adapters poll. Verified keyless/live in the firehose
|
||||
* research appendix (2026-10-09): YouTube channel Atom feed (no key, no
|
||||
* quota), the Telegram Bot API base.
|
||||
*/
|
||||
export const YOUTUBE_FEED_URL = (channelId: string): string =>
|
||||
`https://www.youtube.com/feeds/videos.xml?channel_id=${encodeURIComponent(channelId)}`;
|
||||
|
||||
export const TELEGRAM_API_ORIGIN = "https://api.telegram.org";
|
||||
|
||||
export const YOUTUBE_ORIGIN = "https://www.youtube.com";
|
||||
|
||||
/* eslint-enable no-restricted-syntax */
|
||||
169
apps/api/src/intake/intake-params.ts
Normal file
169
apps/api/src/intake/intake-params.ts
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
/**
|
||||
* Per-kind source params: validation, canonicalization, and the params_hash
|
||||
* identity half of UNIQUE (workspace_id, kind, params_hash) (migration 027).
|
||||
*
|
||||
* The DB only backstops "params is an object" — everything shape-related is
|
||||
* validated here so a bad subscription is a 400/422 at the API edge, never a
|
||||
* CHECK violation at insert time. Canonicalization is deterministic and total
|
||||
* over the validated shape (sorted keys, trimmed strings, normalized URLs), so
|
||||
* two subscriptions that differ only by `https://x/` vs `https://x` collide on
|
||||
* the unique key instead of double-polling one feed.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import { BadRequestException } from "@nestjs/common";
|
||||
import {
|
||||
isIntakeKind, KIND_MASTODON_ACCOUNT, KIND_MASTODON_TAG, KIND_PEERTUBE_CHANNEL,
|
||||
KIND_RSS, KIND_TELEGRAM_CHANNEL, KIND_WEB_WATCH, KIND_YOUTUBE_CHANNEL,
|
||||
TELEGRAM_API_ORIGIN, YOUTUBE_ORIGIN, type IntakeKind,
|
||||
} from "./intake-kinds.js";
|
||||
|
||||
export interface RssParams { url: string }
|
||||
export interface MastodonTagParams { instance: string; tag: string }
|
||||
export interface MastodonAccountParams { instance: string; accountId: string; handle?: string }
|
||||
export interface BlueskyKeywordParams { keywords: string[] }
|
||||
export interface TelegramChannelParams { channelId: string }
|
||||
export interface PeerTubeChannelParams { instance: string; handle: string }
|
||||
export interface YouTubeChannelParams { channelId: string }
|
||||
export interface WebWatchParams { url: string; selector?: string }
|
||||
|
||||
export type IntakeParams =
|
||||
| { kind: typeof KIND_RSS; params: RssParams }
|
||||
| { kind: typeof KIND_MASTODON_TAG; params: MastodonTagParams }
|
||||
| { kind: typeof KIND_MASTODON_ACCOUNT; params: MastodonAccountParams }
|
||||
| { kind: "bluesky_keyword"; params: BlueskyKeywordParams }
|
||||
| { kind: typeof KIND_TELEGRAM_CHANNEL; params: TelegramChannelParams }
|
||||
| { kind: typeof KIND_PEERTUBE_CHANNEL; params: PeerTubeChannelParams }
|
||||
| { kind: typeof KIND_YOUTUBE_CHANNEL; params: YouTubeChannelParams }
|
||||
| { kind: typeof KIND_WEB_WATCH; params: WebWatchParams };
|
||||
|
||||
const URL_MAX = 2_048;
|
||||
const STRING_MAX = 512;
|
||||
const TAG = /^[\p{L}\p{N}_]{1,64}$/u;
|
||||
const ACCOUNT_ID = /^[0-9]{1,19}$/;
|
||||
const TELEGRAM_CHANNEL_ID = /^-?\d{3,}$/;
|
||||
const YOUTUBE_CHANNEL = /^UC[\p{L}\p{N}_-]{20,24}$/u;
|
||||
|
||||
function isObject(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function requireString(value: unknown, field: string, max = STRING_MAX): string {
|
||||
if (typeof value !== "string") throw new BadRequestException(`${field} must be a string`);
|
||||
const trimmed = value.trim();
|
||||
if (trimmed === "") throw new BadRequestException(`${field} must not be empty`);
|
||||
if (trimmed.length > max) throw new BadRequestException(`${field} must be at most ${max} characters`);
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
/** Origin-only instance URL (`https://mastodon.social`) — never a path/query. */
|
||||
function requireInstance(value: unknown, field: string): string {
|
||||
const raw = requireString(value, field, 2_048);
|
||||
const withScheme = /^https?:\/\//i.test(raw) ? raw : `https://${raw}`;
|
||||
let url: URL;
|
||||
try { url = new URL(withScheme); } catch { throw new BadRequestException(`${field} must be a valid host or URL`); }
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") throw new BadRequestException(`${field} must be http(s)`);
|
||||
if (!url.hostname.includes(".")) throw new BadRequestException(`${field} must be a public host`);
|
||||
return `${url.protocol}//${url.hostname.toLowerCase()}${url.port ? `:${url.port}` : ""}`;
|
||||
}
|
||||
|
||||
function requireHttpUrl(value: unknown, field: string): string {
|
||||
const raw = requireString(value, field, URL_MAX);
|
||||
let url: URL;
|
||||
try { url = new URL(raw); } catch { throw new BadRequestException(`${field} must be an absolute URL`); }
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") throw new BadRequestException(`${field} must be an http(s) URL`);
|
||||
if (url.username !== "" || url.password !== "") throw new BadRequestException(`${field} must not carry credentials`);
|
||||
if (!url.hostname.includes(".")) throw new BadRequestException(`${field} must be a public host`);
|
||||
url.hash = "";
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
/** Parse + canonicalize the per-kind params object. Throws BadRequest on shape. */
|
||||
export function parseIntakeParams(kind: unknown, raw: unknown): IntakeParams {
|
||||
if (!isIntakeKind(kind)) throw new BadRequestException("kind must be a supported intake kind");
|
||||
if (!isObject(raw)) throw new BadRequestException("params must be an object");
|
||||
switch (kind) {
|
||||
case KIND_RSS:
|
||||
return { kind, params: { url: requireHttpUrl(raw.url, "params.url") } };
|
||||
case KIND_MASTODON_TAG: {
|
||||
const tag = requireString(raw.tag, "params.tag", 64);
|
||||
if (!TAG.test(tag)) throw new BadRequestException("params.tag must be 1-64 word characters");
|
||||
return { kind, params: { instance: requireInstance(raw.instance, "params.instance"), tag: tag.toLowerCase() } };
|
||||
}
|
||||
case KIND_MASTODON_ACCOUNT: {
|
||||
const accountId = requireString(raw.accountId, "params.accountId", 19);
|
||||
if (!ACCOUNT_ID.test(accountId)) throw new BadRequestException("params.accountId must be the numeric account id");
|
||||
const params: MastodonAccountParams = { instance: requireInstance(raw.instance, "params.instance"), accountId };
|
||||
const handle = raw.handle;
|
||||
if (handle !== undefined && handle !== null && handle !== "") {
|
||||
params.handle = requireString(handle, "params.handle", 64).replace(/^@+/, "").toLowerCase();
|
||||
}
|
||||
return { kind, params };
|
||||
}
|
||||
case "bluesky_keyword": {
|
||||
const keywords = raw.keywords;
|
||||
if (!Array.isArray(keywords) || keywords.length === 0 || keywords.length > 32) {
|
||||
throw new BadRequestException("params.keywords must be a non-empty array (max 32)");
|
||||
}
|
||||
return { kind, params: { keywords: keywords.map((k) => requireString(k, "params.keywords[]", 100).toLowerCase()) } };
|
||||
}
|
||||
case KIND_TELEGRAM_CHANNEL: {
|
||||
const channelId = requireString(raw.channelId, "params.channelId", 32);
|
||||
if (!TELEGRAM_CHANNEL_ID.test(channelId)) throw new BadRequestException("params.channelId must be a numeric Telegram chat id");
|
||||
return { kind, params: { channelId } };
|
||||
}
|
||||
case KIND_PEERTUBE_CHANNEL: {
|
||||
const handle = requireString(raw.handle, "params.handle", 253).replace(/^@+/, "");
|
||||
if (!/^[\p{L}\p{N}_.-]+(@[\p{L}\p{N}.-]+)?$/u.test(handle)) throw new BadRequestException("params.handle must be a channel handle");
|
||||
return { kind, params: { instance: requireInstance(raw.instance, "params.instance"), handle } };
|
||||
}
|
||||
case KIND_YOUTUBE_CHANNEL: {
|
||||
const channelId = requireString(raw.channelId, "params.channelId", 32);
|
||||
if (!YOUTUBE_CHANNEL.test(channelId)) throw new BadRequestException("params.channelId must be a UC… channel id");
|
||||
return { kind, params: { channelId } };
|
||||
}
|
||||
case KIND_WEB_WATCH: {
|
||||
const params: WebWatchParams = { url: requireHttpUrl(raw.url, "params.url") };
|
||||
if (raw.selector !== undefined && raw.selector !== null && raw.selector !== "") {
|
||||
params.selector = requireString(raw.selector, "params.selector", 512);
|
||||
}
|
||||
return { kind, params };
|
||||
}
|
||||
default: {
|
||||
throw new BadRequestException("kind must be a supported intake kind");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Deterministic key-sorted JSON — the canonical form hashing reads. */
|
||||
export function canonicalParams(input: IntakeParams): string {
|
||||
const sorted = (value: unknown): unknown => {
|
||||
if (Array.isArray(value)) return value.map(sorted);
|
||||
if (isObject(value)) {
|
||||
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, sorted(value[key])]));
|
||||
}
|
||||
return value;
|
||||
};
|
||||
return JSON.stringify(sorted({ kind: input.kind, params: input.params }));
|
||||
}
|
||||
|
||||
/** sha256 of the canonical serialization — migration 027's params_hash. */
|
||||
export function paramsHash(input: IntakeParams): string {
|
||||
return createHash("sha256").update(canonicalParams(input)).digest("hex");
|
||||
}
|
||||
|
||||
/** Parse params already stored on a row (trusted shape, still coerced). */
|
||||
export function storedParams(kind: IntakeKind, raw: unknown): IntakeParams["params"] {
|
||||
return parseIntakeParams(kind, isObject(raw) ? raw : {}).params;
|
||||
}
|
||||
|
||||
/** The outbound host a kind+params pair polls — the egress budget key base. */
|
||||
export function egressOrigin(input: IntakeParams): string {
|
||||
const params = input.params;
|
||||
if ("instance" in params && typeof params.instance === "string") return params.instance;
|
||||
if ("url" in params && typeof params.url === "string") return originOf(params.url);
|
||||
return input.kind === KIND_YOUTUBE_CHANNEL ? YOUTUBE_ORIGIN : TELEGRAM_API_ORIGIN;
|
||||
}
|
||||
|
||||
function originOf(url: string): string {
|
||||
try { return new URL(url).origin; } catch { return "https://invalid.example"; }
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue