Tactical Media Broadcast Hub — initial release
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
This commit is contained in:
commit
aa5e262f3a
880 changed files with 111042 additions and 0 deletions
358
apps/api/src/analytics/analytics.integration.test.ts
Normal file
358
apps/api/src/analytics/analytics.integration.test.ts
Normal file
|
|
@ -0,0 +1,358 @@
|
|||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { join } from "node:path";
|
||||
import { Pool, type PoolClient } from "pg";
|
||||
|
||||
/**
|
||||
* WP3 analytics snapshots against a live PostgreSQL, as sp_app (FORCE RLS
|
||||
* binds). Uses its OWN database (ANALYTICS_IT_DATABASE_URL, plus
|
||||
* ANALYTICS_IT_SUPER_DATABASE_URL for migrations/seeding) for the same reason
|
||||
* as groups.integration.test.ts: tenancy.integration.test.ts wipes every
|
||||
* workspace in the shared DB and vitest runs files in parallel.
|
||||
* Skipped cleanly when either env var is unset.
|
||||
*/
|
||||
const appUrl = process.env.ANALYTICS_IT_DATABASE_URL;
|
||||
const ownerUrl = process.env.ANALYTICS_IT_SUPER_DATABASE_URL;
|
||||
|
||||
type Modules = {
|
||||
service: typeof import("./snapshot.service.js");
|
||||
controller: typeof import("./analytics.controller.js");
|
||||
guard: typeof import("../rbac/rbac.guard.js");
|
||||
session: typeof import("../auth/session.js");
|
||||
db: typeof import("../db/client.js");
|
||||
};
|
||||
|
||||
describe.skipIf(appUrl === undefined || ownerUrl === undefined)("analytics snapshots (live DB)", () => {
|
||||
let m: Modules;
|
||||
let ownerPool: Pool;
|
||||
const suffix = `${Date.now()}${Math.floor(Math.random() * 1e6)}`;
|
||||
const currentMonth = new Date().toISOString().slice(0, 7) + "-01";
|
||||
const prevDate = new Date(`${currentMonth}T00:00:00Z`);
|
||||
prevDate.setUTCMonth(prevDate.getUTCMonth() - 1);
|
||||
const previousMonth = prevDate.toISOString().slice(0, 10);
|
||||
let wsA: string;
|
||||
let wsB: string;
|
||||
let user: string;
|
||||
|
||||
async function asOwner<T>(fn: (client: PoolClient) => Promise<T>): Promise<T> {
|
||||
const client = await ownerPool.connect();
|
||||
try {
|
||||
return await fn(client);
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
/** Seed tenant rows honestly: FORCE RLS applies to the owner too. */
|
||||
async function inTenant(client: PoolClient, wsId: string, fn: () => Promise<void>): Promise<void> {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT set_config('app.workspace_id', $1, true)", [wsId]);
|
||||
await fn();
|
||||
await client.query("COMMIT");
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
// The shared pool reads DATABASE_URL at import time: point it at this
|
||||
// isolated DB first, then import the modules under test.
|
||||
process.env.DATABASE_URL = appUrl;
|
||||
process.env.SUPER_DATABASE_URL = ownerUrl;
|
||||
const { runMigrations } = await import("../db/migrate.js");
|
||||
await runMigrations(join(process.cwd(), "migrations"));
|
||||
m = {
|
||||
service: await import("./snapshot.service.js"),
|
||||
controller: await import("./analytics.controller.js"),
|
||||
guard: await import("../rbac/rbac.guard.js"),
|
||||
session: await import("../auth/session.js"),
|
||||
db: await import("../db/client.js"),
|
||||
};
|
||||
ownerPool = new Pool({ connectionString: ownerUrl });
|
||||
|
||||
await asOwner(async (c) => {
|
||||
const ws = async (name: string): Promise<string> =>
|
||||
(await c.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id", [name, `${name}-${suffix}`])).rows[0].id as string;
|
||||
wsA = await ws("analytics-a");
|
||||
wsB = await ws("analytics-b");
|
||||
user = (await c.query(
|
||||
"INSERT INTO users (external_id, username, email, display_name) VALUES ($1, $2, $3, $4) RETURNING id",
|
||||
[`analytics:owner:${suffix}`, `analytics-owner-${suffix}`, `owner-${suffix}@example.com`, "owner"],
|
||||
)).rows[0].id as string;
|
||||
|
||||
const seed = async (wsId: string, archivedCount: number): Promise<void> => {
|
||||
await inTenant(c, wsId, async () => {
|
||||
await c.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, 'Owner')", [wsId, user]);
|
||||
const tag = wsId.slice(0, 8);
|
||||
// 3 connections (mastodon, bluesky, mastodon) + one draft each.
|
||||
const drafts: string[] = [];
|
||||
for (const [i, provider] of ["mastodon", "bluesky", "mastodon"].entries()) {
|
||||
const conn = (await c.query(
|
||||
"INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, $2, $3) RETURNING id",
|
||||
[wsId, provider, `ch-${tag}-${i}`],
|
||||
)).rows[0].id as string;
|
||||
const draft = (await c.query(
|
||||
"INSERT INTO drafts (workspace_id, author_user_id, post) VALUES ($1, $2, $3::jsonb) RETURNING id",
|
||||
[wsId, user, JSON.stringify({ body: "x" })],
|
||||
)).rows[0].id as string;
|
||||
drafts.push(draft);
|
||||
await c.query(
|
||||
`INSERT INTO publications (workspace_id, draft_id, connection_id, idempotency_key, remote_id, published_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6::date + $7::int)`,
|
||||
[wsId, draft, conn, `pub-${tag}-${i}`, `r-${tag}-${i}`, currentMonth, i],
|
||||
);
|
||||
}
|
||||
// One scheduled publication (draft_id is UNIQUE there) carrying
|
||||
// three attempts on the same draft: attempt_no 1..3.
|
||||
const schedule = (await c.query(
|
||||
`INSERT INTO scheduled_publications (workspace_id, draft_id, connection_id, publish_at, idempotency_key)
|
||||
VALUES ($1, $2, (SELECT id FROM connections WHERE workspace_id = $1 LIMIT 1), $3::date, $4) RETURNING id`,
|
||||
[wsId, drafts[0]!, currentMonth, `sch-${tag}`],
|
||||
)).rows[0].id as string;
|
||||
const credential = (await c.query(
|
||||
"INSERT INTO credentials (workspace_id, kind, ciphertext) VALUES ($1, 'test', gen_random_bytes(16)) RETURNING id",
|
||||
[wsId],
|
||||
)).rows[0].id as string;
|
||||
for (const n of [1, 2, 3]) {
|
||||
await c.query(
|
||||
`INSERT INTO publishing_attempts
|
||||
(workspace_id, schedule_id, draft_id, connection_id, attempt_no, idempotency_key, schedule_generation,
|
||||
post_snapshot, post_hash, scheduled_by_user_id, required_permission, approved_revision,
|
||||
reviewer_user_id, credential_id, credential_generation, authorized_epoch, status)
|
||||
VALUES ($1, $2, $3, (SELECT id FROM connections WHERE workspace_id = $1 LIMIT 1), $4, $5, 1,
|
||||
'{}'::jsonb, $6, $7, 'publication.publish', 1, $7, $8, 1, 0, $9)`,
|
||||
[wsId, schedule, drafts[0]!, n, `att-${tag}-${n}`, `hash-${tag}-${n}`, user, credential, n === 3 ? "completed" : "failed"],
|
||||
);
|
||||
}
|
||||
// Archived feed items: `archivedCount` in the current month, one in
|
||||
// the previous month, one in the far past (outside both windows).
|
||||
// remote_ref must equal channel_id || ':' || message_id.
|
||||
// remote_ref must match (@handle|numeric):message_id AND equal
|
||||
// channel_id || ':' || message_id; content_hash is sha256 hex (64).
|
||||
for (let i = 0; i < archivedCount; i += 1) {
|
||||
await c.query(
|
||||
`INSERT INTO archived_feed_items (workspace_id, remote_ref, channel_id, message_id, content_hash, posted_at)
|
||||
VALUES ($1, '@ch' || $2::text || ':' || $3::bigint::text, '@ch' || $2::text, $3::bigint,
|
||||
encode(sha256($4::bytea), 'hex'), $5::date + ($6::int * interval '1 hour'))`,
|
||||
[wsId, wsId.replace(/-/g, "").slice(0, 9), 1000 + i, `${tag}-${i}`, currentMonth, i],
|
||||
);
|
||||
}
|
||||
await c.query(
|
||||
`INSERT INTO archived_feed_items (workspace_id, remote_ref, channel_id, message_id, content_hash, posted_at)
|
||||
VALUES ($1, '@ch' || $2::text || ':9001', '@ch' || $2::text, 9001,
|
||||
encode(sha256($3::bytea), 'hex'), $4::date)`,
|
||||
[wsId, wsId.replace(/-/g, "").slice(0, 9), `${tag}-prev`, previousMonth],
|
||||
);
|
||||
await c.query(
|
||||
`INSERT INTO archived_feed_items (workspace_id, remote_ref, channel_id, message_id, content_hash, posted_at)
|
||||
VALUES ($1, '@ch' || $2::text || ':9002', '@ch' || $2::text, 9002,
|
||||
encode(sha256($3::bytea), 'hex'), '2020-01-01')`,
|
||||
[wsId, wsId.replace(/-/g, "").slice(0, 9), `${tag}-old`],
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
await seed(wsA, 5);
|
||||
await seed(wsB, 10);
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (ownerPool !== undefined) {
|
||||
await asOwner(async (c) => {
|
||||
await c.query("DELETE FROM workspaces WHERE id = ANY($1::uuid[])", [[wsA, wsB]]);
|
||||
await c.query("DELETE FROM users WHERE external_id LIKE $1", [`analytics:%:${suffix}`]);
|
||||
});
|
||||
await ownerPool.end();
|
||||
}
|
||||
await m?.db.pool.end();
|
||||
});
|
||||
|
||||
it("computes per-workspace monthly metrics from fixtures (set-based, no row loops)", async () => {
|
||||
const service = new m.service.SnapshotService();
|
||||
const a = await service.refresh(wsA, currentMonth);
|
||||
expect(a.month).toBe(currentMonth.slice(0, 7));
|
||||
expect(a.publicationsTotal).toBe(3);
|
||||
expect(a.publicationsByProvider).toEqual({ mastodon: 2, bluesky: 1 });
|
||||
expect(a.postsArchivedTotal).toBe(5);
|
||||
expect(a.activeConnections).toBe(3);
|
||||
expect(a.draftsCreatedTotal).toBe(3);
|
||||
expect(a.publishingAttemptsTotal).toBe(3);
|
||||
expect(a.publishFailuresTotal).toBe(2);
|
||||
|
||||
const b = await service.refresh(wsB, currentMonth);
|
||||
expect(b.postsArchivedTotal).toBe(10);
|
||||
expect(b.activeConnections).toBe(3);
|
||||
// Disconnect wsB's connections as sp_app so RLS scopes the update to wsB
|
||||
// (the owner pool is a superuser and bypasses RLS entirely).
|
||||
await m.db.withTenant(wsB, (client) => client.query("UPDATE connections SET disconnected_at = now()"));
|
||||
const bAfter = await service.refresh(wsB, currentMonth);
|
||||
expect(bAfter.activeConnections).toBe(0);
|
||||
});
|
||||
|
||||
it("is idempotent: running twice leaves exactly one row, same id and metrics", async () => {
|
||||
const service = new m.service.SnapshotService();
|
||||
const first = await service.refresh(wsA, currentMonth);
|
||||
const second = await service.refresh(wsA, currentMonth);
|
||||
expect(second.id).toBe(first.id);
|
||||
// Metrics are identical; only the snapshot timestamps move.
|
||||
const stripTimes = (r: typeof first): Record<string, unknown> => {
|
||||
const { snapshotStartedAt, snapshotCompletedAt, ...metrics } = r;
|
||||
void snapshotStartedAt; void snapshotCompletedAt;
|
||||
return metrics;
|
||||
};
|
||||
expect(stripTimes(second)).toEqual(stripTimes(first));
|
||||
expect(second.snapshotCompletedAt >= first.snapshotCompletedAt).toBe(true);
|
||||
// Count through the app pool so FORCE RLS scopes the check to wsA (the
|
||||
// owner pool is a superuser and would see every workspace's rows).
|
||||
const rows = await m.db
|
||||
.withTenant(wsA, (client) => client.query("SELECT id, publications_total FROM analytics_snapshots WHERE month = $1::date", [currentMonth]))
|
||||
.then((r) => r.rows);
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]).toMatchObject({ id: first.id, publications_total: "3" });
|
||||
});
|
||||
|
||||
it("partition pruning: a single-month predicate plans a scan of exactly one leaf", async () => {
|
||||
// Plan-time pruning needs a literal (a $n param defers pruning to run
|
||||
// time); the value is a validated YYYY-MM-01 so inlining is safe.
|
||||
if (!/^\d{4}-\d{2}-01$/.test(currentMonth)) throw new Error("bad month literal");
|
||||
const plan = await asOwner(async (c) => {
|
||||
await c.query("BEGIN");
|
||||
await c.query("SELECT set_config('app.workspace_id', $1, true)", [wsA]);
|
||||
const explained = await c.query<{ "QUERY PLAN": string }>(
|
||||
`EXPLAIN (COSTS OFF) SELECT count(*) FROM analytics_snapshots WHERE month = DATE '${currentMonth}'`,
|
||||
);
|
||||
await c.query("COMMIT");
|
||||
return explained.rows.map((r) => r["QUERY PLAN"]).join("\n");
|
||||
});
|
||||
const scanned = [...plan.matchAll(/Scan on (analytics_snapshots\S*)/g)].map((match) => match[1]!);
|
||||
expect(scanned.filter((r) => /^analytics_snapshots_[0-9]{6}$/.test(r)).length).toBe(1);
|
||||
for (const relation of scanned) expect(relation).not.toContain("default");
|
||||
});
|
||||
|
||||
it("cross-tenant denial: RLS hides other workspaces' snapshots (parent and every leaf)", async () => {
|
||||
const service = new m.service.SnapshotService();
|
||||
await service.refresh(wsB, currentMonth);
|
||||
const aList = await service.list(wsA, { limit: 24 });
|
||||
expect(aList.items.length).toBeGreaterThan(0);
|
||||
expect(aList.items.every((item) => item.workspaceId === wsA)).toBe(true);
|
||||
|
||||
// A raw read as the runtime role under B's context sees only B's rows.
|
||||
const leaked = await m.db.withTenant(wsB, async (client) =>
|
||||
client.query("SELECT workspace_id FROM analytics_snapshots"));
|
||||
expect(leaked.rows.length).toBeGreaterThan(0);
|
||||
expect(new Set(leaked.rows.map((r) => r.workspace_id))).toEqual(new Set([wsB]));
|
||||
|
||||
// Direct leaf access (parent RLS does not cascade to leaves): zero of
|
||||
// A's rows visible from B's context in every partition.
|
||||
const partitions = await asOwner((c) =>
|
||||
c.query<{ relname: string }>(
|
||||
"SELECT c.relname FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'analytics_snapshots'::regclass",
|
||||
));
|
||||
expect(partitions.rows.length).toBeGreaterThan(0);
|
||||
for (const row of partitions.rows) {
|
||||
if (!/^analytics_snapshots_(default|[0-9]{6})$/.test(row.relname)) throw new Error(`unexpected partition ${row.relname}`);
|
||||
const seen = await m.db.withTenant(wsB, (client) => client.query(`SELECT workspace_id FROM ${row.relname}`));
|
||||
expect(seen.rows.every((r) => r.workspace_id === wsB)).toBe(true);
|
||||
}
|
||||
|
||||
// And a write attempt for another workspace is refused by WITH CHECK.
|
||||
await expect(
|
||||
m.db.withTenant(wsA, async (client) =>
|
||||
client.query(
|
||||
`INSERT INTO analytics_snapshots (workspace_id, month) VALUES ($1::uuid, DATE '${currentMonth}')`,
|
||||
[wsB],
|
||||
)),
|
||||
).rejects.toThrow(/row-level security/i);
|
||||
});
|
||||
|
||||
it("keyset pagination: newest first, cursor continues without overlap", async () => {
|
||||
const service = new m.service.SnapshotService();
|
||||
await service.refresh(wsA, previousMonth);
|
||||
const page1 = await service.list(wsA, { limit: 1 });
|
||||
expect(page1.items.map((i) => i.month)).toEqual([currentMonth.slice(0, 7)]);
|
||||
expect(page1.nextCursor).not.toBeNull();
|
||||
const page2 = await service.list(wsA, { cursor: page1.nextCursor!, limit: 1 });
|
||||
expect(page2.items.map((i) => i.month)).toEqual([previousMonth.slice(0, 7)]);
|
||||
expect(page2.nextCursor).toBeNull();
|
||||
await expect(service.list(wsA, { cursor: "!!!not-a-cursor" })).rejects.toMatchObject({ status: 400 });
|
||||
});
|
||||
|
||||
it("summary: current month + previous month with correct deltas", async () => {
|
||||
const service = new m.service.SnapshotService();
|
||||
await service.refresh(wsB, previousMonth);
|
||||
const summary = await service.summary(wsB);
|
||||
expect(summary.currentMonth).toBe(currentMonth.slice(0, 7));
|
||||
expect(summary.previousMonth).toBe(previousMonth.slice(0, 7));
|
||||
expect(summary.previous).not.toBeNull();
|
||||
// previous month: only the one archived item; connections already
|
||||
// disconnected when that snapshot was taken.
|
||||
expect(summary.previous!.postsArchivedTotal).toBe(1);
|
||||
expect(summary.previous!.publicationsTotal).toBe(0);
|
||||
expect(summary.current!.postsArchivedTotal).toBe(10);
|
||||
expect(summary.deltas).toMatchObject({
|
||||
publicationsTotal: 3,
|
||||
postsArchivedTotal: 9,
|
||||
activeConnections: 0,
|
||||
publishFailuresTotal: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("invalid month input is rejected before any query", async () => {
|
||||
await expect(m.service.writeMonthlySnapshot({} as never, "2025-13")).rejects.toMatchObject({ status: 400 });
|
||||
await expect(new m.service.SnapshotService().refresh(wsA, "2025-1")).rejects.toMatchObject({ status: 400 });
|
||||
});
|
||||
|
||||
it("controller + RBAC guard chain: analytics.view required, wrong-workspace token is refused", async () => {
|
||||
const controller = new m.controller.AnalyticsController(new m.service.SnapshotService());
|
||||
const user = (await asOwner((c) => c.query<{ id: string }>("SELECT id FROM users WHERE external_id = $1", [`analytics:owner:${suffix}`]))).rows[0]!.id;
|
||||
|
||||
// Permission gate: a role without analytics.view is forbidden by the
|
||||
// guard. The guard resolves the role from the MEMBERSHIP, not the token,
|
||||
// so an Editor member of wsA is the honest deny-path fixture.
|
||||
const editor = (await asOwner(async (c) => {
|
||||
const id = (await c.query(
|
||||
"INSERT INTO users (external_id, username, email, display_name) VALUES ($1, $2, $3, $4) RETURNING id",
|
||||
[`analytics:editor:${suffix}`, `analytics-editor-${suffix}`, `editor-${suffix}@example.com`, "editor"],
|
||||
)).rows[0].id as string;
|
||||
await c.query("BEGIN");
|
||||
await c.query("SELECT set_config('app.workspace_id', $1, true)", [wsA]);
|
||||
await c.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, 'Editor')", [wsA, id]);
|
||||
await c.query("COMMIT");
|
||||
return id;
|
||||
}));
|
||||
// The membership insert above bumps authz_epoch (continuous authz), so
|
||||
// tokens are minted AFTER it — mirroring a fresh login per epoch.
|
||||
const epochNow = await asOwner(async (c) =>
|
||||
Number((await c.query("SELECT authz_epoch FROM workspaces WHERE id = $1", [wsA])).rows[0]?.authz_epoch));
|
||||
const token = (userId: string, workspaceId: string, role: string): string =>
|
||||
m.session.issueSession({ userId, workspaceId, role, authzEpoch: epochNow });
|
||||
const request = (t: string): { headers: Record<string, string>; session?: unknown } => ({
|
||||
headers: { authorization: `Bearer ${t}` },
|
||||
// In production RbacGuard.canActivate attaches this before the
|
||||
// controller runs; do the same here after verifying the token.
|
||||
session: m.session.verifySession(t) ?? undefined,
|
||||
});
|
||||
|
||||
// Owner of wsA refreshes wsA's snapshot through the real controller.
|
||||
const refreshed = await controller.snapshot(request(token(user, wsA, "Owner")) as never, {});
|
||||
expect(refreshed.workspaceId).toBe(wsA);
|
||||
expect(refreshed.publicationsTotal).toBe(3);
|
||||
|
||||
// A valid token for wsB does not leak wsA: wsB's refresh writes wsB's row.
|
||||
const forB = await controller.snapshot(request(token(user, wsB, "Owner")) as never, {});
|
||||
expect(forB.workspaceId).toBe(wsB);
|
||||
|
||||
const { Reflector } = await import("@nestjs/core");
|
||||
const guard = new m.guard.RbacGuard(new Reflector());
|
||||
const handler = (): void => undefined;
|
||||
Reflect.defineMetadata(m.guard.PERMISSION_KEY, "analytics.view", handler);
|
||||
const ctx = {
|
||||
getHandler: () => handler,
|
||||
switchToHttp: () => ({ getRequest: () => ({ headers: { authorization: `Bearer ${token(editor, wsA, "Editor")}` }, params: {} }) }),
|
||||
} as unknown as import("@nestjs/common").ExecutionContext;
|
||||
await expect(guard.canActivate(ctx)).rejects.toMatchObject({ status: 403 });
|
||||
// ...while the Owner's token passes the same gate.
|
||||
const ownerCtx = {
|
||||
getHandler: () => handler,
|
||||
switchToHttp: () => ({ getRequest: () => ({ headers: { authorization: `Bearer ${token(user, wsA, "Owner")}` }, params: {} }) }),
|
||||
} as unknown as import("@nestjs/common").ExecutionContext;
|
||||
await expect(guard.canActivate(ownerCtx)).resolves.toBe(true);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue