T-HUB/apps/api/src/feeds/columns.integration.test.ts
glass beadian aa5e262f3a Tactical Media Broadcast Hub — initial release
Self-hosted multi-platform broadcast newsroom for activist collectives:
publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style
column wall with hostile-content-safe rendering, live control room with Auto
Director, bridged chat governance with polls and quorum, distributed sandboxed
transcode workers, encrypted content-addressed archive, monthly analytics.

MIT licensed. See README.md and docs/.
2026-10-10 11:56:42 +00:00

376 lines
22 KiB
TypeScript

/**
* Columns P1 integration tests — real Postgres (migrations ≤ 026 applied),
* RLS-faithful via the app pool, following the video.integration.test pattern.
*
* Covers the DELIVER test list:
* - CRUD + reorder, including RBAC 403 for an ungranted role
* - seed idempotency (re-touch creates no duplicates)
* - cooldown enforcement on the column endpoints
* - cursor survival across a cache flush
* - keyset items pagination (stable order, no skips/dupes across pages)
* - dedup: same remote item from two columns sharing an identity → one row,
* visible in both
*/
import "reflect-metadata";
import { randomUUID } from "node:crypto";
import { ForbiddenException, type CallHandler, type ExecutionContext, type INestApplication } from "@nestjs/common";
import { NestFactory, Reflector } from "@nestjs/core";
import { Pool } from "pg";
import { afterAll, beforeAll, afterEach, describe, expect, it, vi } from "vitest";
import { InMemoryCredentials, type ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk";
import type { ConnectorConnection, FeedItem } from "@tactical-media-broadcast-hub/core";
import type { SessionPayload } from "../auth/session.js";
import { PERMISSION_KEY, RbacGuard } from "../rbac/rbac.guard.js";
import { injectHttp } from "../connections/http-test-transport.js";
import { feedRegistry } from "./feed-registry.js";
import { FeedsService } from "./feeds.service.js";
import { FeedsController } from "./feeds.controller.js";
import { ColumnsController, COLUMNS_REGISTRY } from "./columns.controller.js";
import { ColumnStore } from "./column-store.js";
import { ColumnScheduler, type ColumnSchedulerDependencies } from "./column-scheduler.js";
import { MemoryFeedRedis } from "./testing.js";
import type { ConnectionSummary } from "../connections/connections.service.js";
const dbAvailable = process.env.DATABASE_URL !== undefined;
// Setup runs as the table owner (superuser in dev) because memberships has
// FORCE RLS and the seed path needs to insert without a tenant context.
const ownerPool = new Pool({ connectionString: process.env.SUPER_DATABASE_URL ?? process.env.DATABASE_URL });
// The app pool is imported lazily where needed; ColumnStore's own fallback
// deps use withTenant from db/client, but these tests always inject explicit
// deps, so no module-scope side-effectful import is required.
const owner: SessionPayload = { userId: randomUUID(), workspaceId: randomUUID(), role: "Owner", authzEpoch: 1, issuedAt: 0, expiresAt: 9_999_999_999 };
const viewer: SessionPayload = { ...owner, userId: randomUUID(), role: "Moderator" }; // no streams.columns.manage
const item = (remoteId: string, minutesAgo: number): FeedItem => ({
id: remoteId, remoteId, provider: "mock-social", authorHandle: "alice", text: `post ${remoteId}`,
mediaUrls: [], createdAt: new Date(Date.now() - minutesAgo * 60_000), metrics: {},
});
function setupRegistry(items: FeedItem[]): ConnectorRegistry {
const registry = feedRegistry(new InMemoryCredentials());
const mock = registry.get("mock-social");
const original = mock.getFeed;
if (original === undefined) throw new Error("mock-social has no feed");
mock.getFeed = vi.fn(async () => ({ items }));
return registry;
}
describe.skipIf(!dbAvailable)("columns P1 (live DB)", () => {
let app: INestApplication;
let store: ColumnStore;
let scheduler: ColumnScheduler;
let feeds: FeedsService;
let redis: MemoryFeedRedis;
let connections: Array<{ summary: ConnectionSummary; connection: ConnectorConnection }>;
let items: FeedItem[];
let granted = true;
let registry: ConnectorRegistry;
beforeAll(async () => {
// Users first (ids are DB-generated and captured for memberships), then
// workspace + members, in the owner pool which bypasses RLS for setup.
const inserted = await ownerPool.query<{ id: string }>(
`INSERT INTO users (external_id, email, display_name, username)
VALUES ($1,$2,'Columns P1 Owner',$3) RETURNING id`, [`u-${randomUUID()}`, `o-${randomUUID()}@example.com`, `p1o${randomUUID().slice(0, 8)}`]);
(owner as { userId: string }).userId = String(inserted.rows[0]?.id);
const viewerRow = await ownerPool.query<{ id: string }>(
`INSERT INTO users (external_id, email, display_name, username)
VALUES ($1,$2,'Columns P1 Viewer',$3) RETURNING id`, [`u-${randomUUID()}`, `v-${randomUUID()}@example.com`, `p1v${randomUUID().slice(0, 8)}`]);
(viewer as { userId: string }).userId = String(viewerRow.rows[0]?.id);
const workspace = await ownerPool.query(
"INSERT INTO workspaces (name, slug) VALUES ('columns p1', $1) RETURNING id", [`cols-${randomUUID()}`]);
const workspaceId = String(workspace.rows[0]?.id);
// memberships is FORCE RLS: insert inside the workspace's tenant context.
const setupClient = await ownerPool.connect();
try {
await setupClient.query("BEGIN");
await setupClient.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]);
await setupClient.query(
"INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1,$2,'Owner'), ($1,$3,'Moderator')",
[workspaceId, owner.userId, viewer.userId]);
await setupClient.query("COMMIT");
} finally {
setupClient.release();
}
(owner as { workspaceId: string }).workspaceId = workspaceId;
(viewer as { workspaceId: string }).workspaceId = workspaceId;
items = [item("i-1", 1), item("i-2", 5), item("i-3", 10), item("i-4", 30), item("i-5", 60)];
registry = setupRegistry(items);
// Two feed-capable connections in the workspace: the seeding input.
connections = [];
for (let index = 0; index < 2; index += 1) {
// The DB generates the id (connections has no id column we may set);
// RETURNING it keeps the FK target and the summary in sync.
const inserted = await ownerPool.query<{ id: string; created_at: Date }>(
"INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1,'mock-social',$2) RETURNING id, created_at", [owner.workspaceId, `ch-${index}`]);
const id = String(inserted.rows[0]?.id);
const createdAt = inserted.rows[0]?.created_at as Date;
const summary: ConnectionSummary = {
id, provider: "mock-social", channelId: `ch-${index}`, fields: {}, health: "healthy",
credentialRotatedAt: null, recentPublishes: [], createdAt: createdAt.toISOString(),
};
const connection: ConnectorConnection = {
id, workspaceId: owner.workspaceId, provider: "mock-social", channelId: `ch-${index}`,
credentialId: { kind: "credential-ref", id }, fields: {}, createdAt,
};
connections.push({ summary, connection });
}
redis = new MemoryFeedRedis();
store = new ColumnStore({
providerOf: async (_ws, id) => connections.find((row) => row.summary.id === id)?.summary.provider ?? "mock-social",
connections: async (ws) => ws === owner.workspaceId ? connections.map((row) => row.summary) : [],
});
feeds = new FeedsService({
redis, registry, random: () => 0.5, requestsPerSecond: () => 0.5,
workspaces: async () => [owner.workspaceId],
connections: async (ws) => ws === owner.workspaceId ? connections.map((row) => row.summary) : [],
connection: async (ws, id) => { const row = connections.find((c) => c.summary.id === id); if (!row || ws !== owner.workspaceId) throw new Error(); return row.connection; },
visible: async (s) => s.workspaceId === owner.workspaceId ? connections.map((row) => row.summary.id) : [],
audit: async () => undefined,
}, store);
scheduler = new ColumnScheduler(store, {
redis, registry,
connection: async (ws, id) => { const row = connections.find((c) => row_summary(c) === id); if (!row || ws !== owner.workspaceId) throw new Error(); return row.connection; },
cacheHomeFeed: async (ws, id, normalized) => { if (ws === owner.workspaceId) await feeds.cacheHomeFeed(ws, id, normalized); },
hostKeyOf: () => "mock.social:443",
consumeBudget: async () => undefined,
random: () => 0.5,
now: () => Date.now(),
});
// Both poll loops stay off: the feeds discovery timer would contend with
// ensureSeeded's workspace row lock (this suite drives polls explicitly).
vi.spyOn(scheduler, "onApplicationBootstrap").mockImplementation(() => undefined);
vi.spyOn(feeds, "onApplicationBootstrap").mockImplementation(() => undefined);
vi.spyOn(RbacGuard.prototype, "canActivate").mockImplementation(async (context: ExecutionContext) => {
const request = context.switchToHttp().getRequest<{ headers: Record<string, string>; session?: SessionPayload }>();
if (request.headers.authorization !== "Bearer test-session") throw new ForbiddenException("unauthenticated");
const session = request.headers["x-test-user"] === "viewer" ? viewer : owner;
const permission = new Reflector().get<string>(PERMISSION_KEY, context.getHandler());
// The seeded RBAC: streams.columns.manage is Owner/Admin/Editor only.
if (permission === "streams.columns.manage" && !granted) throw new ForbiddenException("missing permission");
request.session = session;
return true;
});
app = await NestFactory.create({
module: class TestModule {},
controllers: [ColumnsController, FeedsController],
providers: [
{ provide: FeedsService, useValue: feeds },
{ provide: ColumnStore, useValue: store },
{ provide: ColumnScheduler, useValue: scheduler },
{ provide: COLUMNS_REGISTRY, useValue: registry },
],
}, { logger: false });
await app.init();
});
afterEach(() => { granted = true; });
afterAll(async () => {
await app?.close();
if (owner.workspaceId !== undefined) await ownerPool.query("DELETE FROM workspaces WHERE id = $1", [owner.workspaceId]);
await ownerPool.end();
});
const request = (path: string, method = "GET", body?: unknown, user: "owner" | "viewer" = "owner") =>
injectHttp(app.getHttpAdapter().getInstance(), `http://api.test${path}`, {
method, headers: { authorization: "Bearer test-session", "x-test-user": user, "content-type": "application/json" },
body: body === undefined ? undefined : JSON.stringify(body),
});
it("seeds one platform_feed column per feed connection, idempotently, with sources", async () => {
const first = await request("/feeds/columns");
expect(first.status).toBe(200);
const seeded = await first.json() as Array<{ id: string; type: string; title: string; sources: Array<{ sourceKind: string; state: string }> }>;
expect(seeded).toHaveLength(2);
expect(seeded.every((column) => column.type === "platform_feed")).toBe(true);
expect(seeded[0]?.sources).toHaveLength(1);
expect(seeded[0]?.sources[0]?.sourceKind).toBe("home");
// Titles reproduce the implicit column the UI renders today.
expect(seeded[0]?.title).toContain("·");
// Second touch seeds nothing.
const second = await request("/feeds/columns");
expect(await second.json()).toHaveLength(2);
// And the DB agrees: two columns, two source rows, one per connection.
const rows = await ownerPool.query("SELECT count(*)::int AS c FROM stream_columns WHERE workspace_id = $1", [owner.workspaceId]);
expect(rows.rows[0]?.c).toBe(2);
const sources = await ownerPool.query("SELECT count(*)::int AS c FROM column_sources WHERE workspace_id = $1", [owner.workspaceId]);
expect(sources.rows[0]?.c).toBe(2);
});
it("enforces streams.columns.manage on writes while reads stay on draft.view", async () => {
granted = false;
expect((await request("/feeds/columns", "POST", { type: "keyword", title: "x", spec: { query: "q", platforms: ["mock-social"] } }, "viewer")).status).toBe(403);
granted = true;
const created = await request("/feeds/columns", "POST", { type: "keyword", title: "news", spec: { query: "anarch", platforms: ["mock-social"] } });
expect(created.status).toBe(201);
const column = await created.json() as { id: string; sources: Array<{ sourceKind: string }> };
expect(column.sources.map((source) => source.sourceKind)).toEqual(["search"]);
// Reads: viewer (no manage permission) still reads columns and items.
expect((await request(`/feeds/columns/${column.id}`, "GET", undefined, "viewer")).status).toBe(200);
granted = false;
expect((await request(`/feeds/columns/${column.id}`, "PATCH", { title: "z" })).status).toBe(403);
expect((await request(`/feeds/columns/${column.id}`, "DELETE")).status).toBe(403);
expect((await request("/feeds/columns/reorder", "POST", { ids: [] })).status).toBe(403);
granted = true;
await request(`/feeds/columns/${column.id}`, "DELETE");
});
it("rejects invalid specs with 422 and never stores them", async () => {
const bad = await request("/feeds/columns", "POST", { type: "keyword", title: "bad", spec: { query: "", platforms: ["mock-social"] } });
expect(bad.status).toBe(422);
const badConnection = await request("/feeds/columns", "POST", { type: "platform_feed", title: "bad", spec: { connectionId: randomUUID() } });
expect(badConnection.status).toBe(422);
const count = await ownerPool.query(
"SELECT count(*)::int AS c FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL", [owner.workspaceId]);
expect(count.rows[0]?.c).toBe(2); // only the two seeded columns (the RBAC test's column was soft-deleted)
});
it("polls due sources, persists items, and fans out to columns sharing an identity", async () => {
const columns = await (await request("/feeds/columns")).json() as Array<{ id: string; sources: Array<{ id: string }> }>;
const first = columns[0]!;
const outcome = await scheduler.pollWorkspaceDue(owner.workspaceId);
expect(outcome.polled).toBeGreaterThanOrEqual(2); // both seeded home sources
// Items landed in the store for the polled column.
const page = await store.columnItems(owner.workspaceId, first.id, { limit: 10 });
expect(page.items.length).toBeGreaterThanOrEqual(1);
expect(page.items.every((stored) => stored.item.remoteId.startsWith("i-"))).toBe(true);
// The source row moved to ready with a next poll scheduled.
const sources = await store.sources(owner.workspaceId, first.id);
expect(sources[0]?.state).toBe("ready");
expect(sources[0]?.nextPollAt).not.toBeNull();
});
it("keyset pagination: pages in stable order, no skips or duplicates, cursor survives cache flush", async () => {
const columns = await (await request("/feeds/columns")).json() as Array<{ id: string }>;
const columnId = columns[0]!.id;
const seen: string[] = [];
let cursor: string | null = null;
// limit 2 over 5 items = 3 pages (2,2,1)
for (let page = 0; page < 3; page += 1) {
const response = await request(`/feeds/columns/${columnId}/items?limit=2${cursor === null ? "" : `&cursor=${cursor}`}`);
expect(response.status).toBe(200);
const body = await response.json() as { items: Array<{ item: { remoteId: string } }>; nextCursor: string | null };
for (const stored of body.items) seen.push(stored.item.remoteId);
cursor = body.nextCursor;
if (cursor === null) break;
}
expect(seen.length).toBe(new Set(seen).size); // no duplicates
expect(seen).toHaveLength(items.length); // no skips
// Cursor survival: flush every cache key, then continue from a cursor
// minted before the flush — the store page must pick up exactly where the
// cache left off because the cursor is a position in the stored order.
redis.entries.clear();
const beforeFlush = await request(`/feeds/columns/${columnId}/items?limit=1`);
const firstCursor = ((await beforeFlush.json()) as { nextCursor: string | null }).nextCursor;
expect(firstCursor).not.toBeNull();
redis.entries.clear();
const afterFlush = await request(`/feeds/columns/${columnId}/items?limit=10&cursor=${firstCursor}`);
expect(afterFlush.status).toBe(200);
const pageItems = (await afterFlush.json()) as { items: Array<{ item: { remoteId: string } }> };
// Everything strictly after the first item, in keyset order.
expect(pageItems.items.map((stored) => stored.item.remoteId)).toEqual(seen.slice(1));
});
it("cooldown blocks a forced refresh from issuing a second poll", async () => {
const columns = await (await request("/feeds/columns")).json() as Array<{ id: string }>;
const columnId = columns[0]!.id;
const before = registry.get("mock-social").getFeed as ReturnType<typeof vi.fn>;
const callsBefore = before.mock.calls.length;
const outcome = await scheduler.pollWorkspaceDue(owner.workspaceId);
expect(outcome.polled).toBe(0); // sources just polled: next_poll_at is in the future
// The HTTP refresh endpoint cannot bypass the persisted floor: the poll
// happened seconds ago, so force-due refuses even though the Redis
// cooldown keys were flushed (cursor-survival test) — next_poll_at is the
// authoritative guard, not the cache.
const refreshed = await request(`/feeds/columns/${columnId}/refresh`, "POST");
expect(refreshed.status).toBe(201);
const body = await refreshed.json() as { refreshed: number };
expect(body.refreshed).toBe(0); // DB floor refused the forced poll
expect(before.mock.calls.length).toBe(callsBefore);
});
it("dedups the same remote item across columns sharing an identity", async () => {
// Second column on the same connection as the first: same identity.
const columns = await (await request("/feeds/columns")).json() as Array<{ id: string; sources: Array<{ id: string }> }>;
const [first, second] = columns;
expect(first && second).toBeTruthy();
// Make both due again, then poll: one identity, one connector call.
await ownerPool.query("UPDATE column_sources SET next_poll_at = now() WHERE workspace_id = $1", [owner.workspaceId]);
const getFeed = registry.get("mock-social").getFeed as ReturnType<typeof vi.fn>;
const before = getFeed.mock.calls.length;
const outcome = await scheduler.pollWorkspaceDue(owner.workspaceId);
expect(outcome.polled).toBe(2);
expect(getFeed.mock.calls.length).toBe(before + 2); // two source rows, two calls (per-source semantics)
// One feed_items row per remote item — the dedup key is workspace-wide.
const stored = await ownerPool.query("SELECT count(*)::int AS c FROM feed_items WHERE workspace_id = $1", [owner.workspaceId]);
expect(stored.rows[0]?.c).toBe(items.length);
// Both columns surface the same items.
const left = await store.columnItems(owner.workspaceId, first!.id, { limit: 10 });
const right = await store.columnItems(owner.workspaceId, second!.id, { limit: 10 });
expect(left.items).toHaveLength(items.length);
expect(right.items).toHaveLength(items.length);
// And a cursor minted on one column pages identically on the other.
expect(left.items.map((row) => row.item.remoteId)).toEqual(right.items.map((row) => row.item.remoteId));
});
it("reorder requires the full id list and rewrites positions densely", async () => {
const columns = await (await request("/feeds/columns")).json() as Array<{ id: string; position: number }>;
const ids = columns.map((column) => column.id);
const partial = await request("/feeds/columns/reorder", "POST", { ids: [ids[0]!] });
expect(partial.status).toBe(400);
const reordered = await request("/feeds/columns/reorder", "POST", { ids: [...ids].reverse() });
expect(reordered.status).toBe(201);
const after = await reordered.json() as Array<{ id: string; position: number }>;
expect(after.map((column) => column.id)).toEqual([...ids].reverse());
expect(after.map((column) => column.position)).toEqual([0, 1]);
});
it("import-saved-searches creates idempotent saved_search columns", async () => {
// No search-capable provider is connected (mock-social declares none), so
// the implicit-platforms path correctly skips everything rather than
// inventing a platform the workspace cannot search.
const skipped = await request("/feeds/columns/import-saved-searches", "POST", { queries: ["anarch"] });
expect(skipped.status).toBe(201);
expect(await skipped.json()).toEqual({ imported: 0, skipped: 1 });
// Explicit platforms still import (the UI knows which it used locally).
const imported = await request("/feeds/columns/import-saved-searches", "POST", { queries: ["anarch", "mutual aid"], platforms: ["mock-social"] });
expect(imported.status).toBe(201);
expect(await imported.json()).toEqual({ imported: 2, skipped: 0 });
// Re-import: idempotent per query.
const again = await request("/feeds/columns/import-saved-searches", "POST", { queries: ["anarch", "squatting"], platforms: ["mock-social"] });
expect(await again.json()).toEqual({ imported: 1, skipped: 1 });
const columns = await (await request("/feeds/columns")).json() as Array<{ type: string; title: string }>;
const searches = columns.filter((column) => column.type === "saved_search");
expect(searches.map((column) => column.title).sort()).toEqual(["anarch", "mutual aid", "squatting"]);
// Unknown platform ids are rejected up front (P0a schema rules).
const bad = await request("/feeds/columns/import-saved-searches", "POST", { queries: ["x"], platforms: ["not-a-provider"] });
expect(bad.status).toBe(422);
});
it("GET /feeds keeps its external shape and gains additive meta.columns", async () => {
const response = await request("/feeds");
expect(response.status).toBe(200);
const feed = await response.json() as { items: unknown[]; nextCursor: string | null; meta: { configuredFeeds: number; columns?: Array<{ id: string }> } };
expect(Array.isArray(feed.items)).toBe(true);
expect(feed.meta.configuredFeeds).toBe(2);
expect(feed.meta.columns?.length).toBeGreaterThanOrEqual(2); // seeded columns surface
});
it("invalid cursor is a 400 and unknown column a 404", async () => {
expect((await request(`/feeds/columns/${randomUUID()}/items`)).status).toBe(404);
expect((await request("/feeds/columns/not-a-uuid/items")).status).toBe(400);
expect((await request("/feeds/columns/00000000-0000-0000-0000-000000000000/items?cursor=garbage")).status).toBe(400);
});
});
function row_summary(row: { summary: ConnectionSummary }): string { return row.summary.id; }