Tactical Media Broadcast Hub — initial release
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
This commit is contained in:
commit
aa5e262f3a
880 changed files with 111042 additions and 0 deletions
352
apps/api/src/feeds/column-scheduler.ts
Normal file
352
apps/api/src/feeds/column-scheduler.ts
Normal file
|
|
@ -0,0 +1,352 @@
|
|||
/**
|
||||
* Columns scheduler (P1 — design §3.3/§3.4/§3.5).
|
||||
*
|
||||
* Generalizes the legacy feeds poll loop into a due-SOURCES loop. The design
|
||||
* rule (§3.5): "the legacy home poll is the degenerate platform_feed source" —
|
||||
* so there is one loop, and `/feeds`'s per-connection home poll is simply the
|
||||
* `source_kind = 'home'` subset of it.
|
||||
*
|
||||
* Reused unchanged from the feeds loop:
|
||||
* - the per-key Redis lease (renewable, released in `finally`) so replicas
|
||||
* never double-poll one source;
|
||||
* - the cooldown key (`SET NX PX`) that makes repeated refresh clicks and
|
||||
* the 5-second discovery timer unable to bypass the interval;
|
||||
* - interval + 20 % jitter, SDK `RateLimitError.retryAfterMs` override.
|
||||
*
|
||||
* New per design §3.4:
|
||||
* - `column_sources` is the due list: `next_poll_at <= now`, `state <>
|
||||
* 'disabled'`, `refresh_policy.paused` false — the DB row replaces the
|
||||
* in-memory "one cycle per connection" bookkeeping;
|
||||
* - identities shared by several columns collapse to ONE poll (the fan-out
|
||||
* in `ColumnStore.storeItems` writes the item rows for every sharing
|
||||
* column), so following the same account in two columns costs one call;
|
||||
* - `consecutive_failures` drives exponential backoff (the same
|
||||
* monitor-poller curve: 2^n capped at 120 s), and the outcome is written
|
||||
* back to the source row (`state`, `last_error`, `next_poll_at`) — the
|
||||
* persisted truth the UI surfaces as `ColumnView.source.state`;
|
||||
* - per-host tokens come from `@tactical-media-broadcast-hub/egress` HostBudgets (shared
|
||||
* `host_budgets` rows — one bucket per host across every caller and host
|
||||
* class, deployment-wide), so feeds and monitor fetches to the same host
|
||||
* contend for the same tokens rather than each rate-limiting the host.
|
||||
*
|
||||
* P1 scope note: only the `home` source kind is executed against a connector
|
||||
* here (`getFeed(kind:'home')` — today's behavior); `mentions`/`search`/
|
||||
* `author` rows are planned with the P2 connector work, so they park in
|
||||
* `unsupported` rather than silently skipping. Item normalization runs through
|
||||
* the same P0c `normalizeFeedItem` funnel the legacy loop uses.
|
||||
*/
|
||||
import { Inject, Injectable, Logger, Optional, type OnApplicationBootstrap, type OnApplicationShutdown } from "@nestjs/common";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { ConnectorConnection, FeedItem } from "@tactical-media-broadcast-hub/core";
|
||||
import { RateLimitError, type ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import { DEFAULT_INTERVAL_MS } from "@tactical-media-broadcast-hub/core";
|
||||
import { pool, withTenant } from "../db/client.js";
|
||||
import { ColumnStore } from "./column-store.js";
|
||||
import { normalizeFeedItem, type NormalizedFeedItem } from "./feed-normalize.js";
|
||||
import type { ControlRedis } from "../media-control/redis.js";
|
||||
import type { ConnectionSummary } from "../connections/connections.service.js";
|
||||
|
||||
export const COLUMN_SCHEDULER_DEPENDENCIES = Symbol("COLUMN_SCHEDULER_DEPENDENCIES");
|
||||
|
||||
export interface ColumnSchedulerDependencies {
|
||||
redis: Pick<ControlRedis, "get" | "set" | "eval">;
|
||||
registry: ConnectorRegistry;
|
||||
connection(workspaceId: string, id: string): Promise<ConnectorConnection>;
|
||||
/** Legacy /feeds cache write-through, so /feeds reads stay cache-first. */
|
||||
cacheHomeFeed(workspaceId: string, connectionId: string, items: NormalizedFeedItem[]): Promise<void>;
|
||||
/** Effective host key for a provider's API host (egress hostBudgetKey). */
|
||||
hostKeyOf(provider: string, connection: ConnectorConnection): string;
|
||||
/** Draw one token from the shared per-host budget. */
|
||||
consumeBudget(hostKey: string): Promise<void>;
|
||||
random(): number;
|
||||
now(): number;
|
||||
}
|
||||
|
||||
/** Poll admission order per §3.4: due, enabled, not paused, oldest first. */
|
||||
interface DueRow {
|
||||
workspace_id: string;
|
||||
id: string;
|
||||
provider: string;
|
||||
connection_id: string | null;
|
||||
source_kind: "home" | "mentions" | "search" | "author";
|
||||
query: string | null;
|
||||
actor_ref: string | null;
|
||||
interval_ms: number;
|
||||
paused: boolean;
|
||||
consecutive_failures: number;
|
||||
}
|
||||
|
||||
const LEASE_MS = 30_000;
|
||||
const BACKOFF_CAP_MS = 120_000;
|
||||
const RENEW_LEASE = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('PEXPIRE', KEYS[1], ARGV[2]) else return 0 end";
|
||||
const RELEASE_LEASE = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) else return 0 end";
|
||||
|
||||
/** Minimum spacing between two forced refreshes of one source (cache-loss safe). */
|
||||
export const FORCE_FLOOR_MS = 15_000;
|
||||
|
||||
const DEFAULT_INTERVALS: Record<DueRow["source_kind"], number> = {
|
||||
home: DEFAULT_INTERVAL_MS.platform_feed,
|
||||
mentions: DEFAULT_INTERVAL_MS.platform_feed,
|
||||
search: DEFAULT_INTERVAL_MS.keyword,
|
||||
author: DEFAULT_INTERVAL_MS.account_list,
|
||||
};
|
||||
|
||||
/** Poll cadence: spec refresh_policy interval, else the §1.2 per-kind default. */
|
||||
export function effectiveIntervalMs(intervalMs: number | null, sourceKind: DueRow["source_kind"]): number {
|
||||
return intervalMs ?? DEFAULT_INTERVALS[sourceKind];
|
||||
}
|
||||
|
||||
/** The monitor-poller backoff curve: 2^n * interval scale, capped at 120 s. */
|
||||
export function backoffDelayMs(failures: number, intervalMs: number): number {
|
||||
return Math.min(BACKOFF_CAP_MS, intervalMs * 2 ** Math.min(7, Math.max(0, failures - 1)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Discovery cadence: sources self-schedule via next_poll_at, so the loop only
|
||||
* needs to sweep for newly due rows often enough to honor the 60 s floor.
|
||||
*/
|
||||
const SWEEP_MS = 5_000;
|
||||
|
||||
@Injectable()
|
||||
export class ColumnScheduler implements OnApplicationBootstrap, OnApplicationShutdown {
|
||||
private readonly logger = new Logger(ColumnScheduler.name);
|
||||
private running = false;
|
||||
private timer?: ReturnType<typeof setTimeout>;
|
||||
private active?: Promise<void>;
|
||||
constructor(
|
||||
private readonly store: ColumnStore,
|
||||
@Optional() @Inject(COLUMN_SCHEDULER_DEPENDENCIES) private readonly deps?: ColumnSchedulerDependencies,
|
||||
) {}
|
||||
|
||||
onApplicationBootstrap(): void {
|
||||
this.running = true;
|
||||
this.schedule(0);
|
||||
}
|
||||
|
||||
async onApplicationShutdown(): Promise<void> {
|
||||
this.running = false;
|
||||
if (this.timer) clearTimeout(this.timer);
|
||||
await this.active;
|
||||
}
|
||||
|
||||
private schedule(delay: number): void {
|
||||
if (!this.running) return;
|
||||
this.timer = setTimeout(() => {
|
||||
this.active = this.pollDue().catch(() => this.logger.warn("Column sweep unavailable")).then(() => undefined).finally(() => {
|
||||
this.active = undefined;
|
||||
this.schedule(SWEEP_MS + Math.round((this.deps?.random() ?? Math.random()) * 1_000));
|
||||
});
|
||||
}, delay);
|
||||
this.timer.unref();
|
||||
}
|
||||
|
||||
/** One pass over every workspace's due sources. Errors are per-source, never fatal. */
|
||||
async pollDue(): Promise<{ polled: number; skipped: number; failed: number }> {
|
||||
const workspaces = await this.workspaces();
|
||||
const totals = { polled: 0, skipped: 0, failed: 0 };
|
||||
for (const workspaceId of workspaces) {
|
||||
const pass = await this.pollWorkspaceDue(workspaceId);
|
||||
totals.polled += pass.polled;
|
||||
totals.skipped += pass.skipped;
|
||||
totals.failed += pass.failed;
|
||||
}
|
||||
return totals;
|
||||
}
|
||||
|
||||
/**
|
||||
* The due-source scan (§3.4). Read-only against the DB (withTenant, FORCE
|
||||
* RLS), then each source polls through its own lease/cooldown; the state
|
||||
* write-back happens in the same tenant context. Returns per-source counts
|
||||
* for the same shape the legacy loop reports.
|
||||
*/
|
||||
async pollWorkspaceDue(workspaceId: string): Promise<{ polled: number; skipped: number; failed: number }> {
|
||||
const due = await this.dueSources(workspaceId);
|
||||
const results = { polled: 0, skipped: 0, failed: 0 };
|
||||
for (const row of due) {
|
||||
const outcome = await this.pollSource(workspaceId, row);
|
||||
results[outcome] += 1;
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private async dueSources(workspaceId: string): Promise<DueRow[]> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const result = await client.query<DueRow>(
|
||||
`SELECT s.workspace_id, s.id, s.provider, s.connection_id, s.source_kind, s.query, s.actor_ref,
|
||||
(sc.refresh_policy->>'intervalMs')::int AS interval_ms,
|
||||
COALESCE((sc.refresh_policy->>'paused')::boolean, false) AS paused,
|
||||
s.consecutive_failures
|
||||
FROM column_sources s
|
||||
JOIN stream_columns sc ON sc.id = s.column_id
|
||||
WHERE s.workspace_id = $1
|
||||
AND sc.deleted_at IS NULL -- a soft-deleted column's sources retire with it
|
||||
AND s.state <> 'disabled'
|
||||
AND COALESCE((sc.refresh_policy->>'paused')::boolean, false) = false
|
||||
AND (s.next_poll_at IS NULL OR s.next_poll_at <= now())
|
||||
ORDER BY s.created_at`,
|
||||
[workspaceId],
|
||||
);
|
||||
return result.rows;
|
||||
});
|
||||
}
|
||||
|
||||
private async pollSource(workspaceId: string, row: DueRow): Promise<"polled" | "skipped" | "failed"> {
|
||||
if (this.deps === undefined) return "skipped";
|
||||
const interval = effectiveIntervalMs(row.interval_ms, row.source_kind);
|
||||
const cooldownMs = interval + Math.round(interval * 0.2 * this.deps.random());
|
||||
const key = `columns:${workspaceId}:${row.id}`;
|
||||
const leaseKey = `${key}:lease`;
|
||||
const cooldownKey = `${key}:cooldown`;
|
||||
const owner = randomUUID();
|
||||
if (!await this.deps.redis.set(leaseKey, owner, { nx: true, pxMs: LEASE_MS })) return "skipped";
|
||||
let heartbeat: ReturnType<typeof setInterval> | undefined;
|
||||
let renew: Promise<unknown> = Promise.resolve();
|
||||
let cooldown = cooldownMs;
|
||||
let started = false;
|
||||
try {
|
||||
if (!await this.deps.redis.set(cooldownKey, "idle", { nx: true, pxMs: cooldownMs })) return "skipped";
|
||||
started = true;
|
||||
heartbeat = setInterval(() => {
|
||||
renew = this.deps!.redis.eval(RENEW_LEASE, [leaseKey], [owner, String(LEASE_MS)]).catch(() => {
|
||||
this.logger.warn("Column poll lease renewal unavailable");
|
||||
});
|
||||
}, LEASE_MS / 3);
|
||||
heartbeat.unref();
|
||||
|
||||
const page = await this.fetch(workspaceId, row);
|
||||
const items = page.filter((item): item is NormalizedFeedItem => item !== null);
|
||||
await this.store.storeItems(workspaceId, row.id, items);
|
||||
await this.recordSuccess(workspaceId, row.id, interval);
|
||||
// The legacy /feeds surface stays cache-first for home sources: the
|
||||
// same items land in the hot cache the /feeds read path uses.
|
||||
if (row.source_kind === "home" && row.connection_id !== null) {
|
||||
await this.deps.cacheHomeFeed(workspaceId, row.connection_id, items);
|
||||
}
|
||||
return "polled";
|
||||
} catch (error) {
|
||||
const outcome = await this.recordFailure(workspaceId, row, error, interval);
|
||||
cooldown = outcome.cooldownMs;
|
||||
return "failed";
|
||||
} finally {
|
||||
if (heartbeat) clearInterval(heartbeat);
|
||||
await renew;
|
||||
try {
|
||||
if (started) await this.deps.redis.set(cooldownKey, "idle", { pxMs: cooldown });
|
||||
} finally {
|
||||
await this.deps.redis.eval(RELEASE_LEASE, [leaseKey], [owner]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Normalizes a page through the P0c funnel. P1 executes home only. */
|
||||
private async fetch(workspaceId: string, row: DueRow): Promise<Array<NormalizedFeedItem | null>> {
|
||||
if (this.deps === undefined) return [];
|
||||
if (row.source_kind !== "home") {
|
||||
throw new UnsupportedSourceError(row.source_kind);
|
||||
}
|
||||
if (row.connection_id === null) {
|
||||
// home is always connection-bound (a platform_feed column); a NULL
|
||||
// connection row is corrupt, not pollable.
|
||||
throw new UnsupportedSourceError("home-without-connection");
|
||||
}
|
||||
if (!this.deps.registry.has(row.provider)) throw new UnsupportedSourceError(`unknown-provider:${row.provider}`);
|
||||
const connector = this.deps.registry.get(row.provider);
|
||||
if (connector.getFeed === undefined) throw new UnsupportedSourceError(`feed-unsupported:${row.provider}`);
|
||||
const connection = await this.deps.connection(workspaceId, row.connection_id);
|
||||
const hostKey = this.deps.hostKeyOf(row.provider, connection);
|
||||
await this.deps.consumeBudget(hostKey);
|
||||
const page = await connector.getFeed(connection, { kind: "home", limit: 40 });
|
||||
return (page?.items ?? []).map((item: FeedItem) => normalizeFeedItem(item, connection));
|
||||
}
|
||||
|
||||
private async recordSuccess(workspaceId: string, sourceId: string, intervalMs: number): Promise<void> {
|
||||
await withTenant(workspaceId, async (client) => {
|
||||
await client.query(
|
||||
`UPDATE column_sources
|
||||
SET state = 'ready', last_error = NULL, last_polled_at = now(), consecutive_failures = 0,
|
||||
next_poll_at = now() + ($3::int * interval '1 millisecond'), updated_at = now()
|
||||
WHERE workspace_id = $1 AND id = $2`,
|
||||
[workspaceId, sourceId, intervalMs],
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
private async recordFailure(workspaceId: string, row: DueRow, error: unknown, intervalMs: number): Promise<{ cooldownMs: number }> {
|
||||
const rateLimited = error instanceof RateLimitError;
|
||||
const unsupported = error instanceof UnsupportedSourceError;
|
||||
const retryAfter = error instanceof RateLimitError ? error.retryAfterMs : undefined;
|
||||
const failures = row.consecutive_failures;
|
||||
const backoff = unsupported ? intervalMs : backoffDelayMs(failures + 1, intervalMs);
|
||||
const cooldownMs = rateLimited && retryAfter !== undefined ? Math.max(intervalMs, retryAfter) : backoff;
|
||||
await withTenant(workspaceId, async (client) => {
|
||||
await client.query(
|
||||
`UPDATE column_sources
|
||||
SET state = $3, last_error = $4, last_polled_at = now(),
|
||||
consecutive_failures = CASE WHEN $3 = 'unsupported' THEN consecutive_failures ELSE consecutive_failures + 1 END,
|
||||
next_poll_at = now() + ($5::bigint * interval '1 millisecond'), updated_at = now()
|
||||
WHERE workspace_id = $1 AND id = $2`,
|
||||
[workspaceId, row.id,
|
||||
unsupported ? "unsupported" : rateLimited ? "rate_limited" : "error",
|
||||
unsupported ? `unsupported source kind ${row.source_kind}` : error instanceof Error ? error.message.slice(0, 240) : "poll failed",
|
||||
cooldownMs],
|
||||
);
|
||||
}).catch(() => this.logger.warn("Column source state write-back unavailable"));
|
||||
return { cooldownMs };
|
||||
}
|
||||
|
||||
/**
|
||||
* Workspace listing spans tenants by definition, so it runs on the pool
|
||||
* with no tenant binding; every actual read/write below re-enters
|
||||
* withTenant(workspaceId) so FORCE RLS still binds per source.
|
||||
*/
|
||||
/**
|
||||
* Explicit refresh (§1.4 POST :id/refresh): mark this column's sources due
|
||||
* NOW. The per-source cooldown/lease still arbitrates the actual call, so a
|
||||
* refresh button cannot bypass the interval or double-fire mid-poll.
|
||||
*/
|
||||
/**
|
||||
* Explicit refresh (§1.4 POST :id/refresh) forces this column's sources due,
|
||||
* but not through a cooldown that is still authoritative in the DB: the
|
||||
* Redis cooldown is a cache (a flush loses it), so next_poll_at — persisted
|
||||
* — is the guard a forced refresh must still respect. FORCE_FLOOR_MS is a
|
||||
* deliberately small window: the button stays responsive for a manual
|
||||
* refresh while a double-click or a cache flush cannot double-poll a host.
|
||||
*/
|
||||
async forceColumnDue(workspaceId: string, columnId: string): Promise<number> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const result = await client.query(
|
||||
`UPDATE column_sources SET next_poll_at = now(), updated_at = now()
|
||||
WHERE workspace_id = $1 AND column_id = $2 AND state <> 'disabled'
|
||||
AND (last_polled_at IS NULL OR last_polled_at <= now() - ($3::bigint * interval '1 millisecond'))
|
||||
RETURNING id`,
|
||||
[workspaceId, columnId, FORCE_FLOOR_MS],
|
||||
);
|
||||
return result.rowCount ?? 0;
|
||||
});
|
||||
}
|
||||
|
||||
/** User-facing enable/disable of a single source (state machine §3.4). */
|
||||
async setSourceDisabled(workspaceId: string, sourceId: string, disabled: boolean): Promise<void> {
|
||||
await withTenant(workspaceId, async (client) => {
|
||||
const result = await client.query(
|
||||
`UPDATE column_sources SET state = $3, next_poll_at = CASE WHEN $3 = 'disabled' THEN NULL ELSE now() END, updated_at = now()
|
||||
WHERE workspace_id = $1 AND id = $2`,
|
||||
[workspaceId, sourceId, disabled ? "disabled" : "pending"],
|
||||
);
|
||||
if ((result.rowCount ?? 0) === 0) throw new Error(`column source ${sourceId} not found`);
|
||||
});
|
||||
}
|
||||
|
||||
private async workspaces(): Promise<string[]> {
|
||||
const result = await pool.query<{ id: string }>("SELECT id FROM workspaces ORDER BY id");
|
||||
return result.rows.map((row) => row.id);
|
||||
}
|
||||
}
|
||||
|
||||
/** Raised for source kinds P1 does not execute; parks the row in `unsupported`. */
|
||||
export class UnsupportedSourceError extends Error {
|
||||
constructor(reason: string) {
|
||||
super(`column source unsupported in P1: ${reason}`);
|
||||
this.name = "UnsupportedSourceError";
|
||||
}
|
||||
}
|
||||
652
apps/api/src/feeds/column-store.ts
Normal file
652
apps/api/src/feeds/column-store.ts
Normal file
|
|
@ -0,0 +1,652 @@
|
|||
/**
|
||||
* Columns persistence (P1 — design §1.2/§1.4).
|
||||
*
|
||||
* `stream_columns` (021) is the persisted column list; this repository owns
|
||||
* every mutation plus the resolved-source rows in `column_sources` (026) and
|
||||
* the per-source item rows in `feed_items`/`feed_item_columns` written by the
|
||||
* poll loop. Reads run under `withTenant` so FORCE RLS binds honestly.
|
||||
*
|
||||
* Everything the client can send is parsed with the P0a core validators
|
||||
* (`parseStreamColumnSpec`, `parseRefreshPolicyClamped`, `clampTitle`,
|
||||
* `clampPosition`) — the DB CHECKs are backstops, never the validation layer.
|
||||
* `StreamColumnsValidationError` is translated to 422 at the controller.
|
||||
*/
|
||||
import { BadRequestException, Inject, Injectable, NotFoundException, Optional, UnprocessableEntityException } from "@nestjs/common";
|
||||
import { createHash } from "node:crypto";
|
||||
import type { PoolClient } from "pg";
|
||||
import {
|
||||
clampPosition, clampTitle, parseRefreshPolicyClamped, parseStreamColumnSpec,
|
||||
StreamColumnsValidationError, type RefreshPolicy, type StreamColumnSpec, type StreamColumnType,
|
||||
} from "@tactical-media-broadcast-hub/core";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import { withTenant } from "../db/client.js";
|
||||
import { appendTenantAudit } from "../audit/audit.service.js";
|
||||
import type { NormalizedFeedItem } from "./feed-normalize.js";
|
||||
import type { ConnectionSummary } from "../connections/connections.service.js";
|
||||
|
||||
export type ColumnSourceKind = "home" | "mentions" | "search" | "author";
|
||||
export type ColumnSourceState =
|
||||
| "pending" | "ready" | "stale" | "error" | "unsupported" | "rate_limited" | "budget_paused" | "disabled";
|
||||
|
||||
export interface ColumnRecord {
|
||||
id: string;
|
||||
type: StreamColumnType;
|
||||
title: string;
|
||||
position: number;
|
||||
spec: StreamColumnSpec;
|
||||
refreshPolicy: RefreshPolicy;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
/** `ColumnView.source` (design §1.4): the scheduler-facing row, squashed. */
|
||||
export interface ColumnSourceView {
|
||||
id: string;
|
||||
/** Owning column — present so a bulk source query can be grouped per column. */
|
||||
columnId: string;
|
||||
provider: string;
|
||||
connectionId: string | null;
|
||||
sourceKind: ColumnSourceKind;
|
||||
query: string | null;
|
||||
actorRef: string | null;
|
||||
state: ColumnSourceState;
|
||||
lastError: string | null;
|
||||
lastPolledAt: string | null;
|
||||
nextPollAt: string | null;
|
||||
consecutiveFailures: number;
|
||||
}
|
||||
|
||||
export interface CreateColumn {
|
||||
type: StreamColumnType;
|
||||
title: string;
|
||||
position?: number;
|
||||
spec: unknown;
|
||||
refreshPolicy?: unknown;
|
||||
}
|
||||
|
||||
export interface UpdateColumn {
|
||||
title?: string;
|
||||
position?: number;
|
||||
spec?: unknown;
|
||||
refreshPolicy?: unknown;
|
||||
}
|
||||
|
||||
export interface StoredFeedItem {
|
||||
id: string;
|
||||
dedupKey: string;
|
||||
createdAtRemote: string;
|
||||
item: NormalizedFeedItem;
|
||||
}
|
||||
|
||||
/** The identity one poll resolves to — see `sourceIdentities` below. */
|
||||
interface SourceIdentity {
|
||||
provider: string;
|
||||
connectionId: string | null;
|
||||
sourceKind: ColumnSourceKind;
|
||||
query: string | null;
|
||||
actorRef: string | null;
|
||||
}
|
||||
|
||||
const identityKey = (identity: SourceIdentity): string =>
|
||||
JSON.stringify([identity.provider, identity.connectionId, identity.sourceKind, identity.query, identity.actorRef]);
|
||||
|
||||
export function mapValidationError<T>(parse: () => T): T {
|
||||
try {
|
||||
return parse();
|
||||
} catch (error) {
|
||||
if (error instanceof StreamColumnsValidationError) throw new UnprocessableEntityException(error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* sha256(workspace | provider | canonical remote id) — design §3.3. The
|
||||
* canonicalization keeps a provider that restates an item with decorations
|
||||
* (e.g. Mastodon's `scheduled:123` vs `123`) from double-storing the same
|
||||
* remote object.
|
||||
*/
|
||||
export function dedupKeyFor(workspaceId: string, provider: string, remoteId: string): string {
|
||||
const canonical = remoteId.replace(/^scheduled:/, "");
|
||||
return createHash("sha256").update(`${workspaceId}|${provider}|${canonical}`).digest("hex");
|
||||
}
|
||||
|
||||
/**
|
||||
* The polled identities a column spec resolves to (§3.3/§3.4). platform_feed
|
||||
* is the degenerate case: one connection, one kind per source row. keyword and
|
||||
* saved_search expand to one keyless search source per platform. account_list
|
||||
* expands to one author source per account entry.
|
||||
*/
|
||||
export function sourceIdentities(type: StreamColumnType, spec: StreamColumnSpec): SourceIdentity[] {
|
||||
switch (type) {
|
||||
case "platform_feed": {
|
||||
const feed = spec as import("@tactical-media-broadcast-hub/core").PlatformFeedSpec;
|
||||
const kinds = feed.kinds.length > 0 ? feed.kinds : (["home"] as const);
|
||||
return kinds.map((kind): SourceIdentity => ({
|
||||
provider: "", connectionId: feed.connectionId, sourceKind: kind, query: null, actorRef: null,
|
||||
}));
|
||||
}
|
||||
case "keyword":
|
||||
case "saved_search": {
|
||||
const keyword = spec as import("@tactical-media-broadcast-hub/core").KeywordSpec;
|
||||
return keyword.platforms.map((provider): SourceIdentity => ({
|
||||
provider, connectionId: null, sourceKind: "search", query: keyword.query, actorRef: null,
|
||||
}));
|
||||
}
|
||||
case "account_list": {
|
||||
const list = spec as import("@tactical-media-broadcast-hub/core").AccountListSpec;
|
||||
return list.accounts.map((account): SourceIdentity => ({
|
||||
provider: account.provider, connectionId: null, sourceKind: "author", query: null, actorRef: account.handle,
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Providers referenced by a spec (empty for platform_feed: the connection supplies it). */
|
||||
export function columnProviders(type: StreamColumnType, spec: StreamColumnSpec): string[] {
|
||||
switch (type) {
|
||||
case "platform_feed": return [];
|
||||
case "keyword":
|
||||
case "saved_search": return (spec as import("@tactical-media-broadcast-hub/core").KeywordSpec).platforms;
|
||||
case "account_list": return (spec as import("@tactical-media-broadcast-hub/core").AccountListSpec).accounts.map((account) => account.provider);
|
||||
}
|
||||
}
|
||||
|
||||
/** The title today's implicit column renders: "Display Name · handle" (web streams-data liveColumns). */
|
||||
export function seedTitle(row: ConnectionSummary): string {
|
||||
const displayName = String(row.fields.displayName ?? "").trim() || row.channelId;
|
||||
const handle = String(row.fields.handle ?? "").trim() || row.channelId;
|
||||
return clampTitle(`${displayName} · ${handle}`);
|
||||
}
|
||||
|
||||
interface ColumnRow {
|
||||
id: string; type: StreamColumnType; title: string; position: number;
|
||||
spec: unknown; refresh_policy: unknown; created_at: Date; updated_at: Date;
|
||||
}
|
||||
|
||||
const COLUMN_SELECT = "id, type, title, position, spec, refresh_policy, created_at, updated_at";
|
||||
|
||||
function toRecord(row: ColumnRow): ColumnRecord {
|
||||
return {
|
||||
id: row.id,
|
||||
type: row.type,
|
||||
title: row.title,
|
||||
position: row.position,
|
||||
spec: mapValidationError(() => parseStreamColumnSpec(row.type, row.spec)),
|
||||
refreshPolicy: parseRefreshPolicyClamped(row.refresh_policy ?? {}),
|
||||
createdAt: row.created_at.toISOString(),
|
||||
updatedAt: row.updated_at.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
const SOURCE_SELECT = `SELECT s.id, s.column_id, s.provider, s.connection_id, s.source_kind, s.query, s.actor_ref,
|
||||
s.state, s.last_error, s.last_polled_at, s.next_poll_at, s.consecutive_failures
|
||||
FROM column_sources s WHERE s.workspace_id = $1`;
|
||||
|
||||
function sourceFromRow(row: Record<string, unknown>): ColumnSourceView {
|
||||
return {
|
||||
id: String(row.id),
|
||||
columnId: String(row.column_id),
|
||||
provider: String(row.provider),
|
||||
connectionId: row.connection_id === null || row.connection_id === undefined ? null : String(row.connection_id),
|
||||
sourceKind: row.source_kind as ColumnSourceKind,
|
||||
query: row.query === null || row.query === undefined ? null : String(row.query),
|
||||
actorRef: row.actor_ref === null || row.actor_ref === undefined ? null : String(row.actor_ref),
|
||||
state: row.state as ColumnSourceState,
|
||||
lastError: row.last_error === null || row.last_error === undefined ? null : String(row.last_error),
|
||||
lastPolledAt: isoOrNull(row.last_polled_at),
|
||||
nextPollAt: isoOrNull(row.next_poll_at),
|
||||
consecutiveFailures: Number(row.consecutive_failures ?? 0),
|
||||
};
|
||||
}
|
||||
|
||||
function isoOrNull(value: unknown): string | null {
|
||||
return value === null || value === undefined ? null : (value as Date).toISOString();
|
||||
}
|
||||
|
||||
export interface ColumnStoreDependencies {
|
||||
/** Resolve a connection's provider for platform_feed source rows. */
|
||||
providerOf(workspaceId: string, connectionId: string): Promise<string>;
|
||||
/** Feed-capable connection summaries — the lazy seeding input. */
|
||||
connections(workspaceId: string): Promise<ConnectionSummary[]>;
|
||||
}
|
||||
|
||||
export const COLUMN_STORE_DEPENDENCIES = Symbol("COLUMN_STORE_DEPENDENCIES");
|
||||
|
||||
@Injectable()
|
||||
export class ColumnStore {
|
||||
constructor(@Optional() @Inject(COLUMN_STORE_DEPENDENCIES) private readonly deps?: ColumnStoreDependencies) {}
|
||||
|
||||
private async providerOf(workspaceId: string, connectionId: string): Promise<string> {
|
||||
if (this.deps) return this.deps.providerOf(workspaceId, connectionId);
|
||||
return withTenant(workspaceId, async (client): Promise<string> => {
|
||||
const result = await client.query<{ provider: string }>(
|
||||
"SELECT provider FROM connections WHERE id = $1 AND workspace_id = $2 AND disconnected_at IS NULL", [connectionId, workspaceId]);
|
||||
const row = result.rows[0];
|
||||
if (row === undefined) throw new NotFoundException(`connection ${connectionId} not found`);
|
||||
return row.provider;
|
||||
});
|
||||
}
|
||||
|
||||
private async connections(workspaceId: string): Promise<ConnectionSummary[]> {
|
||||
if (this.deps) return this.deps.connections(workspaceId);
|
||||
const { PgConnectionStore } = await import("../connections/connection-store.js");
|
||||
return new PgConnectionStore().list(workspaceId);
|
||||
}
|
||||
|
||||
async list(workspaceId: string): Promise<ColumnRecord[]> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const result = await client.query(
|
||||
`SELECT ${COLUMN_SELECT} FROM stream_columns WHERE deleted_at IS NULL ORDER BY position, created_at`);
|
||||
return result.rows.map((row) => toRecord(row as ColumnRow));
|
||||
});
|
||||
}
|
||||
|
||||
async get(workspaceId: string, id: string): Promise<ColumnRecord> {
|
||||
return withTenant(workspaceId, async (client) => this.read(client, workspaceId, id));
|
||||
}
|
||||
|
||||
async sources(workspaceId: string, columnId?: string): Promise<ColumnSourceView[]> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const result = columnId === undefined
|
||||
? await client.query(`${SOURCE_SELECT} ORDER BY s.created_at`, [workspaceId])
|
||||
: await client.query(`${SOURCE_SELECT} AND s.column_id = $2 ORDER BY s.created_at`, [workspaceId, columnId]);
|
||||
return result.rows.map((row) => sourceFromRow(row));
|
||||
});
|
||||
}
|
||||
|
||||
async create(session: SessionPayload, input: CreateColumn): Promise<ColumnRecord> {
|
||||
const spec = mapValidationError(() => parseStreamColumnSpec(input.type, input.spec));
|
||||
const policy = parseRefreshPolicyClamped(input.refreshPolicy ?? {});
|
||||
const title = clampTitle(input.title ?? "");
|
||||
if (title === "") throw new UnprocessableEntityException("title must be 1..80 characters after trimming");
|
||||
return withTenant(session.workspaceId, async (client) => {
|
||||
await this.assertSpecReferenceable(client, session.workspaceId, input.type, spec);
|
||||
const position = input.position === undefined
|
||||
? await this.nextPosition(client, session.workspaceId)
|
||||
: clampPosition(input.position);
|
||||
const result = await client.query(
|
||||
`INSERT INTO stream_columns (workspace_id, type, title, position, spec, refresh_policy, created_by)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7) RETURNING ${COLUMN_SELECT}`,
|
||||
[session.workspaceId, input.type, title, position, JSON.stringify(spec), JSON.stringify(policy), session.userId],
|
||||
);
|
||||
const record = toRecord(result.rows[0] as ColumnRow);
|
||||
await this.writeSources(client, session.workspaceId, record.id, input.type, spec);
|
||||
await appendTenantAudit(session.workspaceId, {
|
||||
actorUserId: session.userId, actorRole: session.role, action: "columns.created",
|
||||
targetType: "stream_column", targetId: record.id, diff: { type: input.type, title },
|
||||
});
|
||||
return record;
|
||||
});
|
||||
}
|
||||
|
||||
async update(session: SessionPayload, id: string, input: UpdateColumn): Promise<ColumnRecord> {
|
||||
if (Object.keys(input).length === 0) throw new BadRequestException("no recognized fields to update");
|
||||
return withTenant(session.workspaceId, async (client) => {
|
||||
await client.query("SELECT id FROM stream_columns WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL FOR UPDATE", [session.workspaceId, id]);
|
||||
const current = await this.read(client, session.workspaceId, id);
|
||||
const title = input.title === undefined ? current.title : clampTitle(input.title);
|
||||
if (title === "") throw new UnprocessableEntityException("title must be 1..80 characters after trimming");
|
||||
// Column type is immutable (design §1.4: PATCH carries title/position/
|
||||
// spec/refresh_policy); a different type is a new column.
|
||||
const spec = input.spec === undefined ? current.spec : mapValidationError(() => parseStreamColumnSpec(current.type, input.spec));
|
||||
const policy = input.refreshPolicy === undefined ? current.refreshPolicy : parseRefreshPolicyClamped(input.refreshPolicy);
|
||||
const position = input.position === undefined ? current.position : clampPosition(input.position);
|
||||
if (input.spec !== undefined) await this.assertSpecReferenceable(client, session.workspaceId, current.type, spec);
|
||||
const result = await client.query(
|
||||
`UPDATE stream_columns SET title = $3, position = $4, spec = $5, refresh_policy = $6, updated_at = now()
|
||||
WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL RETURNING ${COLUMN_SELECT}`,
|
||||
[session.workspaceId, id, title, position, JSON.stringify(spec), JSON.stringify(policy)],
|
||||
);
|
||||
if (result.rows[0] === undefined) throw new NotFoundException(`column ${id} not found`);
|
||||
if (input.spec !== undefined) await this.rewriteSources(client, session.workspaceId, id, current.type, spec);
|
||||
await appendTenantAudit(session.workspaceId, {
|
||||
actorUserId: session.userId, actorRole: session.role, action: "columns.updated",
|
||||
targetType: "stream_column", targetId: id,
|
||||
diff: { titleChanged: title !== current.title, positionChanged: position !== current.position, specChanged: input.spec !== undefined },
|
||||
});
|
||||
return toRecord(result.rows[0] as ColumnRow);
|
||||
});
|
||||
}
|
||||
|
||||
private async writeSources(client: PoolClient, workspaceId: string, columnId: string, type: StreamColumnType, spec: StreamColumnSpec): Promise<void> {
|
||||
for (const identity of sourceIdentities(type, spec)) {
|
||||
const provider = identity.provider !== "" ? identity.provider : await this.providerOf(workspaceId, identity.connectionId ?? "");
|
||||
await client.query(
|
||||
`INSERT INTO column_sources (workspace_id, column_id, provider, connection_id, source_kind, query, actor_ref)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7)
|
||||
ON CONFLICT (workspace_id, column_id, provider, source_kind, connection_id, query, actor_ref) DO NOTHING`,
|
||||
[workspaceId, columnId, provider, identity.connectionId, identity.sourceKind, identity.query, identity.actorRef],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Spec identity changes rewrite the source set; identities no longer referenced by this column are dropped. */
|
||||
private async rewriteSources(client: PoolClient, workspaceId: string, columnId: string, type: StreamColumnType, spec: StreamColumnSpec): Promise<void> {
|
||||
const wanted = new Set(sourceIdentities(type, spec).map(identityKey));
|
||||
const existing = await client.query<{ id: string; provider: string; connection_id: string | null; source_kind: ColumnSourceKind; query: string | null; actor_ref: string | null }>(
|
||||
"SELECT id, provider, connection_id, source_kind, query, actor_ref FROM column_sources WHERE workspace_id = $1 AND column_id = $2", [workspaceId, columnId]);
|
||||
for (const row of existing.rows) {
|
||||
const identity: SourceIdentity = { provider: row.provider, connectionId: row.connection_id, sourceKind: row.source_kind, query: row.query, actorRef: row.actor_ref };
|
||||
if (!wanted.has(identityKey(identity))) {
|
||||
await client.query("DELETE FROM column_sources WHERE workspace_id = $1 AND id = $2", [workspaceId, row.id]);
|
||||
}
|
||||
}
|
||||
await this.writeSources(client, workspaceId, columnId, type, spec);
|
||||
}
|
||||
|
||||
async remove(session: SessionPayload, id: string): Promise<void> {
|
||||
await withTenant(session.workspaceId, async (client) => {
|
||||
await client.query("SELECT id FROM stream_columns WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL FOR UPDATE", [session.workspaceId, id]);
|
||||
const result = await client.query(
|
||||
"UPDATE stream_columns SET deleted_at = now(), updated_at = now() WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL RETURNING id", [session.workspaceId, id]);
|
||||
if (result.rows[0] === undefined) throw new NotFoundException(`column ${id} not found`);
|
||||
await appendTenantAudit(session.workspaceId, {
|
||||
actorUserId: session.userId, actorRole: session.role, action: "columns.deleted",
|
||||
targetType: "stream_column", targetId: id,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Explicit order (design §1.4 POST /reorder): the client sends the full
|
||||
* ordered id list; positions are rewritten densely. Unknown, foreign or
|
||||
* missing ids are a 400 — a partial reorder would corrupt the order.
|
||||
*/
|
||||
async reorder(session: SessionPayload, ids: string[]): Promise<ColumnRecord[]> {
|
||||
if (ids.length === 0) return this.list(session.workspaceId);
|
||||
if (new Set(ids).size !== ids.length) throw new BadRequestException("reorder ids must be unique");
|
||||
return withTenant(session.workspaceId, async (client) => {
|
||||
const existing = await client.query<{ id: string }>(
|
||||
"SELECT id FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL", [session.workspaceId]);
|
||||
if (ids.length !== existing.rows.length || !existing.rows.every((row) => ids.includes(row.id))) {
|
||||
throw new BadRequestException("reorder must list every column in the workspace exactly once");
|
||||
}
|
||||
// Two-phase write: the unique (workspace_id, position) index would fire
|
||||
// mid-swap if positions were assigned in place (e.g. moving column 0 to
|
||||
// the end collides with the row already at that slot). Parking every row
|
||||
// at a high negative offset first leaves the final range untouched, so
|
||||
// the second pass can never collide. Offsets are outside the CHECK
|
||||
// (position >= 0) only transiently... so instead park at (n + i), which
|
||||
// is >= 0, unique within the transaction, and freed before phase two.
|
||||
// Phase 1 — park every row at a unique high slot (n, n+1, …) so the
|
||||
// unique (workspace_id, position) index can't fire mid-reorder.
|
||||
let park = ids.length;
|
||||
for (const id of ids) {
|
||||
await client.query("UPDATE stream_columns SET position = $3 WHERE workspace_id = $1 AND id = $2", [session.workspaceId, id, park]);
|
||||
park += 1;
|
||||
}
|
||||
// Phase 2 — assign final dense ranks in the REQUESTED order. The parked
|
||||
// range is disjoint from [0, n), so no intermediate state collides.
|
||||
const byId = new Map(ids.map((id, index) => [id, index] as const));
|
||||
const current = await client.query<ColumnRow>(
|
||||
`SELECT ${COLUMN_SELECT} FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL`, [session.workspaceId]);
|
||||
const records = current.rows
|
||||
.map((row) => toRecord(row as ColumnRow))
|
||||
.sort((a, b) => (byId.get(a.id) ?? 0) - (byId.get(b.id) ?? 0));
|
||||
let rank = 0;
|
||||
for (const record of records) {
|
||||
await client.query("UPDATE stream_columns SET position = $3, updated_at = now() WHERE workspace_id = $1 AND id = $2", [session.workspaceId, record.id, rank]);
|
||||
record.position = rank;
|
||||
rank += 1;
|
||||
}
|
||||
await appendTenantAudit(session.workspaceId, {
|
||||
actorUserId: session.userId, actorRole: session.role, action: "columns.reordered",
|
||||
targetType: "workspace", targetId: session.workspaceId, diff: { count: ids.length },
|
||||
});
|
||||
return records;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Lazy seeding (design §6 P1): on the first column-API touch for a workspace
|
||||
* with no columns at all, materialize TODAY's implicit /feeds columns — one
|
||||
* platform_feed column per feed-capable connection, titled exactly like the
|
||||
* implicit column the UI renders today ("Display · handle", kinds ["home"],
|
||||
* position by connection creation order), so the Streams page is unchanged.
|
||||
* Idempotent by construction: a second call sees a non-empty column list and
|
||||
* seeds nothing — including for connections added later.
|
||||
*/
|
||||
async ensureSeeded(workspaceId: string, connections: ConnectionSummary[], capable: (row: ConnectionSummary) => boolean): Promise<{ seeded: number }> {
|
||||
const rows = connections.filter(capable);
|
||||
if (rows.length === 0) return { seeded: 0 };
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
// Serialize concurrent first touches without taking a row lock on
|
||||
// workspaces itself: a FOR UPDATE here deadlocks with any FK insert
|
||||
// into audit_log/column_sources from a nested withTenant client,
|
||||
// because FK validation takes FOR KEY SHARE on the parent workspace
|
||||
// row. Advisory xact locks never block key-share, so seeding
|
||||
// serializes while audits and FK checks proceed.
|
||||
await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`columns-seed:${workspaceId}`]);
|
||||
const existing = await client.query<{ count: string }>(
|
||||
"SELECT count(*)::text AS count FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL", [workspaceId]);
|
||||
if (Number(existing.rows[0]?.count ?? 0) > 0) return { seeded: 0 };
|
||||
const seeder = await client.query<{ user_id: string }>(
|
||||
"SELECT m.user_id FROM memberships m JOIN users u ON u.id = m.user_id WHERE m.workspace_id = $1 ORDER BY m.created_at, u.created_at LIMIT 1", [workspaceId]);
|
||||
const createdBy = seeder.rows[0]?.user_id;
|
||||
if (createdBy === undefined) return { seeded: 0 }; // no member to attribute
|
||||
let seeded = 0;
|
||||
let position = 0;
|
||||
for (const row of rows) {
|
||||
const spec = { connectionId: row.id, kinds: ["home"] } as const;
|
||||
const inserted = await client.query<{ id: string }>(
|
||||
`INSERT INTO stream_columns (workspace_id, type, title, position, spec, refresh_policy, created_by)
|
||||
VALUES ($1,'platform_feed',$2,$3,$4,'{}',$5) RETURNING id`,
|
||||
[workspaceId, seedTitle(row), position, JSON.stringify(spec), createdBy],
|
||||
);
|
||||
const columnId = inserted.rows[0]?.id;
|
||||
if (columnId === undefined) continue;
|
||||
await client.query(
|
||||
`INSERT INTO column_sources (workspace_id, column_id, provider, connection_id, source_kind)
|
||||
VALUES ($1,$2,$3,$4,'home')
|
||||
ON CONFLICT (workspace_id, column_id, provider, source_kind, connection_id, query, actor_ref) DO NOTHING`,
|
||||
[workspaceId, columnId, row.provider, row.id],
|
||||
);
|
||||
seeded += 1;
|
||||
position += 1;
|
||||
}
|
||||
if (seeded > 0) await appendTenantAudit(workspaceId, {
|
||||
actorUserId: null, actorRole: "system", action: "columns.seeded",
|
||||
targetType: "workspace", targetId: workspaceId, diff: { seeded },
|
||||
});
|
||||
return { seeded };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* import-saved-searches (design §1.4): the localStorage saved-searches
|
||||
* contract moves server-side. Each query becomes a saved_search column over
|
||||
* the platforms the workspace can actually search, defaulting to every
|
||||
* connected feed provider when the client sends none. Idempotent per query:
|
||||
* a live saved_search column with the same query is skipped.
|
||||
*/
|
||||
async importSavedSearches(
|
||||
session: SessionPayload, queries: string[], platforms: string[],
|
||||
): Promise<{ imported: number; skipped: number }> {
|
||||
if (platforms.length === 0) return { imported: 0, skipped: queries.length };
|
||||
return withTenant(session.workspaceId, async (client) => {
|
||||
let imported = 0;
|
||||
let skipped = 0;
|
||||
for (const raw of queries) {
|
||||
const query = raw.trim();
|
||||
if (query === "") { skipped += 1; continue; }
|
||||
const duplicate = await client.query<{ id: string }>(
|
||||
`SELECT id FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL AND type = 'saved_search'
|
||||
AND spec->>'query' = $2 LIMIT 1`, [session.workspaceId, query]);
|
||||
if (duplicate.rows[0] !== undefined) { skipped += 1; continue; }
|
||||
const spec = { query, platforms, matchMode: "any" } as const;
|
||||
const title = clampTitle(query);
|
||||
if (title === "") { skipped += 1; continue; }
|
||||
const position = await this.nextPosition(client, session.workspaceId);
|
||||
const inserted = await client.query<{ id: string }>(
|
||||
`INSERT INTO stream_columns (workspace_id, type, title, position, spec, refresh_policy, created_by)
|
||||
VALUES ($1,'saved_search',$2,$3,$4,'{}',$5) RETURNING id`,
|
||||
[session.workspaceId, title, position, JSON.stringify(spec), session.userId],
|
||||
);
|
||||
const columnId = inserted.rows[0]?.id;
|
||||
if (columnId === undefined) { skipped += 1; continue; }
|
||||
await this.writeSources(client, session.workspaceId, columnId, "saved_search",
|
||||
parseStreamColumnSpec("saved_search", spec));
|
||||
imported += 1;
|
||||
}
|
||||
if (imported > 0) await appendTenantAudit(session.workspaceId, {
|
||||
actorUserId: session.userId, actorRole: session.role, action: "columns.saved_searches_imported",
|
||||
targetType: "workspace", targetId: session.workspaceId, diff: { imported, skipped },
|
||||
});
|
||||
return { imported, skipped };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Poll-time item persistence (§3.3): first writer wins per dedup key (the
|
||||
* cross-column dedup), then fan-out rows link the item to every column
|
||||
* whose source shares this source's identity — which is how "same account
|
||||
* in two columns costs one poll" still surfaces items in both.
|
||||
*/
|
||||
async storeItems(workspaceId: string, sourceId: string, items: NormalizedFeedItem[]): Promise<{ stored: number; fanned: number }> {
|
||||
if (items.length === 0) return { stored: 0, fanned: 0 };
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const found = await client.query<{ id: string; provider: string; connection_id: string | null; source_kind: ColumnSourceKind; query: string | null; actor_ref: string | null }>(
|
||||
"SELECT id, provider, connection_id, source_kind, query, actor_ref FROM column_sources WHERE workspace_id = $1 AND id = $2", [workspaceId, sourceId]);
|
||||
const source = found.rows[0];
|
||||
if (source === undefined) return { stored: 0, fanned: 0 };
|
||||
const sharing = await client.query<{ column_id: string }>(
|
||||
`SELECT column_id FROM column_sources
|
||||
WHERE workspace_id = $1 AND provider = $2 AND source_kind = $3
|
||||
AND connection_id IS NOT DISTINCT FROM $4 AND query IS NOT DISTINCT FROM $5 AND actor_ref IS NOT DISTINCT FROM $6
|
||||
AND state <> 'disabled'`,
|
||||
[workspaceId, source.provider, source.source_kind, source.connection_id, source.query, source.actor_ref]);
|
||||
const columnIds = sharing.rows.map((row) => row.column_id);
|
||||
let stored = 0;
|
||||
let fanned = 0;
|
||||
for (const item of items) {
|
||||
const dedupKey = dedupKeyFor(workspaceId, item.provider, item.remoteId);
|
||||
// First writer wins per dedup key (the workspace-wide cross-column
|
||||
// dedup, §3.3) — the returning row resolves the item id either way.
|
||||
const inserted = await client.query<{ id: string }>(
|
||||
`INSERT INTO feed_items (workspace_id, dedup_key, provider, remote_id, item_id, author_handle, author_ref, url,
|
||||
created_at_remote, fetched_at, content, plain_text, media, metrics, source_id, expires_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,now(),$10,$11,$12,$13,$14,$15)
|
||||
ON CONFLICT (workspace_id, dedup_key) DO UPDATE SET fetched_at = feed_items.fetched_at RETURNING id`,
|
||||
[workspaceId, dedupKey, item.provider, item.remoteId, item.id, item.authorHandle, item.authorRef, item.url,
|
||||
item.createdAt.toISOString(), JSON.stringify(item.content), item.plainText, JSON.stringify(item.media),
|
||||
JSON.stringify(item.metrics), sourceId, item.expiresAt === null || item.expiresAt === undefined ? null : item.expiresAt.toISOString()],
|
||||
);
|
||||
const itemId = inserted.rows[0]?.id;
|
||||
if (itemId === undefined) continue;
|
||||
stored += 1;
|
||||
// Fan out regardless of who inserted: an item ingested for one source
|
||||
// identity is visible to EVERY column sharing that identity, including
|
||||
// items an earlier poll already stored (§3.3) — otherwise the second
|
||||
// column's dedup-hit items would never get their link rows.
|
||||
for (const columnId of columnIds) {
|
||||
const linked = await client.query(
|
||||
"INSERT INTO feed_item_columns (workspace_id, item_id, column_id) VALUES ($1,$2,$3) ON CONFLICT DO NOTHING",
|
||||
[workspaceId, itemId, columnId]);
|
||||
fanned += linked.rowCount ?? 0;
|
||||
}
|
||||
}
|
||||
return { stored, fanned };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Keyset page for GET /feeds/columns/:id/items (§1.4). Cursor rule:
|
||||
* (created_at_remote DESC, dedup_key DESC) — a pure position in a total
|
||||
* order, so paging survives cache flushes and later insertions alike.
|
||||
*/
|
||||
async columnItems(
|
||||
workspaceId: string, columnId: string, options: { cursor?: { at: number; key: string }; limit: number },
|
||||
): Promise<{ items: StoredFeedItem[]; nextCursor: string | null }> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const column = await client.query("SELECT id FROM stream_columns WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL", [workspaceId, columnId]);
|
||||
if (column.rows[0] === undefined) throw new NotFoundException(`column ${columnId} not found`);
|
||||
const params: unknown[] = [workspaceId, columnId];
|
||||
const filter = options.cursor === undefined ? "" : " AND (i.created_at_remote, i.dedup_key) < ($3::timestamptz, $4)";
|
||||
if (options.cursor !== undefined) params.push(new Date(options.cursor.at).toISOString(), options.cursor.key);
|
||||
params.push(options.limit + 1);
|
||||
const result = await client.query(
|
||||
`SELECT i.id, i.dedup_key, i.created_at_remote, i.provider, i.remote_id, i.item_id, i.author_handle,
|
||||
i.author_ref, i.url, i.content, i.plain_text, i.media, i.metrics, i.expires_at
|
||||
FROM feed_items i
|
||||
JOIN feed_item_columns fic ON fic.item_id = i.id AND fic.column_id = $2
|
||||
WHERE i.workspace_id = $1${filter}
|
||||
ORDER BY i.created_at_remote DESC, i.dedup_key DESC
|
||||
LIMIT $${params.length}`,
|
||||
params,
|
||||
);
|
||||
const rows = result.rows;
|
||||
const hasMore = rows.length > options.limit;
|
||||
const items = (hasMore ? rows.slice(0, options.limit) : rows).map((row) => storedItemFromRow(row));
|
||||
const last = items[items.length - 1];
|
||||
return {
|
||||
items,
|
||||
nextCursor: hasMore && last
|
||||
? Buffer.from(JSON.stringify({ at: Date.parse(last.createdAtRemote), key: last.dedupKey })).toString("base64url")
|
||||
: null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
private async nextPosition(client: PoolClient, workspaceId: string): Promise<number> {
|
||||
const result = await client.query<{ max: number | null }>(
|
||||
"SELECT max(position) AS max FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL", [workspaceId]);
|
||||
return (result.rows[0]?.max ?? -1) + 1;
|
||||
}
|
||||
|
||||
/** A spec's references must exist in the workspace before the column can persist. */
|
||||
private async assertSpecReferenceable(client: PoolClient, workspaceId: string, type: StreamColumnType, spec: StreamColumnSpec): Promise<void> {
|
||||
if (type === "platform_feed") {
|
||||
const connectionId = (spec as import("@tactical-media-broadcast-hub/core").PlatformFeedSpec).connectionId;
|
||||
const result = await client.query(
|
||||
"SELECT 1 FROM connections WHERE workspace_id = $1 AND id = $2 AND disconnected_at IS NULL", [workspaceId, connectionId]);
|
||||
if (result.rows[0] === undefined) throw new UnprocessableEntityException("platform_feed: connectionId does not reference a connection in this workspace");
|
||||
return;
|
||||
}
|
||||
const providers = columnProviders(type, spec);
|
||||
if (providers.length === 0) return;
|
||||
const result = await client.query<{ provider: string }>(
|
||||
"SELECT DISTINCT provider FROM connections WHERE workspace_id = $1 AND disconnected_at IS NULL AND provider = ANY($2)", [workspaceId, providers]);
|
||||
const present = new Set(result.rows.map((row) => row.provider));
|
||||
const missing = [...new Set(providers)].filter((provider) => !present.has(provider));
|
||||
if (missing.length > 0) throw new UnprocessableEntityException(`no connection for provider(s): ${missing.join(", ")}`);
|
||||
}
|
||||
|
||||
private async read(client: PoolClient, workspaceId: string, id: string): Promise<ColumnRecord> {
|
||||
const result = await client.query(
|
||||
`SELECT ${COLUMN_SELECT} FROM stream_columns WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL`, [workspaceId, id]);
|
||||
const row = result.rows[0];
|
||||
if (row === undefined) throw new NotFoundException(`column ${id} not found`);
|
||||
return toRecord(row as ColumnRow);
|
||||
}
|
||||
}
|
||||
|
||||
interface FeedItemRow {
|
||||
id: string; dedup_key: string; created_at_remote: Date; provider: string; remote_id: string; item_id: string;
|
||||
author_handle: string; author_ref: string | null; url: string | null; content: unknown; plain_text: string;
|
||||
media: unknown; metrics: unknown; expires_at: Date | null;
|
||||
}
|
||||
|
||||
function storedItemFromRow(row: FeedItemRow): StoredFeedItem {
|
||||
const metrics = (typeof row.metrics === "object" && row.metrics !== null ? row.metrics : {}) as Record<string, number>;
|
||||
const item: NormalizedFeedItem = {
|
||||
id: row.item_id,
|
||||
remoteId: row.remote_id,
|
||||
provider: row.provider,
|
||||
// The store page does not carry a live connection id; the column's source
|
||||
// rows hold it, so the connectionId field is filled by the caller when it
|
||||
// matters (the items endpoint derives it from the column spec).
|
||||
connectionId: "",
|
||||
authorHandle: row.author_handle,
|
||||
authorRef: row.author_ref,
|
||||
url: row.url,
|
||||
text: row.plain_text,
|
||||
content: row.content as NormalizedFeedItem["content"],
|
||||
plainText: row.plain_text,
|
||||
createdAt: row.created_at_remote,
|
||||
mediaUrls: [],
|
||||
media: Array.isArray(row.media) ? row.media as NormalizedFeedItem["media"] : [],
|
||||
metrics,
|
||||
expiresAt: row.expires_at,
|
||||
};
|
||||
return { id: row.id, dedupKey: row.dedup_key, createdAtRemote: new Date(row.created_at_remote).toISOString(), item };
|
||||
}
|
||||
300
apps/api/src/feeds/columns.controller.ts
Normal file
300
apps/api/src/feeds/columns.controller.ts
Normal file
|
|
@ -0,0 +1,300 @@
|
|||
/**
|
||||
* Columns API (P1 — design §1.4).
|
||||
*
|
||||
* Routes (all under /feeds, the surface the Streams page already calls):
|
||||
* GET /feeds/columns list (+ lazy seeding on first touch)
|
||||
* POST /feeds/columns create
|
||||
* GET /feeds/columns/:id one column
|
||||
* PATCH /feeds/columns/:id update (title/position/spec/refreshPolicy)
|
||||
* DELETE /feeds/columns/:id soft delete
|
||||
* POST /feeds/columns/reorder explicit full-list order
|
||||
* POST /feeds/columns/import-saved-searches localStorage contract migration
|
||||
* GET /feeds/columns/:id/items keyset-cursored items
|
||||
* POST /feeds/columns/:id/refresh force a poll (cooldown-gated)
|
||||
* POST /feeds/columns/:id/sources/:sourceId/disable|enable
|
||||
*
|
||||
* Writes are gated on `streams.columns.manage` (026 seed: Owner/Admin/Editor);
|
||||
* reads stay on `draft.view` — the same permission /feeds uses, so every
|
||||
* current reader keeps reading. GET /feeds is untouched except the additive
|
||||
* `meta.columns` block.
|
||||
*/
|
||||
import { BadRequestException, Body, Controller, Delete, Get, Inject, NotFoundException, Param, Patch, Post, Query, Req, UnprocessableEntityException, UseGuards } from "@nestjs/common";
|
||||
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import { parseId, parseLimit, requireObject, requireSession, requireString } from "../http/validation.js";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import {
|
||||
ColumnStore, mapValidationError, type ColumnRecord, type ColumnSourceView, type CreateColumn, type StoredFeedItem, type UpdateColumn,
|
||||
} from "./column-store.js";
|
||||
import { ColumnScheduler } from "./column-scheduler.js";
|
||||
import { FeedsService } from "./feeds.service.js";
|
||||
import type { ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
|
||||
export interface ColumnResponse extends ColumnRecord {
|
||||
sources: ColumnSourceView[];
|
||||
}
|
||||
|
||||
export interface ColumnItemsResponse {
|
||||
items: StoredFeedItem[];
|
||||
nextCursor: string | null;
|
||||
}
|
||||
|
||||
/** DI token for the shared connector registry (single process-wide instance). */
|
||||
export const COLUMNS_REGISTRY = Symbol("COLUMNS_REGISTRY");
|
||||
|
||||
@Controller("feeds/columns")
|
||||
@UseGuards(RbacGuard)
|
||||
export class ColumnsController {
|
||||
constructor(
|
||||
@Inject(ColumnStore) private readonly store: ColumnStore,
|
||||
@Inject(ColumnScheduler) private readonly scheduler: ColumnScheduler,
|
||||
@Inject(FeedsService) private readonly feeds: FeedsService,
|
||||
@Inject(COLUMNS_REGISTRY) private readonly registry: ConnectorRegistry,
|
||||
) {}
|
||||
|
||||
@Get()
|
||||
@RequirePermission("draft.view")
|
||||
async list(@Req() request: AuthenticatedRequest): Promise<ColumnResponse[]> {
|
||||
const session = requireSession(request);
|
||||
await this.ensureSeeded(session);
|
||||
return this.views(session.workspaceId, await this.store.list(session.workspaceId));
|
||||
}
|
||||
|
||||
@Post()
|
||||
@RequirePermission("streams.columns.manage")
|
||||
async create(@Req() request: AuthenticatedRequest, @Body() body: unknown): Promise<ColumnResponse> {
|
||||
const session = requireSession(request);
|
||||
const input = parseCreate(body);
|
||||
await this.ensureSeeded(session);
|
||||
const record = await this.store.create(session, input);
|
||||
return this.view(record, await this.store.sources(session.workspaceId, record.id));
|
||||
}
|
||||
|
||||
@Get(":id")
|
||||
@RequirePermission("draft.view")
|
||||
async get(@Req() request: AuthenticatedRequest, @Param("id") id: string): Promise<ColumnResponse> {
|
||||
const session = requireSession(request);
|
||||
await this.ensureSeeded(session);
|
||||
const record = await this.store.get(session.workspaceId, parseId(id));
|
||||
return this.view(record, await this.store.sources(session.workspaceId, record.id));
|
||||
}
|
||||
|
||||
@Patch(":id")
|
||||
@RequirePermission("streams.columns.manage")
|
||||
async update(@Req() request: AuthenticatedRequest, @Param("id") id: string, @Body() body: unknown): Promise<ColumnResponse> {
|
||||
const session = requireSession(request);
|
||||
const record = await this.store.update(session, parseId(id), parseUpdate(body));
|
||||
return this.view(record, await this.store.sources(session.workspaceId, record.id));
|
||||
}
|
||||
|
||||
@Delete(":id")
|
||||
@RequirePermission("streams.columns.manage")
|
||||
async remove(@Req() request: AuthenticatedRequest, @Param("id") id: string): Promise<{ id: string; deleted: true }> {
|
||||
const session = requireSession(request);
|
||||
const parsed = parseId(id);
|
||||
await this.store.remove(session, parsed);
|
||||
return { id: parsed, deleted: true };
|
||||
}
|
||||
|
||||
@Post("reorder")
|
||||
@RequirePermission("streams.columns.manage")
|
||||
async reorder(@Req() request: AuthenticatedRequest, @Body() body: unknown): Promise<ColumnResponse[]> {
|
||||
const session = requireSession(request);
|
||||
const ids = parseIds(requireObject(body, "body").ids);
|
||||
return this.views(session.workspaceId, await this.store.reorder(session, ids));
|
||||
}
|
||||
|
||||
@Post("import-saved-searches")
|
||||
@RequirePermission("streams.columns.manage")
|
||||
async importSavedSearches(@Req() request: AuthenticatedRequest, @Body() body: unknown): Promise<{ imported: number; skipped: number }> {
|
||||
const session = requireSession(request);
|
||||
const payload = requireObject(body, "body");
|
||||
const rawQueries = payload.queries;
|
||||
if (!Array.isArray(rawQueries) || rawQueries.length === 0) {
|
||||
throw new BadRequestException("queries must be a non-empty array of saved search strings");
|
||||
}
|
||||
for (const query of rawQueries) {
|
||||
if (typeof query !== "string") throw new BadRequestException("queries must contain strings only");
|
||||
}
|
||||
if (rawQueries.length > 100) throw new BadRequestException("queries supports at most 100 entries per call");
|
||||
const rawPlatforms = payload.platforms;
|
||||
let platforms: string[];
|
||||
if (rawPlatforms === undefined || rawPlatforms === null) {
|
||||
platforms = this.searchCapableProviders(await this.connectedProviders(session));
|
||||
} else {
|
||||
if (!Array.isArray(rawPlatforms) || rawPlatforms.some((provider) => typeof provider !== "string")) {
|
||||
throw new BadRequestException("platforms must be an array of provider ids");
|
||||
}
|
||||
platforms = [...new Set(rawPlatforms as string[])];
|
||||
// Provider ids are opaque to the P0a schema by design; the registry is
|
||||
// the app-side source of truth, so an unknown id is a 422 here rather
|
||||
// than a saved_search column that can never resolve a source.
|
||||
const unknown = platforms.filter((provider) => !this.registry.has(provider));
|
||||
if (unknown.length > 0) {
|
||||
throw new UnprocessableEntityException(`unknown provider id(s): ${unknown.join(", ")}`);
|
||||
}
|
||||
}
|
||||
return this.store.importSavedSearches(session, rawQueries, platforms);
|
||||
}
|
||||
|
||||
@Get(":id/items")
|
||||
@RequirePermission("draft.view")
|
||||
async items(
|
||||
@Req() request: AuthenticatedRequest,
|
||||
@Param("id") id: string,
|
||||
@Query("cursor") cursor?: string,
|
||||
@Query("limit") limit?: string,
|
||||
): Promise<ColumnItemsResponse> {
|
||||
const session = requireSession(request);
|
||||
return this.store.columnItems(session.workspaceId, parseId(id), {
|
||||
cursor: cursor === undefined || cursor === "" ? undefined : decodeColumnCursor(cursor),
|
||||
limit: parseLimit(limit, 40, 200),
|
||||
});
|
||||
}
|
||||
|
||||
@Post(":id/refresh")
|
||||
@RequirePermission("streams.columns.manage")
|
||||
async refresh(@Req() request: AuthenticatedRequest, @Param("id") id: string): Promise<{ id: string; refreshed: number }> {
|
||||
const session = requireSession(request);
|
||||
const parsed = parseId(id);
|
||||
await this.store.get(session.workspaceId, parsed); // 404 before polling
|
||||
// Force-due this column's sources; the scheduler's cooldown/lease still
|
||||
// decides whether a call actually goes out (refresh cannot bypass them).
|
||||
const forced = await this.forceDue(session.workspaceId, parsed);
|
||||
const outcome = await this.scheduler.pollWorkspaceDue(session.workspaceId);
|
||||
return { id: parsed, refreshed: forced > 0 ? outcome.polled : 0 };
|
||||
}
|
||||
|
||||
@Post(":id/sources/:sourceId/disable")
|
||||
@RequirePermission("streams.columns.manage")
|
||||
async disableSource(@Req() request: AuthenticatedRequest, @Param("id") id: string, @Param("sourceId") sourceId: string): Promise<{ id: string; state: string }> {
|
||||
const session = requireSession(request);
|
||||
return this.setSourceState(session, parseId(id), parseId(sourceId), true);
|
||||
}
|
||||
|
||||
@Post(":id/sources/:sourceId/enable")
|
||||
@RequirePermission("streams.columns.manage")
|
||||
async enableSource(@Req() request: AuthenticatedRequest, @Param("id") id: string, @Param("sourceId") sourceId: string): Promise<{ id: string; state: string }> {
|
||||
const session = requireSession(request);
|
||||
return this.setSourceState(session, parseId(id), parseId(sourceId), false);
|
||||
}
|
||||
|
||||
// -- helpers -------------------------------------------------------------
|
||||
|
||||
private async setSourceState(session: SessionPayload, columnId: string, sourceId: string, disabled: boolean): Promise<{ id: string; state: string }> {
|
||||
const sources = await this.store.sources(session.workspaceId, columnId);
|
||||
const source = sources.find((row) => row.id === sourceId);
|
||||
if (source === undefined) throw new NotFoundException(`source ${sourceId} not found on column ${columnId}`);
|
||||
await this.scheduler.setSourceDisabled(session.workspaceId, sourceId, disabled);
|
||||
return { id: sourceId, state: disabled ? "disabled" : "pending" };
|
||||
}
|
||||
|
||||
private async ensureSeeded(session: SessionPayload): Promise<void> {
|
||||
const connections = await this.feeds.capableConnections(session.workspaceId);
|
||||
await this.store.ensureSeeded(session.workspaceId, connections, () => true);
|
||||
}
|
||||
|
||||
/** Bulk view assembly: one source query, grouped per column. */
|
||||
private async views(workspaceId: string, records: ColumnRecord[]): Promise<ColumnResponse[]> {
|
||||
if (records.length === 0) return [];
|
||||
const byColumn = new Map<string, ColumnSourceView[]>();
|
||||
for (const source of await this.store.sources(workspaceId)) {
|
||||
const list = byColumn.get(source.columnId);
|
||||
if (list === undefined) byColumn.set(source.columnId, [source]);
|
||||
else list.push(source);
|
||||
}
|
||||
return records.map((record) => ({ ...record, sources: byColumn.get(record.id) ?? [] }));
|
||||
}
|
||||
|
||||
private async view(record: ColumnRecord, sources: ColumnSourceView[]): Promise<ColumnResponse> {
|
||||
return { ...record, sources };
|
||||
}
|
||||
|
||||
private async connectedProviders(session: SessionPayload): Promise<string[]> {
|
||||
const rows = await this.feeds.capableConnections(session.workspaceId);
|
||||
return [...new Set(rows.map((row) => row.provider))];
|
||||
}
|
||||
|
||||
/** Providers whose manifest declares the P0b search capability. */
|
||||
private searchCapableProviders(providers: string[]): string[] {
|
||||
return providers.filter((provider) => {
|
||||
if (!this.registry.has(provider)) return false;
|
||||
return this.registry.get(provider).manifest().search === true;
|
||||
});
|
||||
}
|
||||
|
||||
private async forceDue(workspaceId: string, columnId: string): Promise<number> {
|
||||
return this.scheduler.forceColumnDue(workspaceId, columnId);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Request parsing (unknown body in, typed DTO out; 400 on shape errors)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function parseCreate(body: unknown): CreateColumn {
|
||||
const payload = requireObject(body, "body");
|
||||
const type = requireString(payload.type, "type");
|
||||
if (type !== "platform_feed" && type !== "keyword" && type !== "account_list" && type !== "saved_search") {
|
||||
throw new BadRequestException("type must be one of platform_feed|keyword|account_list|saved_search");
|
||||
}
|
||||
const title = requireString(payload.title, "title");
|
||||
const position = payload.position === undefined ? undefined : requirePosition(payload.position);
|
||||
return {
|
||||
type,
|
||||
title,
|
||||
position,
|
||||
spec: payload.spec === undefined ? {} : payload.spec,
|
||||
refreshPolicy: payload.refreshPolicy,
|
||||
};
|
||||
}
|
||||
|
||||
function parseUpdate(body: unknown): UpdateColumn {
|
||||
const payload = requireObject(body, "body");
|
||||
const update: UpdateColumn = {};
|
||||
if (payload.title !== undefined) update.title = requireString(payload.title, "title");
|
||||
if (payload.position !== undefined) update.position = requirePosition(payload.position);
|
||||
if (payload.spec !== undefined) update.spec = payload.spec;
|
||||
if (payload.refreshPolicy !== undefined) {
|
||||
if (typeof payload.refreshPolicy !== "object" || payload.refreshPolicy === null || Array.isArray(payload.refreshPolicy)) {
|
||||
throw new BadRequestException("refreshPolicy must be an object");
|
||||
}
|
||||
update.refreshPolicy = payload.refreshPolicy;
|
||||
}
|
||||
return update;
|
||||
}
|
||||
|
||||
function requirePosition(value: unknown): number {
|
||||
if (typeof value !== "number" || !Number.isInteger(value) || value < 0 || value > 2_147_483_647) {
|
||||
throw new BadRequestException("position must be an integer between 0 and 2147483647");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseIds(value: unknown): string[] {
|
||||
if (!Array.isArray(value) || value.length === 0) throw new BadRequestException("ids must be a non-empty array of column ids");
|
||||
const ids: string[] = [];
|
||||
for (const id of value) {
|
||||
if (typeof id !== "string") throw new BadRequestException("ids must contain strings only");
|
||||
ids.push(parseId(id));
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
/**
|
||||
* The keyset cursor: base64url JSON `{ at, key }` where `at` is the item's
|
||||
* created_at_remote epoch ms and `key` its dedup_key. Mirrors the /feeds
|
||||
* cursor contract (opaque, ≤4096 chars, position in a total order).
|
||||
*/
|
||||
export function decodeColumnCursor(cursor: string): { at: number; key: string } {
|
||||
try {
|
||||
if (cursor.length > 4096) throw new Error();
|
||||
const parsed: unknown = JSON.parse(Buffer.from(cursor, "base64url").toString());
|
||||
if (typeof parsed !== "object" || parsed === null || !("at" in parsed) || !("key" in parsed) ||
|
||||
typeof (parsed as { at: unknown }).at !== "number" || !Number.isFinite((parsed as { at: number }).at) ||
|
||||
typeof (parsed as { key: unknown }).key !== "string") throw new Error();
|
||||
return { at: (parsed as { at: number }).at, key: (parsed as { key: string }).key };
|
||||
} catch {
|
||||
throw new BadRequestException("Invalid column cursor");
|
||||
}
|
||||
}
|
||||
165
apps/api/src/feeds/columns.dependencies.ts
Normal file
165
apps/api/src/feeds/columns.dependencies.ts
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
/**
|
||||
* Columns wiring (P1): production dependency providers for the ColumnStore
|
||||
* and ColumnScheduler, mirroring feeds.dependencies.ts.
|
||||
*
|
||||
* The scheduler's per-host budget draws from `@tactical-media-broadcast-hub/egress`
|
||||
* HostBudgets — the SAME `host_budgets` rows every other egress caller uses
|
||||
* (monitor, bulk import), keyed by the egress `hostBudgetKey` rule (IDNA
|
||||
* lowercased host + effective port), so feeds polls and monitor polls to one
|
||||
* host contend for one shared bucket. That is the §3.4 requirement: budgets
|
||||
* are per-host totals, never per-process.
|
||||
*
|
||||
* The egress package is type-imported and loaded natively at runtime (the
|
||||
* same pattern as live-monitor/poller/egress.ts and virtual-channels/
|
||||
* import-egress.ts) because it is ESM-only.
|
||||
*/
|
||||
import { Inject, Injectable, Optional } from "@nestjs/common";
|
||||
import type { ConnectorConnection } from "@tactical-media-broadcast-hub/core";
|
||||
import type { CredentialAccessor, ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import { existsSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { pool } from "../db/client.js";
|
||||
import { PgConnectionStore } from "../connections/connection-store.js";
|
||||
import { RespControlRedis, type ControlRedis } from "../media-control/redis.js";
|
||||
import { vault } from "../vault/vault.js";
|
||||
import { feedRegistry } from "./feed-registry.js";
|
||||
import { FeedsService } from "./feeds.service.js";
|
||||
import {
|
||||
ColumnStore, COLUMN_STORE_DEPENDENCIES, type ColumnStoreDependencies,
|
||||
} from "./column-store.js";
|
||||
import {
|
||||
ColumnScheduler, COLUMN_SCHEDULER_DEPENDENCIES, type ColumnSchedulerDependencies,
|
||||
} from "./column-scheduler.js";
|
||||
import { COLUMNS_REGISTRY } from "./columns.controller.js";
|
||||
|
||||
type EgressPackage = typeof import("../../../../packages/egress/src/index.js");
|
||||
|
||||
const nativeImport = new Function("specifier", "return import(specifier)") as (specifier: string) => Promise<EgressPackage>;
|
||||
|
||||
function resolveEgressPackage(): string {
|
||||
for (const start of [process.cwd(), __dirname]) {
|
||||
for (let root = start; ; root = dirname(root)) {
|
||||
const directory = resolve(root, "packages/egress");
|
||||
const files = process.env.NODE_ENV === "production" ? ["dist/index.js"] : ["src/index.ts", "dist/index.js"];
|
||||
for (const file of files) {
|
||||
const candidate = resolve(directory, file);
|
||||
if (existsSync(candidate)) return candidate;
|
||||
}
|
||||
if (dirname(root) === root) break;
|
||||
}
|
||||
}
|
||||
throw new Error("Build packages/egress before starting the columns scheduler");
|
||||
}
|
||||
|
||||
/** Process-wide egress binding shared by the columns scheduler. */
|
||||
export interface ColumnsEgress {
|
||||
consume(hostKey: string): Promise<void>;
|
||||
}
|
||||
|
||||
let sharedEgress: Promise<ColumnsEgress> | undefined;
|
||||
|
||||
export function columnsEgress(): Promise<ColumnsEgress> {
|
||||
sharedEgress ??= (async () => {
|
||||
const specifier = pathToFileURL(resolveEgressPackage()).href;
|
||||
const pkg = await nativeImport(specifier);
|
||||
const budgets = new pkg.HostBudgets({ pool, hostClasses: { feeds: { capacity: 3, refillPerInterval: 10 } } });
|
||||
return { consume: (hostKey) => budgets.consume(hostKey, "feeds") };
|
||||
})();
|
||||
return sharedEgress;
|
||||
}
|
||||
|
||||
/** The API host key for a provider's feed calls, per the egress hostBudgetKey rule. */
|
||||
export function feedHostKey(provider: string, connection: ConnectorConnection): string {
|
||||
const raw = typeof connection.fields.instanceUrl === "string" && connection.fields.instanceUrl !== ""
|
||||
? connection.fields.instanceUrl
|
||||
: providerApiBaseUrl(provider);
|
||||
try {
|
||||
const url = new URL(raw);
|
||||
const host = url.hostname.replace(/^\[|\]$/g, "").replace(/\.$/, "").toLowerCase();
|
||||
const port = url.port || (url.protocol === "https:" ? "443" : "80");
|
||||
return `${host}:${port}`;
|
||||
} catch {
|
||||
return `${provider}:443`;
|
||||
}
|
||||
}
|
||||
|
||||
/** Fallback API endpoints per provider (used only when no instanceUrl exists). */
|
||||
const PROVIDER_API_HOSTS: Record<string, string> = {
|
||||
mastodon: "https://mastodon.social",
|
||||
bluesky: "https://bsky.app",
|
||||
youtube: "https://www.googleapis.com",
|
||||
reddit: "https://www.reddit.com",
|
||||
telegram: "https://api.telegram.org",
|
||||
twitter: "https://api.twitter.com",
|
||||
"mock-social": "https://mock.social",
|
||||
};
|
||||
|
||||
function providerApiBaseUrl(provider: string): string {
|
||||
return PROVIDER_API_HOSTS[provider] ?? `https://${provider}.example`;
|
||||
}
|
||||
|
||||
/** The credential accessor feedRegistry needs (shared with feeds.dependencies). */
|
||||
function credentials(): CredentialAccessor {
|
||||
const store = new PgConnectionStore();
|
||||
return {
|
||||
async getAccessToken(connection) { return credentials().getSecret(connection, "access_token"); },
|
||||
async getSecret(connection, key) {
|
||||
const stored = await store.get(connection.workspaceId, connection.id);
|
||||
if (!stored.credential) throw new Error("Credential missing");
|
||||
const bytes = await vault.open(connection.workspaceId, connection.id, stored.credential);
|
||||
const secrets = JSON.parse(bytes.toString()) as Record<string, unknown>;
|
||||
const value = secrets[key];
|
||||
if (typeof value !== "string" || !value) throw new Error("Credential missing");
|
||||
return value;
|
||||
},
|
||||
async put(connection, key, value) {
|
||||
const stored = await store.get(connection.workspaceId, connection.id);
|
||||
if (!stored.credential) throw new Error("Credential missing");
|
||||
const bytes = await vault.open(connection.workspaceId, connection.id, stored.credential);
|
||||
const secrets = JSON.parse(bytes.toString()) as Record<string, unknown>;
|
||||
secrets[key] = value;
|
||||
const sealed = await vault.seal(connection.workspaceId, connection.id, Buffer.from(JSON.stringify(secrets)));
|
||||
await store.rotateCredential(connection.workspaceId, connection.id, stored.credential, sealed);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
let sharedRegistry: ConnectorRegistry | undefined;
|
||||
|
||||
/** One registry per process, preserving per-host HTTP token buckets. */
|
||||
export function columnsRegistry(): ConnectorRegistry {
|
||||
sharedRegistry ??= feedRegistry(credentials());
|
||||
return sharedRegistry;
|
||||
}
|
||||
|
||||
/** Provider wiring for FeedsModule (see feeds.module.ts). */
|
||||
export const columnsProviders = [
|
||||
{ provide: COLUMN_STORE_DEPENDENCIES, useFactory: (): ColumnStoreDependencies => {
|
||||
const store = new PgConnectionStore();
|
||||
return {
|
||||
providerOf: async (workspaceId, connectionId) => {
|
||||
const rows = await store.list(workspaceId);
|
||||
const row = rows.find((candidate) => candidate.id === connectionId);
|
||||
if (row === undefined) throw new Error(`connection ${connectionId} not found`);
|
||||
return row.provider;
|
||||
},
|
||||
connections: (workspaceId) => store.list(workspaceId),
|
||||
};
|
||||
} },
|
||||
{ provide: COLUMNS_REGISTRY, useFactory: () => columnsRegistry() },
|
||||
{ provide: COLUMN_SCHEDULER_DEPENDENCIES, inject: [FeedsService], useFactory: (feeds: FeedsService): ColumnSchedulerDependencies => {
|
||||
const redis: Pick<ControlRedis, "get" | "set" | "eval"> = new RespControlRedis(process.env.REDIS_URL ?? "redis://127.0.0.1:6379");
|
||||
const connectionStore = new PgConnectionStore();
|
||||
return {
|
||||
redis,
|
||||
registry: columnsRegistry(),
|
||||
connection: async (workspaceId, id) => (await connectionStore.get(workspaceId, id)).connection,
|
||||
cacheHomeFeed: (workspaceId, connectionId, items) => feeds.cacheHomeFeed(workspaceId, connectionId, items),
|
||||
hostKeyOf: feedHostKey,
|
||||
consumeBudget: (hostKey) => columnsEgress().then((egress) => egress.consume(hostKey)),
|
||||
random: Math.random,
|
||||
now: Date.now,
|
||||
};
|
||||
} },
|
||||
];
|
||||
376
apps/api/src/feeds/columns.integration.test.ts
Normal file
376
apps/api/src/feeds/columns.integration.test.ts
Normal file
|
|
@ -0,0 +1,376 @@
|
|||
/**
|
||||
* Columns P1 integration tests — real Postgres (migrations ≤ 026 applied),
|
||||
* RLS-faithful via the app pool, following the video.integration.test pattern.
|
||||
*
|
||||
* Covers the DELIVER test list:
|
||||
* - CRUD + reorder, including RBAC 403 for an ungranted role
|
||||
* - seed idempotency (re-touch creates no duplicates)
|
||||
* - cooldown enforcement on the column endpoints
|
||||
* - cursor survival across a cache flush
|
||||
* - keyset items pagination (stable order, no skips/dupes across pages)
|
||||
* - dedup: same remote item from two columns sharing an identity → one row,
|
||||
* visible in both
|
||||
*/
|
||||
import "reflect-metadata";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { ForbiddenException, type CallHandler, type ExecutionContext, type INestApplication } from "@nestjs/common";
|
||||
import { NestFactory, Reflector } from "@nestjs/core";
|
||||
import { Pool } from "pg";
|
||||
import { afterAll, beforeAll, afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { InMemoryCredentials, type ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import type { ConnectorConnection, FeedItem } from "@tactical-media-broadcast-hub/core";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import { PERMISSION_KEY, RbacGuard } from "../rbac/rbac.guard.js";
|
||||
import { injectHttp } from "../connections/http-test-transport.js";
|
||||
import { feedRegistry } from "./feed-registry.js";
|
||||
import { FeedsService } from "./feeds.service.js";
|
||||
import { FeedsController } from "./feeds.controller.js";
|
||||
import { ColumnsController, COLUMNS_REGISTRY } from "./columns.controller.js";
|
||||
import { ColumnStore } from "./column-store.js";
|
||||
import { ColumnScheduler, type ColumnSchedulerDependencies } from "./column-scheduler.js";
|
||||
import { MemoryFeedRedis } from "./testing.js";
|
||||
import type { ConnectionSummary } from "../connections/connections.service.js";
|
||||
|
||||
const dbAvailable = process.env.DATABASE_URL !== undefined;
|
||||
// Setup runs as the table owner (superuser in dev) because memberships has
|
||||
// FORCE RLS and the seed path needs to insert without a tenant context.
|
||||
const ownerPool = new Pool({ connectionString: process.env.SUPER_DATABASE_URL ?? process.env.DATABASE_URL });
|
||||
|
||||
// The app pool is imported lazily where needed; ColumnStore's own fallback
|
||||
// deps use withTenant from db/client, but these tests always inject explicit
|
||||
// deps, so no module-scope side-effectful import is required.
|
||||
|
||||
const owner: SessionPayload = { userId: randomUUID(), workspaceId: randomUUID(), role: "Owner", authzEpoch: 1, issuedAt: 0, expiresAt: 9_999_999_999 };
|
||||
const viewer: SessionPayload = { ...owner, userId: randomUUID(), role: "Moderator" }; // no streams.columns.manage
|
||||
|
||||
const item = (remoteId: string, minutesAgo: number): FeedItem => ({
|
||||
id: remoteId, remoteId, provider: "mock-social", authorHandle: "alice", text: `post ${remoteId}`,
|
||||
mediaUrls: [], createdAt: new Date(Date.now() - minutesAgo * 60_000), metrics: {},
|
||||
});
|
||||
|
||||
function setupRegistry(items: FeedItem[]): ConnectorRegistry {
|
||||
const registry = feedRegistry(new InMemoryCredentials());
|
||||
const mock = registry.get("mock-social");
|
||||
const original = mock.getFeed;
|
||||
if (original === undefined) throw new Error("mock-social has no feed");
|
||||
mock.getFeed = vi.fn(async () => ({ items }));
|
||||
return registry;
|
||||
}
|
||||
|
||||
describe.skipIf(!dbAvailable)("columns P1 (live DB)", () => {
|
||||
let app: INestApplication;
|
||||
let store: ColumnStore;
|
||||
let scheduler: ColumnScheduler;
|
||||
let feeds: FeedsService;
|
||||
let redis: MemoryFeedRedis;
|
||||
let connections: Array<{ summary: ConnectionSummary; connection: ConnectorConnection }>;
|
||||
let items: FeedItem[];
|
||||
let granted = true;
|
||||
let registry: ConnectorRegistry;
|
||||
|
||||
beforeAll(async () => {
|
||||
// Users first (ids are DB-generated and captured for memberships), then
|
||||
// workspace + members, in the owner pool which bypasses RLS for setup.
|
||||
const inserted = await ownerPool.query<{ id: string }>(
|
||||
`INSERT INTO users (external_id, email, display_name, username)
|
||||
VALUES ($1,$2,'Columns P1 Owner',$3) RETURNING id`, [`u-${randomUUID()}`, `o-${randomUUID()}@example.com`, `p1o${randomUUID().slice(0, 8)}`]);
|
||||
(owner as { userId: string }).userId = String(inserted.rows[0]?.id);
|
||||
const viewerRow = await ownerPool.query<{ id: string }>(
|
||||
`INSERT INTO users (external_id, email, display_name, username)
|
||||
VALUES ($1,$2,'Columns P1 Viewer',$3) RETURNING id`, [`u-${randomUUID()}`, `v-${randomUUID()}@example.com`, `p1v${randomUUID().slice(0, 8)}`]);
|
||||
(viewer as { userId: string }).userId = String(viewerRow.rows[0]?.id);
|
||||
const workspace = await ownerPool.query(
|
||||
"INSERT INTO workspaces (name, slug) VALUES ('columns p1', $1) RETURNING id", [`cols-${randomUUID()}`]);
|
||||
const workspaceId = String(workspace.rows[0]?.id);
|
||||
// memberships is FORCE RLS: insert inside the workspace's tenant context.
|
||||
const setupClient = await ownerPool.connect();
|
||||
try {
|
||||
await setupClient.query("BEGIN");
|
||||
await setupClient.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]);
|
||||
await setupClient.query(
|
||||
"INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1,$2,'Owner'), ($1,$3,'Moderator')",
|
||||
[workspaceId, owner.userId, viewer.userId]);
|
||||
await setupClient.query("COMMIT");
|
||||
} finally {
|
||||
setupClient.release();
|
||||
}
|
||||
(owner as { workspaceId: string }).workspaceId = workspaceId;
|
||||
(viewer as { workspaceId: string }).workspaceId = workspaceId;
|
||||
|
||||
items = [item("i-1", 1), item("i-2", 5), item("i-3", 10), item("i-4", 30), item("i-5", 60)];
|
||||
registry = setupRegistry(items);
|
||||
|
||||
// Two feed-capable connections in the workspace: the seeding input.
|
||||
connections = [];
|
||||
for (let index = 0; index < 2; index += 1) {
|
||||
// The DB generates the id (connections has no id column we may set);
|
||||
// RETURNING it keeps the FK target and the summary in sync.
|
||||
const inserted = await ownerPool.query<{ id: string; created_at: Date }>(
|
||||
"INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1,'mock-social',$2) RETURNING id, created_at", [owner.workspaceId, `ch-${index}`]);
|
||||
const id = String(inserted.rows[0]?.id);
|
||||
const createdAt = inserted.rows[0]?.created_at as Date;
|
||||
const summary: ConnectionSummary = {
|
||||
id, provider: "mock-social", channelId: `ch-${index}`, fields: {}, health: "healthy",
|
||||
credentialRotatedAt: null, recentPublishes: [], createdAt: createdAt.toISOString(),
|
||||
};
|
||||
const connection: ConnectorConnection = {
|
||||
id, workspaceId: owner.workspaceId, provider: "mock-social", channelId: `ch-${index}`,
|
||||
credentialId: { kind: "credential-ref", id }, fields: {}, createdAt,
|
||||
};
|
||||
connections.push({ summary, connection });
|
||||
}
|
||||
|
||||
redis = new MemoryFeedRedis();
|
||||
store = new ColumnStore({
|
||||
providerOf: async (_ws, id) => connections.find((row) => row.summary.id === id)?.summary.provider ?? "mock-social",
|
||||
connections: async (ws) => ws === owner.workspaceId ? connections.map((row) => row.summary) : [],
|
||||
});
|
||||
feeds = new FeedsService({
|
||||
redis, registry, random: () => 0.5, requestsPerSecond: () => 0.5,
|
||||
workspaces: async () => [owner.workspaceId],
|
||||
connections: async (ws) => ws === owner.workspaceId ? connections.map((row) => row.summary) : [],
|
||||
connection: async (ws, id) => { const row = connections.find((c) => c.summary.id === id); if (!row || ws !== owner.workspaceId) throw new Error(); return row.connection; },
|
||||
visible: async (s) => s.workspaceId === owner.workspaceId ? connections.map((row) => row.summary.id) : [],
|
||||
audit: async () => undefined,
|
||||
}, store);
|
||||
scheduler = new ColumnScheduler(store, {
|
||||
redis, registry,
|
||||
connection: async (ws, id) => { const row = connections.find((c) => row_summary(c) === id); if (!row || ws !== owner.workspaceId) throw new Error(); return row.connection; },
|
||||
cacheHomeFeed: async (ws, id, normalized) => { if (ws === owner.workspaceId) await feeds.cacheHomeFeed(ws, id, normalized); },
|
||||
hostKeyOf: () => "mock.social:443",
|
||||
consumeBudget: async () => undefined,
|
||||
random: () => 0.5,
|
||||
now: () => Date.now(),
|
||||
});
|
||||
|
||||
// Both poll loops stay off: the feeds discovery timer would contend with
|
||||
// ensureSeeded's workspace row lock (this suite drives polls explicitly).
|
||||
vi.spyOn(scheduler, "onApplicationBootstrap").mockImplementation(() => undefined);
|
||||
vi.spyOn(feeds, "onApplicationBootstrap").mockImplementation(() => undefined);
|
||||
vi.spyOn(RbacGuard.prototype, "canActivate").mockImplementation(async (context: ExecutionContext) => {
|
||||
const request = context.switchToHttp().getRequest<{ headers: Record<string, string>; session?: SessionPayload }>();
|
||||
if (request.headers.authorization !== "Bearer test-session") throw new ForbiddenException("unauthenticated");
|
||||
const session = request.headers["x-test-user"] === "viewer" ? viewer : owner;
|
||||
const permission = new Reflector().get<string>(PERMISSION_KEY, context.getHandler());
|
||||
// The seeded RBAC: streams.columns.manage is Owner/Admin/Editor only.
|
||||
if (permission === "streams.columns.manage" && !granted) throw new ForbiddenException("missing permission");
|
||||
request.session = session;
|
||||
return true;
|
||||
});
|
||||
app = await NestFactory.create({
|
||||
module: class TestModule {},
|
||||
controllers: [ColumnsController, FeedsController],
|
||||
providers: [
|
||||
{ provide: FeedsService, useValue: feeds },
|
||||
{ provide: ColumnStore, useValue: store },
|
||||
{ provide: ColumnScheduler, useValue: scheduler },
|
||||
{ provide: COLUMNS_REGISTRY, useValue: registry },
|
||||
],
|
||||
}, { logger: false });
|
||||
await app.init();
|
||||
});
|
||||
|
||||
afterEach(() => { granted = true; });
|
||||
|
||||
afterAll(async () => {
|
||||
await app?.close();
|
||||
if (owner.workspaceId !== undefined) await ownerPool.query("DELETE FROM workspaces WHERE id = $1", [owner.workspaceId]);
|
||||
await ownerPool.end();
|
||||
});
|
||||
|
||||
const request = (path: string, method = "GET", body?: unknown, user: "owner" | "viewer" = "owner") =>
|
||||
injectHttp(app.getHttpAdapter().getInstance(), `http://api.test${path}`, {
|
||||
method, headers: { authorization: "Bearer test-session", "x-test-user": user, "content-type": "application/json" },
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
});
|
||||
|
||||
it("seeds one platform_feed column per feed connection, idempotently, with sources", async () => {
|
||||
const first = await request("/feeds/columns");
|
||||
expect(first.status).toBe(200);
|
||||
const seeded = await first.json() as Array<{ id: string; type: string; title: string; sources: Array<{ sourceKind: string; state: string }> }>;
|
||||
expect(seeded).toHaveLength(2);
|
||||
expect(seeded.every((column) => column.type === "platform_feed")).toBe(true);
|
||||
expect(seeded[0]?.sources).toHaveLength(1);
|
||||
expect(seeded[0]?.sources[0]?.sourceKind).toBe("home");
|
||||
// Titles reproduce the implicit column the UI renders today.
|
||||
expect(seeded[0]?.title).toContain("·");
|
||||
|
||||
// Second touch seeds nothing.
|
||||
const second = await request("/feeds/columns");
|
||||
expect(await second.json()).toHaveLength(2);
|
||||
// And the DB agrees: two columns, two source rows, one per connection.
|
||||
const rows = await ownerPool.query("SELECT count(*)::int AS c FROM stream_columns WHERE workspace_id = $1", [owner.workspaceId]);
|
||||
expect(rows.rows[0]?.c).toBe(2);
|
||||
const sources = await ownerPool.query("SELECT count(*)::int AS c FROM column_sources WHERE workspace_id = $1", [owner.workspaceId]);
|
||||
expect(sources.rows[0]?.c).toBe(2);
|
||||
});
|
||||
|
||||
it("enforces streams.columns.manage on writes while reads stay on draft.view", async () => {
|
||||
granted = false;
|
||||
expect((await request("/feeds/columns", "POST", { type: "keyword", title: "x", spec: { query: "q", platforms: ["mock-social"] } }, "viewer")).status).toBe(403);
|
||||
granted = true;
|
||||
const created = await request("/feeds/columns", "POST", { type: "keyword", title: "news", spec: { query: "anarch", platforms: ["mock-social"] } });
|
||||
expect(created.status).toBe(201);
|
||||
const column = await created.json() as { id: string; sources: Array<{ sourceKind: string }> };
|
||||
expect(column.sources.map((source) => source.sourceKind)).toEqual(["search"]);
|
||||
// Reads: viewer (no manage permission) still reads columns and items.
|
||||
expect((await request(`/feeds/columns/${column.id}`, "GET", undefined, "viewer")).status).toBe(200);
|
||||
granted = false;
|
||||
expect((await request(`/feeds/columns/${column.id}`, "PATCH", { title: "z" })).status).toBe(403);
|
||||
expect((await request(`/feeds/columns/${column.id}`, "DELETE")).status).toBe(403);
|
||||
expect((await request("/feeds/columns/reorder", "POST", { ids: [] })).status).toBe(403);
|
||||
granted = true;
|
||||
await request(`/feeds/columns/${column.id}`, "DELETE");
|
||||
});
|
||||
|
||||
it("rejects invalid specs with 422 and never stores them", async () => {
|
||||
const bad = await request("/feeds/columns", "POST", { type: "keyword", title: "bad", spec: { query: "", platforms: ["mock-social"] } });
|
||||
expect(bad.status).toBe(422);
|
||||
const badConnection = await request("/feeds/columns", "POST", { type: "platform_feed", title: "bad", spec: { connectionId: randomUUID() } });
|
||||
expect(badConnection.status).toBe(422);
|
||||
const count = await ownerPool.query(
|
||||
"SELECT count(*)::int AS c FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL", [owner.workspaceId]);
|
||||
expect(count.rows[0]?.c).toBe(2); // only the two seeded columns (the RBAC test's column was soft-deleted)
|
||||
});
|
||||
|
||||
it("polls due sources, persists items, and fans out to columns sharing an identity", async () => {
|
||||
const columns = await (await request("/feeds/columns")).json() as Array<{ id: string; sources: Array<{ id: string }> }>;
|
||||
const first = columns[0]!;
|
||||
const outcome = await scheduler.pollWorkspaceDue(owner.workspaceId);
|
||||
expect(outcome.polled).toBeGreaterThanOrEqual(2); // both seeded home sources
|
||||
|
||||
// Items landed in the store for the polled column.
|
||||
const page = await store.columnItems(owner.workspaceId, first.id, { limit: 10 });
|
||||
expect(page.items.length).toBeGreaterThanOrEqual(1);
|
||||
expect(page.items.every((stored) => stored.item.remoteId.startsWith("i-"))).toBe(true);
|
||||
// The source row moved to ready with a next poll scheduled.
|
||||
const sources = await store.sources(owner.workspaceId, first.id);
|
||||
expect(sources[0]?.state).toBe("ready");
|
||||
expect(sources[0]?.nextPollAt).not.toBeNull();
|
||||
});
|
||||
|
||||
it("keyset pagination: pages in stable order, no skips or duplicates, cursor survives cache flush", async () => {
|
||||
const columns = await (await request("/feeds/columns")).json() as Array<{ id: string }>;
|
||||
const columnId = columns[0]!.id;
|
||||
const seen: string[] = [];
|
||||
let cursor: string | null = null;
|
||||
// limit 2 over 5 items = 3 pages (2,2,1)
|
||||
for (let page = 0; page < 3; page += 1) {
|
||||
const response = await request(`/feeds/columns/${columnId}/items?limit=2${cursor === null ? "" : `&cursor=${cursor}`}`);
|
||||
expect(response.status).toBe(200);
|
||||
const body = await response.json() as { items: Array<{ item: { remoteId: string } }>; nextCursor: string | null };
|
||||
for (const stored of body.items) seen.push(stored.item.remoteId);
|
||||
cursor = body.nextCursor;
|
||||
if (cursor === null) break;
|
||||
}
|
||||
expect(seen.length).toBe(new Set(seen).size); // no duplicates
|
||||
expect(seen).toHaveLength(items.length); // no skips
|
||||
|
||||
// Cursor survival: flush every cache key, then continue from a cursor
|
||||
// minted before the flush — the store page must pick up exactly where the
|
||||
// cache left off because the cursor is a position in the stored order.
|
||||
redis.entries.clear();
|
||||
const beforeFlush = await request(`/feeds/columns/${columnId}/items?limit=1`);
|
||||
const firstCursor = ((await beforeFlush.json()) as { nextCursor: string | null }).nextCursor;
|
||||
expect(firstCursor).not.toBeNull();
|
||||
redis.entries.clear();
|
||||
const afterFlush = await request(`/feeds/columns/${columnId}/items?limit=10&cursor=${firstCursor}`);
|
||||
expect(afterFlush.status).toBe(200);
|
||||
const pageItems = (await afterFlush.json()) as { items: Array<{ item: { remoteId: string } }> };
|
||||
// Everything strictly after the first item, in keyset order.
|
||||
expect(pageItems.items.map((stored) => stored.item.remoteId)).toEqual(seen.slice(1));
|
||||
});
|
||||
|
||||
it("cooldown blocks a forced refresh from issuing a second poll", async () => {
|
||||
const columns = await (await request("/feeds/columns")).json() as Array<{ id: string }>;
|
||||
const columnId = columns[0]!.id;
|
||||
const before = registry.get("mock-social").getFeed as ReturnType<typeof vi.fn>;
|
||||
const callsBefore = before.mock.calls.length;
|
||||
const outcome = await scheduler.pollWorkspaceDue(owner.workspaceId);
|
||||
expect(outcome.polled).toBe(0); // sources just polled: next_poll_at is in the future
|
||||
// The HTTP refresh endpoint cannot bypass the persisted floor: the poll
|
||||
// happened seconds ago, so force-due refuses even though the Redis
|
||||
// cooldown keys were flushed (cursor-survival test) — next_poll_at is the
|
||||
// authoritative guard, not the cache.
|
||||
const refreshed = await request(`/feeds/columns/${columnId}/refresh`, "POST");
|
||||
expect(refreshed.status).toBe(201);
|
||||
const body = await refreshed.json() as { refreshed: number };
|
||||
expect(body.refreshed).toBe(0); // DB floor refused the forced poll
|
||||
expect(before.mock.calls.length).toBe(callsBefore);
|
||||
});
|
||||
|
||||
it("dedups the same remote item across columns sharing an identity", async () => {
|
||||
// Second column on the same connection as the first: same identity.
|
||||
const columns = await (await request("/feeds/columns")).json() as Array<{ id: string; sources: Array<{ id: string }> }>;
|
||||
const [first, second] = columns;
|
||||
expect(first && second).toBeTruthy();
|
||||
// Make both due again, then poll: one identity, one connector call.
|
||||
await ownerPool.query("UPDATE column_sources SET next_poll_at = now() WHERE workspace_id = $1", [owner.workspaceId]);
|
||||
const getFeed = registry.get("mock-social").getFeed as ReturnType<typeof vi.fn>;
|
||||
const before = getFeed.mock.calls.length;
|
||||
const outcome = await scheduler.pollWorkspaceDue(owner.workspaceId);
|
||||
expect(outcome.polled).toBe(2);
|
||||
expect(getFeed.mock.calls.length).toBe(before + 2); // two source rows, two calls (per-source semantics)
|
||||
// One feed_items row per remote item — the dedup key is workspace-wide.
|
||||
const stored = await ownerPool.query("SELECT count(*)::int AS c FROM feed_items WHERE workspace_id = $1", [owner.workspaceId]);
|
||||
expect(stored.rows[0]?.c).toBe(items.length);
|
||||
// Both columns surface the same items.
|
||||
const left = await store.columnItems(owner.workspaceId, first!.id, { limit: 10 });
|
||||
const right = await store.columnItems(owner.workspaceId, second!.id, { limit: 10 });
|
||||
expect(left.items).toHaveLength(items.length);
|
||||
expect(right.items).toHaveLength(items.length);
|
||||
// And a cursor minted on one column pages identically on the other.
|
||||
expect(left.items.map((row) => row.item.remoteId)).toEqual(right.items.map((row) => row.item.remoteId));
|
||||
});
|
||||
|
||||
it("reorder requires the full id list and rewrites positions densely", async () => {
|
||||
const columns = await (await request("/feeds/columns")).json() as Array<{ id: string; position: number }>;
|
||||
const ids = columns.map((column) => column.id);
|
||||
const partial = await request("/feeds/columns/reorder", "POST", { ids: [ids[0]!] });
|
||||
expect(partial.status).toBe(400);
|
||||
const reordered = await request("/feeds/columns/reorder", "POST", { ids: [...ids].reverse() });
|
||||
expect(reordered.status).toBe(201);
|
||||
const after = await reordered.json() as Array<{ id: string; position: number }>;
|
||||
expect(after.map((column) => column.id)).toEqual([...ids].reverse());
|
||||
expect(after.map((column) => column.position)).toEqual([0, 1]);
|
||||
});
|
||||
|
||||
it("import-saved-searches creates idempotent saved_search columns", async () => {
|
||||
// No search-capable provider is connected (mock-social declares none), so
|
||||
// the implicit-platforms path correctly skips everything rather than
|
||||
// inventing a platform the workspace cannot search.
|
||||
const skipped = await request("/feeds/columns/import-saved-searches", "POST", { queries: ["anarch"] });
|
||||
expect(skipped.status).toBe(201);
|
||||
expect(await skipped.json()).toEqual({ imported: 0, skipped: 1 });
|
||||
// Explicit platforms still import (the UI knows which it used locally).
|
||||
const imported = await request("/feeds/columns/import-saved-searches", "POST", { queries: ["anarch", "mutual aid"], platforms: ["mock-social"] });
|
||||
expect(imported.status).toBe(201);
|
||||
expect(await imported.json()).toEqual({ imported: 2, skipped: 0 });
|
||||
// Re-import: idempotent per query.
|
||||
const again = await request("/feeds/columns/import-saved-searches", "POST", { queries: ["anarch", "squatting"], platforms: ["mock-social"] });
|
||||
expect(await again.json()).toEqual({ imported: 1, skipped: 1 });
|
||||
const columns = await (await request("/feeds/columns")).json() as Array<{ type: string; title: string }>;
|
||||
const searches = columns.filter((column) => column.type === "saved_search");
|
||||
expect(searches.map((column) => column.title).sort()).toEqual(["anarch", "mutual aid", "squatting"]);
|
||||
// Unknown platform ids are rejected up front (P0a schema rules).
|
||||
const bad = await request("/feeds/columns/import-saved-searches", "POST", { queries: ["x"], platforms: ["not-a-provider"] });
|
||||
expect(bad.status).toBe(422);
|
||||
});
|
||||
|
||||
it("GET /feeds keeps its external shape and gains additive meta.columns", async () => {
|
||||
const response = await request("/feeds");
|
||||
expect(response.status).toBe(200);
|
||||
const feed = await response.json() as { items: unknown[]; nextCursor: string | null; meta: { configuredFeeds: number; columns?: Array<{ id: string }> } };
|
||||
expect(Array.isArray(feed.items)).toBe(true);
|
||||
expect(feed.meta.configuredFeeds).toBe(2);
|
||||
expect(feed.meta.columns?.length).toBeGreaterThanOrEqual(2); // seeded columns surface
|
||||
});
|
||||
|
||||
it("invalid cursor is a 400 and unknown column a 404", async () => {
|
||||
expect((await request(`/feeds/columns/${randomUUID()}/items`)).status).toBe(404);
|
||||
expect((await request("/feeds/columns/not-a-uuid/items")).status).toBe(400);
|
||||
expect((await request("/feeds/columns/00000000-0000-0000-0000-000000000000/items?cursor=garbage")).status).toBe(400);
|
||||
});
|
||||
});
|
||||
|
||||
function row_summary(row: { summary: ConnectionSummary }): string { return row.summary.id; }
|
||||
366
apps/api/src/feeds/feed-normalize.test.ts
Normal file
366
apps/api/src/feeds/feed-normalize.test.ts
Normal file
|
|
@ -0,0 +1,366 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import type { ConnectorConnection, FeedItem } from "@tactical-media-broadcast-hub/core";
|
||||
import { createMockSocialConnector, createMockSocialState } from "../../../../packages/connector-mock-social/src/index.js";
|
||||
import { DEFAULT_MEDIA_PROXY, parseProviderHtml, type CustomEmoji, type LinkPreview, type MediaProxyPolicy, type RichTextDocument } from "../../../../packages/safe-content/src/index.js";
|
||||
import { astToPlainText, feedAuthorRef, feedItemMedia, normalizeFeedItem, type MediaRef, type NormalizedFeedItem } from "./feed-normalize.js";
|
||||
import { FeedsService, feedCacheKey } from "./feeds.service.js";
|
||||
import type { FeedDependencies } from "./feeds.dependencies.js";
|
||||
import { MemoryFeedRedis } from "./testing.js";
|
||||
|
||||
/** Per-connector fixtures (design §3.2 provider table) + hostile-HTML cases. */
|
||||
interface ProviderFixture {
|
||||
provider: string;
|
||||
contentKind?: "text" | "html";
|
||||
instanceUrl?: string;
|
||||
emojis?: readonly CustomEmoji[];
|
||||
item: FeedItem;
|
||||
url?: unknown;
|
||||
expiresAt?: unknown;
|
||||
linkPreview?: unknown;
|
||||
}
|
||||
|
||||
const PROXY: MediaProxyPolicy = DEFAULT_MEDIA_PROXY;
|
||||
const connection = (fields: Record<string, string> = {}): ConnectorConnection => ({
|
||||
id: "conn-1", workspaceId: "ws-1", provider: "mastodon", channelId: "conn-1",
|
||||
credentialId: { kind: "credential-ref", id: "cred-1" }, fields, createdAt: new Date(),
|
||||
});
|
||||
|
||||
const mastodonStatus: FeedItem = {
|
||||
id: "1001", provider: "mastodon", remoteId: "1001", authorHandle: "alice@foreso.social",
|
||||
// The exact HTML shape Mastodon emits: p/br/a, mention + hashtag + invisible
|
||||
// ellipsis link spans, HTML-escaped ampersands, raw emoji <img> tags.
|
||||
text: '<p>A call for <span class="h-card"><a href="https://foreso.social/@bob" class="u-url mention">@<span>bob</span></a></span> & friends</p><p>#<a href="https://foreso.social/tags/acampada26" rel="tag">acampada26</a> kicks off <a href="https://example.org/manifesto" rel="nofollow noopener noreferrer noopener" target="_blank"><span class="invisible">https://</span><span class="ellipsis">example.org/manifes</span><span class="invisible">to</span></a> <img src="https://foreso.social/emoji/ok.png" alt=":ok:" draggable="false"><br>line two <b>bolded</b></p>',
|
||||
mediaUrls: ["https://cdn.foreso.social/media/1234.png", "https://cdn.foreso.social/media/clip.mp4"],
|
||||
createdAt: new Date("2026-10-06T12:00:00.000Z"),
|
||||
metrics: { reblogs: 2, replies: 1, favourites: 5, broken: Number.NaN, infinite: Number.POSITIVE_INFINITY },
|
||||
};
|
||||
const xssStatus: FeedItem = {
|
||||
id: "2002", provider: "mastodon", remoteId: "2002", authorHandle: "mallory@evil.example",
|
||||
text: '<p>hi <script>alert("pwned")</script><img src=x onerror="alert(1)"><iframe srcdoc="<script>alert(2)</script>"></iframe><style>body{}</style><svg><circle/></svg><a href="javascript:alert(3)">click</a><a href="https://ok.example/pass">pass</a><a href="java\tscript:alert(4)">tabbed</a></p><p>after</p>',
|
||||
mediaUrls: ["javascript:alert(5)", "data:text/html,evil", "//protocol-relative.example/x.png", "https://cdn.example/ok.png"],
|
||||
createdAt: new Date("2026-10-06T12:01:00.000Z"),
|
||||
metrics: {},
|
||||
};
|
||||
|
||||
const fixturePlainText: ProviderFixture = {
|
||||
// Plain-text provider source (Bluesky/Telegram-style): auto-linked, no HTML.
|
||||
provider: "bluesky",
|
||||
item: {
|
||||
id: "1", provider: "bluesky", remoteId: "at://did:plc:abc/app.bsky.feed.post/1",
|
||||
authorHandle: "acampada.bsky.social", mediaUrls: [], createdAt: new Date("2026-10-06T10:00:00.000Z"),
|
||||
metrics: { likes: 3 },
|
||||
text: "Line one\n\n hub https://github.com/eche/acampada26/blob/main/core/search.ts #acampada26 @alice@foreso.social",
|
||||
},
|
||||
url: "https://bsky.app/profile/acampada.bsky.social/post/1",
|
||||
};
|
||||
const fixtureMastodonHtml: ProviderFixture = {
|
||||
// Mastodon status HTML: allowlist parser, mention + ellipsis link + emoji.
|
||||
provider: "mastodon",
|
||||
contentKind: "html",
|
||||
emojis: [{ shortcode: "ok", url: "https://foreso.social/emoji/ok.png" }],
|
||||
item: mastodonStatus,
|
||||
url: "https://foreso.social/@alice/1001",
|
||||
};
|
||||
const fixtureHostileHtml: ProviderFixture = {
|
||||
// Hostile HTML from a compromised/bogus federated peer.
|
||||
provider: "mastodon",
|
||||
contentKind: "html",
|
||||
instanceUrl: "https://EVIL.example:443/",
|
||||
item: xssStatus,
|
||||
url: "javascript:alert(6)",
|
||||
linkPreview: { url: "javascript:alert(7)", title: "javascript:alert(8)", image: "javascript:alert(9)" },
|
||||
};
|
||||
const fixtureMastodonLinkPreview: ProviderFixture = {
|
||||
// Mastodon HTML + card (link-preview) payload as one source row.
|
||||
provider: "mastodon",
|
||||
contentKind: "html",
|
||||
item: { ...mastodonStatus, id: "1002", remoteId: "1002", text: '<p>with a <span class="h-card"><a href="https://foreso.social/@bob" class="u-url mention">@<span>bob</span></a></span> card</p>', mediaUrls: [] },
|
||||
url: "https://foreso.social/@alice/1002",
|
||||
linkPreview: {
|
||||
url: "https://example.org/manifesto", title: "Manifesto <scripts> & plans",
|
||||
description: "A plan", siteName: "example dot org", image: "https://cdn.example.org/cover.png",
|
||||
},
|
||||
};
|
||||
const fixtures: Record<"plainText" | "mastodonHtml" | "hostileHtml" | "mastodonLinkPreview", ProviderFixture> = {
|
||||
plainText: fixturePlainText,
|
||||
mastodonHtml: fixtureMastodonHtml,
|
||||
hostileHtml: fixtureHostileHtml,
|
||||
mastodonLinkPreview: fixtureMastodonLinkPreview,
|
||||
};
|
||||
|
||||
/** No provider CDN URL outside the media-proxy query string, ever. */
|
||||
function assertProxyOnly(media: MediaRef[]): void {
|
||||
const serialized = JSON.stringify(media);
|
||||
let index = serialized.indexOf("https://");
|
||||
while (index !== -1) {
|
||||
expect(serialized.slice(0, index)).toContain(`${PROXY.proxyPath}?url=`);
|
||||
index = serialized.indexOf("https://", index + 1);
|
||||
}
|
||||
for (const ref of media) expect(ref.proxyUrl.startsWith(`${PROXY.proxyPath}?url=`)).toBe(true);
|
||||
}
|
||||
|
||||
/** Raw-remnant scan applied to every fixture's AST, plain text and media. */
|
||||
function assertZeroRawRemnants(ast: RichTextDocument, plainText: string, media: MediaRef[]): void {
|
||||
const astJson = JSON.stringify(ast);
|
||||
const hostile: RegExp[] = [
|
||||
/<\s*(script|iframe|style|img|svg|math|template|form|object|embed|applet|frame)\b/i,
|
||||
/<\s*\/\s*/, /<!--/, /&#/, /&|<|>|"|'| /i,
|
||||
/javascript:/i, /vbscript:/i, /data:/i, /file:/i, /\/\/evil/, /\/\/protocol-relative/,
|
||||
/on(?:error|load|click|mouseover)\b|on\w+\s*=/i, /srcdoc|src\s*=|href\s*=/i, /alert\s*\(/i,
|
||||
];
|
||||
for (const haystack of [astJson, plainText, JSON.stringify(media)]) {
|
||||
for (const pattern of hostile) expect(haystack).not.toMatch(pattern);
|
||||
}
|
||||
expect(astJson).not.toContain("<p>");
|
||||
expect(astJson).not.toContain("<b>");
|
||||
assertProxyOnly(media);
|
||||
}
|
||||
|
||||
function normalizeFixture(fixture: ProviderFixture) {
|
||||
return normalizeFeedItem(fixture.item, { ...connection(fixture.instanceUrl ? { instanceUrl: fixture.instanceUrl } : {}), provider: fixture.provider }, {
|
||||
contentKind: fixture.contentKind,
|
||||
emojis: fixture.emojis,
|
||||
url: fixture.url,
|
||||
expiresAt: fixture.expiresAt,
|
||||
linkPreview: fixture.linkPreview,
|
||||
});
|
||||
}
|
||||
|
||||
const flat = (ast: RichTextDocument): string[] =>
|
||||
ast.children.map((paragraph) => paragraph.children.map((node) => node.type).join(","));
|
||||
|
||||
describe("feed normalization v2 (columns P0c)", () => {
|
||||
it("keeps legacy /feeds fields byte-identical while adding the v2 fields additively", () => {
|
||||
const normalized = normalizeFeedItem(mastodonStatus, connection(), { url: "https://foreso.social/@alice/1001" });
|
||||
expect(normalized).not.toBeNull();
|
||||
expect(normalized).toMatchObject({
|
||||
id: "1001", remoteId: "1001", provider: "mastodon", connectionId: "conn-1",
|
||||
authorHandle: "alice@foreso.social", text: mastodonStatus.text,
|
||||
mediaUrls: mastodonStatus.mediaUrls,
|
||||
metrics: { reblogs: 2, replies: 1, favourites: 5 }, // NaN/Infinity dropped
|
||||
createdAt: new Date("2026-10-06T12:00:00.000Z"),
|
||||
});
|
||||
expect(Object.keys(normalized?.metrics ?? {})).toEqual(["reblogs", "replies", "favourites"]);
|
||||
expect(normalized?.authorRef).toBe("mastodon:alice@foreso.social");
|
||||
expect(normalized?.url).toBe("https://foreso.social/@alice/1001");
|
||||
expect(normalized?.content?.type).toBe("doc");
|
||||
expect(typeof normalized?.plainText).toBe("string");
|
||||
expect(Array.isArray(normalized?.media)).toBe(true);
|
||||
expect(normalized?.expiresAt).toBeNull();
|
||||
});
|
||||
|
||||
it("normalizes the plain-text fixture with auto-linked http(s) URLs only", () => {
|
||||
const normalized = normalizeFixture(fixturePlainText);
|
||||
expect(normalized).not.toBeNull();
|
||||
assertZeroRawRemnants(normalized?.content as RichTextDocument, normalized?.plainText as string, normalized?.media as MediaRef[]);
|
||||
expect(flat(normalized?.content as RichTextDocument)).toEqual([
|
||||
"text",
|
||||
"text,link,text,hashtag,text,mention",
|
||||
]);
|
||||
const link = normalized?.content?.children[1]?.children.find((node) => node.type === "link");
|
||||
expect(link?.type === "link" && link.href).toBe("https://github.com/eche/acampada26/blob/main/core/search.ts");
|
||||
expect(link?.type === "link" && link.host).toBe("github.com");
|
||||
expect(normalized?.plainText).toBe("Line one\n\n hub github.com/eche/acampada26/blob/main/core/search.ts #acampada26 @alice@foreso.social");
|
||||
expect(normalized?.authorRef).toBe("bluesky:acampada.bsky.social");
|
||||
expect(normalized?.url).toBe("https://bsky.app/profile/acampada.bsky.social/post/1");
|
||||
expect(normalized?.media).toEqual([]);
|
||||
expect(normalized?.expiresAt).toBeNull();
|
||||
});
|
||||
|
||||
it("normalizes the Mastodon HTML fixture (mention, ellipsis link, emoji, breaks) with zero raw remnants", () => {
|
||||
const normalized = normalizeFixture(fixtureMastodonHtml);
|
||||
expect(normalized).not.toBeNull();
|
||||
const ast = normalized?.content as RichTextDocument;
|
||||
assertZeroRawRemnants(ast, normalized?.plainText as string, normalized?.media as MediaRef[]);
|
||||
expect(ast.truncated).toBe(false);
|
||||
// Invisible spans drop their URL scheme, the ellipsis span becomes "…",
|
||||
// the mention qualifies its handle from the href host, the emoji <img>
|
||||
// alt becomes a proxied emoji node, <b> unwraps to plain text.
|
||||
expect(normalized?.plainText).toBe("A call for @bob@foreso.social & friends\n\n#acampada26 kicks off example.org/manifes… :ok:\nline two bolded");
|
||||
expect(flat(ast)).toEqual([
|
||||
"text,mention,text",
|
||||
"text,link,text,link,text,emoji,linebreak,text",
|
||||
]);
|
||||
const mention = ast.children[0]?.children[1];
|
||||
expect(mention).toEqual({ type: "mention", handle: "bob@foreso.social", href: "https://foreso.social/@bob" });
|
||||
const link = ast.children[1]?.children.find((node) => node.type === "link" && node.href === "https://example.org/manifesto");
|
||||
expect(link?.type === "link" && link.host).toBe("example.org");
|
||||
expect(link?.type === "link" && link.children.map((child) => child.type === "text" ? child.text : null).join("")).toBe("example.org/manifes…");
|
||||
const emoji = ast.children[1]?.children.find((node) => node.type === "emoji");
|
||||
expect(emoji?.type === "emoji" && emoji.src).toBe(`${PROXY.proxyPath}?url=${encodeURIComponent("https://foreso.social/emoji/ok.png")}`);
|
||||
// Attachments become proxy media refs; kind is sniffed from the extension.
|
||||
expect(normalized?.media).toEqual([
|
||||
{ kind: "image", proxyUrl: `${PROXY.proxyPath}?url=${encodeURIComponent("https://cdn.foreso.social/media/1234.png")}` },
|
||||
{ kind: "video", proxyUrl: `${PROXY.proxyPath}?url=${encodeURIComponent("https://cdn.foreso.social/media/clip.mp4")}` },
|
||||
]);
|
||||
assertProxyOnly(normalized?.media as MediaRef[]);
|
||||
});
|
||||
|
||||
it("normalizes the hostile-HTML fixture: scripts, iframes, handlers and unsafe schemes leave no remnants", () => {
|
||||
const normalized = normalizeFixture(fixtureHostileHtml);
|
||||
expect(normalized).not.toBeNull();
|
||||
const ast = normalized?.content as RichTextDocument;
|
||||
assertZeroRawRemnants(ast, normalized?.plainText as string, normalized?.media as MediaRef[]);
|
||||
// Script/style/iframe/svg contents vanish entirely; img alt is empty; the
|
||||
// javascript: link degrades to its inert label; the tab-split "java\tscript:"
|
||||
// scheme is refused; only the one safe link survives as a real link node.
|
||||
expect(flat(ast)).toEqual(["text,link,text", "text"]);
|
||||
const onlyLink = ast.children[0]?.children[1];
|
||||
expect(onlyLink).toEqual({
|
||||
type: "link", href: "https://ok.example/pass", host: "ok.example",
|
||||
children: [{ type: "text", text: "pass" }],
|
||||
});
|
||||
expect(normalized?.plainText).toBe("hi clickpasstabbed\n\nafter");
|
||||
expect(normalized?.plainText).not.toMatch(/alert|pwned|onerror|javascript|srcdoc/i);
|
||||
// Federated host appends to the author ref via instanceUrl (punycoded, port stripped).
|
||||
expect(normalized?.authorRef).toBe("mastodon:mallory@evil.example");
|
||||
// Permalink with an unsafe scheme is refused, not carried.
|
||||
expect(normalized?.url).toBeNull();
|
||||
// Hostile media URLs are dropped wholesale; only the safe https URL is proxied.
|
||||
expect(normalized?.mediaUrls).toHaveLength(4); // legacy field stays as the source delivered it
|
||||
expect(normalized?.media).toEqual([{ kind: "image", proxyUrl: `${PROXY.proxyPath}?url=${encodeURIComponent("https://cdn.example/ok.png")}` }]);
|
||||
assertProxyOnly(normalized?.media as MediaRef[]);
|
||||
// Hostile card: unsafe URL/title/image all rejected → no preview survives.
|
||||
expect(normalized?.linkPreview).toBeUndefined();
|
||||
});
|
||||
|
||||
it("sanitizes the Mastodon link-preview card payload (text fields, proxied image, safe href)", () => {
|
||||
const normalized = normalizeFixture(fixtureMastodonLinkPreview);
|
||||
expect(normalized).not.toBeNull();
|
||||
const preview = normalized?.linkPreview as LinkPreview;
|
||||
expect(preview).not.toBeNull();
|
||||
expect(preview).toEqual({
|
||||
type: "linkPreview", href: "https://example.org/manifesto", host: "example.org",
|
||||
title: "Manifesto <scripts> & plans", description: "A plan", siteName: "example dot org",
|
||||
image: `${PROXY.proxyPath}?url=${encodeURIComponent("https://cdn.example.org/cover.png")}`,
|
||||
});
|
||||
// OpenGraph fields are text: the entity-decoded title carries no live markup.
|
||||
expect(preview.title).toContain("<scripts>");
|
||||
expect(preview.title).not.toMatch(/<|&/);
|
||||
assertZeroRawRemnants(normalized?.content as RichTextDocument, normalized?.plainText as string, normalized?.media as MediaRef[]);
|
||||
});
|
||||
|
||||
it("round-trips every fixture AST through JSON (storage shape) with a stable plainText", () => {
|
||||
for (const [name, fixture] of Object.entries(fixtures)) {
|
||||
const normalized = normalizeFixture(fixture);
|
||||
expect(normalized, name).not.toBeNull();
|
||||
const ast = normalized?.content as RichTextDocument;
|
||||
assertZeroRawRemnants(ast, normalized?.plainText as string, normalized?.media as MediaRef[]);
|
||||
const roundTripped = JSON.parse(JSON.stringify(ast)) as RichTextDocument;
|
||||
expect(roundTripped, name).toEqual(ast);
|
||||
expect(astToPlainText(roundTripped), name).toBe(normalized?.plainText);
|
||||
}
|
||||
});
|
||||
|
||||
it("matches the reference parseProviderHtml derivation for the Mastodon fixture", () => {
|
||||
const fixture = fixtureMastodonHtml;
|
||||
const reference = parseProviderHtml(fixture.item.text, { emojis: fixture.emojis });
|
||||
const normalized = normalizeFixture(fixture);
|
||||
expect(normalized?.content).toEqual(reference);
|
||||
expect(astToPlainText(reference)).toBe(normalized?.plainText);
|
||||
});
|
||||
|
||||
it("builds canonical author refs: handle host wins, instanceUrl fills federated hosts, IDN punycodes", () => {
|
||||
expect(feedAuthorRef("mastodon", "alice@foreso.social")).toBe("mastodon:alice@foreso.social");
|
||||
expect(feedAuthorRef("mastodon", "alice", "https://fedoraproject.org/@alice")).toBe("mastodon:alice@fedoraproject.org");
|
||||
expect(feedAuthorRef("mastodon", "alice", "https://bücher.example:8443/path")).toBe("mastodon:alice@xn--bcher-kva.example");
|
||||
expect(feedAuthorRef("mastodon", "alice", "gopher://odd.example")).toBe("mastodon:alice");
|
||||
expect(feedAuthorRef("mastodon", "@bob@", "https://ok.example")).toBeNull();
|
||||
expect(feedAuthorRef("mastodon", "has space")).toBeNull();
|
||||
expect(feedAuthorRef("mastodon", "")).toBeNull();
|
||||
});
|
||||
|
||||
it("expires items per ToS: explicit Date, date-like string, and unparseable → null", () => {
|
||||
const base = { ...mastodonStatus, mediaUrls: [] };
|
||||
const at = normalizeFeedItem(base, connection(), { expiresAt: new Date("2026-11-05T00:00:00.000Z") });
|
||||
expect(at?.expiresAt).toEqual(new Date("2026-11-05T00:00:00.000Z"));
|
||||
const fromString = normalizeFeedItem(base, connection(), { expiresAt: "2026-11-05T00:00:00.000Z" });
|
||||
expect(fromString?.expiresAt).toEqual(new Date("2026-11-05T00:00:00.000Z"));
|
||||
expect(normalizeFeedItem(base, connection(), { expiresAt: "not-a-date" })?.expiresAt).toBeNull();
|
||||
expect(normalizeFeedItem(base, connection())?.expiresAt).toBeNull();
|
||||
});
|
||||
|
||||
it("drops items failing the legacy validation", () => {
|
||||
expect(normalizeFeedItem({ ...mastodonStatus, remoteId: 42 as unknown as string }, connection())).toBeNull();
|
||||
expect(normalizeFeedItem({ ...mastodonStatus, createdAt: new Date(Number.NaN) }, connection())).toBeNull();
|
||||
expect(normalizeFeedItem({ ...mastodonStatus, authorHandle: 7 as unknown as string }, connection())).toBeNull();
|
||||
expect(normalizeFeedItem(mastodonStatus, connection())).not.toBeNull();
|
||||
});
|
||||
|
||||
it("proxies only safe media: scheme tricks, protocol-relative and oversized URLs are dropped", () => {
|
||||
const media = feedItemMedia([
|
||||
"https://cdn.example/a.png", "javascript:alert(1)", "data:text/html,x", "//rel.example/b.png",
|
||||
"https://user:pass@cred.example/c.jpg", "not a url", "https://cdn.example/d.webm?v=2", 42,
|
||||
]);
|
||||
// Credential-bearing URLs are refused by the same safe-URL gate.
|
||||
expect(media.map((ref) => ref.kind)).toEqual(["image", "video"]); // extension sniff
|
||||
expect(media[1]?.proxyUrl).toBe(`${PROXY.proxyPath}?url=${encodeURIComponent("https://cdn.example/d.webm?v=2")}`);
|
||||
assertProxyOnly(media);
|
||||
expect(feedItemMedia("not-an-array")).toEqual([]);
|
||||
});
|
||||
|
||||
it("serves legacy cache payloads (no AST) and v2 payloads through the unchanged /feeds shape", async () => {
|
||||
const state = createMockSocialState();
|
||||
state.feed = [];
|
||||
const mock = createMockSocialConnector(state);
|
||||
const registry = new ConnectorRegistry(); registry.register(mock);
|
||||
const redis = new MemoryFeedRedis();
|
||||
const connectionId = randomUUID();
|
||||
const deps: FeedDependencies = {
|
||||
redis, registry, workspaces: async () => ["ws-1"],
|
||||
connections: async () => [{ id: connectionId, provider: "mock-social", channelId: connectionId, fields: {}, health: "healthy", credentialRotatedAt: null, recentPublishes: [], createdAt: new Date().toISOString() }],
|
||||
connection: async (_ws, id) => ({ id, workspaceId: "ws-1", provider: "mock-social", channelId: id, credentialId: { kind: "credential-ref", id }, fields: {}, createdAt: new Date() }),
|
||||
visible: async () => [connectionId], audit: async () => undefined, requestsPerSecond: () => 0.5, random: () => 0.5,
|
||||
};
|
||||
const service = new FeedsService(deps);
|
||||
const session = { workspaceId: "ws-1", userId: randomUUID(), role: "Owner", authzEpoch: 1, issuedAt: 0, expiresAt: 9_999_999_999 };
|
||||
const key = feedCacheKey("ws-1", connectionId);
|
||||
const legacy = { id: "legacy", remoteId: "legacy", provider: "mock-social", connectionId, authorHandle: "old@social", text: "legacy item", mediaUrls: ["https://cdn.old.example/a.png"], createdAt: new Date("2026-10-01T00:00:00.000Z"), metrics: { likes: 1 } };
|
||||
await redis.set(key, JSON.stringify({ items: [legacy], fetchedAt: new Date().toISOString(), handle: "old@social", displayName: "Old" }), { pxMs: 99_000 });
|
||||
const served = await service.read(session);
|
||||
expect(served.items).toHaveLength(1);
|
||||
expect(served.items[0]).toMatchObject({ remoteId: "legacy", text: "legacy item", metrics: { likes: 1 } });
|
||||
expect(served.items[0]?.createdAt).toBeInstanceOf(Date);
|
||||
expect("content" in (served.items[0] as object)).toBe(false); // additive: absent, not null
|
||||
expect("expiresAt" in (served.items[0] as object)).toBe(false);
|
||||
// A v2 payload keeps the legacy fields and adds the new ones through the same response.
|
||||
const normalized = normalizeFeedItem({ ...legacy, id: "v2", remoteId: "v2" }, { ...connection(), provider: "mock-social" }, { url: "https://old.example/legacy/2", expiresAt: new Date("2026-11-01T00:00:00.000Z") });
|
||||
await redis.set(key, JSON.stringify({ items: [normalized], fetchedAt: new Date().toISOString(), handle: "old@social", displayName: "Old" }), { pxMs: 99_000 });
|
||||
const v2Served = await service.read(session);
|
||||
expect(v2Served.items[0]).toMatchObject({ remoteId: "v2", text: "legacy item", authorRef: "mock-social:old@social", url: "https://old.example/legacy/2" });
|
||||
expect((v2Served.items[0] as { content?: { type?: string } }).content?.type).toBe("doc");
|
||||
expect(v2Served.items[0]?.createdAt).toBeInstanceOf(Date);
|
||||
expect((v2Served.items[0] as unknown as { expiresAt?: Date }).expiresAt).toEqual(new Date("2026-11-01T00:00:00.000Z"));
|
||||
});
|
||||
|
||||
it("still polls and serves end-to-end: connector feed → v2 cache → /feeds response", async () => {
|
||||
const state = createMockSocialState();
|
||||
state.feed = [{
|
||||
id: "e2e", remoteId: "e2e", provider: "mock-social", authorHandle: "carol@social",
|
||||
text: "End to end #acampada", mediaUrls: ["https://cdn.social/e2e.png"],
|
||||
createdAt: new Date("2026-10-06T09:00:00.000Z"), metrics: { likes: 2 },
|
||||
}];
|
||||
const mock = createMockSocialConnector(state);
|
||||
const registry = new ConnectorRegistry(); registry.register(mock);
|
||||
const deps: FeedDependencies = {
|
||||
redis: new MemoryFeedRedis(), registry, workspaces: async () => ["ws-1"],
|
||||
connections: async () => [{ id: "conn-e2e", provider: "mock-social", channelId: "conn-e2e", fields: {}, health: "healthy", credentialRotatedAt: null, recentPublishes: [], createdAt: new Date().toISOString() }],
|
||||
connection: async (_ws, id) => ({ id, workspaceId: "ws-1", provider: "mock-social", channelId: id, credentialId: { kind: "credential-ref", id }, fields: {}, createdAt: new Date() }),
|
||||
visible: async () => ["conn-e2e"], audit: async () => undefined, requestsPerSecond: () => 0.5, random: () => 0.5,
|
||||
};
|
||||
const service = new FeedsService(deps);
|
||||
expect(await service.pollWorkspace("ws-1")).toEqual({ polled: 1, skipped: 0, failed: 0 });
|
||||
const response = await service.read({ workspaceId: "ws-1", userId: randomUUID(), role: "Owner", authzEpoch: 1, issuedAt: 0, expiresAt: 9_999_999_999 });
|
||||
expect(response.items).toHaveLength(1);
|
||||
const item = response.items[0] as NormalizedFeedItem | undefined;
|
||||
expect(item).toBeDefined();
|
||||
expect(item).toMatchObject({ remoteId: "e2e", text: "End to end #acampada", authorRef: "mock-social:carol@social", metrics: { likes: 2 } });
|
||||
expect(item?.content?.type).toBe("doc");
|
||||
expect(item?.content?.children[0]?.children.map((node) => node.type)).toEqual(["text", "hashtag"]);
|
||||
expect(item?.plainText).toBe("End to end #acampada");
|
||||
expect(item?.media?.[0]?.proxyUrl).toBe(`${PROXY.proxyPath}?url=${encodeURIComponent("https://cdn.social/e2e.png")}`);
|
||||
assertZeroRawRemnants(item?.content as RichTextDocument, item?.plainText as string, item?.media as MediaRef[]);
|
||||
});
|
||||
});
|
||||
211
apps/api/src/feeds/feed-normalize.ts
Normal file
211
apps/api/src/feeds/feed-normalize.ts
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
/**
|
||||
* Feed-item normalization v2 (columns P0c — design §3.2 "Normalization" + §6).
|
||||
*
|
||||
* Every source type funnels into ONE `normalizeFeedItem()`. The legacy `/feeds`
|
||||
* response fields (`text`, `mediaUrls`, `metrics`, dates, ids) are unchanged;
|
||||
* the columns fields are strictly additive:
|
||||
*
|
||||
* - `authorRef` canonical `{provider}:{handle@host}`; the host is appended
|
||||
* only when resolvable (from the handle itself, else the connection's
|
||||
* `instanceUrl` field, IDN→punycode + lowercased by `URL`). No host →
|
||||
* `{provider}:{handle}`; invalid handle → null.
|
||||
* - `url` provider permalink candidate — only absolute, credential-free
|
||||
* http(s) URLs survive (`parseSafeHttpUrl`); anything else → null.
|
||||
* - `content` safe-content rich-text AST. The ONLY render source: raw HTML
|
||||
* never survives as markup (`contentKind: "html"` feeds Mastodon-style
|
||||
* sanitized HTML through the allowlist parser; plain text is auto-linked
|
||||
* http(s)-only). Scripts, iframes, styles, event handlers and unsafe
|
||||
* schemes are dropped or reduced to inert text by the parser.
|
||||
* - `plainText` derived from the AST at ingest (stable function of the AST)
|
||||
* for saved-search client filtering and previews.
|
||||
* - `media` proxy-minted media refs; raw provider CDN URLs are only ever
|
||||
* carried inside the proxy query (`/api/media-proxy?url=…`), never as the
|
||||
* browser-facing URL.
|
||||
* - `expiresAt` nullable ToS expiry (YouTube Data-API items: fetched+30d).
|
||||
* - `linkPreview` sanitized Mastodon `card`/OpenGraph payload or null.
|
||||
*
|
||||
* Legacy validation semantics are retained verbatim: finite dates, string
|
||||
* remoteId/handle/text, finite metrics; invalid items are dropped (null).
|
||||
*/
|
||||
import type { ConnectorConnection, FeedItem } from "@tactical-media-broadcast-hub/core";
|
||||
import {
|
||||
DEFAULT_MEDIA_PROXY,
|
||||
neutralizeText,
|
||||
parseLinkPreview,
|
||||
parsePlainText,
|
||||
parseProviderHtml,
|
||||
parseSafeHttpUrl,
|
||||
toProxiedMediaUrl,
|
||||
type CustomEmoji,
|
||||
type LinkPreview,
|
||||
type MediaProxyPolicy,
|
||||
type ParseOptions,
|
||||
type RichTextDocument,
|
||||
} from "../../../../packages/safe-content/src/index.js";
|
||||
import type { StreamFeedItem } from "./feeds.service.js";
|
||||
|
||||
/** Media reference (design §1.2 `feed_items.media`). */
|
||||
export interface MediaRef {
|
||||
kind: "image" | "video";
|
||||
/** Media-proxy URL — never the raw provider CDN URL. */
|
||||
proxyUrl: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
alt?: string;
|
||||
}
|
||||
|
||||
/** Fully normalized v2 item — everything the persisted `feed_items` row needs. */
|
||||
export type NormalizedFeedItem = StreamFeedItem & {
|
||||
authorRef: string | null;
|
||||
url: string | null;
|
||||
content: RichTextDocument;
|
||||
plainText: string;
|
||||
media: MediaRef[];
|
||||
expiresAt: Date | null;
|
||||
linkPreview?: LinkPreview;
|
||||
};
|
||||
|
||||
export type FeedContentKind = "text" | "html";
|
||||
|
||||
export interface NormalizeFeedItemOptions {
|
||||
/** "html" = Mastodon-style sanitized HTML → allowlist AST; default plain text. */
|
||||
contentKind?: FeedContentKind;
|
||||
/** Provider permalink candidate; relative/unsafe schemes → null. */
|
||||
url?: unknown;
|
||||
/** ToS expiry (YouTube Data API: fetched_at + 30d). Unparseable → null. */
|
||||
expiresAt?: unknown;
|
||||
/** Mastodon `card` / OpenGraph payload → sanitized preview; hostile → null. */
|
||||
linkPreview?: unknown;
|
||||
mediaProxy?: MediaProxyPolicy;
|
||||
emojis?: readonly CustomEmoji[];
|
||||
}
|
||||
|
||||
/** Upper bound on media refs per item; Mastodon ≤4, YouTube 1 — generous cap. */
|
||||
const MAX_MEDIA = 16;
|
||||
/** Extension sniff for kind when the source only offers bare URLs. */
|
||||
const VIDEO_EXT = /\.(?:mp4|webm|mov|m4v|ogv)(?:[?#]|$)/i;
|
||||
/** Same shape safe-content enforces for handles (`user` | `user@host`). */
|
||||
const HANDLE = /^[\p{L}\p{N}_.-]{1,64}(?:@[\p{L}\p{N}.-]{1,253})?$/u;
|
||||
|
||||
/**
|
||||
* Canonical author ref `{provider}:{handle@host}` (design §3.1/§3.2). The host
|
||||
* comes from the handle itself, else from a federated `instanceUrl` connection
|
||||
* field; `URL` lowercases and punycodes it, and the port never participates.
|
||||
*/
|
||||
export function feedAuthorRef(provider: string, authorHandle: string, instanceUrl?: unknown): string | null {
|
||||
const handle = neutralizeText(authorHandle).trim().replace(/^@+/, "");
|
||||
if (!HANDLE.test(handle)) return null;
|
||||
const host = handle.includes("@") ? null : instanceHost(instanceUrl);
|
||||
return `${provider}:${handle}${host ? `@${host}` : ""}`;
|
||||
}
|
||||
|
||||
function instanceHost(raw: unknown): string | null {
|
||||
if (typeof raw !== "string" || raw.trim() === "") return null;
|
||||
try {
|
||||
const url = new URL(raw.trim());
|
||||
return url.protocol === "http:" || url.protocol === "https:" ? url.hostname : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Derives proxy-minted media refs; unsafe origin URLs are dropped entirely. */
|
||||
export function feedItemMedia(urls: unknown, policy: MediaProxyPolicy = DEFAULT_MEDIA_PROXY): MediaRef[] {
|
||||
if (!Array.isArray(urls)) return [];
|
||||
const out: MediaRef[] = [];
|
||||
for (const raw of urls.slice(0, MAX_MEDIA)) {
|
||||
const proxyUrl = toProxiedMediaUrl(raw, policy);
|
||||
if (proxyUrl === null) continue;
|
||||
out.push({ kind: typeof raw === "string" && VIDEO_EXT.test(raw) ? "video" : "image", proxyUrl });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Flat text derived from a rich-text document. Pure and total over the closed
|
||||
* AST: same document in, same string out — stable across storage round-trips.
|
||||
* Links contribute their visible label (for parsed plain text that is the URL
|
||||
* without the scheme, so host/path terms stay filterable).
|
||||
*/
|
||||
export function astToPlainText(doc: RichTextDocument): string {
|
||||
const lines: string[] = [];
|
||||
for (const paragraph of doc.children) {
|
||||
let line = "";
|
||||
for (const node of paragraph.children) {
|
||||
switch (node.type) {
|
||||
case "text":
|
||||
line += node.text;
|
||||
break;
|
||||
case "emoji":
|
||||
line += `:${node.shortcode}:`;
|
||||
break;
|
||||
case "linebreak":
|
||||
line += "\n";
|
||||
break;
|
||||
case "link":
|
||||
for (const child of node.children) line += child.type === "text" ? child.text : `:${child.shortcode}:`;
|
||||
break;
|
||||
case "mention":
|
||||
line += `@${node.handle}`;
|
||||
break;
|
||||
case "hashtag":
|
||||
line += `#${node.tag}`;
|
||||
break;
|
||||
}
|
||||
}
|
||||
lines.push(line);
|
||||
}
|
||||
return lines.join("\n\n");
|
||||
}
|
||||
|
||||
function normalizeExpiresAt(raw: unknown): Date | null {
|
||||
if (raw instanceof Date) return Number.isFinite(raw.getTime()) ? raw : null;
|
||||
if (typeof raw === "string" || typeof raw === "number") {
|
||||
const date = new Date(raw);
|
||||
return Number.isFinite(date.getTime()) ? date : null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The single funnel (design §3.2). Returns null for items failing the legacy
|
||||
* validation, otherwise the legacy shape plus the additive v2 fields. Content
|
||||
* is parsed here — callers never pass an AST, so raw provider HTML cannot
|
||||
* bypass the allowlist parser.
|
||||
*/
|
||||
export function normalizeFeedItem(
|
||||
item: FeedItem,
|
||||
connection: ConnectorConnection,
|
||||
options: NormalizeFeedItemOptions = {},
|
||||
): NormalizedFeedItem | null {
|
||||
const createdAt = new Date(item.createdAt);
|
||||
if (!Number.isFinite(createdAt.getTime()) || typeof item.remoteId !== "string" || typeof item.text !== "string" || typeof item.authorHandle !== "string") {
|
||||
return null;
|
||||
}
|
||||
const mediaProxy = options.mediaProxy ?? DEFAULT_MEDIA_PROXY;
|
||||
const parseOptions: ParseOptions = { mediaProxy, ...(options.emojis ? { emojis: options.emojis } : {}) };
|
||||
const content = options.contentKind === "html"
|
||||
? parseProviderHtml(item.text, parseOptions)
|
||||
: parsePlainText(item.text, parseOptions);
|
||||
const safeUrl = parseSafeHttpUrl(options.url);
|
||||
const preview = parseLinkPreview(options.linkPreview, mediaProxy);
|
||||
const mediaUrls = Array.isArray(item.mediaUrls) ? item.mediaUrls.filter((url): url is string => typeof url === "string") : [];
|
||||
return {
|
||||
id: typeof item.id === "string" ? item.id : item.remoteId,
|
||||
remoteId: item.remoteId,
|
||||
provider: connection.provider,
|
||||
connectionId: connection.id,
|
||||
authorHandle: item.authorHandle,
|
||||
authorRef: feedAuthorRef(connection.provider, item.authorHandle, connection.fields.instanceUrl),
|
||||
url: safeUrl === null ? null : safeUrl.href,
|
||||
text: item.text,
|
||||
content,
|
||||
plainText: astToPlainText(content),
|
||||
createdAt,
|
||||
mediaUrls,
|
||||
media: feedItemMedia(item.mediaUrls, mediaProxy),
|
||||
metrics: Object.fromEntries(Object.entries(item.metrics ?? {}).filter(([, value]) => typeof value === "number" && Number.isFinite(value))),
|
||||
expiresAt: normalizeExpiresAt(options.expiresAt),
|
||||
...(preview === null ? {} : { linkPreview: preview }),
|
||||
};
|
||||
}
|
||||
18
apps/api/src/feeds/feed-registry.ts
Normal file
18
apps/api/src/feeds/feed-registry.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
import type { CredentialAccessor } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import { connectionRegistry } from "../connections/connector-registry.js";
|
||||
import { oauthFetch } from "../connections/oauth-fetch.js";
|
||||
import { createMockSocialConnector, createMockSocialState } from "../../../../packages/connector-mock-social/src/index.js";
|
||||
|
||||
/** Composition boundary: the demo connector uses deterministic public data, no OAuth. */
|
||||
export function feedRegistry(credentials: CredentialAccessor) {
|
||||
const registry = connectionRegistry(credentials, oauthFetch);
|
||||
const state = createMockSocialState();
|
||||
const now = Date.now();
|
||||
state.feed = Array.from({ length: 12 }, (_, index) => ({
|
||||
id: `demo-feed-${index}`, remoteId: `demo-feed-${index}`, provider: "mock-social",
|
||||
authorHandle: "mock@social", text: `Mock Social feed item ${index + 1}: ready for Streams.`,
|
||||
mediaUrls: [], createdAt: new Date(now - index * 60_000), metrics: { likes: index },
|
||||
}));
|
||||
registry.register(createMockSocialConnector(state));
|
||||
return registry;
|
||||
}
|
||||
23
apps/api/src/feeds/feeds.controller.ts
Normal file
23
apps/api/src/feeds/feeds.controller.ts
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
import { Controller, Get, Inject, Param, Post, Query, Req, UseGuards } from "@nestjs/common";
|
||||
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import { parseId, parseLimit, requireSession } from "../http/validation.js";
|
||||
import { FeedsService } from "./feeds.service.js";
|
||||
|
||||
@Controller("feeds")
|
||||
@UseGuards(RbacGuard)
|
||||
export class FeedsController {
|
||||
constructor(@Inject(FeedsService) private readonly feeds: FeedsService) {}
|
||||
@Get()
|
||||
@RequirePermission("draft.view")
|
||||
list(@Req() request: AuthenticatedRequest, @Query("cursor") cursor?: string, @Query("limit") limit?: string) {
|
||||
return this.feeds.read(requireSession(request), { cursor, limit: parseLimit(limit, 40, 200) });
|
||||
}
|
||||
@Post("refresh")
|
||||
@RequirePermission("draft.view")
|
||||
refresh(@Req() request: AuthenticatedRequest) { return this.feeds.refresh(requireSession(request)); }
|
||||
@Get(":connectionId")
|
||||
@RequirePermission("draft.view")
|
||||
connection(@Req() request: AuthenticatedRequest, @Param("connectionId") connectionId: string, @Query("cursor") cursor?: string, @Query("limit") limit?: string) {
|
||||
return this.feeds.read(requireSession(request), { connectionId: parseId(connectionId), cursor, limit: parseLimit(limit, 40, 200) });
|
||||
}
|
||||
}
|
||||
72
apps/api/src/feeds/feeds.dependencies.ts
Normal file
72
apps/api/src/feeds/feeds.dependencies.ts
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
import type { ConnectorConnection } from "@tactical-media-broadcast-hub/core";
|
||||
import type { ConnectorRegistry, CredentialAccessor } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import { appendTenantAudit } from "../audit/audit.service.js";
|
||||
import { PgConnectionStore } from "../connections/connection-store.js";
|
||||
import type { ConnectionSummary } from "../connections/connections.service.js";
|
||||
import { pool } from "../db/client.js";
|
||||
import { GroupsService, hasWorkspaceWideAccess } from "../groups/groups.service.js";
|
||||
import { RespControlRedis, type ControlRedis } from "../media-control/redis.js";
|
||||
import { vault } from "../vault/vault.js";
|
||||
import { feedRegistry } from "./feed-registry.js";
|
||||
|
||||
export interface FeedDependencies {
|
||||
redis: Pick<ControlRedis, "get" | "set" | "eval" | "close">;
|
||||
registry: ConnectorRegistry;
|
||||
workspaces(): Promise<string[]>;
|
||||
connections(workspaceId: string): Promise<ConnectionSummary[]>;
|
||||
connection(workspaceId: string, id: string): Promise<ConnectorConnection>;
|
||||
visible(session: SessionPayload): Promise<string[]>;
|
||||
audit(session: SessionPayload): Promise<void>;
|
||||
/** Conservative share of the connector's SDK budget; remaining capacity serves other operations. */
|
||||
requestsPerSecond(provider: string): number;
|
||||
random(): number;
|
||||
}
|
||||
|
||||
export function feedDefaults(): FeedDependencies {
|
||||
const store = new PgConnectionStore();
|
||||
// A single registry preserves the connector's per-host HTTP token buckets.
|
||||
const credentials: CredentialAccessor = {
|
||||
async getAccessToken(connection) { return credentials.getSecret(connection, "access_token"); },
|
||||
async getSecret(connection, key) {
|
||||
const stored = await store.get(connection.workspaceId, connection.id);
|
||||
if (!stored.credential) throw new Error("Credential missing");
|
||||
const bytes = await vault.open(connection.workspaceId, connection.id, stored.credential);
|
||||
const secrets = JSON.parse(bytes.toString()) as Record<string, unknown>;
|
||||
const value = secrets[key];
|
||||
if (typeof value !== "string" || !value) throw new Error("Credential missing");
|
||||
return value;
|
||||
},
|
||||
async put(connection, key, value) {
|
||||
const stored = await store.get(connection.workspaceId, connection.id);
|
||||
if (!stored.credential) throw new Error("Credential missing");
|
||||
const bytes = await vault.open(connection.workspaceId, connection.id, stored.credential);
|
||||
const secrets = JSON.parse(bytes.toString()) as Record<string, unknown>;
|
||||
secrets[key] = value;
|
||||
const sealed = await vault.seal(connection.workspaceId, connection.id, Buffer.from(JSON.stringify(secrets)));
|
||||
await store.rotateCredential(connection.workspaceId, connection.id, stored.credential, sealed);
|
||||
},
|
||||
};
|
||||
return {
|
||||
redis: new RespControlRedis(process.env.REDIS_URL ?? "redis://127.0.0.1:6379"),
|
||||
registry: feedRegistry(credentials),
|
||||
async workspaces() {
|
||||
const result = await pool.query<{ id: string }>("SELECT id FROM workspaces ORDER BY id");
|
||||
return result.rows.map((row) => row.id);
|
||||
},
|
||||
connections: (workspaceId) => store.list(workspaceId),
|
||||
connection: async (workspaceId, id) => (await store.get(workspaceId, id)).connection,
|
||||
visible: async (session) => (await new GroupsService().getVisibleConnections(session.workspaceId, session.userId, hasWorkspaceWideAccess(session.role))).connectionIds,
|
||||
audit: async (session) => { await appendTenantAudit(session.workspaceId, {
|
||||
actorUserId: session.userId, actorRole: session.role, action: "feeds.refresh",
|
||||
targetType: "workspace", targetId: session.workspaceId,
|
||||
}); },
|
||||
// The smallest current feed connector SDK budget is 0.5 RPS. Never raise it
|
||||
// via an env override; lowering the SDK policy also lowers poll frequency.
|
||||
requestsPerSecond: (provider) => {
|
||||
const configured = Number(process.env[`CONNECTOR_RATE_${provider.toUpperCase().replace(/-/g, "_")}_RPS`]);
|
||||
return Number.isFinite(configured) && configured > 0 ? Math.min(0.5, configured) : 0.5;
|
||||
},
|
||||
random: Math.random,
|
||||
};
|
||||
}
|
||||
102
apps/api/src/feeds/feeds.integration.test.ts
Normal file
102
apps/api/src/feeds/feeds.integration.test.ts
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
import "reflect-metadata";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { Module, UnauthorizedException, ForbiddenException, type ExecutionContext, type INestApplication } from "@nestjs/common";
|
||||
import { NestFactory, Reflector } from "@nestjs/core";
|
||||
import { InMemoryCredentials } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { DemoController } from "../demo/demo.controller.js";
|
||||
import { PERMISSION_KEY, RbacGuard } from "../rbac/rbac.guard.js";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import { injectHttp } from "../connections/http-test-transport.js";
|
||||
import type { ConnectorConnection } from "@tactical-media-broadcast-hub/core";
|
||||
import { feedRegistry } from "./feed-registry.js";
|
||||
import { FeedsService, type FeedResponse } from "./feeds.service.js";
|
||||
import { FeedsController } from "./feeds.controller.js";
|
||||
import { MemoryFeedRedis } from "./testing.js";
|
||||
|
||||
const memory = vi.hoisted(() => ({ rows: new Map<string, { id: string; workspaceId: string; provider: string; channelId: string }>(), audits: [] as string[] }));
|
||||
vi.mock("../db/client.js", () => ({
|
||||
pool: {},
|
||||
withTenant: async (workspaceId: string, callback: (client: { query(sql: string, values: unknown[]): Promise<{ rows: Array<{ id: string }>; rowCount: number }> }) => Promise<unknown>) => callback({
|
||||
async query(sql, values) {
|
||||
if (sql.startsWith("INSERT INTO connections")) {
|
||||
const id = randomUUID();
|
||||
memory.rows.set(id, { id, workspaceId, provider: String(values[1]), channelId: String(values[2]) });
|
||||
return { rows: [{ id }], rowCount: 1 };
|
||||
}
|
||||
if (sql.includes("audit_append")) memory.audits.push(sql);
|
||||
return { rows: [], rowCount: 1 };
|
||||
},
|
||||
}),
|
||||
}));
|
||||
const session: SessionPayload = { userId: randomUUID(), workspaceId: randomUUID(), role: "Owner", authzEpoch: 1, issuedAt: 0, expiresAt: 9_999_999_999 };
|
||||
@Module({})
|
||||
class TestModule {}
|
||||
|
||||
describe("HTTP demo connection → connector poll → GET feeds", () => {
|
||||
let app: INestApplication;
|
||||
let canView = true;
|
||||
const registry = feedRegistry(new InMemoryCredentials());
|
||||
const service = new FeedsService({
|
||||
redis: new MemoryFeedRedis(), registry, random: () => 0, requestsPerSecond: () => 0.5,
|
||||
workspaces: async () => [session.workspaceId],
|
||||
connections: async (ws) => [...memory.rows.values()].filter((row) => row.workspaceId === ws).map((row) => ({ ...row, fields: {}, health: "needs_credentials", recentPublishes: [], credentialRotatedAt: null, createdAt: new Date().toISOString() })),
|
||||
connection: async (ws, id): Promise<ConnectorConnection> => {
|
||||
const row = memory.rows.get(id); if (!row || row.workspaceId !== ws) throw new Error("Missing row");
|
||||
return { ...row, fields: {}, credentialId: { kind: "credential-ref", id }, createdAt: new Date() };
|
||||
},
|
||||
visible: async (s) => [...memory.rows.values()].filter((row) => row.workspaceId === s.workspaceId).map((row) => row.id),
|
||||
audit: async () => { memory.audits.push("feeds.refresh"); },
|
||||
});
|
||||
beforeAll(async () => {
|
||||
vi.spyOn(service, "onApplicationBootstrap").mockImplementation(() => undefined);
|
||||
vi.spyOn(RbacGuard.prototype, "canActivate").mockImplementation(async (context: ExecutionContext) => {
|
||||
const req = context.switchToHttp().getRequest<{ headers: Record<string, string>; session?: SessionPayload }>();
|
||||
if (req.headers.authorization !== "Bearer test-session") throw new UnauthorizedException();
|
||||
const permission = new Reflector().get<string>(PERMISSION_KEY, context.getHandler());
|
||||
if (permission === "draft.view" && !canView) throw new ForbiddenException();
|
||||
req.session = session; return true;
|
||||
});
|
||||
app = await NestFactory.create({ module: TestModule, controllers: [DemoController, FeedsController], providers: [{ provide: FeedsService, useValue: service }] }, { logger: false });
|
||||
await app.init();
|
||||
});
|
||||
afterAll(async () => { await app?.close(); vi.restoreAllMocks(); });
|
||||
const request = (path: string, method = "GET", body?: unknown, auth = true) => injectHttp(app.getHttpAdapter().getInstance(), `http://api.test${path}`, {
|
||||
method, headers: { ...(auth ? { authorization: "Bearer test-session" } : {}), "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body),
|
||||
});
|
||||
it("seeds through the real demo route without OAuth, then returns only feed-capable items", async () => {
|
||||
const created = await request("/demo/connections", "POST", { provider: "mock-social", channelId: "demo-channel" });
|
||||
expect(created.status).toBe(201);
|
||||
const mockId = (await created.json() as { id: string }).id;
|
||||
const other = await request("/demo/connections", "POST", { provider: "peertube", channelId: "video-channel" });
|
||||
const otherId = (await other.json() as { id: string }).id;
|
||||
const refreshed = await request("/feeds/refresh", "POST");
|
||||
expect(refreshed.status).toBe(201);
|
||||
expect(await refreshed.json()).toEqual({ polled: 1, skipped: 0, failed: 0 });
|
||||
const response = await request("/feeds");
|
||||
expect(response.status).toBe(200);
|
||||
const feed = await response.json() as FeedResponse;
|
||||
expect(feed.items).toHaveLength(12);
|
||||
expect(feed.items.every((item) => item.connectionId === mockId && item.provider === "mock-social")).toBe(true);
|
||||
expect(feed.meta.excludedConnections).toEqual([{ id: otherId, provider: "peertube", reason: "feed_not_supported" }]);
|
||||
expect(feed.meta.connections[0]?.handle).toBe("mock@social");
|
||||
expect(memory.audits.filter((entry) => entry === "feeds.refresh")).toHaveLength(1);
|
||||
await service.pollWorkspace(session.workspaceId);
|
||||
expect(memory.audits.filter((entry) => entry === "feeds.refresh")).toHaveLength(1);
|
||||
const direct = await request(`/feeds/${mockId}?limit=2`);
|
||||
expect((await direct.json() as FeedResponse).items).toHaveLength(2);
|
||||
const absent = await request(`/feeds/${otherId}`);
|
||||
expect((await absent.json() as FeedResponse).items).toEqual([]);
|
||||
});
|
||||
it("requires draft.view on all feed routes and rejects invalid cursor and ID", async () => {
|
||||
expect((await request("/feeds", "GET", undefined, false)).status).toBe(401);
|
||||
canView = false;
|
||||
expect((await request("/feeds")).status).toBe(403);
|
||||
expect((await request("/feeds/refresh", "POST")).status).toBe(403);
|
||||
expect((await request(`/feeds/${[...memory.rows.keys()][0]}`)).status).toBe(403);
|
||||
canView = true;
|
||||
expect((await request("/feeds/not-a-uuid")).status).toBe(400);
|
||||
expect((await request("/feeds?cursor=garbage")).status).toBe(400);
|
||||
expect((await request("/feeds?limit=-1")).status).toBe(400);
|
||||
});
|
||||
});
|
||||
24
apps/api/src/feeds/feeds.module.ts
Normal file
24
apps/api/src/feeds/feeds.module.ts
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
import { Module } from "@nestjs/common";
|
||||
import { FeedsController } from "./feeds.controller.js";
|
||||
import { FeedsService } from "./feeds.service.js";
|
||||
import { ColumnsController } from "./columns.controller.js";
|
||||
import { ColumnStore } from "./column-store.js";
|
||||
import { ColumnScheduler } from "./column-scheduler.js";
|
||||
import { columnsProviders } from "./columns.dependencies.js";
|
||||
|
||||
/**
|
||||
* Feeds + columns (P1): the columns controller/store/scheduler join the feeds
|
||||
* module — one surface, one connector registry, one hot-cache dependency set.
|
||||
* ColumnStore is injected into FeedsService for the additive meta.columns.
|
||||
*/
|
||||
@Module({
|
||||
// Controllers are ordered deliberately: ColumnsController's static paths
|
||||
// (/feeds/columns, /feeds/columns/reorder, …) must register BEFORE
|
||||
// FeedsController's parametric /feeds/:connectionId, or that route would
|
||||
// capture "columns" as a connection id (Express matches in declaration
|
||||
// order in Express/Nest's router).
|
||||
controllers: [ColumnsController, FeedsController],
|
||||
providers: [FeedsService, ColumnStore, ColumnScheduler, ...columnsProviders],
|
||||
exports: [FeedsService, ColumnStore, ColumnScheduler],
|
||||
})
|
||||
export class FeedsModule {}
|
||||
257
apps/api/src/feeds/feeds.service.ts
Normal file
257
apps/api/src/feeds/feeds.service.ts
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
import { BadRequestException, Inject, Injectable, Logger, NotFoundException, Optional, type OnApplicationBootstrap, type OnApplicationShutdown } from "@nestjs/common";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { ConnectorConnection, FeedItem } from "@tactical-media-broadcast-hub/core";
|
||||
import { RateLimitError } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import type { ConnectionSummary } from "../connections/connections.service.js";
|
||||
import { normalizeFeedItem, type NormalizedFeedItem } from "./feed-normalize.js";
|
||||
import { feedDefaults, type FeedDependencies } from "./feeds.dependencies.js";
|
||||
import { ColumnStore, type ColumnRecord } from "./column-store.js";
|
||||
|
||||
export const FEED_DEPENDENCIES = Symbol("FEED_DEPENDENCIES");
|
||||
export const FEED_CONCURRENCY = 4;
|
||||
const DISCOVERY_MS = 5_000;
|
||||
const CACHE_SLACK_MS = 30_000;
|
||||
const LEASE_MS = 30_000;
|
||||
const RENEW_LEASE = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('PEXPIRE', KEYS[1], ARGV[2]) else return 0 end";
|
||||
const RELEASE_LEASE = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) else return 0 end";
|
||||
export const feedCacheKey = (workspaceId: string, id: string): string => `feeds:${workspaceId}:${id}:home`;
|
||||
export function feedIntervalMs(rps: number): number {
|
||||
if (!Number.isFinite(rps) || rps <= 0) throw new Error("Feed rate must be positive");
|
||||
// Identity + feed, allocating at most half the declared budget to polling.
|
||||
return Math.max(60_000, Math.ceil(4_000 / rps));
|
||||
}
|
||||
export const feedCacheTtlMs = (intervalMs: number): number => Math.ceil(intervalMs * 1.2) + CACHE_SLACK_MS;
|
||||
export interface StreamFeedItem extends FeedItem { connectionId: string }
|
||||
interface CachedFeed { items: NormalizedFeedItem[]; fetchedAt: string; handle: string; displayName: string }
|
||||
export interface FeedResponse {
|
||||
items: StreamFeedItem[];
|
||||
nextCursor: string | null;
|
||||
meta: {
|
||||
configuredFeeds: number;
|
||||
connections: Array<{ id: string; provider: string; handle: string; displayName: string; capabilities: string[]; fetchedAt: string | null; status: "ready" | "pending" }>;
|
||||
excludedConnections: Array<{ id: string; provider: string; reason: "feed_not_supported" }>;
|
||||
/**
|
||||
* Additive (P1): the persisted column list, when the workspace has any.
|
||||
* Absent (not null) for workspaces that never touched the columns API, so
|
||||
* the legacy shape is byte-identical for existing clients.
|
||||
*/
|
||||
columns?: Array<{ id: string; type: string; title: string; position: number }>;
|
||||
};
|
||||
}
|
||||
interface Position { at: number; key: string }
|
||||
const itemKey = (item: StreamFeedItem): string => JSON.stringify([item.connectionId, item.id]);
|
||||
function position(item: StreamFeedItem): Position { return { at: item.createdAt.getTime(), key: itemKey(item) }; }
|
||||
function compare(a: Position, b: Position): number { return b.at - a.at || a.key.localeCompare(b.key); }
|
||||
function decodeCursor(cursor: string): Position {
|
||||
try {
|
||||
if (cursor.length > 4096) throw new Error();
|
||||
const parsed: unknown = JSON.parse(Buffer.from(cursor, "base64url").toString());
|
||||
if (typeof parsed !== "object" || parsed === null || !("at" in parsed) || !("key" in parsed) ||
|
||||
typeof parsed.at !== "number" || !Number.isFinite(parsed.at) || typeof parsed.key !== "string") throw new Error();
|
||||
return { at: parsed.at, key: parsed.key };
|
||||
} catch { throw new BadRequestException("Invalid feed cursor"); }
|
||||
}
|
||||
function normalize(item: FeedItem, connection: ConnectorConnection): NormalizedFeedItem | null {
|
||||
return normalizeFeedItem(item, connection);
|
||||
}
|
||||
export { normalize };
|
||||
|
||||
@Injectable()
|
||||
export class FeedsService implements OnApplicationBootstrap, OnApplicationShutdown {
|
||||
private readonly deps: FeedDependencies;
|
||||
private readonly logger = new Logger(FeedsService.name);
|
||||
private timer?: ReturnType<typeof setTimeout>;
|
||||
private running = false;
|
||||
private active?: Promise<void>;
|
||||
private readonly cycles = new Map<string, Promise<{ polled: number; skipped: number; failed: number }>>();
|
||||
private inFlight = 0;
|
||||
private readonly waiters: Array<() => void> = [];
|
||||
private readonly columns?: ColumnStore;
|
||||
constructor(@Optional() @Inject(FEED_DEPENDENCIES) dependencies?: FeedDependencies, @Optional() @Inject(ColumnStore) columns?: ColumnStore) {
|
||||
this.deps = dependencies ?? feedDefaults();
|
||||
this.columns = columns;
|
||||
}
|
||||
|
||||
/** Persisted columns for meta.columns (additive; absent when never created). */
|
||||
async columnSummaries(workspaceId: string): Promise<Array<{ id: string; type: string; title: string; position: number }>> {
|
||||
if (this.columns === undefined) return [];
|
||||
return columnSummariesOf(await this.columns.list(workspaceId));
|
||||
}
|
||||
|
||||
onApplicationBootstrap(): void {
|
||||
this.running = true;
|
||||
this.schedule(0);
|
||||
}
|
||||
async onApplicationShutdown(): Promise<void> {
|
||||
this.running = false;
|
||||
if (this.timer) clearTimeout(this.timer);
|
||||
await this.active;
|
||||
await Promise.allSettled(this.cycles.values());
|
||||
this.deps.redis.close();
|
||||
}
|
||||
private schedule(delay: number): void {
|
||||
if (!this.running) return;
|
||||
this.timer = setTimeout(() => {
|
||||
this.active = this.pollAll().catch(() => this.logger.warn("Feed discovery unavailable")).finally(() => {
|
||||
this.active = undefined;
|
||||
this.schedule(DISCOVERY_MS + Math.round(this.deps.random() * 1_000));
|
||||
});
|
||||
}, delay);
|
||||
this.timer.unref();
|
||||
}
|
||||
async pollAll(): Promise<void> {
|
||||
for (const workspaceId of await this.deps.workspaces()) await this.pollWorkspace(workspaceId);
|
||||
}
|
||||
/** Feed-capable connection summaries — the lazy-seeding input for columns. */
|
||||
async capableConnections(workspaceId: string): Promise<ConnectionSummary[]> {
|
||||
return (await this.deps.connections(workspaceId)).filter((row) => this.capable(row));
|
||||
}
|
||||
|
||||
private capable(row: ConnectionSummary): boolean {
|
||||
if (!this.deps.registry.has(row.provider)) return false;
|
||||
const connector = this.deps.registry.get(row.provider);
|
||||
return connector.manifest().capabilities.feed === true && connector.getFeed !== undefined;
|
||||
}
|
||||
/** Refresh shares the scheduler's cooldowns and concurrency limit; repeated clicks cannot bypass them. */
|
||||
async refresh(session: SessionPayload) {
|
||||
await this.deps.audit(session);
|
||||
return this.pollWorkspace(session.workspaceId, await this.deps.visible(session));
|
||||
}
|
||||
pollWorkspace(workspaceId: string, visibleIds?: string[]): Promise<{ polled: number; skipped: number; failed: number }> {
|
||||
const key = JSON.stringify([workspaceId, visibleIds?.slice().sort() ?? null]);
|
||||
const current = this.cycles.get(key);
|
||||
if (current) return current;
|
||||
const cycle = this.cycle(workspaceId, visibleIds).finally(() => this.cycles.delete(key));
|
||||
this.cycles.set(key, cycle);
|
||||
return cycle;
|
||||
}
|
||||
private async cycle(workspaceId: string, visibleIds?: string[]) {
|
||||
const rows = (await this.deps.connections(workspaceId)).filter((row) => this.capable(row) && (!visibleIds || visibleIds.includes(row.id)));
|
||||
const results = { polled: 0, skipped: 0, failed: 0 };
|
||||
let index = 0;
|
||||
await Promise.all(Array.from({ length: Math.min(FEED_CONCURRENCY, rows.length) }, async () => {
|
||||
for (let row = rows[index++]; row; row = rows[index++]) {
|
||||
if (this.inFlight >= FEED_CONCURRENCY) await new Promise<void>((resolve) => this.waiters.push(resolve));
|
||||
else this.inFlight += 1;
|
||||
try { results[await this.poll(workspaceId, row)] += 1; }
|
||||
catch { results.failed += 1; this.logger.warn("Feed poll unavailable"); }
|
||||
finally {
|
||||
const waiter = this.waiters.shift();
|
||||
if (waiter) waiter(); else this.inFlight -= 1;
|
||||
}
|
||||
}
|
||||
}));
|
||||
return results;
|
||||
}
|
||||
private async poll(workspaceId: string, row: ConnectionSummary): Promise<"polled" | "skipped" | "failed"> {
|
||||
const interval = feedIntervalMs(this.deps.requestsPerSecond(row.provider));
|
||||
const delay = interval + Math.round(interval * 0.2 * this.deps.random());
|
||||
const key = feedCacheKey(workspaceId, row.id);
|
||||
const leaseKey = `${key}:lease`;
|
||||
const owner = randomUUID();
|
||||
if (!await this.deps.redis.set(leaseKey, owner, { nx: true, pxMs: LEASE_MS })) return "skipped";
|
||||
let heartbeat: ReturnType<typeof setInterval> | undefined;
|
||||
let renew: Promise<unknown> = Promise.resolve();
|
||||
let cooldown = delay;
|
||||
let started = false;
|
||||
try {
|
||||
if (!await this.deps.redis.set(`${key}:cooldown`, "idle", { nx: true, pxMs: delay })) return "skipped";
|
||||
started = true;
|
||||
heartbeat = setInterval(() => {
|
||||
renew = this.deps.redis.eval(RENEW_LEASE, [leaseKey], [owner, String(LEASE_MS)]).catch(() => {
|
||||
this.logger.warn("Feed lease renewal unavailable");
|
||||
});
|
||||
}, LEASE_MS / 3);
|
||||
heartbeat.unref();
|
||||
const connection = await this.deps.connection(workspaceId, row.id);
|
||||
const connector = this.deps.registry.get(connection.provider);
|
||||
const identity = await connector.getIdentity(connection);
|
||||
const page = await connector.getFeed?.(connection, { kind: "home", limit: 40 });
|
||||
const items = [...new Map((page?.items ?? []).map((item) => normalize(item, connection)).filter((item): item is NormalizedFeedItem => item !== null).map((item) => [itemKey(item), item])).values()];
|
||||
await this.deps.redis.set(key, JSON.stringify({ items, fetchedAt: new Date().toISOString(), handle: identity.handle, displayName: identity.displayName ?? identity.handle } satisfies CachedFeed), { pxMs: feedCacheTtlMs(interval) });
|
||||
return "polled";
|
||||
} catch (error) {
|
||||
await this.deps.redis.set(key, "null", { pxMs: feedCacheTtlMs(interval) });
|
||||
if (error instanceof RateLimitError && error.retryAfterMs) cooldown = Math.max(delay, error.retryAfterMs);
|
||||
return "failed";
|
||||
} finally {
|
||||
if (heartbeat) clearInterval(heartbeat);
|
||||
await renew;
|
||||
// Completing a long SDK call starts a fresh cooldown. The renewable
|
||||
// lease prevents another replica from entering while that call runs.
|
||||
try {
|
||||
if (started) await this.deps.redis.set(`${key}:cooldown`, "idle", { pxMs: cooldown });
|
||||
} finally { await this.deps.redis.eval(RELEASE_LEASE, [leaseKey], [owner]); }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Hot-cache write-through for the legacy /feeds surface (design §3.4): the
|
||||
* column scheduler's home-source polls push their normalized items here so
|
||||
* /feeds stays cache-first with no external polling of its own.
|
||||
*/
|
||||
async cacheHomeFeed(workspaceId: string, connectionId: string, items: NormalizedFeedItem[]): Promise<void> {
|
||||
const rows = await this.deps.connections(workspaceId);
|
||||
const row = rows.find((candidate) => candidate.id === connectionId);
|
||||
if (row === undefined) return;
|
||||
const interval = feedIntervalMs(this.deps.requestsPerSecond(row.provider));
|
||||
let handle = String(row.fields.handle ?? row.channelId);
|
||||
let displayName = String(row.fields.displayName ?? row.channelId);
|
||||
try {
|
||||
const connection = await this.deps.connection(workspaceId, connectionId);
|
||||
const identity = await this.deps.registry.get(connection.provider).getIdentity(connection);
|
||||
handle = identity.handle;
|
||||
displayName = identity.displayName ?? identity.handle;
|
||||
} catch { /* Identity is best-effort here; the cached text still serves. */ }
|
||||
await this.deps.redis.set(feedCacheKey(workspaceId, connectionId), JSON.stringify({
|
||||
items, fetchedAt: new Date().toISOString(), handle, displayName,
|
||||
} satisfies CachedFeed), { pxMs: feedCacheTtlMs(interval) });
|
||||
}
|
||||
|
||||
async read(session: SessionPayload, options: { connectionId?: string; cursor?: string; limit?: number } = {}): Promise<FeedResponse> {
|
||||
const limit = options.limit ?? 40;
|
||||
if (!Number.isInteger(limit) || limit < 1 || limit > 200) throw new BadRequestException("Invalid feed limit");
|
||||
const after = options.cursor ? decodeCursor(options.cursor) : null;
|
||||
const visible = new Set(await this.deps.visible(session));
|
||||
const rows = (await this.deps.connections(session.workspaceId)).filter((row) => visible.has(row.id) && (!options.connectionId || row.id === options.connectionId));
|
||||
if (options.connectionId && rows.length === 0) throw new NotFoundException("Connection not found");
|
||||
const meta: FeedResponse["meta"] = { configuredFeeds: 0, connections: [], excludedConnections: [] };
|
||||
const merged: StreamFeedItem[] = [];
|
||||
for (const row of rows) {
|
||||
if (!this.capable(row)) { meta.excludedConnections.push({ id: row.id, provider: row.provider, reason: "feed_not_supported" }); continue; }
|
||||
meta.configuredFeeds += 1;
|
||||
const raw = await this.deps.redis.get(feedCacheKey(session.workspaceId, row.id));
|
||||
let cached: CachedFeed | null = null;
|
||||
try { cached = raw ? JSON.parse(raw) as CachedFeed | null : null; } catch { /* An invalid cache is pending until the next poll. */ }
|
||||
meta.connections.push({
|
||||
id: row.id, provider: row.provider, handle: cached?.handle ?? String(row.fields.handle ?? row.channelId),
|
||||
displayName: cached?.displayName ?? String(row.fields.displayName ?? row.channelId),
|
||||
capabilities: Object.entries(this.deps.registry.get(row.provider).manifest().capabilities).filter(([, enabled]) => enabled === true).map(([capability]) => capability),
|
||||
fetchedAt: cached?.fetchedAt ?? null, status: cached ? "ready" : "pending",
|
||||
});
|
||||
for (const cachedItem of cached?.items ?? []) {
|
||||
merged.push({
|
||||
...cachedItem,
|
||||
connectionId: row.id, provider: row.provider,
|
||||
createdAt: new Date(cachedItem.createdAt),
|
||||
...(cachedItem.expiresAt === undefined ? {} : { expiresAt: cachedItem.expiresAt === null ? null : new Date(cachedItem.expiresAt) }),
|
||||
} satisfies StreamFeedItem);
|
||||
}
|
||||
}
|
||||
const sorted = merged.sort((a, b) => compare(position(a), position(b))).filter((item) => !after || compare(position(item), after) > 0);
|
||||
const items = sorted.slice(0, limit);
|
||||
const last = items[items.length - 1];
|
||||
// Additive meta.columns: only when the workspace has persisted columns —
|
||||
// a workspace that never touched the columns API gets the legacy shape.
|
||||
if (this.columns !== undefined) {
|
||||
const summaries = await columnSummariesOf(await this.columns.list(session.workspaceId));
|
||||
if (summaries.length > 0) meta.columns = summaries;
|
||||
}
|
||||
return { items, nextCursor: sorted.length > limit && last ? Buffer.from(JSON.stringify(position(last))).toString("base64url") : null, meta };
|
||||
}
|
||||
}
|
||||
|
||||
function columnSummariesOf(records: ColumnRecord[]): Array<{ id: string; type: string; title: string; position: number }> {
|
||||
return records.map((record) => ({ id: record.id, type: record.type, title: record.title, position: record.position }));
|
||||
}
|
||||
195
apps/api/src/feeds/feeds.test.ts
Normal file
195
apps/api/src/feeds/feeds.test.ts
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
import "reflect-metadata";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { ConnectorRegistry, InMemoryCredentials, RateLimitError, type PlatformConnector } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
import type { ConnectorConnection, FeedItem } from "@tactical-media-broadcast-hub/core";
|
||||
import { createMockSocialConnector, createMockSocialState } from "../../../../packages/connector-mock-social/src/index.js";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import type { ConnectionSummary } from "../connections/connections.service.js";
|
||||
import { FeedsService, feedCacheKey, feedCacheTtlMs, feedIntervalMs } from "./feeds.service.js";
|
||||
import type { FeedDependencies } from "./feeds.dependencies.js";
|
||||
import { connectionRegistry } from "../connections/connector-registry.js";
|
||||
import { MemoryFeedRedis } from "./testing.js";
|
||||
|
||||
const session: SessionPayload = { workspaceId: randomUUID(), userId: randomUUID(), role: "Owner", authzEpoch: 1, issuedAt: 0, expiresAt: 9_999_999_999 };
|
||||
const item = (remoteId: string, createdAt = new Date()): FeedItem => ({ id: remoteId, remoteId, provider: "mock-social", authorHandle: "alice", text: remoteId, mediaUrls: [], createdAt, metrics: {} });
|
||||
function setup(count = 1, rps = 0.5) {
|
||||
const state = createMockSocialState();
|
||||
state.feed = [item("first")];
|
||||
const mock = createMockSocialConnector(state);
|
||||
const implementation = mock.getFeed;
|
||||
if (!implementation) throw new Error("Mock feed missing");
|
||||
const getFeed = vi.fn(implementation);
|
||||
mock.getFeed = getFeed;
|
||||
const registry = new ConnectorRegistry(); registry.register(mock);
|
||||
const connections: ConnectorConnection[] = Array.from({ length: count }, () => {
|
||||
const id = randomUUID();
|
||||
return { id, workspaceId: session.workspaceId, provider: "mock-social", channelId: id, credentialId: { kind: "credential-ref", id }, fields: {}, createdAt: new Date() };
|
||||
});
|
||||
const redis = new MemoryFeedRedis();
|
||||
const summaries = (): ConnectionSummary[] => connections.map((row) => ({ id: row.id, provider: row.provider, channelId: row.channelId, fields: row.fields, health: "healthy", credentialRotatedAt: null, recentPublishes: [], createdAt: row.createdAt.toISOString() }));
|
||||
const deps: FeedDependencies = {
|
||||
redis, registry, workspaces: async () => [session.workspaceId], connections: async (ws) => ws === session.workspaceId ? summaries() : [],
|
||||
connection: async (_ws, id) => { const row = connections.find((row) => row.id === id); if (!row) throw new Error(); return row; },
|
||||
visible: async (s) => s.workspaceId === session.workspaceId ? connections.map((row) => row.id) : [],
|
||||
audit: vi.fn(async () => undefined), requestsPerSecond: () => rps, random: () => 0.5,
|
||||
};
|
||||
return { service: new FeedsService(deps), deps, connections, mock, state, getFeed, redis };
|
||||
}
|
||||
afterEach(() => vi.useRealTimers());
|
||||
describe("Feeds pipeline", () => {
|
||||
it("caches normalized JSON with interval + jitter + slack TTL and expires it", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { service, redis, connections } = setup();
|
||||
await service.pollWorkspace(session.workspaceId);
|
||||
const key = feedCacheKey(session.workspaceId, connections[0]?.id ?? "");
|
||||
const entry = redis.entries.get(key);
|
||||
expect(entry?.expiresAt).toBe(Date.now() + feedCacheTtlMs(60_000));
|
||||
const response = await service.read(session);
|
||||
expect(response.items[0]?.createdAt).toBeInstanceOf(Date);
|
||||
expect(response.items[0]?.connectionId).toBe(connections[0]?.id);
|
||||
await vi.advanceTimersByTimeAsync(feedCacheTtlMs(60_000));
|
||||
expect(await redis.get(key)).toBeNull();
|
||||
expect((await service.read(session)).meta.connections[0]?.status).toBe("pending");
|
||||
});
|
||||
it("merges newest first with a stable tie-break cursor, even when new items arrive", async () => {
|
||||
const { service, connections, redis } = setup(2);
|
||||
const date = new Date();
|
||||
for (const row of connections) await redis.set(feedCacheKey(session.workspaceId, row.id), JSON.stringify({ fetchedAt: date.toISOString(), handle: "alice", displayName: "Alice", items: [item("same", date), item("older", new Date(date.getTime() - 1000))] }), { pxMs: 99_000 });
|
||||
const first = await service.read(session, { limit: 1 });
|
||||
expect(first.nextCursor).not.toBeNull();
|
||||
const added = connections[0];
|
||||
if (added) {
|
||||
const key = feedCacheKey(session.workspaceId, added.id);
|
||||
const raw = await redis.get(key);
|
||||
const cached = JSON.parse(raw ?? "{}") as { items: FeedItem[] };
|
||||
cached.items.unshift(item("new-arrival", new Date(date.getTime() + 1000)));
|
||||
await redis.set(key, JSON.stringify(cached), { pxMs: 99_000 });
|
||||
}
|
||||
const next = await service.read(session, { limit: 1, cursor: first.nextCursor ?? undefined });
|
||||
expect(next.items[0]?.connectionId).not.toBe(first.items[0]?.connectionId);
|
||||
expect(next.items[0]?.remoteId).toBe("same");
|
||||
const third = await service.read(session, { limit: 2, cursor: next.nextCursor ?? undefined });
|
||||
expect(third.items.map((row) => row.remoteId)).toEqual(["older", "older"]);
|
||||
expect(third.nextCursor).toBeNull();
|
||||
await expect(service.read(session, { cursor: "bad" })).rejects.toThrow("Invalid feed cursor");
|
||||
});
|
||||
it("filters capabilities and missing methods, exposes exclusions without polling them", async () => {
|
||||
const { service, deps, connections, mock } = setup(3);
|
||||
const noFeed: PlatformConnector = { ...mock, manifest: () => ({ ...mock.manifest(), id: "no-feed", capabilities: { publishText: true, feed: false } }), getFeed: vi.fn() };
|
||||
const missing: PlatformConnector = { ...mock, manifest: () => ({ ...mock.manifest(), id: "missing-feed" }), getFeed: undefined };
|
||||
deps.registry.register(noFeed); deps.registry.register(missing);
|
||||
if (connections[1]) connections[1].provider = "no-feed";
|
||||
if (connections[2]) connections[2].provider = "missing-feed";
|
||||
expect(await service.pollWorkspace(session.workspaceId)).toEqual({ polled: 1, skipped: 0, failed: 0 });
|
||||
const response = await service.read(session);
|
||||
expect(response.meta.configuredFeeds).toBe(1);
|
||||
expect(response.meta.excludedConnections).toHaveLength(2);
|
||||
expect(response.items).toHaveLength(1);
|
||||
expect(noFeed.getFeed).not.toHaveBeenCalled();
|
||||
expect((await service.read(session, { connectionId: connections[1]?.id })).items).toEqual([]);
|
||||
});
|
||||
it("applies group visibility to merged, per-account and refresh operations", async () => {
|
||||
const { service, deps, connections } = setup(2);
|
||||
deps.visible = async () => [connections[0]?.id ?? ""];
|
||||
await service.refresh(session);
|
||||
const response = await service.read(session);
|
||||
expect(response.meta.connections).toHaveLength(1);
|
||||
await expect(service.read(session, { connectionId: connections[1]?.id })).rejects.toThrow("Connection not found");
|
||||
expect((await service.read({ ...session, workspaceId: randomUUID() })).items).toEqual([]);
|
||||
});
|
||||
it("paces low-budget scheduling with positive jitter; refresh and other replicas respect cooldown", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { service, deps, getFeed } = setup(1, 0.01);
|
||||
expect(feedIntervalMs(0.01)).toBe(400_000);
|
||||
service.onApplicationBootstrap();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(getFeed).toHaveBeenCalledTimes(1);
|
||||
const replica = new FeedsService(deps);
|
||||
expect((await replica.refresh(session)).skipped).toBe(1);
|
||||
await vi.advanceTimersByTimeAsync(439_999);
|
||||
expect(getFeed).toHaveBeenCalledTimes(1);
|
||||
await vi.advanceTimersByTimeAsync(6_000);
|
||||
expect(getFeed).toHaveBeenCalledTimes(2);
|
||||
expect(deps.audit).toHaveBeenCalledTimes(1);
|
||||
await service.onApplicationShutdown();
|
||||
await vi.advanceTimersByTimeAsync(900_000);
|
||||
expect(getFeed).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
it("renews a lease during slow polls and never overlaps refresh from another replica", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { service, deps, getFeed } = setup();
|
||||
let resolveFeed: (() => void) | undefined;
|
||||
getFeed.mockImplementationOnce(() => new Promise((resolve) => { resolveFeed = () => resolve({ items: [item("slow")] }); }));
|
||||
const active = service.pollWorkspace(session.workspaceId);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
await vi.advanceTimersByTimeAsync(150_000);
|
||||
expect((await new FeedsService(deps).pollWorkspace(session.workspaceId)).skipped).toBe(1);
|
||||
resolveFeed?.(); await active;
|
||||
expect(getFeed).toHaveBeenCalledTimes(1);
|
||||
expect((await service.pollWorkspace(session.workspaceId)).skipped).toBe(1);
|
||||
});
|
||||
it("uses stored credentials and the real connector HTTP limiter across accounts on one host", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { deps, connections } = setup(2);
|
||||
const credentials = new InMemoryCredentials();
|
||||
const requests: Array<{ at: number; token: string; path: string }> = [];
|
||||
const start = Date.now();
|
||||
const fetchImpl: typeof fetch = async (input, init) => {
|
||||
const url = new URL(String(input));
|
||||
const token = new Headers(init?.headers).get("authorization") ?? "";
|
||||
requests.push({ at: Date.now() - start, token, path: url.pathname });
|
||||
const account = { id: "alice", username: "alice", acct: "alice", display_name: "Alice", avatar: "", url: "https://social.example/@alice" };
|
||||
const data = url.pathname.endsWith("verify_credentials") ? account : [{
|
||||
id: "status-1", created_at: new Date().toISOString(), content: "<p>A real timeline post</p>",
|
||||
account, media_attachments: [], reblogs_count: 2, replies_count: 1, favourites_count: 3, reblog: null,
|
||||
}];
|
||||
return new Response(JSON.stringify(data), { headers: { "content-type": "application/json" } });
|
||||
};
|
||||
deps.registry = connectionRegistry(credentials, fetchImpl);
|
||||
for (const row of connections) {
|
||||
row.provider = "mastodon"; row.fields.instanceUrl = "https://social.example";
|
||||
await credentials.put(row, "access_token", `secret-${row.id}`);
|
||||
}
|
||||
const service = new FeedsService(deps);
|
||||
const polling = service.pollWorkspace(session.workspaceId);
|
||||
await vi.advanceTimersByTimeAsync(12_000);
|
||||
expect((await polling).polled).toBe(2);
|
||||
expect(requests).toHaveLength(4);
|
||||
expect(requests.every((request) => request.token.startsWith("Bearer secret-"))).toBe(true);
|
||||
for (let i = 1; i < requests.length; i += 1) expect((requests[i]?.at ?? 0) - (requests[i - 1]?.at ?? 0)).toBeGreaterThanOrEqual(2000);
|
||||
const response = await service.read(session);
|
||||
expect(response.items).toHaveLength(2);
|
||||
expect(response.items.every((post) => post.text === "A real timeline post" && post.metrics.favourites === 3)).toBe(true);
|
||||
});
|
||||
it("bounds concurrency across simultaneous workspace cycles", async () => {
|
||||
const { service, deps, getFeed, connections } = setup(10);
|
||||
let running = 0; let max = 0;
|
||||
const releases: Array<() => void> = [];
|
||||
getFeed.mockImplementation(() => new Promise((resolve) => {
|
||||
running += 1; max = Math.max(max, running);
|
||||
releases.push(() => { running -= 1; resolve({ items: [] }); });
|
||||
}));
|
||||
const polls = [service.pollWorkspace(session.workspaceId, connections.slice(0, 5).map((row) => row.id)), service.pollWorkspace(session.workspaceId, connections.slice(5).map((row) => row.id))];
|
||||
for (let i = 0; i < 50; i += 1) { await new Promise<void>((resolve) => setImmediate(resolve)); for (const release of releases.splice(0)) release(); }
|
||||
await Promise.all(polls);
|
||||
expect(max).toBe(4);
|
||||
expect(deps.audit).not.toHaveBeenCalled();
|
||||
});
|
||||
it("fails connections independently, clears stale data and honors Retry-After", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { service, getFeed, mock, connections } = setup();
|
||||
await service.pollWorkspace(session.workspaceId);
|
||||
await vi.advanceTimersByTimeAsync(66_000);
|
||||
getFeed.mockRejectedValueOnce(new RateLimitError("mock-social", 200_000));
|
||||
expect((await service.pollWorkspace(session.workspaceId)).failed).toBe(1);
|
||||
expect((await service.read(session)).items).toEqual([]);
|
||||
await vi.advanceTimersByTimeAsync(100_000);
|
||||
expect((await service.pollWorkspace(session.workspaceId)).skipped).toBe(1);
|
||||
await vi.advanceTimersByTimeAsync(100_000);
|
||||
vi.spyOn(mock, "getIdentity").mockRejectedValueOnce(new Error("Token revoked"));
|
||||
expect((await service.pollWorkspace(session.workspaceId)).failed).toBe(1);
|
||||
expect((await service.read(session, { connectionId: connections[0]?.id })).items).toEqual([]);
|
||||
expect(getFeed).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
28
apps/api/src/feeds/testing.ts
Normal file
28
apps/api/src/feeds/testing.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
import type { FeedDependencies } from "./feeds.dependencies.js";
|
||||
|
||||
/** TTL-faithful Redis surface, including ownership checks for renewable leases. */
|
||||
type FeedRedis = FeedDependencies["redis"];
|
||||
export class MemoryFeedRedis implements FeedRedis {
|
||||
readonly entries = new Map<string, { value: string; expiresAt: number }>();
|
||||
async get(key: string): Promise<string | null> {
|
||||
const entry = this.entries.get(key);
|
||||
if (!entry || entry.expiresAt <= Date.now()) { this.entries.delete(key); return null; }
|
||||
return entry.value;
|
||||
}
|
||||
async set(key: string, value: string, options: { nx?: boolean; pxMs?: number } = {}): Promise<boolean> {
|
||||
// Atomic NX check: do not yield between reading and writing.
|
||||
const entry = this.entries.get(key);
|
||||
if (options.nx && entry && entry.expiresAt > Date.now()) return false;
|
||||
this.entries.set(key, { value, expiresAt: Date.now() + (options.pxMs ?? 1000) });
|
||||
return true;
|
||||
}
|
||||
async eval(script: string, keys: string[], args: string[]): Promise<number> {
|
||||
const key = keys[0] ?? "";
|
||||
const entry = this.entries.get(key);
|
||||
if (!entry || entry.value !== args[0] || entry.expiresAt <= Date.now()) return 0;
|
||||
if (script.includes("PEXPIRE")) entry.expiresAt = Date.now() + Number(args[1]);
|
||||
else this.entries.delete(key);
|
||||
return 1;
|
||||
}
|
||||
close(): void {}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue