Tactical Media Broadcast Hub — initial release
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
This commit is contained in:
commit
aa5e262f3a
880 changed files with 111042 additions and 0 deletions
193
apps/api/src/intake/intake-search.ts
Normal file
193
apps/api/src/intake/intake-search.ts
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
/**
|
||||
* Intake search + listing (task requirement 5) — the 025 archive-search
|
||||
* pattern applied to intake_items: Postgres FTS `websearch_to_tsquery`
|
||||
* ORed with pg_trgm similarity per token for typo tolerance, RLS-scoped via
|
||||
* withTenant (FORCE binds — there is no system path), keyset pagination with
|
||||
* a base64url JSON cursor and an N+1 fetch.
|
||||
*
|
||||
* Ranking: ts_rank dominates (weight-A title hits outrank body hits via the
|
||||
* 027 trigger's setweight), trigram similarity breaks ties. The no-query
|
||||
* path keysets on (published_at, id).
|
||||
*/
|
||||
import { BadRequestException } from "@nestjs/common";
|
||||
import { withTenant } from "../db/client.js";
|
||||
import type { StorageService } from "../storage/storage.js";
|
||||
import { searchTokens } from "../archive/archive-search.js";
|
||||
|
||||
export interface IntakeQueryFilters {
|
||||
q: string | null;
|
||||
sourceId: string | null;
|
||||
kind: string | null;
|
||||
cursor: string | undefined;
|
||||
limit: number;
|
||||
}
|
||||
|
||||
export const SEARCH_DEFAULT_LIMIT = 20;
|
||||
export const SEARCH_MAX_LIMIT = 100;
|
||||
|
||||
export interface IntakeItemRow {
|
||||
id: string;
|
||||
source_id: string;
|
||||
remote_ref: string;
|
||||
content_hash: string;
|
||||
published_at: Date;
|
||||
text: string;
|
||||
content: Record<string, unknown>;
|
||||
media_refs: Array<{ storageKey?: string; proxyUrl?: string; kind?: string }>;
|
||||
attribution: Record<string, unknown>;
|
||||
fetched_at: Date;
|
||||
created_at: Date;
|
||||
source_kind: string;
|
||||
rank?: number;
|
||||
}
|
||||
|
||||
const UUID_SHAPE = /^[0-9a-f-]{36}$/i;
|
||||
const ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?$/;
|
||||
|
||||
interface Params {
|
||||
values: unknown[];
|
||||
bind(value: unknown): number;
|
||||
}
|
||||
|
||||
function params(): Params {
|
||||
const values: unknown[] = [];
|
||||
return {
|
||||
values,
|
||||
bind(value: unknown): number { values.push(value); return values.length; },
|
||||
};
|
||||
}
|
||||
|
||||
function decodeCursor<T>(cursor: string | undefined, what: string, shape: (v: Record<string, unknown>) => T): T {
|
||||
if (cursor === undefined) throw new BadRequestException(`invalid cursor: ${what}`);
|
||||
let value: unknown;
|
||||
try { value = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")); }
|
||||
catch { throw new BadRequestException(`invalid cursor: ${what}`); }
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
||||
throw new BadRequestException(`invalid cursor: ${what}`);
|
||||
}
|
||||
return shape(value as Record<string, unknown>);
|
||||
}
|
||||
|
||||
const ITEM_FIELDS = "intake_items.id, intake_items.source_id, intake_items.remote_ref, intake_items.content_hash, intake_items.published_at, intake_items.text, intake_items.content, intake_items.media_refs, intake_items.attribution, intake_items.fetched_at, intake_items.created_at";
|
||||
|
||||
function filterConds(p: Params, filters: IntakeQueryFilters): string[] {
|
||||
const conds: string[] = [];
|
||||
if (filters.sourceId !== null) conds.push(`source_id = $${p.bind(filters.sourceId)}::uuid`);
|
||||
if (filters.kind !== null) conds.push(`s.kind = $${p.bind(filters.kind)}`);
|
||||
return conds;
|
||||
}
|
||||
|
||||
/**
|
||||
* Word-level fuzzy predicate. `text % $token` compares the WHOLE corpus
|
||||
* string to the token, so a single misspelled word inside a long post scores
|
||||
* ~0.1 and never clears pg_trgm's 0.3 threshold. Splitting first makes the
|
||||
* comparison word-vs-word, where a one-letter typo still scores ~0.67 —
|
||||
* that is the typo tolerance the spec asks pg_trgm for.
|
||||
*/
|
||||
function fuzzyPredicate(p: Params, tokens: string[]): string {
|
||||
if (tokens.length === 0) return "false";
|
||||
return tokens.map((token) => {
|
||||
const ti = p.bind(token);
|
||||
return `(EXISTS (SELECT 1 FROM regexp_split_to_table(intake_items.text, ' ') w WHERE w % $${ti})`
|
||||
+ ` OR coalesce(intake_items.attribution->>'title', '') % $${ti})`;
|
||||
}).join(" OR ");
|
||||
}
|
||||
|
||||
function rankExpression(p: Params, q: string, tokens: string[]): string {
|
||||
const qi = p.bind(q);
|
||||
const textSims = tokens.map((token) => `(SELECT max(similarity(w, $${p.bind(token)})) FROM regexp_split_to_table(intake_items.text, ' ') w)`);
|
||||
const titleSims = tokens.map((token) => `similarity(coalesce(intake_items.attribution->>'title',''), $${p.bind(token)})`);
|
||||
return `(ts_rank(intake_items.search_vector, websearch_to_tsquery('simple', $${qi})) * 2`
|
||||
+ (textSims.length > 0 ? ` + GREATEST(${textSims.join(", ")})` : "")
|
||||
+ (titleSims.length > 0 ? ` + GREATEST(${titleSims.join(", ")})` : "")
|
||||
+ `)`;
|
||||
}
|
||||
|
||||
export function searchIntake(workspaceId: string, filters: IntakeQueryFilters,
|
||||
storage: StorageService): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
|
||||
return runQuery(workspaceId, filters, storage, true);
|
||||
}
|
||||
|
||||
export function listIntakeItems(workspaceId: string, filters: IntakeQueryFilters,
|
||||
storage: StorageService): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
|
||||
// Listing without q is the plain keyset path; a q present upgrades it to search.
|
||||
return runQuery(workspaceId, filters, storage, filters.q !== null);
|
||||
}
|
||||
|
||||
async function runQuery(workspaceId: string, filters: IntakeQueryFilters,
|
||||
storage: StorageService, ranked: boolean): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
|
||||
const p = params();
|
||||
const tokens = filters.q !== null ? searchTokens(filters.q) : [];
|
||||
let sql: string;
|
||||
let cursorValue: { rk: number; id: string } | null = null;
|
||||
let cursorPosted: { at: string; id: string } | null = null;
|
||||
if (ranked && filters.q !== null) {
|
||||
const rank = rankExpression(p, filters.q, tokens);
|
||||
const conds = filterConds(p, filters);
|
||||
const qi = p.bind(filters.q);
|
||||
conds.push(`(intake_items.search_vector @@ websearch_to_tsquery('simple', $${qi}) OR ${fuzzyPredicate(p, tokens)})`);
|
||||
if (filters.cursor !== undefined) {
|
||||
cursorValue = decodeCursor(filters.cursor, "rank", (v) => {
|
||||
if (typeof v.rk !== "number" || !Number.isFinite(v.rk) || typeof v.id !== "string" || !UUID_SHAPE.test(v.id)) {
|
||||
throw new BadRequestException("invalid cursor: rank");
|
||||
}
|
||||
return { rk: v.rk, id: v.id };
|
||||
});
|
||||
conds.push(`(${rank}, intake_items.id) < ($${p.bind(cursorValue.rk)}::real, $${p.bind(cursorValue.id)}::uuid)`);
|
||||
}
|
||||
sql = `SELECT ranked.*, s.kind AS source_kind FROM
|
||||
(SELECT ${ITEM_FIELDS}, ${rank} AS rank
|
||||
FROM intake_items JOIN intake_sources s ON s.id = intake_items.source_id
|
||||
WHERE ${conds.join(" AND ")}) ranked
|
||||
JOIN intake_sources s ON s.id = ranked.source_id
|
||||
ORDER BY ranked.rank DESC, ranked.id ASC LIMIT $${p.bind(filters.limit + 1)}`;
|
||||
} else {
|
||||
const conds = filterConds(p, filters);
|
||||
if (filters.cursor !== undefined) {
|
||||
cursorPosted = decodeCursor(filters.cursor, "published", (v) => {
|
||||
if (typeof v.at !== "string" || !ISO_TIMESTAMP.test(v.at) || !Number.isFinite(Date.parse(v.at))
|
||||
|| typeof v.id !== "string" || !UUID_SHAPE.test(v.id)) {
|
||||
throw new BadRequestException("invalid cursor: published");
|
||||
}
|
||||
return { at: v.at, id: v.id };
|
||||
});
|
||||
conds.push(`(intake_items.published_at, intake_items.id) < ($${p.bind(cursorPosted.at)}::timestamptz, $${p.bind(cursorPosted.id)}::uuid)`);
|
||||
}
|
||||
sql = `SELECT ${ITEM_FIELDS}, s.kind AS source_kind FROM intake_items
|
||||
JOIN intake_sources s ON s.id = intake_items.source_id
|
||||
WHERE ${conds.length > 0 ? conds.join(" AND ") : "true"}
|
||||
ORDER BY intake_items.published_at DESC, intake_items.id DESC LIMIT $${p.bind(filters.limit + 1)}`;
|
||||
}
|
||||
const result = await withTenant(workspaceId, (client) => client.query<IntakeItemRow>(sql, p.values));
|
||||
const page = result.rows.slice(0, filters.limit);
|
||||
const last = page.at(-1);
|
||||
const items = await Promise.all(page.map(async (row) => {
|
||||
const media = await Promise.all((row.media_refs ?? []).map(async (media) => ({
|
||||
...media,
|
||||
thumbUrl: media.storageKey ? await storage.presign(media.storageKey, 900) : null,
|
||||
})));
|
||||
return {
|
||||
id: row.id,
|
||||
sourceId: row.source_id,
|
||||
remoteRef: row.remote_ref,
|
||||
url: typeof row.attribution.url === "string" ? row.attribution.url : null,
|
||||
contentHash: row.content_hash,
|
||||
publishedAt: row.published_at.toISOString(),
|
||||
text: row.text,
|
||||
content: row.content,
|
||||
media,
|
||||
attribution: row.attribution,
|
||||
kind: row.source_kind,
|
||||
fetchedAt: row.fetched_at.toISOString(),
|
||||
...(row.rank !== undefined ? { rank: row.rank } : {}),
|
||||
};
|
||||
}));
|
||||
const nextCursor = result.rows.length > filters.limit && last
|
||||
? Buffer.from(JSON.stringify(cursorShape(last, ranked && filters.q !== null))).toString("base64url")
|
||||
: null;
|
||||
return { items, nextCursor };
|
||||
}
|
||||
|
||||
function cursorShape(last: IntakeItemRow, ranked: boolean): Record<string, string | number> {
|
||||
return ranked ? { rk: last.rank ?? 0, id: last.id } : { at: last.published_at.toISOString(), id: last.id };
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue