/** * Intake search + listing (task requirement 5) — the 025 archive-search * pattern applied to intake_items: Postgres FTS `websearch_to_tsquery` * ORed with pg_trgm similarity per token for typo tolerance, RLS-scoped via * withTenant (FORCE binds — there is no system path), keyset pagination with * a base64url JSON cursor and an N+1 fetch. * * Ranking: ts_rank dominates (weight-A title hits outrank body hits via the * 027 trigger's setweight), trigram similarity breaks ties. The no-query * path keysets on (published_at, id). */ import { BadRequestException } from "@nestjs/common"; import { withTenant } from "../db/client.js"; import type { StorageService } from "../storage/storage.js"; import { searchTokens } from "../archive/archive-search.js"; export interface IntakeQueryFilters { q: string | null; sourceId: string | null; kind: string | null; cursor: string | undefined; limit: number; } export const SEARCH_DEFAULT_LIMIT = 20; export const SEARCH_MAX_LIMIT = 100; export interface IntakeItemRow { id: string; source_id: string; remote_ref: string; content_hash: string; published_at: Date; text: string; content: Record; media_refs: Array<{ storageKey?: string; proxyUrl?: string; kind?: string }>; attribution: Record; fetched_at: Date; created_at: Date; source_kind: string; rank?: number; } const UUID_SHAPE = /^[0-9a-f-]{36}$/i; const ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?$/; interface Params { values: unknown[]; bind(value: unknown): number; } function params(): Params { const values: unknown[] = []; return { values, bind(value: unknown): number { values.push(value); return values.length; }, }; } function decodeCursor(cursor: string | undefined, what: string, shape: (v: Record) => T): T { if (cursor === undefined) throw new BadRequestException(`invalid cursor: ${what}`); let value: unknown; try { value = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")); } catch { throw new BadRequestException(`invalid cursor: ${what}`); } if (typeof value !== "object" || value === null || Array.isArray(value)) { throw new BadRequestException(`invalid cursor: ${what}`); } return shape(value as Record); } const ITEM_FIELDS = "intake_items.id, intake_items.source_id, intake_items.remote_ref, intake_items.content_hash, intake_items.published_at, intake_items.text, intake_items.content, intake_items.media_refs, intake_items.attribution, intake_items.fetched_at, intake_items.created_at"; function filterConds(p: Params, filters: IntakeQueryFilters): string[] { const conds: string[] = []; if (filters.sourceId !== null) conds.push(`source_id = $${p.bind(filters.sourceId)}::uuid`); if (filters.kind !== null) conds.push(`s.kind = $${p.bind(filters.kind)}`); return conds; } /** * Word-level fuzzy predicate. `text % $token` compares the WHOLE corpus * string to the token, so a single misspelled word inside a long post scores * ~0.1 and never clears pg_trgm's 0.3 threshold. Splitting first makes the * comparison word-vs-word, where a one-letter typo still scores ~0.67 — * that is the typo tolerance the spec asks pg_trgm for. */ function fuzzyPredicate(p: Params, tokens: string[]): string { if (tokens.length === 0) return "false"; return tokens.map((token) => { const ti = p.bind(token); return `(EXISTS (SELECT 1 FROM regexp_split_to_table(intake_items.text, ' ') w WHERE w % $${ti})` + ` OR coalesce(intake_items.attribution->>'title', '') % $${ti})`; }).join(" OR "); } function rankExpression(p: Params, q: string, tokens: string[]): string { const qi = p.bind(q); const textSims = tokens.map((token) => `(SELECT max(similarity(w, $${p.bind(token)})) FROM regexp_split_to_table(intake_items.text, ' ') w)`); const titleSims = tokens.map((token) => `similarity(coalesce(intake_items.attribution->>'title',''), $${p.bind(token)})`); return `(ts_rank(intake_items.search_vector, websearch_to_tsquery('simple', $${qi})) * 2` + (textSims.length > 0 ? ` + GREATEST(${textSims.join(", ")})` : "") + (titleSims.length > 0 ? ` + GREATEST(${titleSims.join(", ")})` : "") + `)`; } export function searchIntake(workspaceId: string, filters: IntakeQueryFilters, storage: StorageService): Promise<{ items: Record[]; nextCursor: string | null }> { return runQuery(workspaceId, filters, storage, true); } export function listIntakeItems(workspaceId: string, filters: IntakeQueryFilters, storage: StorageService): Promise<{ items: Record[]; nextCursor: string | null }> { // Listing without q is the plain keyset path; a q present upgrades it to search. return runQuery(workspaceId, filters, storage, filters.q !== null); } async function runQuery(workspaceId: string, filters: IntakeQueryFilters, storage: StorageService, ranked: boolean): Promise<{ items: Record[]; nextCursor: string | null }> { const p = params(); const tokens = filters.q !== null ? searchTokens(filters.q) : []; let sql: string; let cursorValue: { rk: number; id: string } | null = null; let cursorPosted: { at: string; id: string } | null = null; if (ranked && filters.q !== null) { const rank = rankExpression(p, filters.q, tokens); const conds = filterConds(p, filters); const qi = p.bind(filters.q); conds.push(`(intake_items.search_vector @@ websearch_to_tsquery('simple', $${qi}) OR ${fuzzyPredicate(p, tokens)})`); if (filters.cursor !== undefined) { cursorValue = decodeCursor(filters.cursor, "rank", (v) => { if (typeof v.rk !== "number" || !Number.isFinite(v.rk) || typeof v.id !== "string" || !UUID_SHAPE.test(v.id)) { throw new BadRequestException("invalid cursor: rank"); } return { rk: v.rk, id: v.id }; }); conds.push(`(${rank}, intake_items.id) < ($${p.bind(cursorValue.rk)}::real, $${p.bind(cursorValue.id)}::uuid)`); } sql = `SELECT ranked.*, s.kind AS source_kind FROM (SELECT ${ITEM_FIELDS}, ${rank} AS rank FROM intake_items JOIN intake_sources s ON s.id = intake_items.source_id WHERE ${conds.join(" AND ")}) ranked JOIN intake_sources s ON s.id = ranked.source_id ORDER BY ranked.rank DESC, ranked.id ASC LIMIT $${p.bind(filters.limit + 1)}`; } else { const conds = filterConds(p, filters); if (filters.cursor !== undefined) { cursorPosted = decodeCursor(filters.cursor, "published", (v) => { if (typeof v.at !== "string" || !ISO_TIMESTAMP.test(v.at) || !Number.isFinite(Date.parse(v.at)) || typeof v.id !== "string" || !UUID_SHAPE.test(v.id)) { throw new BadRequestException("invalid cursor: published"); } return { at: v.at, id: v.id }; }); conds.push(`(intake_items.published_at, intake_items.id) < ($${p.bind(cursorPosted.at)}::timestamptz, $${p.bind(cursorPosted.id)}::uuid)`); } sql = `SELECT ${ITEM_FIELDS}, s.kind AS source_kind FROM intake_items JOIN intake_sources s ON s.id = intake_items.source_id WHERE ${conds.length > 0 ? conds.join(" AND ") : "true"} ORDER BY intake_items.published_at DESC, intake_items.id DESC LIMIT $${p.bind(filters.limit + 1)}`; } const result = await withTenant(workspaceId, (client) => client.query(sql, p.values)); const page = result.rows.slice(0, filters.limit); const last = page.at(-1); const items = await Promise.all(page.map(async (row) => { const media = await Promise.all((row.media_refs ?? []).map(async (media) => ({ ...media, thumbUrl: media.storageKey ? await storage.presign(media.storageKey, 900) : null, }))); return { id: row.id, sourceId: row.source_id, remoteRef: row.remote_ref, url: typeof row.attribution.url === "string" ? row.attribution.url : null, contentHash: row.content_hash, publishedAt: row.published_at.toISOString(), text: row.text, content: row.content, media, attribution: row.attribution, kind: row.source_kind, fetchedAt: row.fetched_at.toISOString(), ...(row.rank !== undefined ? { rank: row.rank } : {}), }; })); const nextCursor = result.rows.length > filters.limit && last ? Buffer.from(JSON.stringify(cursorShape(last, ranked && filters.q !== null))).toString("base64url") : null; return { items, nextCursor }; } function cursorShape(last: IntakeItemRow, ranked: boolean): Record { return ranked ? { rk: last.rank ?? 0, id: last.id } : { at: last.published_at.toISOString(), id: last.id }; }