T-HUB/apps/api/src/intake/intake-search.ts

/**
 * Intake search + listing (task requirement 5) — the 025 archive-search
 * pattern applied to intake_items: Postgres FTS `websearch_to_tsquery`
 * ORed with pg_trgm similarity per token for typo tolerance, RLS-scoped via
 * withTenant (FORCE binds — there is no system path), keyset pagination with
 * a base64url JSON cursor and an N+1 fetch.
 *
 * Ranking: ts_rank dominates (weight-A title hits outrank body hits via the
 * 027 trigger's setweight), trigram similarity breaks ties. The no-query
 * path keysets on (published_at, id).
 */
import { BadRequestException } from "@nestjs/common";
import { withTenant } from "../db/client.js";
import type { StorageService } from "../storage/storage.js";
import { searchTokens } from "../archive/archive-search.js";

export interface IntakeQueryFilters {
  q: string | null;
  sourceId: string | null;
  kind: string | null;
  cursor: string | undefined;
  limit: number;
}

export const SEARCH_DEFAULT_LIMIT = 20;
export const SEARCH_MAX_LIMIT = 100;

export interface IntakeItemRow {
  id: string;
  source_id: string;
  remote_ref: string;
  content_hash: string;
  published_at: Date;
  text: string;
  content: Record<string, unknown>;
  media_refs: Array<{ storageKey?: string; proxyUrl?: string; kind?: string }>;
  attribution: Record<string, unknown>;
  fetched_at: Date;
  created_at: Date;
  source_kind: string;
  rank?: number;
}

const UUID_SHAPE = /^[0-9a-f-]{36}$/i;
const ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?$/;

interface Params {
  values: unknown[];
  bind(value: unknown): number;
}

function params(): Params {
  const values: unknown[] = [];
  return {
    values,
    bind(value: unknown): number { values.push(value); return values.length; },
  };
}

function decodeCursor<T>(cursor: string | undefined, what: string, shape: (v: Record<string, unknown>) => T): T {
  if (cursor === undefined) throw new BadRequestException(`invalid cursor: ${what}`);
  let value: unknown;
  try { value = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")); }
  catch { throw new BadRequestException(`invalid cursor: ${what}`); }
  if (typeof value !== "object" || value === null || Array.isArray(value)) {
    throw new BadRequestException(`invalid cursor: ${what}`);
  }
  return shape(value as Record<string, unknown>);
}

const ITEM_FIELDS = "intake_items.id, intake_items.source_id, intake_items.remote_ref, intake_items.content_hash, intake_items.published_at, intake_items.text, intake_items.content, intake_items.media_refs, intake_items.attribution, intake_items.fetched_at, intake_items.created_at";

function filterConds(p: Params, filters: IntakeQueryFilters): string[] {
  const conds: string[] = [];
  if (filters.sourceId !== null) conds.push(`source_id = $${p.bind(filters.sourceId)}::uuid`);
  if (filters.kind !== null) conds.push(`s.kind = $${p.bind(filters.kind)}`);
  return conds;
}

/**
 * Word-level fuzzy predicate. `text % $token` compares the WHOLE corpus
 * string to the token, so a single misspelled word inside a long post scores
 * ~0.1 and never clears pg_trgm's 0.3 threshold. Splitting first makes the
 * comparison word-vs-word, where a one-letter typo still scores ~0.67 —
 * that is the typo tolerance the spec asks pg_trgm for.
 */
function fuzzyPredicate(p: Params, tokens: string[]): string {
  if (tokens.length === 0) return "false";
  return tokens.map((token) => {
    const ti = p.bind(token);
    return `(EXISTS (SELECT 1 FROM regexp_split_to_table(intake_items.text, ' ') w WHERE w % $${ti})`
      + ` OR coalesce(intake_items.attribution->>'title', '') % $${ti})`;
  }).join(" OR ");
}

function rankExpression(p: Params, q: string, tokens: string[]): string {
  const qi = p.bind(q);
  const textSims = tokens.map((token) => `(SELECT max(similarity(w, $${p.bind(token)})) FROM regexp_split_to_table(intake_items.text, ' ') w)`);
  const titleSims = tokens.map((token) => `similarity(coalesce(intake_items.attribution->>'title',''), $${p.bind(token)})`);
  return `(ts_rank(intake_items.search_vector, websearch_to_tsquery('simple', $${qi})) * 2`
    + (textSims.length > 0 ? ` + GREATEST(${textSims.join(", ")})` : "")
    + (titleSims.length > 0 ? ` + GREATEST(${titleSims.join(", ")})` : "")
    + `)`;
}

export function searchIntake(workspaceId: string, filters: IntakeQueryFilters,
  storage: StorageService): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
  return runQuery(workspaceId, filters, storage, true);
}

export function listIntakeItems(workspaceId: string, filters: IntakeQueryFilters,
  storage: StorageService): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
  // Listing without q is the plain keyset path; a q present upgrades it to search.
  return runQuery(workspaceId, filters, storage, filters.q !== null);
}

async function runQuery(workspaceId: string, filters: IntakeQueryFilters,
  storage: StorageService, ranked: boolean): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
  const p = params();
  const tokens = filters.q !== null ? searchTokens(filters.q) : [];
  let sql: string;
  let cursorValue: { rk: number; id: string } | null = null;
  let cursorPosted: { at: string; id: string } | null = null;
  if (ranked && filters.q !== null) {
    const rank = rankExpression(p, filters.q, tokens);
    const conds = filterConds(p, filters);
    const qi = p.bind(filters.q);
    conds.push(`(intake_items.search_vector @@ websearch_to_tsquery('simple', $${qi}) OR ${fuzzyPredicate(p, tokens)})`);
    if (filters.cursor !== undefined) {
      cursorValue = decodeCursor(filters.cursor, "rank", (v) => {
        if (typeof v.rk !== "number" || !Number.isFinite(v.rk) || typeof v.id !== "string" || !UUID_SHAPE.test(v.id)) {
          throw new BadRequestException("invalid cursor: rank");
        }
        return { rk: v.rk, id: v.id };
      });
      conds.push(`(${rank}, intake_items.id) < ($${p.bind(cursorValue.rk)}::real, $${p.bind(cursorValue.id)}::uuid)`);
    }
    sql = `SELECT ranked.*, s.kind AS source_kind FROM
             (SELECT ${ITEM_FIELDS}, ${rank} AS rank
                FROM intake_items JOIN intake_sources s ON s.id = intake_items.source_id
               WHERE ${conds.join(" AND ")}) ranked
             JOIN intake_sources s ON s.id = ranked.source_id
           ORDER BY ranked.rank DESC, ranked.id ASC LIMIT $${p.bind(filters.limit + 1)}`;
  } else {
    const conds = filterConds(p, filters);
    if (filters.cursor !== undefined) {
      cursorPosted = decodeCursor(filters.cursor, "published", (v) => {
        if (typeof v.at !== "string" || !ISO_TIMESTAMP.test(v.at) || !Number.isFinite(Date.parse(v.at))
          || typeof v.id !== "string" || !UUID_SHAPE.test(v.id)) {
          throw new BadRequestException("invalid cursor: published");
        }
        return { at: v.at, id: v.id };
      });
      conds.push(`(intake_items.published_at, intake_items.id) < ($${p.bind(cursorPosted.at)}::timestamptz, $${p.bind(cursorPosted.id)}::uuid)`);
    }
    sql = `SELECT ${ITEM_FIELDS}, s.kind AS source_kind FROM intake_items
             JOIN intake_sources s ON s.id = intake_items.source_id
            WHERE ${conds.length > 0 ? conds.join(" AND ") : "true"}
            ORDER BY intake_items.published_at DESC, intake_items.id DESC LIMIT $${p.bind(filters.limit + 1)}`;
  }
  const result = await withTenant(workspaceId, (client) => client.query<IntakeItemRow>(sql, p.values));
  const page = result.rows.slice(0, filters.limit);
  const last = page.at(-1);
  const items = await Promise.all(page.map(async (row) => {
    const media = await Promise.all((row.media_refs ?? []).map(async (media) => ({
      ...media,
      thumbUrl: media.storageKey ? await storage.presign(media.storageKey, 900) : null,
    })));
    return {
      id: row.id,
      sourceId: row.source_id,
      remoteRef: row.remote_ref,
      url: typeof row.attribution.url === "string" ? row.attribution.url : null,
      contentHash: row.content_hash,
      publishedAt: row.published_at.toISOString(),
      text: row.text,
      content: row.content,
      media,
      attribution: row.attribution,
      kind: row.source_kind,
      fetchedAt: row.fetched_at.toISOString(),
      ...(row.rank !== undefined ? { rank: row.rank } : {}),
    };
  }));
  const nextCursor = result.rows.length > filters.limit && last
    ? Buffer.from(JSON.stringify(cursorShape(last, ranked && filters.q !== null))).toString("base64url")
    : null;
  return { items, nextCursor };
}

function cursorShape(last: IntakeItemRow, ranked: boolean): Record<string, string | number> {
  return ranked ? { rk: last.rank ?? 0, id: last.id } : { at: last.published_at.toISOString(), id: last.id };
}