import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { join } from "node:path"; import { Pool, type PoolClient } from "pg"; /** * WP15 archive search + export against a live PostgreSQL, as sp_app (FORCE * RLS binds). Uses its OWN database (ARCHIVE_SEARCH_IT_DATABASE_URL, plus * ARCHIVE_SEARCH_IT_SUPER_DATABASE_URL for migrations/seeding) per the * local-db-sol pattern: tenancy.integration.test.ts wipes every workspace in * the shared DB and vitest runs files in parallel. Skipped when unset. */ const appUrl = process.env.ARCHIVE_SEARCH_IT_DATABASE_URL; const ownerUrl = process.env.ARCHIVE_SEARCH_IT_SUPER_DATABASE_URL; type Modules = { service: typeof import("./archive.service.js"); "search-helpers": typeof import("./archive-search.js"); db: typeof import("../db/client.js"); }; describe.skipIf(appUrl === undefined || ownerUrl === undefined)("archive search + export (live DB)", () => { let m: Modules; let ownerPool: Pool; const suffix = `${Date.now()}${Math.floor(Math.random() * 1e6)}`; let wsA: string; let wsB: string; let chA = ""; const chB = `@chB${suffix}`; async function asOwner(fn: (client: PoolClient) => Promise): Promise { const client = await ownerPool.connect(); try { return await fn(client); } finally { client.release(); } } /** Seed tenant rows honestly: FORCE RLS applies to the owner too. */ async function inTenant(client: PoolClient, wsId: string, fn: () => Promise): Promise { await client.query("BEGIN"); await client.query("SELECT set_config('app.workspace_id', $1, true)", [wsId]); await fn(); await client.query("COMMIT"); } let seedSeq = 0; /** remote_ref must equal channel_id || ':' || message_id; content_hash is sha256 hex. */ async function seedItem(client: PoolClient, wsId: string, item: { channel: string; message: number; text: string; postedAt: Date; attribution?: Record; }): Promise { seedSeq += 1; await client.query( `INSERT INTO archived_feed_items (workspace_id, remote_ref, channel_id, message_id, text_content, content_hash, posted_at, attribution) VALUES ($1, $2 || ':' || $3::bigint::text, $2, $3::bigint, $4, encode(sha256($5::bytea), 'hex'), $6, $7::jsonb)`, [wsId, item.channel, item.message, item.text, `seed-${suffix}-${seedSeq}`, item.postedAt, JSON.stringify(item.attribution ?? {})], ); } beforeAll(async () => { process.env.DATABASE_URL = appUrl; process.env.SUPER_DATABASE_URL = ownerUrl; const { runMigrations } = await import("../db/migrate.js"); await runMigrations(join(process.cwd(), "migrations")); m = { service: await import("./archive.service.js"), "search-helpers": await import("./archive-search.js"), db: await import("../db/client.js"), }; ownerPool = new Pool({ connectionString: ownerUrl }); await asOwner(async (c) => { const ws = async (name: string): Promise => (await c.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id", [name, `${name}-${suffix}`])).rows[0].id as string; wsA = await ws("search-a"); wsB = await ws("search-b"); }); // Tenant data: distinct texts per tenant, overlap only inside wsA. chA = `@chA${suffix}`; await asOwner((c) => inTenant(c, wsA, async () => { await seedItem(c, wsA, { channel: chA, message: 1001, text: "Water rights are a community responsibility, not a commodity for markets.", postedAt: new Date("2026-01-10T09:00:00Z"), attribution: { channelUsername: chA.slice(1), author: "Water Working Group" }, }); await seedItem(c, wsA, { channel: chA, message: 1002, text: "Totally unrelated post about bicycle repair cooperatives and tools.", postedAt: new Date("2026-01-11T09:00:00Z"), }); await seedItem(c, wsA, { channel: chA, message: 1003, text: "Water rights update: the hearing is postponed to February.", postedAt: new Date("2026-01-12T09:00:00Z"), }); })); await asOwner((c) => inTenant(c, wsB, async () => { await seedItem(c, wsB, { channel: chB, message: 2001, text: "Water rights are a community responsibility, not a commodity for markets.", postedAt: new Date("2026-01-05T09:00:00Z"), }); })); }); afterAll(async () => { if (ownerPool !== undefined) { await asOwner(async (c) => { await c.query("DELETE FROM workspaces WHERE id = ANY($1::uuid[])", [[wsA, wsB]]); }); await ownerPool.end(); } await m?.db.pool.end(); }); it("full-text search matches archived text (websearch_to_tsquery)", async () => { const service = new m.service.ArchiveService(null as never); const result = await service.search(wsA, { q: "water rights" }); expect(result.items).toHaveLength(2); for (const item of result.items) { expect(String(item.text)).toMatch(/water/i); expect(item.remoteRef).toBeDefined(); } // Ranked path: both matches, the denser one first is NOT guaranteed, but // the update (1003) must not shadow the older match. expect(result.items.map((i) => i.messageId).sort()).toEqual([1001, 1003]); }); it("fuzzy typo tolerance: misspelled token still matches via pg_trgm", async () => { const service = new m.service.ArchiveService(null as never); const result = await service.search(wsA, { q: "watter rghts" }); expect(result.items.length).toBeGreaterThanOrEqual(1); expect(result.items.map((i) => i.messageId).sort()).toEqual([1001, 1003]); }); it("metadata search: channel id and attribution text are searchable", async () => { const service = new m.service.ArchiveService(null as never); const byChannel = await service.search(wsA, { q: chA }); expect(byChannel.items).toHaveLength(3); const byAuthor = await service.search(wsA, { q: "Working Group" }); expect(byAuthor.items.map((i) => i.messageId)).toEqual([1001]); }); it("RLS: search never crosses tenants even with shared text", async () => { const service = new m.service.ArchiveService(null as never); const a = await service.search(wsA, { q: "water rights" }); const b = await service.search(wsB, { q: "water rights" }); expect(a.items).toHaveLength(2); expect(b.items).toHaveLength(1); expect(b.items.every((i) => String(i.remoteRef).startsWith(chB))).toBe(true); // Direct DB-level cross-tenant denial as sp_app. const leaked = await m.db.withTenant(wsB, (client) => client.query( "SELECT count(*)::int AS n FROM archived_feed_items WHERE workspace_id = $1", [wsA])); expect(leaked.rows[0]?.n).toBe(0); }); it("date + channel filters narrow the result set", async () => { const service = new m.service.ArchiveService(null as never); const from = await service.search(wsA, { q: "water", from: "2026-01-12" }); expect(from.items.map((i) => i.messageId)).toEqual([1003]); const to = await service.search(wsA, { q: "water", to: "2026-01-10T23:59:59Z" }); expect(to.items.map((i) => i.messageId)).toEqual([1001]); const channel = await service.search(wsA, { q: "bicycle", channelId: chA }); expect(channel.items.map((i) => i.messageId)).toEqual([1002]); const none = await service.search(wsA, { q: "bicycle", channelId: `@nonesuch${suffix}` }); expect(none.items).toHaveLength(0); }); it("keyset pagination walks the ranked result set without repeats or gaps", async () => { const service = new m.service.ArchiveService(null as never); const page1 = await service.search(wsA, { q: "water rights", limit: "1" }); expect(page1.items).toHaveLength(1); expect(page1.nextCursor).not.toBeNull(); const page2 = await service.search(wsA, { q: "water rights", limit: "1", cursor: page1.nextCursor ?? "" }); expect(page2.items).toHaveLength(1); expect(page2.nextCursor).toBeNull(); const seen = [...page1.items, ...page2.items].map((i) => i.messageId).sort(); expect(seen).toEqual([1001, 1003]); // Bogus cursor is rejected, not silently ignored. await expect(service.search(wsA, { q: "water", cursor: "bogus-cursor" })) .rejects.toThrow(/invalid cursor/); }); it("export streams every matching item with faithful content (json + csv)", async () => { const service = new m.service.ArchiveService(null as never); const json: string[] = []; const format = await service.exportItems(wsA, { q: "water rights" }, (chunk) => json.push(chunk)); expect(format).toBe("json"); const parsed = JSON.parse(json.join("")) as Array>; expect(parsed).toHaveLength(2); expect(parsed.map((i) => i.messageId).sort()).toEqual([1001, 1003]); expect(parsed.every((i) => typeof i.postedAt === "string" && typeof i.remoteRef === "string")).toBe(true); const csv: string[] = []; const csvFormat = await service.exportItems(wsA, { q: "water", format: "csv" }, (chunk) => csv.push(chunk)); expect(csvFormat).toBe("csv"); const csvText = csv.join(""); const lines = csvText.split("\r\n").filter((line) => line !== ""); expect(lines).toHaveLength(3); // header + 2 rows const header = lines[0]!.split(","); expect(header.slice(0, 4)).toEqual(["id", "remote_ref", "channel_id", "message_id"]); const rows = lines.slice(1).map((line) => line.split(",")); for (const row of rows) { expect(row).toHaveLength(header.length); expect(UUID_SHAPE.test(row[0]!)).toBe(true); expect(row[1]).toMatch(new RegExp(`^${chA}:[0-9]+$`)); } const textCol = header.indexOf("text"); const texts = rows.map((row) => row[textCol]!).sort(); expect(texts[1]).toContain("hearing is postponed"); }); it("export CSV escaping: commas, quotes and newlines round-trip", async () => { const service = new m.service.ArchiveService(null as never); const hostile = 'He said "water, rights", then\nleft'; await asOwner((c) => inTenant(c, wsA, async () => { await seedItem(c, wsA, { channel: chA, message: 1004, text: hostile, postedAt: new Date("2026-01-13T09:00:00Z"), }); })); const csv: string[] = []; await service.exportItems(wsA, { channelId: chA, format: "csv" }, (chunk) => csv.push(chunk)); const lines = csv.join("").split("\r\n").filter((l) => l !== ""); const header = lines[0]!.split(","); const textCol = header.indexOf("text"); const rows = lines.slice(1).map((line) => line.split(",")); const hostileRow = rows.find((row) => row[textCol] === hostile); expect(hostileRow).toBeDefined(); expect(hostileRow![textCol]).toBe(hostile); expect(rows.every((row) => row.length === header.length)).toBe(true); }); it("filter validation rejects garbage bounds, limits and formats", async () => { const service = new m.service.ArchiveService(null as never); await expect(service.search(wsA, { q: "water", from: "not-a-date" })).rejects.toThrow(/ISO-8601/); await expect(service.search(wsA, { q: "water", from: "2026-02-01", to: "2026-01-01" })).rejects.toThrow(/after/); await expect(service.search(wsA, { q: "water", limit: "0" })).rejects.toThrow(/between 1 and/); await expect(service.search(wsA, { q: "water", limit: "1000" })).rejects.toThrow(/between 1 and/); await expect(service.search(wsA, { q: "water", cursor: "bogus" })).rejects.toThrow(/invalid cursor/); await expect(service.exportItems(wsA, { format: "xml" }, () => undefined)).rejects.toThrow(/csv or json/); }); it("unindexed helper behavior: tokenization + CSV header shape", () => { const helpers = m["search-helpers"]; expect(helpers.searchTokens("Watter, Rghts! @channel123")).toEqual(["watter", "rghts", "@channel123"]); expect(helpers.searchTokens("ab xx ")).toEqual([]); expect(helpers.ARCHIVE_CSV_HEADER).toMatch(/^id,remote_ref,channel_id,message_id,posted_at,revision,protected_content,text,storage_keys,attribution\r\n$/); }); }); const UUID_SHAPE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;