glass beadian
aa5e262f3a
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
256 lines
12 KiB
TypeScript
256 lines
12 KiB
TypeScript
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
import { join } from "node:path";
|
|
import { Pool, type PoolClient } from "pg";
|
|
|
|
/**
|
|
* WP15 archive search + export against a live PostgreSQL, as sp_app (FORCE
|
|
* RLS binds). Uses its OWN database (ARCHIVE_SEARCH_IT_DATABASE_URL, plus
|
|
* ARCHIVE_SEARCH_IT_SUPER_DATABASE_URL for migrations/seeding) per the
|
|
* local-db-sol pattern: tenancy.integration.test.ts wipes every workspace in
|
|
* the shared DB and vitest runs files in parallel. Skipped when unset.
|
|
*/
|
|
const appUrl = process.env.ARCHIVE_SEARCH_IT_DATABASE_URL;
|
|
const ownerUrl = process.env.ARCHIVE_SEARCH_IT_SUPER_DATABASE_URL;
|
|
|
|
type Modules = {
|
|
service: typeof import("./archive.service.js");
|
|
"search-helpers": typeof import("./archive-search.js");
|
|
db: typeof import("../db/client.js");
|
|
};
|
|
|
|
describe.skipIf(appUrl === undefined || ownerUrl === undefined)("archive search + export (live DB)", () => {
|
|
let m: Modules;
|
|
let ownerPool: Pool;
|
|
const suffix = `${Date.now()}${Math.floor(Math.random() * 1e6)}`;
|
|
let wsA: string;
|
|
let wsB: string;
|
|
let chA = "";
|
|
const chB = `@chB${suffix}`;
|
|
|
|
async function asOwner<T>(fn: (client: PoolClient) => Promise<T>): Promise<T> {
|
|
const client = await ownerPool.connect();
|
|
try {
|
|
return await fn(client);
|
|
} finally {
|
|
client.release();
|
|
}
|
|
}
|
|
|
|
/** Seed tenant rows honestly: FORCE RLS applies to the owner too. */
|
|
async function inTenant(client: PoolClient, wsId: string, fn: () => Promise<void>): Promise<void> {
|
|
await client.query("BEGIN");
|
|
await client.query("SELECT set_config('app.workspace_id', $1, true)", [wsId]);
|
|
await fn();
|
|
await client.query("COMMIT");
|
|
}
|
|
|
|
let seedSeq = 0;
|
|
/** remote_ref must equal channel_id || ':' || message_id; content_hash is sha256 hex. */
|
|
async function seedItem(client: PoolClient, wsId: string, item: {
|
|
channel: string; message: number; text: string; postedAt: Date; attribution?: Record<string, unknown>;
|
|
}): Promise<void> {
|
|
seedSeq += 1;
|
|
await client.query(
|
|
`INSERT INTO archived_feed_items
|
|
(workspace_id, remote_ref, channel_id, message_id, text_content, content_hash, posted_at, attribution)
|
|
VALUES ($1, $2 || ':' || $3::bigint::text, $2, $3::bigint, $4, encode(sha256($5::bytea), 'hex'), $6, $7::jsonb)`,
|
|
[wsId, item.channel, item.message, item.text, `seed-${suffix}-${seedSeq}`, item.postedAt,
|
|
JSON.stringify(item.attribution ?? {})],
|
|
);
|
|
}
|
|
|
|
beforeAll(async () => {
|
|
process.env.DATABASE_URL = appUrl;
|
|
process.env.SUPER_DATABASE_URL = ownerUrl;
|
|
const { runMigrations } = await import("../db/migrate.js");
|
|
await runMigrations(join(process.cwd(), "migrations"));
|
|
m = {
|
|
service: await import("./archive.service.js"),
|
|
"search-helpers": await import("./archive-search.js"),
|
|
db: await import("../db/client.js"),
|
|
};
|
|
ownerPool = new Pool({ connectionString: ownerUrl });
|
|
|
|
await asOwner(async (c) => {
|
|
const ws = async (name: string): Promise<string> =>
|
|
(await c.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id",
|
|
[name, `${name}-${suffix}`])).rows[0].id as string;
|
|
wsA = await ws("search-a");
|
|
wsB = await ws("search-b");
|
|
});
|
|
|
|
// Tenant data: distinct texts per tenant, overlap only inside wsA.
|
|
chA = `@chA${suffix}`;
|
|
await asOwner((c) => inTenant(c, wsA, async () => {
|
|
await seedItem(c, wsA, {
|
|
channel: chA, message: 1001,
|
|
text: "Water rights are a community responsibility, not a commodity for markets.",
|
|
postedAt: new Date("2026-01-10T09:00:00Z"),
|
|
attribution: { channelUsername: chA.slice(1), author: "Water Working Group" },
|
|
});
|
|
await seedItem(c, wsA, {
|
|
channel: chA, message: 1002,
|
|
text: "Totally unrelated post about bicycle repair cooperatives and tools.",
|
|
postedAt: new Date("2026-01-11T09:00:00Z"),
|
|
});
|
|
await seedItem(c, wsA, {
|
|
channel: chA, message: 1003,
|
|
text: "Water rights update: the hearing is postponed to February.",
|
|
postedAt: new Date("2026-01-12T09:00:00Z"),
|
|
});
|
|
}));
|
|
await asOwner((c) => inTenant(c, wsB, async () => {
|
|
await seedItem(c, wsB, {
|
|
channel: chB, message: 2001,
|
|
text: "Water rights are a community responsibility, not a commodity for markets.",
|
|
postedAt: new Date("2026-01-05T09:00:00Z"),
|
|
});
|
|
}));
|
|
});
|
|
|
|
afterAll(async () => {
|
|
if (ownerPool !== undefined) {
|
|
await asOwner(async (c) => {
|
|
await c.query("DELETE FROM workspaces WHERE id = ANY($1::uuid[])", [[wsA, wsB]]);
|
|
});
|
|
await ownerPool.end();
|
|
}
|
|
await m?.db.pool.end();
|
|
});
|
|
|
|
it("full-text search matches archived text (websearch_to_tsquery)", async () => {
|
|
const service = new m.service.ArchiveService(null as never);
|
|
const result = await service.search(wsA, { q: "water rights" });
|
|
expect(result.items).toHaveLength(2);
|
|
for (const item of result.items) {
|
|
expect(String(item.text)).toMatch(/water/i);
|
|
expect(item.remoteRef).toBeDefined();
|
|
}
|
|
// Ranked path: both matches, the denser one first is NOT guaranteed, but
|
|
// the update (1003) must not shadow the older match.
|
|
expect(result.items.map((i) => i.messageId).sort()).toEqual([1001, 1003]);
|
|
});
|
|
|
|
it("fuzzy typo tolerance: misspelled token still matches via pg_trgm", async () => {
|
|
const service = new m.service.ArchiveService(null as never);
|
|
const result = await service.search(wsA, { q: "watter rghts" });
|
|
expect(result.items.length).toBeGreaterThanOrEqual(1);
|
|
expect(result.items.map((i) => i.messageId).sort()).toEqual([1001, 1003]);
|
|
});
|
|
|
|
it("metadata search: channel id and attribution text are searchable", async () => {
|
|
const service = new m.service.ArchiveService(null as never);
|
|
const byChannel = await service.search(wsA, { q: chA });
|
|
expect(byChannel.items).toHaveLength(3);
|
|
const byAuthor = await service.search(wsA, { q: "Working Group" });
|
|
expect(byAuthor.items.map((i) => i.messageId)).toEqual([1001]);
|
|
});
|
|
|
|
it("RLS: search never crosses tenants even with shared text", async () => {
|
|
const service = new m.service.ArchiveService(null as never);
|
|
const a = await service.search(wsA, { q: "water rights" });
|
|
const b = await service.search(wsB, { q: "water rights" });
|
|
expect(a.items).toHaveLength(2);
|
|
expect(b.items).toHaveLength(1);
|
|
expect(b.items.every((i) => String(i.remoteRef).startsWith(chB))).toBe(true);
|
|
// Direct DB-level cross-tenant denial as sp_app.
|
|
const leaked = await m.db.withTenant(wsB, (client) => client.query(
|
|
"SELECT count(*)::int AS n FROM archived_feed_items WHERE workspace_id = $1", [wsA]));
|
|
expect(leaked.rows[0]?.n).toBe(0);
|
|
});
|
|
|
|
it("date + channel filters narrow the result set", async () => {
|
|
const service = new m.service.ArchiveService(null as never);
|
|
const from = await service.search(wsA, { q: "water", from: "2026-01-12" });
|
|
expect(from.items.map((i) => i.messageId)).toEqual([1003]);
|
|
const to = await service.search(wsA, { q: "water", to: "2026-01-10T23:59:59Z" });
|
|
expect(to.items.map((i) => i.messageId)).toEqual([1001]);
|
|
const channel = await service.search(wsA, { q: "bicycle", channelId: chA });
|
|
expect(channel.items.map((i) => i.messageId)).toEqual([1002]);
|
|
const none = await service.search(wsA, { q: "bicycle", channelId: `@nonesuch${suffix}` });
|
|
expect(none.items).toHaveLength(0);
|
|
});
|
|
|
|
it("keyset pagination walks the ranked result set without repeats or gaps", async () => {
|
|
const service = new m.service.ArchiveService(null as never);
|
|
const page1 = await service.search(wsA, { q: "water rights", limit: "1" });
|
|
expect(page1.items).toHaveLength(1);
|
|
expect(page1.nextCursor).not.toBeNull();
|
|
const page2 = await service.search(wsA, { q: "water rights", limit: "1", cursor: page1.nextCursor ?? "" });
|
|
expect(page2.items).toHaveLength(1);
|
|
expect(page2.nextCursor).toBeNull();
|
|
const seen = [...page1.items, ...page2.items].map((i) => i.messageId).sort();
|
|
expect(seen).toEqual([1001, 1003]);
|
|
// Bogus cursor is rejected, not silently ignored.
|
|
await expect(service.search(wsA, { q: "water", cursor: "bogus-cursor" }))
|
|
.rejects.toThrow(/invalid cursor/);
|
|
});
|
|
|
|
it("export streams every matching item with faithful content (json + csv)", async () => {
|
|
const service = new m.service.ArchiveService(null as never);
|
|
const json: string[] = [];
|
|
const format = await service.exportItems(wsA, { q: "water rights" }, (chunk) => json.push(chunk));
|
|
expect(format).toBe("json");
|
|
const parsed = JSON.parse(json.join("")) as Array<Record<string, unknown>>;
|
|
expect(parsed).toHaveLength(2);
|
|
expect(parsed.map((i) => i.messageId).sort()).toEqual([1001, 1003]);
|
|
expect(parsed.every((i) => typeof i.postedAt === "string" && typeof i.remoteRef === "string")).toBe(true);
|
|
|
|
const csv: string[] = [];
|
|
const csvFormat = await service.exportItems(wsA, { q: "water", format: "csv" }, (chunk) => csv.push(chunk));
|
|
expect(csvFormat).toBe("csv");
|
|
const csvText = csv.join("");
|
|
const lines = csvText.split("\r\n").filter((line) => line !== "");
|
|
expect(lines).toHaveLength(3); // header + 2 rows
|
|
const header = lines[0]!.split(",");
|
|
expect(header.slice(0, 4)).toEqual(["id", "remote_ref", "channel_id", "message_id"]);
|
|
const rows = lines.slice(1).map((line) => line.split(","));
|
|
for (const row of rows) {
|
|
expect(row).toHaveLength(header.length);
|
|
expect(UUID_SHAPE.test(row[0]!)).toBe(true);
|
|
expect(row[1]).toMatch(new RegExp(`^${chA}:[0-9]+$`));
|
|
}
|
|
const textCol = header.indexOf("text");
|
|
const texts = rows.map((row) => row[textCol]!).sort();
|
|
expect(texts[1]).toContain("hearing is postponed");
|
|
});
|
|
|
|
it("export CSV escaping: commas, quotes and newlines round-trip", async () => {
|
|
const service = new m.service.ArchiveService(null as never);
|
|
const hostile = 'He said "water, rights", then\nleft';
|
|
await asOwner((c) => inTenant(c, wsA, async () => {
|
|
await seedItem(c, wsA, {
|
|
channel: chA, message: 1004, text: hostile, postedAt: new Date("2026-01-13T09:00:00Z"),
|
|
});
|
|
}));
|
|
const csv: string[] = [];
|
|
await service.exportItems(wsA, { channelId: chA, format: "csv" }, (chunk) => csv.push(chunk));
|
|
const lines = csv.join("").split("\r\n").filter((l) => l !== "");
|
|
const header = lines[0]!.split(",");
|
|
const textCol = header.indexOf("text");
|
|
const rows = lines.slice(1).map((line) => line.split(","));
|
|
const hostileRow = rows.find((row) => row[textCol] === hostile);
|
|
expect(hostileRow).toBeDefined();
|
|
expect(hostileRow![textCol]).toBe(hostile);
|
|
expect(rows.every((row) => row.length === header.length)).toBe(true);
|
|
});
|
|
|
|
it("filter validation rejects garbage bounds, limits and formats", async () => {
|
|
const service = new m.service.ArchiveService(null as never);
|
|
await expect(service.search(wsA, { q: "water", from: "not-a-date" })).rejects.toThrow(/ISO-8601/);
|
|
await expect(service.search(wsA, { q: "water", from: "2026-02-01", to: "2026-01-01" })).rejects.toThrow(/after/);
|
|
await expect(service.search(wsA, { q: "water", limit: "0" })).rejects.toThrow(/between 1 and/);
|
|
await expect(service.search(wsA, { q: "water", limit: "1000" })).rejects.toThrow(/between 1 and/);
|
|
await expect(service.search(wsA, { q: "water", cursor: "bogus" })).rejects.toThrow(/invalid cursor/);
|
|
await expect(service.exportItems(wsA, { format: "xml" }, () => undefined)).rejects.toThrow(/csv or json/);
|
|
});
|
|
|
|
it("unindexed helper behavior: tokenization + CSV header shape", () => {
|
|
const helpers = m["search-helpers"];
|
|
expect(helpers.searchTokens("Watter, Rghts! @channel123")).toEqual(["watter", "rghts", "@channel123"]);
|
|
expect(helpers.searchTokens("ab xx ")).toEqual([]);
|
|
expect(helpers.ARCHIVE_CSV_HEADER).toMatch(/^id,remote_ref,channel_id,message_id,posted_at,revision,protected_content,text,storage_keys,attribution\r\n$/);
|
|
});
|
|
});
|
|
|
|
const UUID_SHAPE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|