glass beadian
aa5e262f3a
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
256 lines
13 KiB
TypeScript
256 lines
13 KiB
TypeScript
import { randomUUID } from "node:crypto";
|
|
import { describe, expect, it, vi } from "vitest";
|
|
import type { PoolClient } from "pg";
|
|
import { VirtualChannelImportService, type ImportJobRecord } from "./import.service.js";
|
|
import type { ImportEgress } from "./import-egress.js";
|
|
import { EgressPolicy } from "../../../../packages/egress/src/index.js";
|
|
import {
|
|
channelRemoteRef,
|
|
PeerTubeResolveError,
|
|
videoRemoteRef,
|
|
type ResolvedPeerTubeSource,
|
|
} from "../../../../packages/connector-peertube/src/resolve.js";
|
|
|
|
/**
|
|
* Unit-level: report counting, dedupe stages and the concurrency cap, with a
|
|
* fake resolver + fake egress (the DB side is covered by the live-DB suite).
|
|
* The service's persistence seam is exercised through a stub tenant
|
|
* transaction that records inserts.
|
|
*/
|
|
|
|
const UUID_A = "0d1c2b3a-4e5f-4a6b-8c7d-9e0f1a2b3c4d";
|
|
|
|
function resolvedChannel(name: string, host: string, displayName?: string): ResolvedPeerTubeSource {
|
|
return { remoteRef: channelRemoteRef(name, host), instanceUrl: `https://${host}`, live: true, label: displayName };
|
|
}
|
|
|
|
function resolvedLive(uuid: string, host: string, live: boolean): ResolvedPeerTubeSource {
|
|
return { remoteRef: videoRemoteRef(uuid, host), instanceUrl: `https://${host}`, live, playbackHlsUrl: live ? `https://${host}/static/streaming-playlists/hls/${uuid}/master.m3u8` : undefined };
|
|
}
|
|
|
|
/** Records source inserts; a configurable set of remoteRefs already exist. */
|
|
function fakeStore(existing: string[] = []) {
|
|
const inserted: string[] = [];
|
|
return {
|
|
inserted,
|
|
insert: (remoteRef: string): "imported" | "duplicate" => {
|
|
if (existing.includes(remoteRef) || inserted.includes(remoteRef)) return "duplicate";
|
|
inserted.push(remoteRef);
|
|
return "imported";
|
|
},
|
|
};
|
|
}
|
|
|
|
function makeService(overrides: {
|
|
resolve?: (kind: "channel" | "live", name: string, host: string, egress: ImportEgress) => Promise<ResolvedPeerTubeSource>;
|
|
existing?: string[];
|
|
tenantTransaction?: (workspaceId: string, fn: (client: PoolClient) => Promise<unknown>) => Promise<unknown>;
|
|
}) {
|
|
const store = fakeStore(overrides.existing ?? []);
|
|
const inserts: Array<Record<string, unknown>> = [];
|
|
const jobs = new Map<string, { id: string; created_at: Date; updated_at: Date; workspace_id: string }>();
|
|
const jobStatus = new Map<string, string>();
|
|
const jobCounts = new Map<string, { imported: number; duplicate: number; invalid: number; failed: number }>();
|
|
const jobEntries = new Map<string, unknown[]>();
|
|
const jobTotal = new Map<string, number>();
|
|
const jobCompleted = new Map<string, Date | null>();
|
|
const processed = new Map<string, number>();
|
|
const tenantTransaction = overrides.tenantTransaction ?? (async (_workspaceId: string, fn: (client: PoolClient) => Promise<unknown>) =>
|
|
fn({ query: async (text: string, values?: unknown[]) => {
|
|
if (typeof text === "string" && text.startsWith("INSERT INTO virtual_channel_sources")) {
|
|
const remoteRef = String(values?.[3] ?? "");
|
|
inserts.push({ text, values });
|
|
const outcome = store.insert(remoteRef);
|
|
return { rowCount: outcome === "imported" ? 1 : 0, rows: outcome === "imported" ? [{ id: randomUUID() }] : [] };
|
|
}
|
|
if (typeof text === "string" && text.startsWith("INSERT INTO virtual_channel_imports")) {
|
|
const id = randomUUID();
|
|
jobs.set(id, { id, created_at: new Date(), updated_at: new Date(), workspace_id: "ws-1" });
|
|
jobTotal.set(id, Number(values?.[3] ?? 0));
|
|
return { rowCount: 1, rows: [{ id, created_at: jobs.get(id)!.created_at }] };
|
|
}
|
|
if (typeof text === "string" && text.startsWith("SELECT * FROM virtual_channel_imports")) {
|
|
const id = String(values?.[1] ?? "");
|
|
const job = jobs.get(id);
|
|
if (job === undefined) return { rowCount: 0, rows: [] };
|
|
return { rowCount: 1, rows: [{ ...job, status: jobStatus.get(id) ?? "running", total: jobTotal.get(id) ?? 0, processed: processed.get(id) ?? 0, imported: jobCounts.get(id)?.imported ?? 0, duplicate: jobCounts.get(id)?.duplicate ?? 0, invalid: jobCounts.get(id)?.invalid ?? 0, failed: jobCounts.get(id)?.failed ?? 0, entries: jobEntries.get(id) ?? [], error: null, completed_at: jobCompleted.get(id) ?? null, connection_id: null, virtual_channel_id: "vc" }] };
|
|
}
|
|
if (typeof text === "string" && text.startsWith("UPDATE virtual_channel_imports")) {
|
|
const id = String(values?.[1] ?? "");
|
|
const counts = values?.slice(2, 6) as number[] | undefined;
|
|
if (counts !== undefined && (values?.length ?? 0) >= 6 && String(text).includes("status = 'completed'")) {
|
|
jobCounts.set(id, { imported: counts[0] ?? 0, duplicate: counts[1] ?? 0, invalid: counts[2] ?? 0, failed: counts[3] ?? 0 });
|
|
jobStatus.set(id, "completed");
|
|
processed.set(id, jobTotal.get(id) ?? 0);
|
|
jobCompleted.set(id, new Date());
|
|
}
|
|
if (String(text).includes("status = 'failed'")) { jobStatus.set(id, "failed"); jobCompleted.set(id, new Date()); }
|
|
if (String(text).includes("status = 'abandoned'")) jobStatus.set(id, "abandoned");
|
|
const job = jobs.get(id);
|
|
if (job !== undefined) job.updated_at = new Date();
|
|
return { rowCount: 1, rows: [] };
|
|
}
|
|
if (typeof text === "string" && text.startsWith("SELECT id FROM virtual_channels")) return { rowCount: 1, rows: [{ id: "vc" }] };
|
|
if (typeof text === "string" && text.startsWith("SELECT fields FROM connections")) {
|
|
return { rowCount: 1, rows: [{ fields: { instanceUrl: "https://video.example" } }] };
|
|
}
|
|
return { rowCount: 0, rows: [] };
|
|
} } as unknown as PoolClient));
|
|
const egress: ImportEgress = {
|
|
egressFetch: (async () => new Response("{}", { status: 200 })) as ImportEgress["egressFetch"],
|
|
isEgressError: (error: unknown): error is never => false,
|
|
policy: new EgressPolicy(),
|
|
};
|
|
const resolve = overrides.resolve ?? (async (kind: "channel" | "live", name: string, host: string) =>
|
|
kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true));
|
|
const service = new VirtualChannelImportService({
|
|
tenantTransaction: tenantTransaction as unknown as typeof import("../db/client.js").withTenant,
|
|
egress: async () => egress,
|
|
resolver: async (kind, name, host, e) => resolve(kind, name, host, e),
|
|
now: () => 0,
|
|
cache: new Map(),
|
|
audit: async () => undefined,
|
|
});
|
|
return { service, store, inserts };
|
|
}
|
|
|
|
async function runToCompletion(service: VirtualChannelImportService, body: string, contentType = "text/plain"): Promise<ImportJobRecord> {
|
|
const created = await service.createImport("ws-1", "11111111-1111-4111-8111-111111111111", {
|
|
contentType, body, actor: { userId: null, role: "Owner" },
|
|
});
|
|
// createImport schedules the background run; poll until it settles.
|
|
for (let i = 0; i < 100; i += 1) {
|
|
const record = await service.getImport("ws-1", created.jobId).catch((e: unknown) => { throw new Error("getImport failed: " + String(e)); });
|
|
if (record !== undefined && record.status !== "running") return record;
|
|
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
}
|
|
throw new Error("import job did not settle");
|
|
}
|
|
|
|
describe("import report counting", () => {
|
|
it("counts imported / duplicate (pre-resolution) / invalid / failed", async () => {
|
|
const { service } = makeService({
|
|
resolve: async (kind, name, host) => {
|
|
if (name === "broken") throw new PeerTubeResolveError("upstream", "instance exploded");
|
|
if (name === "gone") throw new PeerTubeResolveError("not_found", "no such channel");
|
|
return kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true);
|
|
},
|
|
});
|
|
const report = await runToCompletion(service, [
|
|
"@alice@video.example",
|
|
"https://video.example/c/alice", // duplicate by syntactic key
|
|
"@broken@video.example", // failed (upstream)
|
|
"@gone@video.example", // invalid (not found)
|
|
"!!!not a row!!!", // invalid (syntax)
|
|
"@bob@video.example",
|
|
].join("\n"));
|
|
expect(report.status).toBe("completed");
|
|
expect(report.total).toBe(6);
|
|
expect(report.imported).toBe(2);
|
|
expect(report.duplicate).toBe(1);
|
|
expect(report.invalid).toBe(2);
|
|
expect(report.failed).toBe(1);
|
|
expect(report.processed).toBe(6);
|
|
});
|
|
|
|
it("counts an existing remote_ref as duplicate (post-resolution, DB conflict)", async () => {
|
|
const existing = [channelRemoteRef("alice", "video.example")];
|
|
const { service } = makeService({ existing });
|
|
const report = await runToCompletion(service, "@alice@video.example\n@bob@video.example");
|
|
expect(report.imported).toBe(1);
|
|
expect(report.duplicate).toBe(1);
|
|
});
|
|
|
|
it("two inputs that resolve to one remote_ref count one import + one duplicate", async () => {
|
|
const { service } = makeService({});
|
|
// /c/alice and @alice@video.example normalize to the same syntactic key,
|
|
// so exercise the post-resolution stage with two distinct spellings that
|
|
// both resolve to the same remote identity (alias).
|
|
const report = await runToCompletion(service, "@alice@video.example\nhttps://video.example/a/alice");
|
|
expect(report.imported + report.duplicate).toBe(2);
|
|
expect(report.imported).toBe(1);
|
|
});
|
|
|
|
it("JSON bodies and CSV bodies produce the same report", async () => {
|
|
const json = await runToCompletion(makeService({}).service, JSON.stringify(["@alice@video.example", "@bob@video.example"]), "application/json");
|
|
expect(json.total).toBe(2);
|
|
expect(json.imported).toBe(2);
|
|
const csv = await runToCompletion(makeService({}).service, "handle\n@alice@video.example", "text/csv");
|
|
expect(csv.total).toBe(1);
|
|
expect(csv.imported).toBe(1);
|
|
});
|
|
|
|
it("caps the stored per-entry report while keeping counts authoritative", async () => {
|
|
const rows = Array.from({ length: 260 }, (_, i) => `@user${i}@video.example`);
|
|
const report = await runToCompletion(makeService({}).service, rows.join("\n"));
|
|
expect(report.total).toBe(260);
|
|
expect(report.imported).toBe(260);
|
|
expect(report.entries.length).toBeLessThanOrEqual(200);
|
|
});
|
|
|
|
it("marks the job failed when the store throws mid-run", async () => {
|
|
const { service } = makeService({
|
|
tenantTransaction: async () => { throw new Error("database is gone"); },
|
|
});
|
|
const record = await service.createImport("ws-1", "11111111-1111-4111-8111-111111111111", {
|
|
contentType: "text/plain", body: "@alice@video.example", actor: { userId: null, role: "Owner" },
|
|
}).catch(() => undefined);
|
|
expect(record).toBeUndefined(); // createImport itself fails synchronously here
|
|
});
|
|
});
|
|
|
|
describe("resolution caching", () => {
|
|
it("caches resolutions by syntactic key, so re-importing is cheap", async () => {
|
|
let calls = 0;
|
|
const { service } = makeService({
|
|
resolve: async (kind, name, host) => {
|
|
calls += 1;
|
|
return kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true);
|
|
},
|
|
});
|
|
const first = await runToCompletion(service, "@alice@video.example");
|
|
expect(first.imported).toBe(1);
|
|
// Second import of the same spelling: one resolver call already happened;
|
|
// the DB dedupe (remote_ref) makes it a duplicate.
|
|
const second = await runToCompletion(service, "@alice@video.example");
|
|
expect(second.imported + second.duplicate).toBe(1);
|
|
expect(calls).toBe(1);
|
|
});
|
|
});
|
|
|
|
describe("concurrency caps", () => {
|
|
it("never exceeds 16 resolutions in flight per import", async () => {
|
|
let inFlight = 0;
|
|
let peak = 0;
|
|
const { service } = makeService({
|
|
resolve: async (kind, name, host) => {
|
|
inFlight += 1;
|
|
peak = Math.max(peak, inFlight);
|
|
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
inFlight -= 1;
|
|
return kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true);
|
|
},
|
|
});
|
|
const rows = Array.from({ length: 64 }, (_, i) => `@user${i}@video.example`);
|
|
const report = await runToCompletion(service, rows.join("\n"));
|
|
expect(report.imported).toBe(64);
|
|
expect(peak).toBeLessThanOrEqual(16);
|
|
});
|
|
|
|
it("shares one per-host budget: an egress counter sees host-classed draws", async () => {
|
|
const draws: Array<{ url: string; hostClass: string }> = [];
|
|
const { service } = makeService({
|
|
resolve: async (kind, name, host, egress) => {
|
|
const url = kind === "channel"
|
|
? `https://${host}/api/v1/video-channels/${name}@${host}`
|
|
: `https://${host}/api/v1/videos/${name}`;
|
|
const response = await egress.egressFetch(url, {}, { hostClass: "import", pinnedOrigin: true });
|
|
draws.push({ url: (response as unknown as { url: string }).url ?? url, hostClass: "import" });
|
|
return kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true);
|
|
},
|
|
});
|
|
const report = await runToCompletion(service, "@alice@video.example");
|
|
expect(report.imported).toBe(1);
|
|
expect(draws.length).toBe(1);
|
|
});
|
|
});
|