import { randomUUID } from "node:crypto"; import { describe, expect, it, vi } from "vitest"; import type { PoolClient } from "pg"; import { VirtualChannelImportService, type ImportJobRecord } from "./import.service.js"; import type { ImportEgress } from "./import-egress.js"; import { EgressPolicy } from "../../../../packages/egress/src/index.js"; import { channelRemoteRef, PeerTubeResolveError, videoRemoteRef, type ResolvedPeerTubeSource, } from "../../../../packages/connector-peertube/src/resolve.js"; /** * Unit-level: report counting, dedupe stages and the concurrency cap, with a * fake resolver + fake egress (the DB side is covered by the live-DB suite). * The service's persistence seam is exercised through a stub tenant * transaction that records inserts. */ const UUID_A = "0d1c2b3a-4e5f-4a6b-8c7d-9e0f1a2b3c4d"; function resolvedChannel(name: string, host: string, displayName?: string): ResolvedPeerTubeSource { return { remoteRef: channelRemoteRef(name, host), instanceUrl: `https://${host}`, live: true, label: displayName }; } function resolvedLive(uuid: string, host: string, live: boolean): ResolvedPeerTubeSource { return { remoteRef: videoRemoteRef(uuid, host), instanceUrl: `https://${host}`, live, playbackHlsUrl: live ? `https://${host}/static/streaming-playlists/hls/${uuid}/master.m3u8` : undefined }; } /** Records source inserts; a configurable set of remoteRefs already exist. */ function fakeStore(existing: string[] = []) { const inserted: string[] = []; return { inserted, insert: (remoteRef: string): "imported" | "duplicate" => { if (existing.includes(remoteRef) || inserted.includes(remoteRef)) return "duplicate"; inserted.push(remoteRef); return "imported"; }, }; } function makeService(overrides: { resolve?: (kind: "channel" | "live", name: string, host: string, egress: ImportEgress) => Promise; existing?: string[]; tenantTransaction?: (workspaceId: string, fn: (client: PoolClient) => Promise) => Promise; }) { const store = fakeStore(overrides.existing ?? []); const inserts: Array> = []; const jobs = new Map(); const jobStatus = new Map(); const jobCounts = new Map(); const jobEntries = new Map(); const jobTotal = new Map(); const jobCompleted = new Map(); const processed = new Map(); const tenantTransaction = overrides.tenantTransaction ?? (async (_workspaceId: string, fn: (client: PoolClient) => Promise) => fn({ query: async (text: string, values?: unknown[]) => { if (typeof text === "string" && text.startsWith("INSERT INTO virtual_channel_sources")) { const remoteRef = String(values?.[3] ?? ""); inserts.push({ text, values }); const outcome = store.insert(remoteRef); return { rowCount: outcome === "imported" ? 1 : 0, rows: outcome === "imported" ? [{ id: randomUUID() }] : [] }; } if (typeof text === "string" && text.startsWith("INSERT INTO virtual_channel_imports")) { const id = randomUUID(); jobs.set(id, { id, created_at: new Date(), updated_at: new Date(), workspace_id: "ws-1" }); jobTotal.set(id, Number(values?.[3] ?? 0)); return { rowCount: 1, rows: [{ id, created_at: jobs.get(id)!.created_at }] }; } if (typeof text === "string" && text.startsWith("SELECT * FROM virtual_channel_imports")) { const id = String(values?.[1] ?? ""); const job = jobs.get(id); if (job === undefined) return { rowCount: 0, rows: [] }; return { rowCount: 1, rows: [{ ...job, status: jobStatus.get(id) ?? "running", total: jobTotal.get(id) ?? 0, processed: processed.get(id) ?? 0, imported: jobCounts.get(id)?.imported ?? 0, duplicate: jobCounts.get(id)?.duplicate ?? 0, invalid: jobCounts.get(id)?.invalid ?? 0, failed: jobCounts.get(id)?.failed ?? 0, entries: jobEntries.get(id) ?? [], error: null, completed_at: jobCompleted.get(id) ?? null, connection_id: null, virtual_channel_id: "vc" }] }; } if (typeof text === "string" && text.startsWith("UPDATE virtual_channel_imports")) { const id = String(values?.[1] ?? ""); const counts = values?.slice(2, 6) as number[] | undefined; if (counts !== undefined && (values?.length ?? 0) >= 6 && String(text).includes("status = 'completed'")) { jobCounts.set(id, { imported: counts[0] ?? 0, duplicate: counts[1] ?? 0, invalid: counts[2] ?? 0, failed: counts[3] ?? 0 }); jobStatus.set(id, "completed"); processed.set(id, jobTotal.get(id) ?? 0); jobCompleted.set(id, new Date()); } if (String(text).includes("status = 'failed'")) { jobStatus.set(id, "failed"); jobCompleted.set(id, new Date()); } if (String(text).includes("status = 'abandoned'")) jobStatus.set(id, "abandoned"); const job = jobs.get(id); if (job !== undefined) job.updated_at = new Date(); return { rowCount: 1, rows: [] }; } if (typeof text === "string" && text.startsWith("SELECT id FROM virtual_channels")) return { rowCount: 1, rows: [{ id: "vc" }] }; if (typeof text === "string" && text.startsWith("SELECT fields FROM connections")) { return { rowCount: 1, rows: [{ fields: { instanceUrl: "https://video.example" } }] }; } return { rowCount: 0, rows: [] }; } } as unknown as PoolClient)); const egress: ImportEgress = { egressFetch: (async () => new Response("{}", { status: 200 })) as ImportEgress["egressFetch"], isEgressError: (error: unknown): error is never => false, policy: new EgressPolicy(), }; const resolve = overrides.resolve ?? (async (kind: "channel" | "live", name: string, host: string) => kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true)); const service = new VirtualChannelImportService({ tenantTransaction: tenantTransaction as unknown as typeof import("../db/client.js").withTenant, egress: async () => egress, resolver: async (kind, name, host, e) => resolve(kind, name, host, e), now: () => 0, cache: new Map(), audit: async () => undefined, }); return { service, store, inserts }; } async function runToCompletion(service: VirtualChannelImportService, body: string, contentType = "text/plain"): Promise { const created = await service.createImport("ws-1", "11111111-1111-4111-8111-111111111111", { contentType, body, actor: { userId: null, role: "Owner" }, }); // createImport schedules the background run; poll until it settles. for (let i = 0; i < 100; i += 1) { const record = await service.getImport("ws-1", created.jobId).catch((e: unknown) => { throw new Error("getImport failed: " + String(e)); }); if (record !== undefined && record.status !== "running") return record; await new Promise((resolve) => setTimeout(resolve, 5)); } throw new Error("import job did not settle"); } describe("import report counting", () => { it("counts imported / duplicate (pre-resolution) / invalid / failed", async () => { const { service } = makeService({ resolve: async (kind, name, host) => { if (name === "broken") throw new PeerTubeResolveError("upstream", "instance exploded"); if (name === "gone") throw new PeerTubeResolveError("not_found", "no such channel"); return kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true); }, }); const report = await runToCompletion(service, [ "@alice@video.example", "https://video.example/c/alice", // duplicate by syntactic key "@broken@video.example", // failed (upstream) "@gone@video.example", // invalid (not found) "!!!not a row!!!", // invalid (syntax) "@bob@video.example", ].join("\n")); expect(report.status).toBe("completed"); expect(report.total).toBe(6); expect(report.imported).toBe(2); expect(report.duplicate).toBe(1); expect(report.invalid).toBe(2); expect(report.failed).toBe(1); expect(report.processed).toBe(6); }); it("counts an existing remote_ref as duplicate (post-resolution, DB conflict)", async () => { const existing = [channelRemoteRef("alice", "video.example")]; const { service } = makeService({ existing }); const report = await runToCompletion(service, "@alice@video.example\n@bob@video.example"); expect(report.imported).toBe(1); expect(report.duplicate).toBe(1); }); it("two inputs that resolve to one remote_ref count one import + one duplicate", async () => { const { service } = makeService({}); // /c/alice and @alice@video.example normalize to the same syntactic key, // so exercise the post-resolution stage with two distinct spellings that // both resolve to the same remote identity (alias). const report = await runToCompletion(service, "@alice@video.example\nhttps://video.example/a/alice"); expect(report.imported + report.duplicate).toBe(2); expect(report.imported).toBe(1); }); it("JSON bodies and CSV bodies produce the same report", async () => { const json = await runToCompletion(makeService({}).service, JSON.stringify(["@alice@video.example", "@bob@video.example"]), "application/json"); expect(json.total).toBe(2); expect(json.imported).toBe(2); const csv = await runToCompletion(makeService({}).service, "handle\n@alice@video.example", "text/csv"); expect(csv.total).toBe(1); expect(csv.imported).toBe(1); }); it("caps the stored per-entry report while keeping counts authoritative", async () => { const rows = Array.from({ length: 260 }, (_, i) => `@user${i}@video.example`); const report = await runToCompletion(makeService({}).service, rows.join("\n")); expect(report.total).toBe(260); expect(report.imported).toBe(260); expect(report.entries.length).toBeLessThanOrEqual(200); }); it("marks the job failed when the store throws mid-run", async () => { const { service } = makeService({ tenantTransaction: async () => { throw new Error("database is gone"); }, }); const record = await service.createImport("ws-1", "11111111-1111-4111-8111-111111111111", { contentType: "text/plain", body: "@alice@video.example", actor: { userId: null, role: "Owner" }, }).catch(() => undefined); expect(record).toBeUndefined(); // createImport itself fails synchronously here }); }); describe("resolution caching", () => { it("caches resolutions by syntactic key, so re-importing is cheap", async () => { let calls = 0; const { service } = makeService({ resolve: async (kind, name, host) => { calls += 1; return kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true); }, }); const first = await runToCompletion(service, "@alice@video.example"); expect(first.imported).toBe(1); // Second import of the same spelling: one resolver call already happened; // the DB dedupe (remote_ref) makes it a duplicate. const second = await runToCompletion(service, "@alice@video.example"); expect(second.imported + second.duplicate).toBe(1); expect(calls).toBe(1); }); }); describe("concurrency caps", () => { it("never exceeds 16 resolutions in flight per import", async () => { let inFlight = 0; let peak = 0; const { service } = makeService({ resolve: async (kind, name, host) => { inFlight += 1; peak = Math.max(peak, inFlight); await new Promise((resolve) => setTimeout(resolve, 5)); inFlight -= 1; return kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true); }, }); const rows = Array.from({ length: 64 }, (_, i) => `@user${i}@video.example`); const report = await runToCompletion(service, rows.join("\n")); expect(report.imported).toBe(64); expect(peak).toBeLessThanOrEqual(16); }); it("shares one per-host budget: an egress counter sees host-classed draws", async () => { const draws: Array<{ url: string; hostClass: string }> = []; const { service } = makeService({ resolve: async (kind, name, host, egress) => { const url = kind === "channel" ? `https://${host}/api/v1/video-channels/${name}@${host}` : `https://${host}/api/v1/videos/${name}`; const response = await egress.egressFetch(url, {}, { hostClass: "import", pinnedOrigin: true }); draws.push({ url: (response as unknown as { url: string }).url ?? url, hostClass: "import" }); return kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true); }, }); const report = await runToCompletion(service, "@alice@video.example"); expect(report.imported).toBe(1); expect(draws.length).toBe(1); }); });