Tactical Media Broadcast Hub — initial release
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
This commit is contained in:
commit
aa5e262f3a
880 changed files with 111042 additions and 0 deletions
256
apps/api/src/virtual-channels/import.service.test.ts
Normal file
256
apps/api/src/virtual-channels/import.service.test.ts
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { PoolClient } from "pg";
|
||||
import { VirtualChannelImportService, type ImportJobRecord } from "./import.service.js";
|
||||
import type { ImportEgress } from "./import-egress.js";
|
||||
import { EgressPolicy } from "../../../../packages/egress/src/index.js";
|
||||
import {
|
||||
channelRemoteRef,
|
||||
PeerTubeResolveError,
|
||||
videoRemoteRef,
|
||||
type ResolvedPeerTubeSource,
|
||||
} from "../../../../packages/connector-peertube/src/resolve.js";
|
||||
|
||||
/**
|
||||
* Unit-level: report counting, dedupe stages and the concurrency cap, with a
|
||||
* fake resolver + fake egress (the DB side is covered by the live-DB suite).
|
||||
* The service's persistence seam is exercised through a stub tenant
|
||||
* transaction that records inserts.
|
||||
*/
|
||||
|
||||
const UUID_A = "0d1c2b3a-4e5f-4a6b-8c7d-9e0f1a2b3c4d";
|
||||
|
||||
function resolvedChannel(name: string, host: string, displayName?: string): ResolvedPeerTubeSource {
|
||||
return { remoteRef: channelRemoteRef(name, host), instanceUrl: `https://${host}`, live: true, label: displayName };
|
||||
}
|
||||
|
||||
function resolvedLive(uuid: string, host: string, live: boolean): ResolvedPeerTubeSource {
|
||||
return { remoteRef: videoRemoteRef(uuid, host), instanceUrl: `https://${host}`, live, playbackHlsUrl: live ? `https://${host}/static/streaming-playlists/hls/${uuid}/master.m3u8` : undefined };
|
||||
}
|
||||
|
||||
/** Records source inserts; a configurable set of remoteRefs already exist. */
|
||||
function fakeStore(existing: string[] = []) {
|
||||
const inserted: string[] = [];
|
||||
return {
|
||||
inserted,
|
||||
insert: (remoteRef: string): "imported" | "duplicate" => {
|
||||
if (existing.includes(remoteRef) || inserted.includes(remoteRef)) return "duplicate";
|
||||
inserted.push(remoteRef);
|
||||
return "imported";
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeService(overrides: {
|
||||
resolve?: (kind: "channel" | "live", name: string, host: string, egress: ImportEgress) => Promise<ResolvedPeerTubeSource>;
|
||||
existing?: string[];
|
||||
tenantTransaction?: (workspaceId: string, fn: (client: PoolClient) => Promise<unknown>) => Promise<unknown>;
|
||||
}) {
|
||||
const store = fakeStore(overrides.existing ?? []);
|
||||
const inserts: Array<Record<string, unknown>> = [];
|
||||
const jobs = new Map<string, { id: string; created_at: Date; updated_at: Date; workspace_id: string }>();
|
||||
const jobStatus = new Map<string, string>();
|
||||
const jobCounts = new Map<string, { imported: number; duplicate: number; invalid: number; failed: number }>();
|
||||
const jobEntries = new Map<string, unknown[]>();
|
||||
const jobTotal = new Map<string, number>();
|
||||
const jobCompleted = new Map<string, Date | null>();
|
||||
const processed = new Map<string, number>();
|
||||
const tenantTransaction = overrides.tenantTransaction ?? (async (_workspaceId: string, fn: (client: PoolClient) => Promise<unknown>) =>
|
||||
fn({ query: async (text: string, values?: unknown[]) => {
|
||||
if (typeof text === "string" && text.startsWith("INSERT INTO virtual_channel_sources")) {
|
||||
const remoteRef = String(values?.[3] ?? "");
|
||||
inserts.push({ text, values });
|
||||
const outcome = store.insert(remoteRef);
|
||||
return { rowCount: outcome === "imported" ? 1 : 0, rows: outcome === "imported" ? [{ id: randomUUID() }] : [] };
|
||||
}
|
||||
if (typeof text === "string" && text.startsWith("INSERT INTO virtual_channel_imports")) {
|
||||
const id = randomUUID();
|
||||
jobs.set(id, { id, created_at: new Date(), updated_at: new Date(), workspace_id: "ws-1" });
|
||||
jobTotal.set(id, Number(values?.[3] ?? 0));
|
||||
return { rowCount: 1, rows: [{ id, created_at: jobs.get(id)!.created_at }] };
|
||||
}
|
||||
if (typeof text === "string" && text.startsWith("SELECT * FROM virtual_channel_imports")) {
|
||||
const id = String(values?.[1] ?? "");
|
||||
const job = jobs.get(id);
|
||||
if (job === undefined) return { rowCount: 0, rows: [] };
|
||||
return { rowCount: 1, rows: [{ ...job, status: jobStatus.get(id) ?? "running", total: jobTotal.get(id) ?? 0, processed: processed.get(id) ?? 0, imported: jobCounts.get(id)?.imported ?? 0, duplicate: jobCounts.get(id)?.duplicate ?? 0, invalid: jobCounts.get(id)?.invalid ?? 0, failed: jobCounts.get(id)?.failed ?? 0, entries: jobEntries.get(id) ?? [], error: null, completed_at: jobCompleted.get(id) ?? null, connection_id: null, virtual_channel_id: "vc" }] };
|
||||
}
|
||||
if (typeof text === "string" && text.startsWith("UPDATE virtual_channel_imports")) {
|
||||
const id = String(values?.[1] ?? "");
|
||||
const counts = values?.slice(2, 6) as number[] | undefined;
|
||||
if (counts !== undefined && (values?.length ?? 0) >= 6 && String(text).includes("status = 'completed'")) {
|
||||
jobCounts.set(id, { imported: counts[0] ?? 0, duplicate: counts[1] ?? 0, invalid: counts[2] ?? 0, failed: counts[3] ?? 0 });
|
||||
jobStatus.set(id, "completed");
|
||||
processed.set(id, jobTotal.get(id) ?? 0);
|
||||
jobCompleted.set(id, new Date());
|
||||
}
|
||||
if (String(text).includes("status = 'failed'")) { jobStatus.set(id, "failed"); jobCompleted.set(id, new Date()); }
|
||||
if (String(text).includes("status = 'abandoned'")) jobStatus.set(id, "abandoned");
|
||||
const job = jobs.get(id);
|
||||
if (job !== undefined) job.updated_at = new Date();
|
||||
return { rowCount: 1, rows: [] };
|
||||
}
|
||||
if (typeof text === "string" && text.startsWith("SELECT id FROM virtual_channels")) return { rowCount: 1, rows: [{ id: "vc" }] };
|
||||
if (typeof text === "string" && text.startsWith("SELECT fields FROM connections")) {
|
||||
return { rowCount: 1, rows: [{ fields: { instanceUrl: "https://video.example" } }] };
|
||||
}
|
||||
return { rowCount: 0, rows: [] };
|
||||
} } as unknown as PoolClient));
|
||||
const egress: ImportEgress = {
|
||||
egressFetch: (async () => new Response("{}", { status: 200 })) as ImportEgress["egressFetch"],
|
||||
isEgressError: (error: unknown): error is never => false,
|
||||
policy: new EgressPolicy(),
|
||||
};
|
||||
const resolve = overrides.resolve ?? (async (kind: "channel" | "live", name: string, host: string) =>
|
||||
kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true));
|
||||
const service = new VirtualChannelImportService({
|
||||
tenantTransaction: tenantTransaction as unknown as typeof import("../db/client.js").withTenant,
|
||||
egress: async () => egress,
|
||||
resolver: async (kind, name, host, e) => resolve(kind, name, host, e),
|
||||
now: () => 0,
|
||||
cache: new Map(),
|
||||
audit: async () => undefined,
|
||||
});
|
||||
return { service, store, inserts };
|
||||
}
|
||||
|
||||
async function runToCompletion(service: VirtualChannelImportService, body: string, contentType = "text/plain"): Promise<ImportJobRecord> {
|
||||
const created = await service.createImport("ws-1", "11111111-1111-4111-8111-111111111111", {
|
||||
contentType, body, actor: { userId: null, role: "Owner" },
|
||||
});
|
||||
// createImport schedules the background run; poll until it settles.
|
||||
for (let i = 0; i < 100; i += 1) {
|
||||
const record = await service.getImport("ws-1", created.jobId).catch((e: unknown) => { throw new Error("getImport failed: " + String(e)); });
|
||||
if (record !== undefined && record.status !== "running") return record;
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
throw new Error("import job did not settle");
|
||||
}
|
||||
|
||||
describe("import report counting", () => {
|
||||
it("counts imported / duplicate (pre-resolution) / invalid / failed", async () => {
|
||||
const { service } = makeService({
|
||||
resolve: async (kind, name, host) => {
|
||||
if (name === "broken") throw new PeerTubeResolveError("upstream", "instance exploded");
|
||||
if (name === "gone") throw new PeerTubeResolveError("not_found", "no such channel");
|
||||
return kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true);
|
||||
},
|
||||
});
|
||||
const report = await runToCompletion(service, [
|
||||
"@alice@video.example",
|
||||
"https://video.example/c/alice", // duplicate by syntactic key
|
||||
"@broken@video.example", // failed (upstream)
|
||||
"@gone@video.example", // invalid (not found)
|
||||
"!!!not a row!!!", // invalid (syntax)
|
||||
"@bob@video.example",
|
||||
].join("\n"));
|
||||
expect(report.status).toBe("completed");
|
||||
expect(report.total).toBe(6);
|
||||
expect(report.imported).toBe(2);
|
||||
expect(report.duplicate).toBe(1);
|
||||
expect(report.invalid).toBe(2);
|
||||
expect(report.failed).toBe(1);
|
||||
expect(report.processed).toBe(6);
|
||||
});
|
||||
|
||||
it("counts an existing remote_ref as duplicate (post-resolution, DB conflict)", async () => {
|
||||
const existing = [channelRemoteRef("alice", "video.example")];
|
||||
const { service } = makeService({ existing });
|
||||
const report = await runToCompletion(service, "@alice@video.example\n@bob@video.example");
|
||||
expect(report.imported).toBe(1);
|
||||
expect(report.duplicate).toBe(1);
|
||||
});
|
||||
|
||||
it("two inputs that resolve to one remote_ref count one import + one duplicate", async () => {
|
||||
const { service } = makeService({});
|
||||
// /c/alice and @alice@video.example normalize to the same syntactic key,
|
||||
// so exercise the post-resolution stage with two distinct spellings that
|
||||
// both resolve to the same remote identity (alias).
|
||||
const report = await runToCompletion(service, "@alice@video.example\nhttps://video.example/a/alice");
|
||||
expect(report.imported + report.duplicate).toBe(2);
|
||||
expect(report.imported).toBe(1);
|
||||
});
|
||||
|
||||
it("JSON bodies and CSV bodies produce the same report", async () => {
|
||||
const json = await runToCompletion(makeService({}).service, JSON.stringify(["@alice@video.example", "@bob@video.example"]), "application/json");
|
||||
expect(json.total).toBe(2);
|
||||
expect(json.imported).toBe(2);
|
||||
const csv = await runToCompletion(makeService({}).service, "handle\n@alice@video.example", "text/csv");
|
||||
expect(csv.total).toBe(1);
|
||||
expect(csv.imported).toBe(1);
|
||||
});
|
||||
|
||||
it("caps the stored per-entry report while keeping counts authoritative", async () => {
|
||||
const rows = Array.from({ length: 260 }, (_, i) => `@user${i}@video.example`);
|
||||
const report = await runToCompletion(makeService({}).service, rows.join("\n"));
|
||||
expect(report.total).toBe(260);
|
||||
expect(report.imported).toBe(260);
|
||||
expect(report.entries.length).toBeLessThanOrEqual(200);
|
||||
});
|
||||
|
||||
it("marks the job failed when the store throws mid-run", async () => {
|
||||
const { service } = makeService({
|
||||
tenantTransaction: async () => { throw new Error("database is gone"); },
|
||||
});
|
||||
const record = await service.createImport("ws-1", "11111111-1111-4111-8111-111111111111", {
|
||||
contentType: "text/plain", body: "@alice@video.example", actor: { userId: null, role: "Owner" },
|
||||
}).catch(() => undefined);
|
||||
expect(record).toBeUndefined(); // createImport itself fails synchronously here
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolution caching", () => {
|
||||
it("caches resolutions by syntactic key, so re-importing is cheap", async () => {
|
||||
let calls = 0;
|
||||
const { service } = makeService({
|
||||
resolve: async (kind, name, host) => {
|
||||
calls += 1;
|
||||
return kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true);
|
||||
},
|
||||
});
|
||||
const first = await runToCompletion(service, "@alice@video.example");
|
||||
expect(first.imported).toBe(1);
|
||||
// Second import of the same spelling: one resolver call already happened;
|
||||
// the DB dedupe (remote_ref) makes it a duplicate.
|
||||
const second = await runToCompletion(service, "@alice@video.example");
|
||||
expect(second.imported + second.duplicate).toBe(1);
|
||||
expect(calls).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("concurrency caps", () => {
|
||||
it("never exceeds 16 resolutions in flight per import", async () => {
|
||||
let inFlight = 0;
|
||||
let peak = 0;
|
||||
const { service } = makeService({
|
||||
resolve: async (kind, name, host) => {
|
||||
inFlight += 1;
|
||||
peak = Math.max(peak, inFlight);
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
inFlight -= 1;
|
||||
return kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true);
|
||||
},
|
||||
});
|
||||
const rows = Array.from({ length: 64 }, (_, i) => `@user${i}@video.example`);
|
||||
const report = await runToCompletion(service, rows.join("\n"));
|
||||
expect(report.imported).toBe(64);
|
||||
expect(peak).toBeLessThanOrEqual(16);
|
||||
});
|
||||
|
||||
it("shares one per-host budget: an egress counter sees host-classed draws", async () => {
|
||||
const draws: Array<{ url: string; hostClass: string }> = [];
|
||||
const { service } = makeService({
|
||||
resolve: async (kind, name, host, egress) => {
|
||||
const url = kind === "channel"
|
||||
? `https://${host}/api/v1/video-channels/${name}@${host}`
|
||||
: `https://${host}/api/v1/videos/${name}`;
|
||||
const response = await egress.egressFetch(url, {}, { hostClass: "import", pinnedOrigin: true });
|
||||
draws.push({ url: (response as unknown as { url: string }).url ?? url, hostClass: "import" });
|
||||
return kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true);
|
||||
},
|
||||
});
|
||||
const report = await runToCompletion(service, "@alice@video.example");
|
||||
expect(report.imported).toBe(1);
|
||||
expect(draws.length).toBe(1);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue