/** * Bulk source import: input parsing + normalization + the syntactic identity * key (WP7, proposal §4.4 "Bulk source import"). * * The pipeline is deliberately staged so no network is needed to dedupe: * parse input rows (URL / handle / multiline / CSV / JSON) * -> normalize (strip leading `@`, lowercase, IDNA/punycode host, * strip default port) * -> syntactic validation + egress policy check (validateUrl — no DNS, * no budget draw) * -> pre-resolution dedupe on the syntactic key * `kind:{name-or-id}@{canonical host}` * Only the unique surviving keys are resolved remotely; the resolved * `remote_ref` is the instance-provided identity and dedupes again * post-resolution, and the DB's UNIQUE (workspace_id, virtual_channel_id, * remote_ref) is the final authority. */ import { BadRequestException } from "@nestjs/common"; import { domainToASCII } from "node:url"; import { isIP } from "node:net"; import type { EgressPolicy } from "../../../../packages/egress/src/index.js"; /** Hard request cap (proposal §4.4): larger lists are a 413, never silently cut. */ export const IMPORT_ROW_LIMIT = 5_000; export type ImportRowKind = "channel" | "live"; /** One normalized input row. `key` is the pre-resolution dedupe key. */ export interface NormalizedRow { kind: ImportRowKind; /** Lower-cased, leading-`@`-stripped name (channel) or uuid/shortUUID (live). */ name: string; /** Lower-cased IDNA host, default port stripped, no brackets. */ host: string; key: string; label: string; /** Original input line, for the report. */ raw: string; } /** Row-level failure with a stable reason code for the import report. */ export class InvalidRowError extends Error { constructor(readonly reason: string, message: string) { super(message); this.name = "InvalidRowError"; } } const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; /** PeerTube short ids: base64url, 6-12 chars. */ const SHORT_ID = /^[0-9a-zA-Z_-]{6,12}$/; /** PeerTube usernames/channels: lowercase letters, digits, `_`, 1..50. */ const CHANNEL_NAME = /^[a-z0-9_]{1,50}$/; /** Latent claim: anything else that could be an account/channel handle. */ const HANDLE_LIKE = /^@?[a-z0-9_][a-z0-9_.-]{0,64}$/i; const CONTROL = /[\u0000-\u001f\u007f]/; /** Strip a trailing DNS dot and lowercase; returns "" for unusable hosts. */ function canonicalHost(raw: string): string { const host = raw.replace(/^\[|\]$/g, "").replace(/\.$/, "").toLowerCase(); if (host === "") return ""; if (isIP(host)) return host; const ascii = domainToASCII(host); if (ascii === "" || ascii.includes("\u0000")) return ""; return ascii; } /** Port stripped when it is the scheme default; anything else is rejected later. */ function stripDefaultPort(host: string, port: string, scheme: string): string { if (port === "") return host; const effective = scheme === "https:" ? "443" : "80"; return port === effective ? host : `${host}:${port}`; } interface NormalizedParts { kind: ImportRowKind; name: string; host: string; } /** https://Host/w/UUID | /videos/watch/UUID | /videos/embed/UUID | /embed/UUID | /c/NAME | /a/NAME | /video-channels/NAME */ function rowFromUrl(raw: string, url: URL): NormalizedParts { const host = stripDefaultPort( canonicalHost(url.hostname), url.port, url.protocol, ); if (host === "") throw new InvalidRowError("invalid_host", "Hostname could not be canonicalized"); const segments = url.pathname.split("/").filter((part) => part !== ""); const head = segments[0]; // Watch/embed URLs name a live (or a VOD): the uuid is the identity. // /w/{id} · /videos/watch/{id} · /videos/embed/{id} · /embed/{id} if (head === "w" && segments.length === 2) { return liveRow(segments[1]!.split(";")[0]!, host); } if (head === "videos" && (segments[1] === "watch" || segments[1] === "embed") && segments.length === 3) { return liveRow(segments[2]!.split(";")[0]!, host); } if (head === "embed" && segments.length === 2) { return liveRow(segments[1]!.split(";")[0]!, host); } // Channel-ish pages. if ((head === "c" || head === "a" || head === "video-channels") && segments.length === 2) { return channelRow(decodeURIComponent(segments[1]!), host); } throw new InvalidRowError("unrecognized_url", "URL does not name a PeerTube channel or video"); } function liveRow(id: string, host: string): NormalizedParts { const name = id.toLowerCase(); if (!UUID.test(name) && !SHORT_ID.test(name)) { throw new InvalidRowError("invalid_video_id", "Video id is not a uuid or short id"); } return { kind: "live", name, host }; } function channelRow(name: string, host: string): NormalizedParts { const normalized = name.replace(/^@+/, "").toLowerCase(); if (!CHANNEL_NAME.test(normalized)) { throw new InvalidRowError("invalid_channel_name", "Channel name is not a PeerTube handle segment"); } return { kind: "channel", name: normalized, host }; } /** * Normalize one input row. `defaultHost` (the import connection's instance * host, already canonical) is used for bare handles/ids that name no host. * `policy` performs the syntactic egress check on URL inputs — no DNS, no * budget consumption (validateUrl is the policy-only entry point). */ export function normalizeImportRow( raw: string, defaultHost: string | undefined, policy: Pick, ): NormalizedRow { const text = raw.trim(); if (text === "") throw new InvalidRowError("empty", "Row is empty"); if (text.length > 2048) throw new InvalidRowError("too_long", "Row exceeds 2048 characters"); if (CONTROL.test(text)) throw new InvalidRowError("control_characters", "Row contains control characters"); // URL forms go through the shared egress policy first (scheme/port/private // literals denied before any network work), then identity extraction. if (/^[a-z][a-z0-9+.-]*:\/\//i.test(text) || text.includes("://")) { let url: URL; try { url = policy.validateUrl(text); } catch { // Surface the policy taxonomy without echoing the raw string. throw new InvalidRowError("denied_url", "URL rejected by the outbound request policy"); } return finish(rowFromUrl(text, url), text); } // @handle@host (activitypub style) — strip the leading @ before splitting. const handleMatch = /^@?([^\s@/]+)@([^\s@/]+)$/.exec(text); if (handleMatch) { const host = canonicalHost(handleMatch[2]!); if (host === "") throw new InvalidRowError("invalid_host", "Hostname could not be canonicalized"); const name = handleMatch[1]!; // name@host where name is a uuid => a live named by handle. if (UUID.test(name.toLowerCase())) return finish(liveRow(name, host), text); return finish(channelRow(name, host), text); } // Bare uuid / short id: only importable against a default host. if (UUID.test(text.toLowerCase()) || SHORT_ID.test(text)) { if (defaultHost === undefined) throw new InvalidRowError("no_host", "A bare video id needs an instance host"); return finish(liveRow(text, defaultHost), text); } // Bare channel name (no host). if (HANDLE_LIKE.test(text.replace(/^@/, ""))) { if (defaultHost === undefined) throw new InvalidRowError("no_host", "A bare handle needs an instance host"); return finish(channelRow(text.replace(/^@+/, ""), defaultHost), text); } throw new InvalidRowError("unrecognized", "Row is not a URL, handle or video id"); } function finish(parts: NormalizedParts, raw: string): NormalizedRow { const key = `${parts.kind}:${parts.name}@${parts.host}`; const label = parts.kind === "channel" ? `@${parts.name}@${parts.host}` : `live:${parts.name}@${parts.host}`; return { ...parts, key, label, raw }; } export interface ParsedImport { rows: string[]; /** How the body was interpreted, for the report. */ format: "url" | "handle" | "multiline" | "csv" | "json"; } /** * Accepts the four documented body shapes and yields raw string rows: * - JSON array (of strings, or objects with a url/handle/id/ref field) * - CSV (one value per row, first column used; a header row is skipped) * - one URL or handle (a bare string body) * - multiline text (one row per line) */ export function parseImportBody(contentType: string | undefined, body: string): ParsedImport { const type = (contentType ?? "").split(";")[0]?.trim().toLowerCase(); if (type === "application/json" || body.trimStart().startsWith("[")) { let parsed: unknown; try { parsed = JSON.parse(body); } catch { throw new BadRequestException("body is not valid JSON"); } if (!Array.isArray(parsed)) throw new BadRequestException("JSON import body must be an array"); const rows: string[] = []; for (const entry of parsed) { if (typeof entry === "string") rows.push(entry); else if (entry !== null && typeof entry === "object") { const record = entry as Record; const value = record.url ?? record.handle ?? record.id ?? record.ref ?? record.value; if (typeof value !== "string") throw new BadRequestException("JSON entries must be strings or objects with url/handle/id"); rows.push(value); } else throw new BadRequestException("JSON entries must be strings or objects with url/handle/id"); } return { rows, format: "json" }; } if (type === "text/csv" || type === "application/csv") { const lines = body.split(/\r?\n/).filter((line) => line.trim() !== ""); if (lines.length === 0) return { rows: [], format: "csv" }; const first = lines[0]!.split(",")[0]?.trim().toLowerCase() ?? ""; // A header row is skipped; data rows use the first column only. const data = /^(url|handle|source|id|ref)$/.test(first) ? lines.slice(1) : lines; const rows = data .map((line) => { const cell = line.split(",")[0] ?? ""; // CSV cells may be quoted; strip one pair of double quotes. return cell.replace(/^"(.*)"$/, "$1").trim(); }) .filter((cell) => cell !== ""); return { rows, format: "csv" }; } const lines = body.split(/\r?\n/).map((line) => line.trim()); const rows = lines.filter((line) => line !== ""); if (rows.length === 1 && !/\s/.test(rows[0]!)) { return { rows, format: /^https?:\/\//i.test(rows[0]!) ? "url" : "handle" }; } return { rows, format: "multiline" }; } /** Pre-resolution dedupe: first occurrence of a syntactic key wins. */ export function dedupeSyntactic(rows: readonly NormalizedRow[]): { unique: NormalizedRow[]; duplicateRows: number } { const seen = new Set(); const unique: NormalizedRow[] = []; let duplicateRows = 0; for (const row of rows) { if (seen.has(row.key)) { duplicateRows += 1; continue; } seen.add(row.key); unique.push(row); } return { unique, duplicateRows }; }