glass beadian
aa5e262f3a
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
258 lines
11 KiB
TypeScript
258 lines
11 KiB
TypeScript
/**
|
|
* Bulk source import: input parsing + normalization + the syntactic identity
|
|
* key (WP7, proposal §4.4 "Bulk source import").
|
|
*
|
|
* The pipeline is deliberately staged so no network is needed to dedupe:
|
|
* parse input rows (URL / handle / multiline / CSV / JSON)
|
|
* -> normalize (strip leading `@`, lowercase, IDNA/punycode host,
|
|
* strip default port)
|
|
* -> syntactic validation + egress policy check (validateUrl — no DNS,
|
|
* no budget draw)
|
|
* -> pre-resolution dedupe on the syntactic key
|
|
* `kind:{name-or-id}@{canonical host}`
|
|
* Only the unique surviving keys are resolved remotely; the resolved
|
|
* `remote_ref` is the instance-provided identity and dedupes again
|
|
* post-resolution, and the DB's UNIQUE (workspace_id, virtual_channel_id,
|
|
* remote_ref) is the final authority.
|
|
*/
|
|
|
|
import { BadRequestException } from "@nestjs/common";
|
|
import { domainToASCII } from "node:url";
|
|
import { isIP } from "node:net";
|
|
import type { EgressPolicy } from "../../../../packages/egress/src/index.js";
|
|
|
|
/** Hard request cap (proposal §4.4): larger lists are a 413, never silently cut. */
|
|
export const IMPORT_ROW_LIMIT = 5_000;
|
|
|
|
export type ImportRowKind = "channel" | "live";
|
|
|
|
/** One normalized input row. `key` is the pre-resolution dedupe key. */
|
|
export interface NormalizedRow {
|
|
kind: ImportRowKind;
|
|
/** Lower-cased, leading-`@`-stripped name (channel) or uuid/shortUUID (live). */
|
|
name: string;
|
|
/** Lower-cased IDNA host, default port stripped, no brackets. */
|
|
host: string;
|
|
key: string;
|
|
label: string;
|
|
/** Original input line, for the report. */
|
|
raw: string;
|
|
}
|
|
|
|
/** Row-level failure with a stable reason code for the import report. */
|
|
export class InvalidRowError extends Error {
|
|
constructor(readonly reason: string, message: string) {
|
|
super(message);
|
|
this.name = "InvalidRowError";
|
|
}
|
|
}
|
|
|
|
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
/** PeerTube short ids: base64url, 6-12 chars. */
|
|
const SHORT_ID = /^[0-9a-zA-Z_-]{6,12}$/;
|
|
/** PeerTube usernames/channels: lowercase letters, digits, `_`, 1..50. */
|
|
const CHANNEL_NAME = /^[a-z0-9_]{1,50}$/;
|
|
/** Latent claim: anything else that could be an account/channel handle. */
|
|
const HANDLE_LIKE = /^@?[a-z0-9_][a-z0-9_.-]{0,64}$/i;
|
|
const CONTROL = /[\u0000-\u001f\u007f]/;
|
|
|
|
/** Strip a trailing DNS dot and lowercase; returns "" for unusable hosts. */
|
|
function canonicalHost(raw: string): string {
|
|
const host = raw.replace(/^\[|\]$/g, "").replace(/\.$/, "").toLowerCase();
|
|
if (host === "") return "";
|
|
if (isIP(host)) return host;
|
|
const ascii = domainToASCII(host);
|
|
if (ascii === "" || ascii.includes("\u0000")) return "";
|
|
return ascii;
|
|
}
|
|
|
|
/** Port stripped when it is the scheme default; anything else is rejected later. */
|
|
function stripDefaultPort(host: string, port: string, scheme: string): string {
|
|
if (port === "") return host;
|
|
const effective = scheme === "https:" ? "443" : "80";
|
|
return port === effective ? host : `${host}:${port}`;
|
|
}
|
|
|
|
interface NormalizedParts {
|
|
kind: ImportRowKind;
|
|
name: string;
|
|
host: string;
|
|
}
|
|
|
|
/** https://Host/w/UUID | /videos/watch/UUID | /videos/embed/UUID | /embed/UUID | /c/NAME | /a/NAME | /video-channels/NAME */
|
|
function rowFromUrl(raw: string, url: URL): NormalizedParts {
|
|
const host = stripDefaultPort(
|
|
canonicalHost(url.hostname),
|
|
url.port,
|
|
url.protocol,
|
|
);
|
|
if (host === "") throw new InvalidRowError("invalid_host", "Hostname could not be canonicalized");
|
|
const segments = url.pathname.split("/").filter((part) => part !== "");
|
|
const head = segments[0];
|
|
|
|
// Watch/embed URLs name a live (or a VOD): the uuid is the identity.
|
|
// /w/{id} · /videos/watch/{id} · /videos/embed/{id} · /embed/{id}
|
|
if (head === "w" && segments.length === 2) {
|
|
return liveRow(segments[1]!.split(";")[0]!, host);
|
|
}
|
|
if (head === "videos" && (segments[1] === "watch" || segments[1] === "embed") && segments.length === 3) {
|
|
return liveRow(segments[2]!.split(";")[0]!, host);
|
|
}
|
|
if (head === "embed" && segments.length === 2) {
|
|
return liveRow(segments[1]!.split(";")[0]!, host);
|
|
}
|
|
// Channel-ish pages.
|
|
if ((head === "c" || head === "a" || head === "video-channels") && segments.length === 2) {
|
|
return channelRow(decodeURIComponent(segments[1]!), host);
|
|
}
|
|
throw new InvalidRowError("unrecognized_url", "URL does not name a PeerTube channel or video");
|
|
}
|
|
|
|
function liveRow(id: string, host: string): NormalizedParts {
|
|
const name = id.toLowerCase();
|
|
if (!UUID.test(name) && !SHORT_ID.test(name)) {
|
|
throw new InvalidRowError("invalid_video_id", "Video id is not a uuid or short id");
|
|
}
|
|
return { kind: "live", name, host };
|
|
}
|
|
|
|
function channelRow(name: string, host: string): NormalizedParts {
|
|
const normalized = name.replace(/^@+/, "").toLowerCase();
|
|
if (!CHANNEL_NAME.test(normalized)) {
|
|
throw new InvalidRowError("invalid_channel_name", "Channel name is not a PeerTube handle segment");
|
|
}
|
|
return { kind: "channel", name: normalized, host };
|
|
}
|
|
|
|
/**
|
|
* Normalize one input row. `defaultHost` (the import connection's instance
|
|
* host, already canonical) is used for bare handles/ids that name no host.
|
|
* `policy` performs the syntactic egress check on URL inputs — no DNS, no
|
|
* budget consumption (validateUrl is the policy-only entry point).
|
|
*/
|
|
export function normalizeImportRow(
|
|
raw: string,
|
|
defaultHost: string | undefined,
|
|
policy: Pick<EgressPolicy, "validateUrl">,
|
|
): NormalizedRow {
|
|
const text = raw.trim();
|
|
if (text === "") throw new InvalidRowError("empty", "Row is empty");
|
|
if (text.length > 2048) throw new InvalidRowError("too_long", "Row exceeds 2048 characters");
|
|
if (CONTROL.test(text)) throw new InvalidRowError("control_characters", "Row contains control characters");
|
|
|
|
// URL forms go through the shared egress policy first (scheme/port/private
|
|
// literals denied before any network work), then identity extraction.
|
|
if (/^[a-z][a-z0-9+.-]*:\/\//i.test(text) || text.includes("://")) {
|
|
let url: URL;
|
|
try {
|
|
url = policy.validateUrl(text);
|
|
} catch {
|
|
// Surface the policy taxonomy without echoing the raw string.
|
|
throw new InvalidRowError("denied_url", "URL rejected by the outbound request policy");
|
|
}
|
|
return finish(rowFromUrl(text, url), text);
|
|
}
|
|
|
|
// @handle@host (activitypub style) — strip the leading @ before splitting.
|
|
const handleMatch = /^@?([^\s@/]+)@([^\s@/]+)$/.exec(text);
|
|
if (handleMatch) {
|
|
const host = canonicalHost(handleMatch[2]!);
|
|
if (host === "") throw new InvalidRowError("invalid_host", "Hostname could not be canonicalized");
|
|
const name = handleMatch[1]!;
|
|
// name@host where name is a uuid => a live named by handle.
|
|
if (UUID.test(name.toLowerCase())) return finish(liveRow(name, host), text);
|
|
return finish(channelRow(name, host), text);
|
|
}
|
|
|
|
// Bare uuid / short id: only importable against a default host.
|
|
if (UUID.test(text.toLowerCase()) || SHORT_ID.test(text)) {
|
|
if (defaultHost === undefined) throw new InvalidRowError("no_host", "A bare video id needs an instance host");
|
|
return finish(liveRow(text, defaultHost), text);
|
|
}
|
|
// Bare channel name (no host).
|
|
if (HANDLE_LIKE.test(text.replace(/^@/, ""))) {
|
|
if (defaultHost === undefined) throw new InvalidRowError("no_host", "A bare handle needs an instance host");
|
|
return finish(channelRow(text.replace(/^@+/, ""), defaultHost), text);
|
|
}
|
|
throw new InvalidRowError("unrecognized", "Row is not a URL, handle or video id");
|
|
}
|
|
|
|
function finish(parts: NormalizedParts, raw: string): NormalizedRow {
|
|
const key = `${parts.kind}:${parts.name}@${parts.host}`;
|
|
const label = parts.kind === "channel" ? `@${parts.name}@${parts.host}` : `live:${parts.name}@${parts.host}`;
|
|
return { ...parts, key, label, raw };
|
|
}
|
|
|
|
export interface ParsedImport {
|
|
rows: string[];
|
|
/** How the body was interpreted, for the report. */
|
|
format: "url" | "handle" | "multiline" | "csv" | "json";
|
|
}
|
|
|
|
/**
|
|
* Accepts the four documented body shapes and yields raw string rows:
|
|
* - JSON array (of strings, or objects with a url/handle/id/ref field)
|
|
* - CSV (one value per row, first column used; a header row is skipped)
|
|
* - one URL or handle (a bare string body)
|
|
* - multiline text (one row per line)
|
|
*/
|
|
export function parseImportBody(contentType: string | undefined, body: string): ParsedImport {
|
|
const type = (contentType ?? "").split(";")[0]?.trim().toLowerCase();
|
|
if (type === "application/json" || body.trimStart().startsWith("[")) {
|
|
let parsed: unknown;
|
|
try {
|
|
parsed = JSON.parse(body);
|
|
} catch {
|
|
throw new BadRequestException("body is not valid JSON");
|
|
}
|
|
if (!Array.isArray(parsed)) throw new BadRequestException("JSON import body must be an array");
|
|
const rows: string[] = [];
|
|
for (const entry of parsed) {
|
|
if (typeof entry === "string") rows.push(entry);
|
|
else if (entry !== null && typeof entry === "object") {
|
|
const record = entry as Record<string, unknown>;
|
|
const value = record.url ?? record.handle ?? record.id ?? record.ref ?? record.value;
|
|
if (typeof value !== "string") throw new BadRequestException("JSON entries must be strings or objects with url/handle/id");
|
|
rows.push(value);
|
|
} else throw new BadRequestException("JSON entries must be strings or objects with url/handle/id");
|
|
}
|
|
return { rows, format: "json" };
|
|
}
|
|
if (type === "text/csv" || type === "application/csv") {
|
|
const lines = body.split(/\r?\n/).filter((line) => line.trim() !== "");
|
|
if (lines.length === 0) return { rows: [], format: "csv" };
|
|
const first = lines[0]!.split(",")[0]?.trim().toLowerCase() ?? "";
|
|
// A header row is skipped; data rows use the first column only.
|
|
const data = /^(url|handle|source|id|ref)$/.test(first) ? lines.slice(1) : lines;
|
|
const rows = data
|
|
.map((line) => {
|
|
const cell = line.split(",")[0] ?? "";
|
|
// CSV cells may be quoted; strip one pair of double quotes.
|
|
return cell.replace(/^"(.*)"$/, "$1").trim();
|
|
})
|
|
.filter((cell) => cell !== "");
|
|
return { rows, format: "csv" };
|
|
}
|
|
const lines = body.split(/\r?\n/).map((line) => line.trim());
|
|
const rows = lines.filter((line) => line !== "");
|
|
if (rows.length === 1 && !/\s/.test(rows[0]!)) {
|
|
return { rows, format: /^https?:\/\//i.test(rows[0]!) ? "url" : "handle" };
|
|
}
|
|
return { rows, format: "multiline" };
|
|
}
|
|
|
|
/** Pre-resolution dedupe: first occurrence of a syntactic key wins. */
|
|
export function dedupeSyntactic(rows: readonly NormalizedRow[]): { unique: NormalizedRow[]; duplicateRows: number } {
|
|
const seen = new Set<string>();
|
|
const unique: NormalizedRow[] = [];
|
|
let duplicateRows = 0;
|
|
for (const row of rows) {
|
|
if (seen.has(row.key)) {
|
|
duplicateRows += 1;
|
|
continue;
|
|
}
|
|
seen.add(row.key);
|
|
unique.push(row);
|
|
}
|
|
return { unique, duplicateRows };
|
|
}
|