Add group cloud: files index api, web UI, archive sync bot
The Telegram-like group file contract (packages/core shared-files.ts) becomes a working subsystem, built in three parallel tracks: API (NestJS): migration 029 group_files (chat_media_archive metadata-only index with partial replay-unique + RLS, group_file_shares with sha256 token persistence, archive_sync_state/outbox durable-claim seam incl. a unique claim_key index), group-files module (list/search with cursor pagination, direct upload ≤20 MiB streamed+hashed to oCIS WebDAV, download with ACL + retention rechecks, share links with expiry, retention modes incl. legalHold) — RBAC group-membership authz + audit chain events. Web: Files section per group — group picker, drag-drop upload (20 MiB cap), search + MIME/origin filters, cursor pagination, download via authenticated blob fetch, share dialog with expiry + shown-once token, admin retention dialog; bilingual EN/ES (wiki i18n pattern); nav sidebar/palette/bottom-nav wiring; MIME classes aligned to the DB CHECK (document/other). Archive bot (apps/archive-bot): plain sync-loop Matrix bot mirroring attachments from explicitly-mapped unencrypted archive rooms — raw /sync with durable since-cursor, outbox-claimed ingestion (hash-verified stream, no temp files), magic-byte MIME sniffing, safe-name normalization, WebDAV upload, indexed + replied with link, redaction→unavailable; health endpoint; Dockerfile. Tests 74/74, typecheck clean across api/web/bot.
This commit is contained in:
parent
eb1b60f67d
commit
d58db63fe6
58 changed files with 5543 additions and 4 deletions
17
apps/archive-bot/Dockerfile
Normal file
17
apps/archive-bot/Dockerfile
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
# Hub archive sync bot. Build from the monorepo root so workspace:* deps resolve:
|
||||
# docker build -f apps/archive-bot/Dockerfile -t hub-archive-bot .
|
||||
FROM node:22-slim AS build
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
RUN NODE_ENV=development npx --yes pnpm@9.12.0 install --frozen-lockfile
|
||||
RUN npx --yes pnpm@9.12.0 --filter @tactical-media-broadcast-hub/archive-bot... build
|
||||
|
||||
FROM node:22-slim
|
||||
ENV NODE_ENV=production
|
||||
WORKDIR /app
|
||||
# Workspace node_modules are relative symlinks; copy the built tree whole.
|
||||
COPY --from=build --chown=node:node /app /app
|
||||
# Non-root, and the bot never writes payloads to disk (streams straight to WebDAV).
|
||||
USER node
|
||||
EXPOSE 8085
|
||||
CMD ["node", "/app/apps/archive-bot/dist/main.js"]
|
||||
29
apps/archive-bot/package.json
Normal file
29
apps/archive-bot/package.json
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
{
|
||||
"name": "@tactical-media-broadcast-hub/archive-bot",
|
||||
"private": true,
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "tsx watch src/main.ts",
|
||||
"build": "tsc -p tsconfig.build.json",
|
||||
"start": "node dist/main.js",
|
||||
"lint": "eslint src",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit",
|
||||
"test": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"@tactical-media-broadcast-hub/core": "workspace:*",
|
||||
"pg": "^8.23.1",
|
||||
"pino": "^9.3.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.5.0",
|
||||
"@types/pg": "^8.23.1",
|
||||
"tsx": "^4.17.0",
|
||||
"typescript": "^5.5.4",
|
||||
"vitest": "^2.0.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22.13.0"
|
||||
}
|
||||
}
|
||||
90
apps/archive-bot/src/archiver.ts
Normal file
90
apps/archive-bot/src/archiver.ts
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
import type { GroupFile } from "@tactical-media-broadcast-hub/core";
|
||||
import { toGroupFileMapping, type RoomMapping } from "./config.js";
|
||||
import type { AttachmentWork } from "./events.js";
|
||||
import { inspectStream } from "./inspector.js";
|
||||
import { replyTxnId } from "./keys.js";
|
||||
import type { MatrixPort } from "./matrix.js";
|
||||
import { safeFileName } from "./names.js";
|
||||
import type { ArchiveStore, OutboxRow } from "./store.js";
|
||||
import type { LibraryPort } from "./webdav.js";
|
||||
|
||||
export interface ArchivedAttachment {
|
||||
file: GroupFile;
|
||||
safeName: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Steps 2–3 of the processing contract for one claimed attachment:
|
||||
* authenticated download → in-flight size/hash/MIME → staged WebDAV PUT →
|
||||
* content-addressed MOVE → index row + outbox done in one transaction.
|
||||
* The payload is never written to local disk.
|
||||
*/
|
||||
export class Archiver {
|
||||
constructor(
|
||||
private readonly matrix: MatrixPort,
|
||||
private readonly library: LibraryPort,
|
||||
private readonly store: ArchiveStore,
|
||||
) {}
|
||||
|
||||
async archive(
|
||||
mapping: RoomMapping,
|
||||
work: AttachmentWork,
|
||||
outbox: OutboxRow,
|
||||
claimKey: string,
|
||||
signal: AbortSignal,
|
||||
): Promise<ArchivedAttachment> {
|
||||
const { candidate, event } = work;
|
||||
const download = await this.matrix.downloadMedia(candidate.mxcUri, signal);
|
||||
const { stream, result } = inspectStream(download.body, download.contentLength);
|
||||
const dir = ["Library", mapping.workspaceId, mapping.groupId];
|
||||
// Deterministic per claim: a retried upload overwrites its own partial staging copy.
|
||||
const stagingName = `${replyTxnId(claimKey)}.part`;
|
||||
await this.library.putStaging(dir, stagingName, stream, signal);
|
||||
const payload = await result;
|
||||
|
||||
const safeName = safeFileName(work.displayName, payload.sniffed.extension);
|
||||
const resource = await this.library.commit(dir, stagingName, `${payload.sha256}-${safeName}`, signal);
|
||||
const createdAt = new Date(event.origin_server_ts ?? Date.now());
|
||||
const indexed = await this.store.commitArchived({
|
||||
workspaceId: mapping.workspaceId,
|
||||
groupId: mapping.groupId,
|
||||
roomId: mapping.roomId,
|
||||
eventId: event.event_id,
|
||||
senderMxid: event.sender,
|
||||
matrixEventJson: event,
|
||||
originalName: work.displayName,
|
||||
safeName,
|
||||
mime: payload.sniffed.mime,
|
||||
mimeClass: payload.sniffed.mimeClass,
|
||||
byteSize: payload.byteSize,
|
||||
sha256: payload.sha256,
|
||||
ocisResourceId: resource.resourceId,
|
||||
ocisResourceVersion: resource.version,
|
||||
createdAt,
|
||||
}, outbox.id);
|
||||
|
||||
const group = toGroupFileMapping(mapping);
|
||||
return {
|
||||
safeName,
|
||||
file: {
|
||||
id: indexed.id,
|
||||
workspaceId: mapping.workspaceId,
|
||||
groupId: mapping.groupId,
|
||||
roomId: mapping.roomId,
|
||||
senderId: event.sender,
|
||||
eventId: event.event_id,
|
||||
origin: { kind: "matrix", mxcUri: candidate.mxcUri },
|
||||
name: safeName,
|
||||
mimeType: payload.sniffed.mime,
|
||||
sizeBytes: payload.byteSize,
|
||||
checksum: { algorithm: "sha256", value: payload.sha256 },
|
||||
createdAt: createdAt.toISOString(),
|
||||
indexedAt: indexed.indexedAt.toISOString(),
|
||||
file: { spaceId: group.spaceId, resourceId: resource.resourceId, version: resource.version },
|
||||
shareLink: { url: resource.url, access: "members", expiresAt: null },
|
||||
retention: group.retention,
|
||||
status: "available",
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
14
apps/archive-bot/src/backoff.ts
Normal file
14
apps/archive-bot/src/backoff.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
export const BACKOFF_BASE_MS = 30_000;
|
||||
export const BACKOFF_MAX_MS = 60 * 60 * 1000;
|
||||
/** After this many attempts a row is marked failed (a durable, terminal retry record). */
|
||||
export const MAX_ATTEMPTS = 12;
|
||||
|
||||
/** Exponential per-row delay after `attempts` failures, capped at one hour. */
|
||||
export function backoffMs(attempts: number): number {
|
||||
if (attempts <= 0) return 0;
|
||||
return Math.min(BACKOFF_BASE_MS * 2 ** (attempts - 1), BACKOFF_MAX_MS);
|
||||
}
|
||||
|
||||
export function nextAttemptAt(attempts: number, lastAttemptAt: Date): Date {
|
||||
return new Date(lastAttemptAt.getTime() + backoffMs(attempts));
|
||||
}
|
||||
56
apps/archive-bot/src/config.test.ts
Normal file
56
apps/archive-bot/src/config.test.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { ConfigError, loadConfig, parseRoomMappings, toGroupFileMapping } from "./config.js";
|
||||
|
||||
const WS = "11111111-1111-4111-8111-111111111111";
|
||||
const GROUP = "22222222-2222-4222-8222-222222222222";
|
||||
|
||||
describe("parseRoomMappings", () => {
|
||||
it("accepts immutable room IDs with UUID workspace/group", () => {
|
||||
expect(parseRoomMappings(JSON.stringify([{ roomId: "!abc:example.org", workspaceId: WS, groupId: GROUP }])))
|
||||
.toEqual([{ roomId: "!abc:example.org", workspaceId: WS, groupId: GROUP }]);
|
||||
});
|
||||
|
||||
it("rejects aliases, non-UUIDs, duplicates, empty and malformed input", () => {
|
||||
expect(() => parseRoomMappings(JSON.stringify([{ roomId: "#archive:example.org", workspaceId: WS, groupId: GROUP }])))
|
||||
.toThrow(ConfigError);
|
||||
expect(() => parseRoomMappings(JSON.stringify([{ roomId: "!a:x", workspaceId: "ws", groupId: GROUP }]))).toThrow(ConfigError);
|
||||
const dup = { roomId: "!a:x", workspaceId: WS, groupId: GROUP };
|
||||
expect(() => parseRoomMappings(JSON.stringify([dup, dup]))).toThrow(/duplicate/);
|
||||
expect(() => parseRoomMappings("[]")).toThrow(ConfigError);
|
||||
expect(() => parseRoomMappings("{nope")).toThrow(ConfigError);
|
||||
});
|
||||
|
||||
it("maps to an unencrypted-archive GroupFileMapping kept forever", () => {
|
||||
const mapping = toGroupFileMapping({ roomId: "!a:x", workspaceId: WS, groupId: GROUP });
|
||||
expect(mapping.archivePolicy).toBe("unencrypted-archive");
|
||||
expect(mapping.retention).toEqual({ legalHold: false, mode: "keep" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("loadConfig", () => {
|
||||
const env = {
|
||||
DATABASE_URL: "postgres://bot@db/hub",
|
||||
MATRIX_HOMESERVER_URL: "https://matrix.example.org",
|
||||
MATRIX_USER: "archive-bot",
|
||||
MATRIX_PASSWORD: "pw",
|
||||
ROOM_MAPPINGS: JSON.stringify([{ roomId: "!a:x", workspaceId: WS, groupId: GROUP }]),
|
||||
FILES_OCID_WEBDAV_URL: "https://files.example.org",
|
||||
FILES_OCID_USER: "svc",
|
||||
FILES_OCID_PASSWORD: "pw2",
|
||||
ARCHIVE_BOT_HEALTH_PORT: "9100",
|
||||
MATRIX_MEDIA_REDIRECT_HOSTS: " R2.example.com ,",
|
||||
};
|
||||
|
||||
it("parses the full env contract", () => {
|
||||
const config = loadConfig(env);
|
||||
expect(config.healthPort).toBe(9100);
|
||||
expect(config.mediaRedirectHosts).toEqual(["r2.example.com"]);
|
||||
expect(config.matrixDeviceId).toBe("HUB_ARCHIVE_BOT");
|
||||
});
|
||||
|
||||
it("fails on missing secrets or a bad port", () => {
|
||||
expect(() => loadConfig({ ...env, MATRIX_PASSWORD: "" })).toThrow(/MATRIX_PASSWORD/);
|
||||
expect(() => loadConfig({ ...env, ARCHIVE_BOT_HEALTH_PORT: "70000" })).toThrow(/PORT/);
|
||||
expect(() => loadConfig({ ...env, FILES_OCID_WEBDAV_URL: "ftp://x" })).toThrow(/http/);
|
||||
});
|
||||
});
|
||||
110
apps/archive-bot/src/config.ts
Normal file
110
apps/archive-bot/src/config.ts
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
import type { GroupFileMapping } from "@tactical-media-broadcast-hub/core";
|
||||
|
||||
/** Explicit, immutable room → library mapping. Only rooms listed here are ever read. */
|
||||
export interface RoomMapping {
|
||||
roomId: string;
|
||||
workspaceId: string;
|
||||
groupId: string;
|
||||
}
|
||||
|
||||
export interface BotConfig {
|
||||
databaseUrl: string;
|
||||
homeserverUrl: URL;
|
||||
matrixUser: string;
|
||||
matrixPassword: string;
|
||||
/** Stable device so Matrix transaction-ID idempotency survives restarts. */
|
||||
matrixDeviceId: string;
|
||||
/** Extra hosts (e.g. the R2 media endpoint) media redirects may target; auth is always dropped. */
|
||||
mediaRedirectHosts: string[];
|
||||
mappings: RoomMapping[];
|
||||
webdav: { baseUrl: URL; user: string; password: string };
|
||||
healthPort: number;
|
||||
}
|
||||
|
||||
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||
/** Room IDs only: `!opaque:server`. Aliases (`#…`) are rejected — mappings must be immutable. */
|
||||
const ROOM_ID = /^![^:\s]+:[^\s]+$/;
|
||||
|
||||
export class ConfigError extends Error {}
|
||||
|
||||
function required(env: NodeJS.ProcessEnv, name: string): string {
|
||||
const value = env[name];
|
||||
if (value === undefined || value.trim() === "") throw new ConfigError(`${name} is required`);
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
function httpUrl(name: string, raw: string): URL {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(raw);
|
||||
} catch {
|
||||
throw new ConfigError(`${name} must be an absolute URL`);
|
||||
}
|
||||
if (url.protocol !== "https:" && url.protocol !== "http:") throw new ConfigError(`${name} must be http(s)`);
|
||||
return url;
|
||||
}
|
||||
|
||||
export function parseRoomMappings(raw: string): RoomMapping[] {
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch {
|
||||
throw new ConfigError("ROOM_MAPPINGS must be JSON");
|
||||
}
|
||||
if (!Array.isArray(parsed) || parsed.length === 0) throw new ConfigError("ROOM_MAPPINGS must be a non-empty array");
|
||||
const seen = new Set<string>();
|
||||
return parsed.map((entry: unknown, index) => {
|
||||
if (typeof entry !== "object" || entry === null) throw new ConfigError(`ROOM_MAPPINGS[${index}] must be an object`);
|
||||
const { roomId, workspaceId, groupId } = entry as Record<string, unknown>;
|
||||
if (typeof roomId !== "string" || !ROOM_ID.test(roomId)) {
|
||||
throw new ConfigError(`ROOM_MAPPINGS[${index}].roomId must be a room ID (!id:server), not an alias`);
|
||||
}
|
||||
if (typeof workspaceId !== "string" || !UUID.test(workspaceId)) {
|
||||
throw new ConfigError(`ROOM_MAPPINGS[${index}].workspaceId must be a UUID`);
|
||||
}
|
||||
if (typeof groupId !== "string" || !UUID.test(groupId)) {
|
||||
throw new ConfigError(`ROOM_MAPPINGS[${index}].groupId must be a UUID`);
|
||||
}
|
||||
if (seen.has(roomId)) throw new ConfigError(`ROOM_MAPPINGS has duplicate roomId ${roomId}`);
|
||||
seen.add(roomId);
|
||||
return { roomId, workspaceId: workspaceId.toLowerCase(), groupId: groupId.toLowerCase() };
|
||||
});
|
||||
}
|
||||
|
||||
/** The env mapping only exists for rooms explicitly enabled as unencrypted archive rooms. */
|
||||
export function toGroupFileMapping(mapping: RoomMapping): GroupFileMapping {
|
||||
return {
|
||||
workspaceId: mapping.workspaceId,
|
||||
groupId: mapping.groupId,
|
||||
roomId: mapping.roomId,
|
||||
spaceId: mapping.workspaceId,
|
||||
folderId: `${mapping.workspaceId}/${mapping.groupId}`,
|
||||
archivePolicy: "unencrypted-archive",
|
||||
retention: { legalHold: false, mode: "keep" },
|
||||
};
|
||||
}
|
||||
|
||||
export function loadConfig(env: NodeJS.ProcessEnv = process.env): BotConfig {
|
||||
const healthPort = Number(env.ARCHIVE_BOT_HEALTH_PORT ?? 8085);
|
||||
if (!Number.isInteger(healthPort) || healthPort <= 0 || healthPort > 65535) {
|
||||
throw new ConfigError("ARCHIVE_BOT_HEALTH_PORT must be a TCP port");
|
||||
}
|
||||
return {
|
||||
databaseUrl: required(env, "DATABASE_URL"),
|
||||
homeserverUrl: httpUrl("MATRIX_HOMESERVER_URL", required(env, "MATRIX_HOMESERVER_URL")),
|
||||
matrixUser: required(env, "MATRIX_USER"),
|
||||
matrixPassword: required(env, "MATRIX_PASSWORD"),
|
||||
matrixDeviceId: env.MATRIX_DEVICE_ID?.trim() || "HUB_ARCHIVE_BOT",
|
||||
mediaRedirectHosts: (env.MATRIX_MEDIA_REDIRECT_HOSTS ?? "")
|
||||
.split(",")
|
||||
.map((h) => h.trim().toLowerCase())
|
||||
.filter((h) => h !== ""),
|
||||
mappings: parseRoomMappings(required(env, "ROOM_MAPPINGS")),
|
||||
webdav: {
|
||||
baseUrl: httpUrl("FILES_OCID_WEBDAV_URL", required(env, "FILES_OCID_WEBDAV_URL")),
|
||||
user: required(env, "FILES_OCID_USER"),
|
||||
password: required(env, "FILES_OCID_PASSWORD"),
|
||||
},
|
||||
healthPort,
|
||||
};
|
||||
}
|
||||
71
apps/archive-bot/src/events.test.ts
Normal file
71
apps/archive-bot/src/events.test.ts
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import type { RoomMapping } from "./config.js";
|
||||
import { classifyEvent, parseMxc, redactionTarget, type MatrixEvent } from "./events.js";
|
||||
|
||||
const mapping: RoomMapping = {
|
||||
roomId: "!room:example.org",
|
||||
workspaceId: "11111111-1111-4111-8111-111111111111",
|
||||
groupId: "22222222-2222-4222-8222-222222222222",
|
||||
};
|
||||
const BOT = "@archive:example.org";
|
||||
|
||||
function message(content: Record<string, unknown>, extra: Partial<MatrixEvent> = {}): MatrixEvent {
|
||||
return { type: "m.room.message", event_id: "$ev1", sender: "@alice:example.org", content, ...extra };
|
||||
}
|
||||
const image = { msgtype: "m.image", body: "cat.png", url: "mxc://example.org/abc123" };
|
||||
|
||||
describe("classifyEvent", () => {
|
||||
it("accepts an unencrypted m.image with an mxc URI", () => {
|
||||
const result = classifyEvent(message(image), mapping, "unencrypted", BOT);
|
||||
expect(result.kind).toBe("attachment");
|
||||
if (result.kind !== "attachment") return;
|
||||
expect(result.candidate).toMatchObject({ roomId: mapping.roomId, eventId: "$ev1", msgtype: "m.image", mxcUri: image.url });
|
||||
expect(result.displayName).toBe("cat.png");
|
||||
});
|
||||
|
||||
it("prefers content.filename over body (caption) for the display name", () => {
|
||||
const result = classifyEvent(message({ ...image, filename: "real.png", body: "a caption" }), mapping, "unencrypted", BOT);
|
||||
expect(result.kind === "attachment" && result.displayName).toBe("real.png");
|
||||
});
|
||||
|
||||
it.each(["m.file", "m.audio", "m.video"])("accepts %s", (msgtype) => {
|
||||
expect(classifyEvent(message({ ...image, msgtype }), mapping, "unencrypted", BOT).kind).toBe("attachment");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["own-event", message(image, { sender: BOT }), "unencrypted"],
|
||||
["room-encrypted", message(image), "encrypted"],
|
||||
["room-unknown", message(image), "unknown"],
|
||||
["encrypted-event", { ...message(image), type: "m.room.encrypted" }, "unencrypted"],
|
||||
["encrypted-file", message({ msgtype: "m.file", body: "x", file: { url: "mxc://example.org/enc" } }), "unencrypted"],
|
||||
["edit", message({ ...image, "m.relates_to": { rel_type: "m.replace", event_id: "$old" } }), "unencrypted"],
|
||||
["edit", message({ ...image, "m.new_content": image }), "unencrypted"],
|
||||
["redacted", message(image, { unsigned: { redacted_because: {} } }), "unencrypted"],
|
||||
["not-an-attachment", message({ msgtype: "m.text", body: "hi" }), "unencrypted"],
|
||||
["not-a-message", { ...message(image), type: "m.sticker" }, "unencrypted"],
|
||||
["ineligible", message({ ...image, url: "https://evil.example/x.png" }), "unencrypted"],
|
||||
["ineligible", message({ ...image, url: "mxc://example.org/a/b" }), "unencrypted"],
|
||||
["ineligible", message({ msgtype: "m.image", body: "x" }), "unencrypted"],
|
||||
] as const)("skips %s", (reason, event, encryption) => {
|
||||
const result = classifyEvent(event, mapping, encryption, BOT);
|
||||
expect(result).toMatchObject({ kind: "skip", reason });
|
||||
});
|
||||
|
||||
it("classifies redactions regardless of room encryption (v11 content.redacts and legacy top-level)", () => {
|
||||
const v11: MatrixEvent = { type: "m.room.redaction", event_id: "$r", sender: "@mod:x", content: { redacts: "$ev1" } };
|
||||
const legacy: MatrixEvent = { type: "m.room.redaction", event_id: "$r", sender: "@mod:x", content: {}, redacts: "$ev2" };
|
||||
expect(classifyEvent(v11, mapping, "encrypted", BOT)).toMatchObject({ kind: "redaction", redacts: "$ev1" });
|
||||
expect(redactionTarget(legacy)).toBe("$ev2");
|
||||
expect(classifyEvent({ ...v11, content: {} }, mapping, "unencrypted", BOT).kind).toBe("skip");
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseMxc", () => {
|
||||
it("splits server and media id", () => {
|
||||
expect(parseMxc("mxc://example.org/abc")).toEqual({ serverName: "example.org", mediaId: "abc" });
|
||||
});
|
||||
it("rejects anything else", () => {
|
||||
expect(() => parseMxc("https://example.org/abc")).toThrow();
|
||||
expect(() => parseMxc("mxc://example.org/../x")).toThrow();
|
||||
});
|
||||
});
|
||||
109
apps/archive-bot/src/events.ts
Normal file
109
apps/archive-bot/src/events.ts
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
import { canArchiveAttachment, type MatrixAttachmentCandidate } from "@tactical-media-broadcast-hub/core";
|
||||
import { toGroupFileMapping, type RoomMapping } from "./config.js";
|
||||
|
||||
/** Client-server API event shape (only the fields the bot reads). */
|
||||
export interface MatrixEvent {
|
||||
type: string;
|
||||
event_id: string;
|
||||
sender: string;
|
||||
origin_server_ts?: number;
|
||||
state_key?: string;
|
||||
/** Room v1–v10 redactions carry the target here; v11+ moved it into content. */
|
||||
redacts?: string;
|
||||
content: Record<string, unknown>;
|
||||
unsigned?: { redacted_because?: unknown; [key: string]: unknown };
|
||||
}
|
||||
|
||||
export type RoomEncryption = MatrixAttachmentCandidate["roomEncryption"];
|
||||
|
||||
export const ATTACHMENT_MSGTYPES = ["m.file", "m.image", "m.audio", "m.video"] as const;
|
||||
|
||||
export interface AttachmentWork {
|
||||
kind: "attachment";
|
||||
event: MatrixEvent;
|
||||
candidate: MatrixAttachmentCandidate;
|
||||
/** Sender-supplied display name; only ever used after normalisation. */
|
||||
displayName: string;
|
||||
}
|
||||
|
||||
export type Classified =
|
||||
| AttachmentWork
|
||||
| { kind: "redaction"; event: MatrixEvent; redacts: string }
|
||||
| { kind: "skip"; event: MatrixEvent; reason: string };
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
/** Runtime guard for raw JSON from /sync or /messages. */
|
||||
export function isMatrixEvent(value: unknown): value is MatrixEvent {
|
||||
return isRecord(value)
|
||||
&& typeof value.type === "string"
|
||||
&& typeof value.event_id === "string"
|
||||
&& typeof value.sender === "string"
|
||||
&& isRecord(value.content);
|
||||
}
|
||||
|
||||
export function redactionTarget(event: MatrixEvent): string | null {
|
||||
const target = typeof event.content.redacts === "string" ? event.content.redacts : event.redacts;
|
||||
return typeof target === "string" && target.startsWith("$") ? target : null;
|
||||
}
|
||||
|
||||
function isEdit(content: Record<string, unknown>): boolean {
|
||||
const relates = content["m.relates_to"];
|
||||
return (isRecord(relates) && relates.rel_type === "m.replace") || "m.new_content" in content;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide what (if anything) to do with a timeline event in a mapped room.
|
||||
* Every exclusion from the processing contract lives here so it is unit-testable;
|
||||
* the final eligibility gate is the shared `canArchiveAttachment`.
|
||||
*/
|
||||
export function classifyEvent(
|
||||
event: MatrixEvent,
|
||||
mapping: RoomMapping,
|
||||
roomEncryption: RoomEncryption,
|
||||
botUserId: string,
|
||||
): Classified {
|
||||
if (event.sender === botUserId) return { kind: "skip", event, reason: "own-event" };
|
||||
if (event.type === "m.room.redaction") {
|
||||
const redacts = redactionTarget(event);
|
||||
return redacts === null ? { kind: "skip", event, reason: "redaction-without-target" } : { kind: "redaction", event, redacts };
|
||||
}
|
||||
if (event.type === "m.room.encrypted") return { kind: "skip", event, reason: "encrypted-event" };
|
||||
if (event.type !== "m.room.message") return { kind: "skip", event, reason: "not-a-message" };
|
||||
if (event.unsigned?.redacted_because !== undefined) return { kind: "skip", event, reason: "redacted" };
|
||||
const { content } = event;
|
||||
const msgtype = typeof content.msgtype === "string" ? content.msgtype : "";
|
||||
if (!(ATTACHMENT_MSGTYPES as readonly string[]).includes(msgtype)) {
|
||||
return { kind: "skip", event, reason: "not-an-attachment" };
|
||||
}
|
||||
if (isEdit(content)) return { kind: "skip", event, reason: "edit" };
|
||||
if (content.file !== undefined) return { kind: "skip", event, reason: "encrypted-file" };
|
||||
if (roomEncryption !== "unencrypted") return { kind: "skip", event, reason: `room-${roomEncryption}` };
|
||||
|
||||
const candidate: MatrixAttachmentCandidate = {
|
||||
roomId: mapping.roomId,
|
||||
eventId: event.event_id,
|
||||
senderId: event.sender,
|
||||
eventType: event.type,
|
||||
msgtype,
|
||||
roomEncryption,
|
||||
encryptedFile: content.file !== undefined,
|
||||
redacted: false,
|
||||
mxcUri: typeof content.url === "string" ? content.url : "",
|
||||
};
|
||||
if (!canArchiveAttachment(toGroupFileMapping(mapping), candidate)) {
|
||||
return { kind: "skip", event, reason: "ineligible" };
|
||||
}
|
||||
const filename = typeof content.filename === "string" && content.filename.trim() !== "" ? content.filename : null;
|
||||
const body = typeof content.body === "string" ? content.body : "";
|
||||
return { kind: "attachment", event, candidate, displayName: filename ?? body };
|
||||
}
|
||||
|
||||
/** Split a validated `mxc://server/mediaId` into its parts. */
|
||||
export function parseMxc(mxcUri: string): { serverName: string; mediaId: string } {
|
||||
const match = /^mxc:\/\/([^/\s]+)\/([^/\s?#]+)$/.exec(mxcUri);
|
||||
if (match === null || match[1] === undefined || match[2] === undefined) throw new Error("invalid mxc URI");
|
||||
return { serverName: match[1], mediaId: match[2] };
|
||||
}
|
||||
24
apps/archive-bot/src/health.ts
Normal file
24
apps/archive-bot/src/health.ts
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
import { createServer, type Server } from "node:http";
|
||||
import type { Logger } from "pino";
|
||||
import type { HealthSnapshot } from "./processor.js";
|
||||
|
||||
/** GET /health (or /) → 200 {rooms, cursor, pending, lastProcessedAt}; 503 when the DB count fails. */
|
||||
export function startHealthServer(port: number, snapshot: () => Promise<HealthSnapshot>, logger: Logger): Server {
|
||||
const server = createServer((req, res) => {
|
||||
if (req.method !== "GET" || (req.url !== "/" && req.url !== "/health")) {
|
||||
res.writeHead(404).end();
|
||||
return;
|
||||
}
|
||||
snapshot().then(
|
||||
(body) => {
|
||||
res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify(body));
|
||||
},
|
||||
(err: unknown) => {
|
||||
logger.warn({ err: err instanceof Error ? err.message : String(err) }, "health: snapshot failed");
|
||||
res.writeHead(503, { "content-type": "application/json" }).end(JSON.stringify({ status: "down" }));
|
||||
},
|
||||
);
|
||||
});
|
||||
server.listen(port, () => logger.info({ port }, "health: listening"));
|
||||
return server;
|
||||
}
|
||||
81
apps/archive-bot/src/inspector.test.ts
Normal file
81
apps/archive-bot/src/inspector.test.ts
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { inspectStream, parseContentLength, PayloadInspector, PermanentArchiveError } from "./inspector.js";
|
||||
|
||||
const PDF = new TextEncoder().encode("%PDF-1.7\nhello world");
|
||||
|
||||
function streamOf(...chunks: Uint8Array[]): ReadableStream<Uint8Array> {
|
||||
return new ReadableStream({
|
||||
start(controller) {
|
||||
for (const chunk of chunks) controller.enqueue(chunk);
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function drain(stream: ReadableStream<Uint8Array>): Promise<number> {
|
||||
let total = 0;
|
||||
for await (const chunk of stream) total += chunk.byteLength;
|
||||
return total;
|
||||
}
|
||||
|
||||
describe("PayloadInspector", () => {
|
||||
it("hashes, counts and sniffs across chunk boundaries", () => {
|
||||
const inspector = new PayloadInspector(PDF.byteLength);
|
||||
inspector.update(PDF.subarray(0, 2));
|
||||
inspector.update(PDF.subarray(2));
|
||||
expect(inspector.finish()).toEqual({
|
||||
sha256: createHash("sha256").update(PDF).digest("hex"),
|
||||
byteSize: PDF.byteLength,
|
||||
sniffed: { mime: "application/pdf", mimeClass: "document", extension: "pdf" },
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects an oversized declared length before reading", () => {
|
||||
expect(() => new PayloadInspector(21 * 1024 * 1024)).toThrow(PermanentArchiveError);
|
||||
});
|
||||
|
||||
it("rejects streams exceeding the cap without Content-Length", () => {
|
||||
const inspector = new PayloadInspector(null, 10);
|
||||
expect(() => inspector.update(new Uint8Array(11))).toThrow(/exceeds/);
|
||||
});
|
||||
|
||||
it("rejects Content-Length lies in both directions", () => {
|
||||
const longer = new PayloadInspector(4);
|
||||
expect(() => longer.update(new Uint8Array(5))).toThrow(/longer/);
|
||||
const shorter = new PayloadInspector(10);
|
||||
shorter.update(new Uint8Array(4));
|
||||
expect(() => shorter.finish()).toThrow(/shorter/);
|
||||
});
|
||||
|
||||
it("rejects empty payloads", () => {
|
||||
expect(() => new PayloadInspector(null).finish()).toThrow(/empty/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("inspectStream", () => {
|
||||
it("passes bytes through untouched and resolves the result", async () => {
|
||||
const { stream, result } = inspectStream(streamOf(PDF.subarray(0, 5), PDF.subarray(5)), PDF.byteLength);
|
||||
expect(await drain(stream)).toBe(PDF.byteLength);
|
||||
expect((await result).byteSize).toBe(PDF.byteLength);
|
||||
});
|
||||
|
||||
it("errors the stream (aborting the upload) when the source lies about length", async () => {
|
||||
const { stream, result } = inspectStream(streamOf(PDF), PDF.byteLength + 1);
|
||||
await expect(drain(stream)).rejects.toThrow(PermanentArchiveError);
|
||||
await expect(result).rejects.toThrow(/shorter/);
|
||||
});
|
||||
|
||||
it("errors mid-stream once the cap is crossed", async () => {
|
||||
const { stream } = inspectStream(streamOf(new Uint8Array(6), new Uint8Array(6)), null, 10);
|
||||
await expect(drain(stream)).rejects.toThrow(/exceeds/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseContentLength", () => {
|
||||
it("parses digits and treats absence as unknown", () => {
|
||||
expect(parseContentLength("42")).toBe(42);
|
||||
expect(parseContentLength(null)).toBeNull();
|
||||
expect(() => parseContentLength("-1")).toThrow(PermanentArchiveError);
|
||||
});
|
||||
});
|
||||
105
apps/archive-bot/src/inspector.ts
Normal file
105
apps/archive-bot/src/inspector.ts
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
import { createHash, type Hash } from "node:crypto";
|
||||
import { sniffMime, SNIFF_BYTES, type SniffedMime } from "./mime.js";
|
||||
|
||||
export const MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024;
|
||||
|
||||
/** Non-retryable payload problems: the outbox row is marked failed immediately. */
|
||||
export class PermanentArchiveError extends Error {}
|
||||
|
||||
export interface InspectedPayload {
|
||||
sha256: string;
|
||||
byteSize: number;
|
||||
sniffed: SniffedMime;
|
||||
}
|
||||
|
||||
/**
|
||||
* Incremental size/hash/magic-byte inspection. Bytes pass straight through to
|
||||
* the upload; nothing is buffered beyond the first SNIFF_BYTES.
|
||||
*/
|
||||
export class PayloadInspector {
|
||||
private readonly hash: Hash = createHash("sha256");
|
||||
private readonly head = new Uint8Array(SNIFF_BYTES);
|
||||
private headLength = 0;
|
||||
private total = 0;
|
||||
|
||||
constructor(
|
||||
private readonly declaredLength: number | null,
|
||||
private readonly maxBytes = MAX_ATTACHMENT_BYTES,
|
||||
) {
|
||||
if (declaredLength !== null && declaredLength > maxBytes) {
|
||||
throw new PermanentArchiveError(`declared size ${declaredLength} exceeds ${maxBytes} bytes`);
|
||||
}
|
||||
}
|
||||
|
||||
update(chunk: Uint8Array): void {
|
||||
this.total += chunk.byteLength;
|
||||
if (this.total > this.maxBytes) throw new PermanentArchiveError(`payload exceeds ${this.maxBytes} bytes`);
|
||||
if (this.declaredLength !== null && this.total > this.declaredLength) {
|
||||
throw new PermanentArchiveError("payload longer than Content-Length");
|
||||
}
|
||||
if (this.headLength < SNIFF_BYTES) {
|
||||
const take = chunk.subarray(0, SNIFF_BYTES - this.headLength);
|
||||
this.head.set(take, this.headLength);
|
||||
this.headLength += take.byteLength;
|
||||
}
|
||||
this.hash.update(chunk);
|
||||
}
|
||||
|
||||
finish(): InspectedPayload {
|
||||
if (this.declaredLength !== null && this.total !== this.declaredLength) {
|
||||
throw new PermanentArchiveError("payload shorter than Content-Length");
|
||||
}
|
||||
if (this.total === 0) throw new PermanentArchiveError("empty payload");
|
||||
return {
|
||||
sha256: this.hash.digest("hex"),
|
||||
byteSize: this.total,
|
||||
sniffed: sniffMime(this.head.subarray(0, this.headLength)),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap a byte stream so it is inspected in flight. Any violation errors the
|
||||
* stream, which aborts the consuming upload before it can complete.
|
||||
*/
|
||||
export function inspectStream(
|
||||
source: ReadableStream<Uint8Array>,
|
||||
declaredLength: number | null,
|
||||
maxBytes = MAX_ATTACHMENT_BYTES,
|
||||
): { stream: ReadableStream<Uint8Array>; result: Promise<InspectedPayload> } {
|
||||
const inspector = new PayloadInspector(declaredLength, maxBytes);
|
||||
let resolve!: (value: InspectedPayload) => void;
|
||||
let reject!: (reason: unknown) => void;
|
||||
const result = new Promise<InspectedPayload>((res, rej) => {
|
||||
resolve = res;
|
||||
reject = rej;
|
||||
});
|
||||
// The caller awaits the upload first; avoid an unhandled rejection in the meantime.
|
||||
result.catch(() => undefined);
|
||||
const transform = new TransformStream<Uint8Array, Uint8Array>({
|
||||
transform(chunk, controller) {
|
||||
try {
|
||||
inspector.update(chunk);
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
throw err;
|
||||
}
|
||||
controller.enqueue(chunk);
|
||||
},
|
||||
flush() {
|
||||
try {
|
||||
resolve(inspector.finish());
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
});
|
||||
return { stream: source.pipeThrough(transform), result };
|
||||
}
|
||||
|
||||
export function parseContentLength(header: string | null): number | null {
|
||||
if (header === null) return null;
|
||||
if (!/^\d+$/.test(header.trim())) throw new PermanentArchiveError("invalid Content-Length");
|
||||
return Number(header.trim());
|
||||
}
|
||||
33
apps/archive-bot/src/keys.test.ts
Normal file
33
apps/archive-bot/src/keys.test.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { backoffMs, BACKOFF_BASE_MS, BACKOFF_MAX_MS, nextAttemptAt } from "./backoff.js";
|
||||
import { claimKey, replyTxnId } from "./keys.js";
|
||||
|
||||
describe("claimKey", () => {
|
||||
it("is the shared attachmentArchiveKey encoding", () => {
|
||||
expect(claimKey("w", "!r:x", "$e")).toBe(JSON.stringify(["w", "!r:x", "$e"]));
|
||||
});
|
||||
it("is collision-free across separators", () => {
|
||||
expect(claimKey("a", "b|c", "d")).not.toBe(claimKey("a|b", "c", "d"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("replyTxnId", () => {
|
||||
it("is the first 24 hex chars of sha256(claim key) and stable", () => {
|
||||
const key = claimKey("w", "!r:x", "$e");
|
||||
const expected = createHash("sha256").update(key).digest("hex").slice(0, 24);
|
||||
expect(replyTxnId(key)).toBe(expected);
|
||||
expect(replyTxnId(key)).toMatch(/^[0-9a-f]{24}$/);
|
||||
expect(replyTxnId(claimKey("w", "!r:x", "$f"))).not.toBe(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe("backoff", () => {
|
||||
it("grows exponentially and caps at one hour", () => {
|
||||
expect(backoffMs(0)).toBe(0);
|
||||
expect(backoffMs(1)).toBe(BACKOFF_BASE_MS);
|
||||
expect(backoffMs(3)).toBe(BACKOFF_BASE_MS * 4);
|
||||
expect(backoffMs(40)).toBe(BACKOFF_MAX_MS);
|
||||
expect(nextAttemptAt(2, new Date(0)).getTime()).toBe(BACKOFF_BASE_MS * 2);
|
||||
});
|
||||
});
|
||||
12
apps/archive-bot/src/keys.ts
Normal file
12
apps/archive-bot/src/keys.ts
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import { attachmentArchiveKey } from "@tactical-media-broadcast-hub/core";
|
||||
|
||||
/** Durable claim identity; the shared contract owns the encoding. */
|
||||
export function claimKey(workspaceId: string, roomId: string, eventId: string): string {
|
||||
return attachmentArchiveKey(workspaceId, roomId, eventId);
|
||||
}
|
||||
|
||||
/** Stable Matrix transaction ID for the single confirmation reply of a claim. */
|
||||
export function replyTxnId(key: string): string {
|
||||
return createHash("sha256").update(key, "utf8").digest("hex").slice(0, 24);
|
||||
}
|
||||
14
apps/archive-bot/src/logger.ts
Normal file
14
apps/archive-bot/src/logger.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
import pino from "pino";
|
||||
|
||||
export const logger = pino({
|
||||
level: process.env.LOG_LEVEL ?? "info",
|
||||
base: { service: "archive-bot" },
|
||||
redact: {
|
||||
paths: [
|
||||
"password", "accessToken", "access_token", "authorization",
|
||||
"*.password", "*.accessToken", "*.access_token", "*.authorization",
|
||||
"headers.authorization", "*.headers.authorization",
|
||||
],
|
||||
censor: "[REDACTED]",
|
||||
},
|
||||
});
|
||||
50
apps/archive-bot/src/main.ts
Normal file
50
apps/archive-bot/src/main.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
import { Archiver } from "./archiver.js";
|
||||
import { loadConfig } from "./config.js";
|
||||
import { startHealthServer } from "./health.js";
|
||||
import { logger } from "./logger.js";
|
||||
import { MatrixClient } from "./matrix.js";
|
||||
import { ArchiveProcessor } from "./processor.js";
|
||||
import { PgArchiveStore } from "./store.js";
|
||||
import { OcisWebDav } from "./webdav.js";
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const config = loadConfig();
|
||||
const shutdown = new AbortController();
|
||||
for (const signal of ["SIGTERM", "SIGINT"] as const) {
|
||||
process.once(signal, () => {
|
||||
logger.info({ signal }, "archive-bot: shutting down");
|
||||
shutdown.abort();
|
||||
});
|
||||
}
|
||||
|
||||
const matrix = new MatrixClient({
|
||||
homeserver: config.homeserverUrl,
|
||||
user: config.matrixUser,
|
||||
password: config.matrixPassword,
|
||||
deviceId: config.matrixDeviceId,
|
||||
mediaRedirectHosts: config.mediaRedirectHosts,
|
||||
roomIds: config.mappings.map((m) => m.roomId),
|
||||
logger,
|
||||
});
|
||||
await matrix.login(shutdown.signal);
|
||||
logger.info({ userId: matrix.userId, rooms: config.mappings.length }, "archive-bot: logged in");
|
||||
|
||||
const store = PgArchiveStore.create(config.databaseUrl, matrix.userId);
|
||||
const library = new OcisWebDav(config.webdav.baseUrl, config.webdav.user, config.webdav.password);
|
||||
const processor = new ArchiveProcessor(matrix, store, new Archiver(matrix, library, store), config.mappings, logger);
|
||||
await processor.init();
|
||||
|
||||
const health = startHealthServer(config.healthPort, () => processor.health(), logger);
|
||||
try {
|
||||
await processor.run(shutdown.signal, (signal) => matrix.login(signal));
|
||||
} finally {
|
||||
await new Promise<void>((resolve) => health.close(() => resolve()));
|
||||
await store.close();
|
||||
}
|
||||
logger.info("archive-bot: stopped");
|
||||
}
|
||||
|
||||
void main().catch((error: unknown) => {
|
||||
logger.fatal({ err: error instanceof Error ? error.message : String(error) }, "archive-bot: fatal");
|
||||
process.exitCode = 1;
|
||||
});
|
||||
69
apps/archive-bot/src/matrix.test.ts
Normal file
69
apps/archive-bot/src/matrix.test.ts
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { PermanentArchiveError } from "./inspector.js";
|
||||
import { parseSync, resolveRedirect, supportsAuthenticatedMedia, syncFilter } from "./matrix.js";
|
||||
|
||||
const HS = new URL("https://matrix.example.org");
|
||||
|
||||
describe("supportsAuthenticatedMedia", () => {
|
||||
it("requires v1.11+ or the MSC3916 stable flag", () => {
|
||||
expect(supportsAuthenticatedMedia({ versions: ["v1.10", "v1.11"] })).toBe(true);
|
||||
expect(supportsAuthenticatedMedia({ versions: ["v1.12"] })).toBe(true);
|
||||
expect(supportsAuthenticatedMedia({ versions: ["v1.9"], unstable_features: { "org.matrix.msc3916.stable": true } })).toBe(true);
|
||||
expect(supportsAuthenticatedMedia({ versions: ["r0.6.1", "v1.10"] })).toBe(false);
|
||||
expect(supportsAuthenticatedMedia(null)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveRedirect", () => {
|
||||
it("keeps auth only for the homeserver origin", () => {
|
||||
expect(resolveRedirect(HS, "/_matrix/media/x", HS, [])).toMatchObject({ sendAuth: true });
|
||||
});
|
||||
|
||||
it("drops auth for allowlisted object-store hosts", () => {
|
||||
const hop = resolveRedirect(HS, "https://r2.example.com/bucket/obj?sig=1", HS, ["r2.example.com"]);
|
||||
expect(hop).toMatchObject({ sendAuth: false });
|
||||
expect(hop.url.hostname).toBe("r2.example.com");
|
||||
});
|
||||
|
||||
it("refuses unapproved hosts and http downgrades", () => {
|
||||
expect(() => resolveRedirect(HS, "https://evil.example/x", HS, ["r2.example.com"])).toThrow(PermanentArchiveError);
|
||||
expect(() => resolveRedirect(HS, "http://r2.example.com/x", HS, ["r2.example.com"])).toThrow(/non-https/);
|
||||
expect(() => resolveRedirect(HS, "https://matrix.example.org:8448/x", HS, [])).toThrow(PermanentArchiveError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseSync", () => {
|
||||
it("extracts joined timelines, state and invites, dropping malformed events", () => {
|
||||
const result = parseSync({
|
||||
next_batch: "s2",
|
||||
rooms: {
|
||||
join: {
|
||||
"!a:x": {
|
||||
timeline: { events: [{ type: "m.room.message", event_id: "$1", sender: "@u:x", content: {} }, { junk: true }], limited: true, prev_batch: "p1" },
|
||||
state: { events: [{ type: "m.room.encryption", event_id: "$s", sender: "@u:x", state_key: "", content: { algorithm: "m.megolm.v1.aes-sha2" } }] },
|
||||
},
|
||||
},
|
||||
invite: { "!b:x": {} },
|
||||
},
|
||||
});
|
||||
expect(result.nextBatch).toBe("s2");
|
||||
expect(result.invited).toEqual(["!b:x"]);
|
||||
const room = result.joined.get("!a:x");
|
||||
expect(room?.timeline).toMatchObject({ limited: true, prevBatch: "p1" });
|
||||
expect(room?.timeline.events).toHaveLength(1);
|
||||
expect(room?.stateEvents[0]?.type).toBe("m.room.encryption");
|
||||
});
|
||||
|
||||
it("rejects a response without next_batch", () => {
|
||||
expect(() => parseSync({})).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("syncFilter", () => {
|
||||
it("restricts to mapped rooms and the needed event types", () => {
|
||||
const filter = syncFilter(["!a:x"]) as { room: { rooms: string[]; timeline: { types: string[] } } };
|
||||
expect(filter.room.rooms).toEqual(["!a:x"]);
|
||||
expect(filter.room.timeline.types).toContain("m.room.redaction");
|
||||
expect(filter.room.timeline.types).toContain("m.room.encryption");
|
||||
});
|
||||
});
|
||||
298
apps/archive-bot/src/matrix.ts
Normal file
298
apps/archive-bot/src/matrix.ts
Normal file
|
|
@ -0,0 +1,298 @@
|
|||
import type { Logger } from "pino";
|
||||
import { isMatrixEvent, parseMxc, type MatrixEvent, type RoomEncryption } from "./events.js";
|
||||
import { PermanentArchiveError, parseContentLength } from "./inspector.js";
|
||||
|
||||
export class MatrixHttpError extends Error {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
readonly errcode: string | null,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
export interface JoinedRoomSync {
|
||||
timeline: { events: MatrixEvent[]; limited: boolean; prevBatch: string | null };
|
||||
stateEvents: MatrixEvent[];
|
||||
}
|
||||
|
||||
export interface SyncResult {
|
||||
nextBatch: string;
|
||||
joined: Map<string, JoinedRoomSync>;
|
||||
invited: string[];
|
||||
}
|
||||
|
||||
export interface MediaDownload {
|
||||
body: ReadableStream<Uint8Array>;
|
||||
contentLength: number | null;
|
||||
}
|
||||
|
||||
/** The narrow Matrix surface the processor needs; faked in tests. */
|
||||
export interface MatrixPort {
|
||||
readonly userId: string;
|
||||
sync(since: string | null, signal: AbortSignal): Promise<SyncResult>;
|
||||
/** Older events between `from` (a prev_batch) back to `to` (the previous sync token), oldest first. */
|
||||
backfill(roomId: string, from: string, to: string, signal: AbortSignal): Promise<MatrixEvent[]>;
|
||||
roomEncryption(roomId: string, signal: AbortSignal): Promise<RoomEncryption>;
|
||||
joinRoom(roomId: string, signal: AbortSignal): Promise<void>;
|
||||
downloadMedia(mxcUri: string, signal: AbortSignal): Promise<MediaDownload>;
|
||||
sendNotice(roomId: string, txnId: string, content: Record<string, unknown>, signal: AbortSignal): Promise<void>;
|
||||
}
|
||||
|
||||
const SYNC_TIMEOUT_MS = 30_000;
|
||||
const MAX_REDIRECTS = 3;
|
||||
const BACKFILL_PAGES = 20;
|
||||
const TIMELINE_TYPES = ["m.room.message", "m.room.redaction", "m.room.encrypted", "m.room.encryption"];
|
||||
|
||||
/** Client-side filter: mapped rooms only, minimal timeline types, no presence/ephemeral noise. */
|
||||
export function syncFilter(roomIds: readonly string[]): Record<string, unknown> {
|
||||
return {
|
||||
presence: { not_types: ["*"] },
|
||||
account_data: { not_types: ["*"] },
|
||||
room: {
|
||||
rooms: [...roomIds],
|
||||
account_data: { not_types: ["*"] },
|
||||
ephemeral: { not_types: ["*"] },
|
||||
state: { types: ["m.room.encryption"], lazy_load_members: true },
|
||||
timeline: { limit: 50, types: TIMELINE_TYPES },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Authenticated media requires spec v1.11+ (or the MSC3916 stable flag on older advertisers). */
|
||||
export function supportsAuthenticatedMedia(versions: unknown): boolean {
|
||||
if (typeof versions !== "object" || versions === null) return false;
|
||||
const record = versions as { versions?: unknown; unstable_features?: unknown };
|
||||
const list = Array.isArray(record.versions) ? record.versions.filter((v): v is string => typeof v === "string") : [];
|
||||
const modern = list.some((v) => {
|
||||
const match = /^v1\.(\d+)$/.exec(v);
|
||||
return match !== null && Number(match[1]) >= 11;
|
||||
});
|
||||
const features = record.unstable_features;
|
||||
const flagged = typeof features === "object" && features !== null
|
||||
&& (features as Record<string, unknown>)["org.matrix.msc3916.stable"] === true;
|
||||
return modern || flagged;
|
||||
}
|
||||
|
||||
/**
|
||||
* Redirect policy for media downloads. Bearer auth is only ever sent to the
|
||||
* homeserver origin; other hosts must be explicitly allowlisted (e.g. the R2
|
||||
* endpoint) and are fetched without credentials. Downgrades to http are refused.
|
||||
*/
|
||||
export function resolveRedirect(
|
||||
current: URL,
|
||||
location: string,
|
||||
homeserver: URL,
|
||||
allowedHosts: readonly string[],
|
||||
): { url: URL; sendAuth: boolean } {
|
||||
const url = new URL(location, current);
|
||||
if (url.origin === homeserver.origin) return { url, sendAuth: true };
|
||||
if (url.protocol !== "https:") throw new PermanentArchiveError(`refusing non-https media redirect to ${url.host}`);
|
||||
if (!allowedHosts.includes(url.hostname.toLowerCase())) {
|
||||
throw new PermanentArchiveError(`media redirect to unapproved host ${url.hostname}`);
|
||||
}
|
||||
return { url, sendAuth: false };
|
||||
}
|
||||
|
||||
function events(value: unknown): MatrixEvent[] {
|
||||
return Array.isArray(value) ? value.filter(isMatrixEvent) : [];
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
|
||||
}
|
||||
|
||||
export function parseSync(body: unknown): SyncResult {
|
||||
const root = record(body);
|
||||
if (typeof root.next_batch !== "string") throw new Error("sync response missing next_batch");
|
||||
const rooms = record(root.rooms);
|
||||
const joined = new Map<string, JoinedRoomSync>();
|
||||
for (const [roomId, raw] of Object.entries(record(rooms.join))) {
|
||||
const room = record(raw);
|
||||
const timeline = record(room.timeline);
|
||||
joined.set(roomId, {
|
||||
timeline: {
|
||||
events: events(timeline.events),
|
||||
limited: timeline.limited === true,
|
||||
prevBatch: typeof timeline.prev_batch === "string" ? timeline.prev_batch : null,
|
||||
},
|
||||
stateEvents: events(record(room.state).events),
|
||||
});
|
||||
}
|
||||
return { nextBatch: root.next_batch, joined, invited: Object.keys(record(rooms.invite)) };
|
||||
}
|
||||
|
||||
export interface MatrixClientOptions {
|
||||
homeserver: URL;
|
||||
user: string;
|
||||
password: string;
|
||||
deviceId: string;
|
||||
mediaRedirectHosts: string[];
|
||||
roomIds: string[];
|
||||
logger: Logger;
|
||||
}
|
||||
|
||||
/** Plain client-server sync-loop client (not an appservice). No crypto: encrypted content is never touched. */
|
||||
export class MatrixClient implements MatrixPort {
|
||||
private accessToken: string | null = null;
|
||||
private resolvedUserId = "";
|
||||
private readonly filter: string;
|
||||
|
||||
constructor(private readonly opts: MatrixClientOptions) {
|
||||
this.filter = JSON.stringify(syncFilter(opts.roomIds));
|
||||
}
|
||||
|
||||
get userId(): string {
|
||||
return this.resolvedUserId;
|
||||
}
|
||||
|
||||
async login(signal: AbortSignal): Promise<void> {
|
||||
const versions = await this.request("GET", "/_matrix/client/versions", { signal, auth: false });
|
||||
if (!supportsAuthenticatedMedia(versions)) {
|
||||
throw new Error("homeserver does not support authenticated media (v1.11); refusing to start");
|
||||
}
|
||||
const body = record(await this.request("POST", "/_matrix/client/v3/login", {
|
||||
signal,
|
||||
auth: false,
|
||||
json: {
|
||||
type: "m.login.password",
|
||||
identifier: { type: "m.id.user", user: this.opts.user },
|
||||
password: this.opts.password,
|
||||
device_id: this.opts.deviceId,
|
||||
initial_device_display_name: "Hub archive bot",
|
||||
},
|
||||
}));
|
||||
if (typeof body.access_token !== "string" || typeof body.user_id !== "string") {
|
||||
throw new Error("login response missing access_token/user_id");
|
||||
}
|
||||
this.accessToken = body.access_token;
|
||||
this.resolvedUserId = body.user_id;
|
||||
}
|
||||
|
||||
async sync(since: string | null, signal: AbortSignal): Promise<SyncResult> {
|
||||
const query = new URLSearchParams({ filter: this.filter, timeout: String(since === null ? 0 : SYNC_TIMEOUT_MS) });
|
||||
if (since !== null) query.set("since", since);
|
||||
const timeout = AbortSignal.any([signal, AbortSignal.timeout(SYNC_TIMEOUT_MS + 30_000)]);
|
||||
return parseSync(await this.request("GET", `/_matrix/client/v3/sync?${query.toString()}`, { signal: timeout }));
|
||||
}
|
||||
|
||||
async backfill(roomId: string, from: string, to: string, signal: AbortSignal): Promise<MatrixEvent[]> {
|
||||
const collected: MatrixEvent[] = [];
|
||||
let cursor: string | null = from;
|
||||
for (let page = 0; page < BACKFILL_PAGES && cursor !== null; page += 1) {
|
||||
const query = new URLSearchParams({
|
||||
from: cursor,
|
||||
to,
|
||||
dir: "b",
|
||||
limit: "100",
|
||||
filter: JSON.stringify({ types: TIMELINE_TYPES }),
|
||||
});
|
||||
const body = record(await this.request(
|
||||
"GET",
|
||||
`/_matrix/client/v3/rooms/${encodeURIComponent(roomId)}/messages?${query.toString()}`,
|
||||
{ signal },
|
||||
));
|
||||
const chunk = events(body.chunk);
|
||||
collected.push(...chunk);
|
||||
cursor = chunk.length > 0 && typeof body.end === "string" ? body.end : null;
|
||||
}
|
||||
if (cursor !== null) this.opts.logger.warn({ roomId }, "matrix: backfill page budget exhausted; older gap events skipped");
|
||||
return collected.reverse();
|
||||
}
|
||||
|
||||
async roomEncryption(roomId: string, signal: AbortSignal): Promise<RoomEncryption> {
|
||||
try {
|
||||
const body = record(await this.request(
|
||||
"GET",
|
||||
`/_matrix/client/v3/rooms/${encodeURIComponent(roomId)}/state/m.room.encryption/`,
|
||||
{ signal },
|
||||
));
|
||||
// Any present encryption state (even an unknown algorithm) means encrypted.
|
||||
return Object.keys(body).length > 0 ? "encrypted" : "unknown";
|
||||
} catch (err) {
|
||||
if (err instanceof MatrixHttpError && err.status === 404 && err.errcode === "M_NOT_FOUND") return "unencrypted";
|
||||
if (err instanceof MatrixHttpError && (err.status === 403 || err.status === 404)) return "unknown";
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async joinRoom(roomId: string, signal: AbortSignal): Promise<void> {
|
||||
await this.request("POST", `/_matrix/client/v3/join/${encodeURIComponent(roomId)}`, { signal, json: {} });
|
||||
}
|
||||
|
||||
async sendNotice(roomId: string, txnId: string, content: Record<string, unknown>, signal: AbortSignal): Promise<void> {
|
||||
await this.request(
|
||||
"PUT",
|
||||
`/_matrix/client/v3/rooms/${encodeURIComponent(roomId)}/send/m.room.message/${encodeURIComponent(txnId)}`,
|
||||
{ signal, json: content },
|
||||
);
|
||||
}
|
||||
|
||||
/** Authenticated media endpoint only; the URL is derived from a validated mxc URI, never user-supplied HTTP. */
|
||||
async downloadMedia(mxcUri: string, signal: AbortSignal): Promise<MediaDownload> {
|
||||
const { serverName, mediaId } = parseMxc(mxcUri);
|
||||
let url = new URL(
|
||||
`/_matrix/client/v1/media/download/${encodeURIComponent(serverName)}/${encodeURIComponent(mediaId)}?allow_redirect=true`,
|
||||
this.opts.homeserver,
|
||||
);
|
||||
let sendAuth = true;
|
||||
for (let hop = 0; hop <= MAX_REDIRECTS; hop += 1) {
|
||||
const headers: Record<string, string> = {};
|
||||
if (sendAuth) headers.authorization = `Bearer ${this.token()}`;
|
||||
const res = await fetch(url, { headers, redirect: "manual", signal });
|
||||
if (res.status >= 300 && res.status < 400) {
|
||||
const location = res.headers.get("location");
|
||||
await res.body?.cancel();
|
||||
if (location === null) throw new Error("media redirect without Location");
|
||||
({ url, sendAuth } = resolveRedirect(url, location, this.opts.homeserver, this.opts.mediaRedirectHosts));
|
||||
continue;
|
||||
}
|
||||
if (!res.ok || res.body === null) {
|
||||
const error = await this.toError(res);
|
||||
// Missing/forbidden/too-large media will not appear on retry.
|
||||
if (res.status === 404 || res.status === 403 || res.status === 413) throw new PermanentArchiveError(error.message);
|
||||
throw error;
|
||||
}
|
||||
return { body: res.body, contentLength: parseContentLength(res.headers.get("content-length")) };
|
||||
}
|
||||
throw new PermanentArchiveError("too many media redirects");
|
||||
}
|
||||
|
||||
private token(): string {
|
||||
if (this.accessToken === null) throw new Error("matrix client not logged in");
|
||||
return this.accessToken;
|
||||
}
|
||||
|
||||
private async request(
|
||||
method: string,
|
||||
path: string,
|
||||
opts: { signal: AbortSignal; auth?: boolean; json?: unknown },
|
||||
): Promise<unknown> {
|
||||
const headers: Record<string, string> = { accept: "application/json" };
|
||||
if (opts.auth !== false) headers.authorization = `Bearer ${this.token()}`;
|
||||
if (opts.json !== undefined) headers["content-type"] = "application/json";
|
||||
const res = await fetch(new URL(path, this.opts.homeserver), {
|
||||
method,
|
||||
headers,
|
||||
body: opts.json === undefined ? undefined : JSON.stringify(opts.json),
|
||||
redirect: "error",
|
||||
signal: opts.signal,
|
||||
});
|
||||
if (!res.ok) throw await this.toError(res);
|
||||
return res.json() as Promise<unknown>;
|
||||
}
|
||||
|
||||
private async toError(res: Response): Promise<MatrixHttpError> {
|
||||
let errcode: string | null = null;
|
||||
let message = `matrix HTTP ${res.status}`;
|
||||
try {
|
||||
const body = record(await res.json());
|
||||
if (typeof body.errcode === "string") errcode = body.errcode;
|
||||
if (typeof body.error === "string") message = `${message}: ${body.error}`;
|
||||
} catch {
|
||||
// Non-JSON error body; status is enough.
|
||||
}
|
||||
return new MatrixHttpError(res.status, errcode, message);
|
||||
}
|
||||
}
|
||||
27
apps/archive-bot/src/mime.test.ts
Normal file
27
apps/archive-bot/src/mime.test.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { sniffMime } from "./mime.js";
|
||||
|
||||
const bytes = (...parts: (number[] | string)[]): Uint8Array =>
|
||||
Uint8Array.from(parts.flatMap((p) => (typeof p === "string" ? [...p].map((c) => c.charCodeAt(0)) : p)));
|
||||
|
||||
describe("sniffMime", () => {
|
||||
it.each([
|
||||
["png", bytes([0x89], "PNG", [0x0d, 0x0a, 0x1a, 0x0a, 0, 0]), "image/png", "image"],
|
||||
["jpeg", bytes([0xff, 0xd8, 0xff, 0xe0]), "image/jpeg", "image"],
|
||||
["gif87", bytes("GIF87a"), "image/gif", "image"],
|
||||
["gif89", bytes("GIF89a"), "image/gif", "image"],
|
||||
["pdf", bytes("%PDF-1.7"), "application/pdf", "document"],
|
||||
["zip", bytes("PK", [3, 4, 20, 0]), "application/zip", "archive"],
|
||||
["mp4", bytes([0, 0, 0, 0x18], "ftypisom"), "video/mp4", "video"],
|
||||
["webm", bytes([0x1a, 0x45, 0xdf, 0xa3, 0x9f]), "video/webm", "video"],
|
||||
["ogg", bytes("OggS", [0, 2]), "audio/ogg", "audio"],
|
||||
])("%s", (_name, head, mime, mimeClass) => {
|
||||
expect(sniffMime(head)).toMatchObject({ mime, mimeClass });
|
||||
});
|
||||
|
||||
it("ignores declared types and falls back to octet-stream", () => {
|
||||
expect(sniffMime(bytes("<html>"))).toEqual({ mime: "application/octet-stream", mimeClass: "other", extension: null });
|
||||
expect(sniffMime(new Uint8Array())).toMatchObject({ mimeClass: "other" });
|
||||
expect(sniffMime(bytes([0x89], "PN"))).toMatchObject({ mimeClass: "other" });
|
||||
});
|
||||
});
|
||||
43
apps/archive-bot/src/mime.ts
Normal file
43
apps/archive-bot/src/mime.ts
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
export type MimeClass = "image" | "video" | "audio" | "document" | "archive" | "other";
|
||||
|
||||
export interface SniffedMime {
|
||||
mime: string;
|
||||
mimeClass: MimeClass;
|
||||
extension: string | null;
|
||||
}
|
||||
|
||||
/** Bytes needed to recognise every supported signature. */
|
||||
export const SNIFF_BYTES = 16;
|
||||
|
||||
const UNKNOWN: SniffedMime = { mime: "application/octet-stream", mimeClass: "other", extension: null };
|
||||
|
||||
function startsWith(head: Uint8Array, signature: readonly number[], offset = 0): boolean {
|
||||
if (head.length < offset + signature.length) return false;
|
||||
return signature.every((byte, i) => head[offset + i] === byte);
|
||||
}
|
||||
|
||||
function ascii(text: string): number[] {
|
||||
return [...text].map((c) => c.charCodeAt(0));
|
||||
}
|
||||
|
||||
/**
|
||||
* MIME is decided by magic bytes only; the sender's declared `info.mimetype`
|
||||
* is never trusted. Unrecognised payloads are stored as octet-stream/other.
|
||||
*/
|
||||
export function sniffMime(head: Uint8Array): SniffedMime {
|
||||
if (startsWith(head, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) {
|
||||
return { mime: "image/png", mimeClass: "image", extension: "png" };
|
||||
}
|
||||
if (startsWith(head, [0xff, 0xd8, 0xff])) return { mime: "image/jpeg", mimeClass: "image", extension: "jpg" };
|
||||
if (startsWith(head, ascii("GIF87a")) || startsWith(head, ascii("GIF89a"))) {
|
||||
return { mime: "image/gif", mimeClass: "image", extension: "gif" };
|
||||
}
|
||||
if (startsWith(head, ascii("%PDF-"))) return { mime: "application/pdf", mimeClass: "document", extension: "pdf" };
|
||||
if (startsWith(head, [0x50, 0x4b, 0x03, 0x04])) {
|
||||
return { mime: "application/zip", mimeClass: "archive", extension: "zip" };
|
||||
}
|
||||
if (startsWith(head, ascii("ftyp"), 4)) return { mime: "video/mp4", mimeClass: "video", extension: "mp4" };
|
||||
if (startsWith(head, [0x1a, 0x45, 0xdf, 0xa3])) return { mime: "video/webm", mimeClass: "video", extension: "webm" };
|
||||
if (startsWith(head, ascii("OggS"))) return { mime: "audio/ogg", mimeClass: "audio", extension: "ogg" };
|
||||
return UNKNOWN;
|
||||
}
|
||||
40
apps/archive-bot/src/names.test.ts
Normal file
40
apps/archive-bot/src/names.test.ts
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { MAX_SAFE_NAME_BYTES, safeFileName } from "./names.js";
|
||||
|
||||
describe("safeFileName", () => {
|
||||
it("keeps ordinary names", () => {
|
||||
expect(safeFileName("Meeting notes.pdf")).toBe("Meeting notes.pdf");
|
||||
});
|
||||
|
||||
it("strips directories and traversal", () => {
|
||||
expect(safeFileName("../../etc/passwd")).toBe("passwd");
|
||||
expect(safeFileName("C:\\Users\\x\\evil.exe")).toBe("evil.exe");
|
||||
expect(safeFileName("..")).toBe("attachment");
|
||||
});
|
||||
|
||||
it("removes control/bidi characters and reserved punctuation", () => {
|
||||
expect(safeFileName("a\u0000b\u202ecod.exe")).toBe("abcod.exe");
|
||||
expect(safeFileName('we<ir>d:"na|me?*.txt')).toBe("we_ir_d_na_me_.txt");
|
||||
expect(safeFileName(" .hidden ")).toBe("hidden");
|
||||
});
|
||||
|
||||
it("NFKC-normalises and collapses whitespace", () => {
|
||||
expect(safeFileName("ｆｕｌｌ \t width.PNG")).toBe("full width.png");
|
||||
});
|
||||
|
||||
it("guards Windows device names", () => {
|
||||
expect(safeFileName("CON.txt")).toBe("_CON.txt");
|
||||
});
|
||||
|
||||
it("appends the sniffed extension only when none exists", () => {
|
||||
expect(safeFileName("photo", "jpg")).toBe("photo.jpg");
|
||||
expect(safeFileName("photo.jpeg", "jpg")).toBe("photo.jpeg");
|
||||
expect(safeFileName("", "pdf")).toBe("attachment.pdf");
|
||||
});
|
||||
|
||||
it("bounds length in UTF-8 bytes while preserving the extension", () => {
|
||||
const name = safeFileName(`${"ü".repeat(200)}.pdf`);
|
||||
expect(Buffer.byteLength(name, "utf8")).toBeLessThanOrEqual(MAX_SAFE_NAME_BYTES);
|
||||
expect(name.endsWith(".pdf")).toBe(true);
|
||||
});
|
||||
});
|
||||
44
apps/archive-bot/src/names.ts
Normal file
44
apps/archive-bot/src/names.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
export const MAX_SAFE_NAME_BYTES = 120;
|
||||
const FALLBACK = "attachment";
|
||||
const WINDOWS_RESERVED = /^(con|prn|aux|nul|com[0-9]|lpt[0-9])(\..*)?$/i;
|
||||
|
||||
function truncateUtf8(text: string, maxBytes: number): string {
|
||||
let out = "";
|
||||
for (const ch of text) {
|
||||
if (Buffer.byteLength(out + ch, "utf8") > maxBytes) break;
|
||||
out += ch;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Library-safe single path segment: no directories, control or reserved
|
||||
* characters, leading dots, or unbounded length. Extension is preserved when
|
||||
* truncating; a sniffed extension is appended when the name has none.
|
||||
*/
|
||||
export function safeFileName(raw: string, sniffedExtension: string | null = null): string {
|
||||
const basename = raw.normalize("NFKC").split(/[/\\]/).pop() ?? "";
|
||||
let name = basename
|
||||
.replace(/[\p{Cc}\p{Cf}]/gu, "")
|
||||
.replace(/[<>:"|?*#%{}^~[\]`]/g, "_")
|
||||
.replace(/\s+/g, " ")
|
||||
.replace(/_+/g, "_")
|
||||
.trim()
|
||||
.replace(/^[.\s_-]+/, "")
|
||||
.replace(/[.\s]+$/, "");
|
||||
if (name === "") name = FALLBACK;
|
||||
if (WINDOWS_RESERVED.test(name)) name = `_${name}`;
|
||||
|
||||
const dot = name.lastIndexOf(".");
|
||||
let stem = dot > 0 ? name.slice(0, dot) : name;
|
||||
let ext = dot > 0 ? name.slice(dot + 1).toLowerCase() : "";
|
||||
if (ext === "" && sniffedExtension !== null) ext = sniffedExtension;
|
||||
if (Buffer.byteLength(ext, "utf8") > 16) {
|
||||
stem = name;
|
||||
ext = sniffedExtension ?? "";
|
||||
}
|
||||
const suffix = ext === "" ? "" : `.${ext}`;
|
||||
stem = truncateUtf8(stem, MAX_SAFE_NAME_BYTES - Buffer.byteLength(suffix, "utf8")).trim();
|
||||
if (stem === "") stem = FALLBACK;
|
||||
return `${stem}${suffix}`;
|
||||
}
|
||||
267
apps/archive-bot/src/processor.test.ts
Normal file
267
apps/archive-bot/src/processor.test.ts
Normal file
|
|
@ -0,0 +1,267 @@
|
|||
import pino from "pino";
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import { Archiver } from "./archiver.js";
|
||||
import { BACKOFF_BASE_MS } from "./backoff.js";
|
||||
import type { RoomMapping } from "./config.js";
|
||||
import type { MatrixEvent, RoomEncryption } from "./events.js";
|
||||
import { claimKey, replyTxnId } from "./keys.js";
|
||||
import type { MatrixPort, MediaDownload, SyncResult } from "./matrix.js";
|
||||
import { ArchiveProcessor } from "./processor.js";
|
||||
import type { ArchiveRecord, ArchiveStore, OutboxRow } from "./store.js";
|
||||
import type { LibraryPort, LibraryResource } from "./webdav.js";
|
||||
|
||||
const mapping: RoomMapping = {
|
||||
roomId: "!room:example.org",
|
||||
workspaceId: "11111111-1111-4111-8111-111111111111",
|
||||
groupId: "22222222-2222-4222-8222-222222222222",
|
||||
};
|
||||
const BOT = "@archive:example.org";
|
||||
const PNG = Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3]);
|
||||
|
||||
function imageEvent(eventId: string, extra: Partial<MatrixEvent> = {}): MatrixEvent {
|
||||
return {
|
||||
type: "m.room.message",
|
||||
event_id: eventId,
|
||||
sender: "@alice:example.org",
|
||||
origin_server_ts: 1_700_000_000_000,
|
||||
content: { msgtype: "m.image", body: "../cat", url: `mxc://example.org/${eventId.slice(1)}` },
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
class FakeMatrix implements MatrixPort {
|
||||
readonly userId = BOT;
|
||||
batches: SyncResult[] = [];
|
||||
syncedWith: (string | null)[] = [];
|
||||
encryption: RoomEncryption = "unencrypted";
|
||||
notices: { roomId: string; txnId: string; content: Record<string, unknown> }[] = [];
|
||||
failNotices = 0;
|
||||
joined: string[] = [];
|
||||
|
||||
async sync(since: string | null): Promise<SyncResult> {
|
||||
this.syncedWith.push(since);
|
||||
const next = this.batches.shift();
|
||||
if (next === undefined) throw new Error("no batch queued");
|
||||
return next;
|
||||
}
|
||||
async backfill(): Promise<MatrixEvent[]> {
|
||||
return [];
|
||||
}
|
||||
async roomEncryption(): Promise<RoomEncryption> {
|
||||
return this.encryption;
|
||||
}
|
||||
async joinRoom(roomId: string): Promise<void> {
|
||||
this.joined.push(roomId);
|
||||
}
|
||||
async downloadMedia(): Promise<MediaDownload> {
|
||||
return {
|
||||
body: new ReadableStream({ start: (c) => { c.enqueue(PNG); c.close(); } }),
|
||||
contentLength: PNG.byteLength,
|
||||
};
|
||||
}
|
||||
async sendNotice(roomId: string, txnId: string, content: Record<string, unknown>): Promise<void> {
|
||||
if (this.failNotices > 0) {
|
||||
this.failNotices -= 1;
|
||||
throw new Error("notice failed");
|
||||
}
|
||||
this.notices.push({ roomId, txnId, content });
|
||||
}
|
||||
}
|
||||
|
||||
class FakeLibrary implements LibraryPort {
|
||||
failPuts = 0;
|
||||
committed: string[] = [];
|
||||
async putStaging(_dir: string[], _name: string, body: ReadableStream<Uint8Array>): Promise<void> {
|
||||
for await (const chunk of body) void chunk;
|
||||
if (this.failPuts > 0) {
|
||||
this.failPuts -= 1;
|
||||
throw new Error("WebDAV PUT failed: 507");
|
||||
}
|
||||
}
|
||||
async commit(dir: string[], _staging: string, finalName: string): Promise<LibraryResource> {
|
||||
const path = [...dir, finalName].join("/");
|
||||
this.committed.push(path);
|
||||
return { path, url: `https://files/${path}`, resourceId: "rid", version: "v1" };
|
||||
}
|
||||
}
|
||||
|
||||
class FakeStore implements ArchiveStore {
|
||||
cursor: string | null = null;
|
||||
outbox = new Map<string, OutboxRow & { claimKey: string }>();
|
||||
archive: (ArchiveRecord & { available: boolean })[] = [];
|
||||
async loadCursor(): Promise<string | null> {
|
||||
return this.cursor;
|
||||
}
|
||||
async saveCursor(_ws: readonly string[], cursor: string): Promise<void> {
|
||||
this.cursor = cursor;
|
||||
}
|
||||
async claim(_ws: string, _room: string, _event: string, key: string): Promise<OutboxRow> {
|
||||
const existing = this.outbox.get(key);
|
||||
if (existing !== undefined) return { ...existing };
|
||||
const row = { id: `ob${this.outbox.size + 1}`, status: "pending" as const, attempts: 0, lastError: null, updatedAt: new Date(0), claimKey: key };
|
||||
this.outbox.set(key, row);
|
||||
return { ...row };
|
||||
}
|
||||
async commitArchived(record: ArchiveRecord, outboxId: string): Promise<{ id: string; indexedAt: Date }> {
|
||||
this.archive.push({ ...record, available: true });
|
||||
this.byId(outboxId).status = "done";
|
||||
return { id: `file-${this.archive.length}`, indexedAt: new Date(0) };
|
||||
}
|
||||
async recordFailure(_ws: string, outboxId: string, error: string, terminal: boolean): Promise<OutboxRow> {
|
||||
const row = this.byId(outboxId);
|
||||
row.attempts += 1;
|
||||
row.lastError = error;
|
||||
row.status = terminal ? "failed" : "pending";
|
||||
row.updatedAt = clock.now;
|
||||
return { ...row };
|
||||
}
|
||||
async markRedacted(_ws: string, _room: string, eventId: string, key: string): Promise<number> {
|
||||
const rows = this.archive.filter((r) => r.eventId === eventId && r.available);
|
||||
for (const r of rows) r.available = false;
|
||||
const claim = this.outbox.get(key);
|
||||
if (claim?.status === "pending") claim.status = "failed";
|
||||
return rows.length;
|
||||
}
|
||||
async countPending(): Promise<number> {
|
||||
return [...this.outbox.values()].filter((r) => r.status === "pending").length;
|
||||
}
|
||||
private byId(id: string): OutboxRow {
|
||||
const row = [...this.outbox.values()].find((r) => r.id === id);
|
||||
if (row === undefined) throw new Error("no row");
|
||||
return row;
|
||||
}
|
||||
}
|
||||
|
||||
const clock = { now: new Date(1_000_000) };
|
||||
|
||||
function batch(nextBatch: string, events: MatrixEvent[], extra: { stateEvents?: MatrixEvent[]; invited?: string[] } = {}): SyncResult {
|
||||
return {
|
||||
nextBatch,
|
||||
invited: extra.invited ?? [],
|
||||
joined: new Map([[mapping.roomId, { timeline: { events, limited: false, prevBatch: null }, stateEvents: extra.stateEvents ?? [] }]]),
|
||||
};
|
||||
}
|
||||
|
||||
describe("ArchiveProcessor", () => {
|
||||
let matrix: FakeMatrix;
|
||||
let library: FakeLibrary;
|
||||
let store: FakeStore;
|
||||
let processor: ArchiveProcessor;
|
||||
const signal = new AbortController().signal;
|
||||
|
||||
beforeEach(async () => {
|
||||
clock.now = new Date(1_000_000);
|
||||
matrix = new FakeMatrix();
|
||||
library = new FakeLibrary();
|
||||
store = new FakeStore();
|
||||
processor = new ArchiveProcessor(
|
||||
matrix, store, new Archiver(matrix, library, store), [mapping], pino({ level: "silent" }), () => clock.now,
|
||||
);
|
||||
await processor.init();
|
||||
});
|
||||
|
||||
it("archives, indexes, replies once with the stable txn id, then advances the cursor", async () => {
|
||||
matrix.batches.push(batch("s1", [imageEvent("$a")]));
|
||||
expect(await processor.runBatch(signal)).toEqual({ advanced: true, retryAt: null });
|
||||
|
||||
expect(store.cursor).toBe("s1");
|
||||
expect(store.archive).toHaveLength(1);
|
||||
const row = store.archive[0];
|
||||
expect(row).toMatchObject({ mime: "image/png", mimeClass: "image", safeName: "cat.png", originalName: "../cat", byteSize: PNG.byteLength });
|
||||
expect(library.committed[0]).toBe(`Library/${mapping.workspaceId}/${mapping.groupId}/${row?.sha256}-cat.png`);
|
||||
const key = claimKey(mapping.workspaceId, mapping.roomId, "$a");
|
||||
expect(matrix.notices).toEqual([{
|
||||
roomId: mapping.roomId,
|
||||
txnId: replyTxnId(key),
|
||||
content: expect.objectContaining({ body: "Archived to the group library: cat.png" }) as unknown,
|
||||
}]);
|
||||
|
||||
// Replaying the same event (e.g. after a crash before cursor save) neither re-archives nor re-replies.
|
||||
matrix.batches.push(batch("s2", [imageEvent("$a")]));
|
||||
await processor.runBatch(signal);
|
||||
expect(store.archive).toHaveLength(1);
|
||||
expect(matrix.notices).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("holds the cursor on a WebDAV failure, records attempts, and retries after backoff", async () => {
|
||||
library.failPuts = 1;
|
||||
matrix.batches.push(batch("s1", [imageEvent("$a")]));
|
||||
const first = await processor.runBatch(signal);
|
||||
expect(first.advanced).toBe(false);
|
||||
expect(first.retryAt?.getTime()).toBe(clock.now.getTime() + BACKOFF_BASE_MS);
|
||||
expect(store.cursor).toBeNull();
|
||||
const row = [...store.outbox.values()][0];
|
||||
expect(row).toMatchObject({ status: "pending", attempts: 1, lastError: "WebDAV PUT failed: 507" });
|
||||
|
||||
// Re-sync before the row is due: still blocked, nothing uploaded.
|
||||
matrix.batches.push(batch("s1b", [imageEvent("$a")]));
|
||||
expect((await processor.runBatch(signal)).advanced).toBe(false);
|
||||
expect(store.archive).toHaveLength(0);
|
||||
expect(matrix.syncedWith).toEqual([null, null]);
|
||||
|
||||
clock.now = new Date(clock.now.getTime() + BACKOFF_BASE_MS);
|
||||
matrix.batches.push(batch("s1c", [imageEvent("$a")]));
|
||||
expect((await processor.runBatch(signal)).advanced).toBe(true);
|
||||
expect(store.cursor).toBe("s1c");
|
||||
expect(store.archive).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("retries a failed reply without re-archiving", async () => {
|
||||
matrix.failNotices = 1;
|
||||
matrix.batches.push(batch("s1", [imageEvent("$a")]), batch("s2", [imageEvent("$a")]));
|
||||
expect((await processor.runBatch(signal)).advanced).toBe(false);
|
||||
expect(store.archive).toHaveLength(1);
|
||||
expect((await processor.runBatch(signal)).advanced).toBe(true);
|
||||
expect(matrix.notices).toHaveLength(1);
|
||||
expect(store.archive).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("marks permanent payload errors failed and lets the cursor advance", async () => {
|
||||
matrix.downloadMedia = async () => ({
|
||||
body: new ReadableStream({ start: (c) => { c.enqueue(PNG); c.close(); } }),
|
||||
contentLength: PNG.byteLength + 10,
|
||||
});
|
||||
matrix.batches.push(batch("s1", [imageEvent("$a")]));
|
||||
expect((await processor.runBatch(signal)).advanced).toBe(true);
|
||||
expect([...store.outbox.values()][0]).toMatchObject({ status: "failed", attempts: 1 });
|
||||
expect(matrix.notices).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("never mirrors when the room is encrypted or its state is unknown", async () => {
|
||||
matrix.encryption = "unknown";
|
||||
matrix.batches.push(batch("s1", [imageEvent("$a")]));
|
||||
await processor.runBatch(signal);
|
||||
matrix.encryption = "unencrypted";
|
||||
const encryptionEvent: MatrixEvent = {
|
||||
type: "m.room.encryption", event_id: "$enc", sender: "@admin:x", state_key: "", content: { algorithm: "m.megolm.v1.aes-sha2" },
|
||||
};
|
||||
matrix.batches.push(batch("s2", [imageEvent("$b"), encryptionEvent]));
|
||||
await processor.runBatch(signal);
|
||||
expect(store.outbox.size).toBe(0);
|
||||
expect(store.archive).toHaveLength(0);
|
||||
expect(store.cursor).toBe("s2");
|
||||
});
|
||||
|
||||
it("ignores its own events", async () => {
|
||||
matrix.batches.push(batch("s1", [imageEvent("$a", { sender: BOT })]));
|
||||
await processor.runBatch(signal);
|
||||
expect(store.outbox.size).toBe(0);
|
||||
});
|
||||
|
||||
it("marks redacted attachments unavailable without touching the library", async () => {
|
||||
matrix.batches.push(batch("s1", [imageEvent("$a")]));
|
||||
await processor.runBatch(signal);
|
||||
const redaction: MatrixEvent = { type: "m.room.redaction", event_id: "$r", sender: "@mod:x", content: { redacts: "$a" } };
|
||||
matrix.batches.push(batch("s2", [redaction]));
|
||||
await processor.runBatch(signal);
|
||||
expect(store.archive[0]?.available).toBe(false);
|
||||
expect(library.committed).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("joins invites only for mapped rooms and reports health", async () => {
|
||||
matrix.batches.push({ nextBatch: "s1", joined: new Map(), invited: [mapping.roomId, "!other:x"] });
|
||||
await processor.runBatch(signal);
|
||||
expect(matrix.joined).toEqual([mapping.roomId]);
|
||||
expect(await processor.health()).toEqual({ rooms: 1, cursor: "s1", pending: 0, lastProcessedAt: null });
|
||||
});
|
||||
});
|
||||
235
apps/archive-bot/src/processor.ts
Normal file
235
apps/archive-bot/src/processor.ts
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
import type { Logger } from "pino";
|
||||
import type { Archiver } from "./archiver.js";
|
||||
import { backoffMs, BACKOFF_MAX_MS, MAX_ATTEMPTS, nextAttemptAt } from "./backoff.js";
|
||||
import type { RoomMapping } from "./config.js";
|
||||
import { classifyEvent, type AttachmentWork, type MatrixEvent, type RoomEncryption } from "./events.js";
|
||||
import { PermanentArchiveError } from "./inspector.js";
|
||||
import { claimKey, replyTxnId } from "./keys.js";
|
||||
import { MatrixHttpError, type MatrixPort } from "./matrix.js";
|
||||
import type { ArchiveStore } from "./store.js";
|
||||
|
||||
export interface BatchResult {
|
||||
/** True when the sync cursor was durably advanced past this batch. */
|
||||
advanced: boolean;
|
||||
/** Earliest time a blocked row may be retried; null when nothing is blocked. */
|
||||
retryAt: Date | null;
|
||||
}
|
||||
|
||||
export interface HealthSnapshot {
|
||||
rooms: number;
|
||||
cursor: string | null;
|
||||
pending: number;
|
||||
lastProcessedAt: string | null;
|
||||
}
|
||||
|
||||
type Outcome = { blockedUntil: Date } | null;
|
||||
|
||||
export function replyContent(safeName: string, eventId: string): Record<string, unknown> {
|
||||
return {
|
||||
msgtype: "m.notice",
|
||||
body: `Archived to the group library: ${safeName}`,
|
||||
"m.relates_to": { "m.in_reply_to": { event_id: eventId } },
|
||||
};
|
||||
}
|
||||
|
||||
function errorText(err: unknown): string {
|
||||
return err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
|
||||
function abortableSleep(ms: number, signal: AbortSignal): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
if (signal.aborted) return resolve();
|
||||
const timer = setTimeout(done, ms);
|
||||
function done(): void {
|
||||
clearTimeout(timer);
|
||||
signal.removeEventListener("abort", done);
|
||||
resolve();
|
||||
}
|
||||
signal.addEventListener("abort", done, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync-loop orchestration. The cursor is persisted only after every event in
|
||||
* the batch has reached a durable terminal state (done/failed/skipped) and its
|
||||
* reply was sent; otherwise the same `since` is re-synced after the earliest
|
||||
* per-row backoff, so no attachment is lost to a transient failure.
|
||||
*/
|
||||
export class ArchiveProcessor {
|
||||
private cursor: string | null = null;
|
||||
private lastProcessedAt: Date | null = null;
|
||||
/** Committed claims whose single reply has not been delivered yet (key → safe name). */
|
||||
private readonly unsentReplies = new Map<string, string>();
|
||||
private readonly byRoom: Map<string, RoomMapping>;
|
||||
private readonly workspaceIds: string[];
|
||||
|
||||
constructor(
|
||||
private readonly matrix: MatrixPort,
|
||||
private readonly store: ArchiveStore,
|
||||
private readonly archiver: Archiver,
|
||||
mappings: readonly RoomMapping[],
|
||||
private readonly logger: Logger,
|
||||
private readonly now: () => Date = () => new Date(),
|
||||
) {
|
||||
this.byRoom = new Map(mappings.map((m) => [m.roomId, m]));
|
||||
this.workspaceIds = [...new Set(mappings.map((m) => m.workspaceId))];
|
||||
}
|
||||
|
||||
async init(): Promise<void> {
|
||||
this.cursor = await this.store.loadCursor(this.workspaceIds);
|
||||
}
|
||||
|
||||
async health(): Promise<HealthSnapshot> {
|
||||
return {
|
||||
rooms: this.byRoom.size,
|
||||
cursor: this.cursor,
|
||||
pending: await this.store.countPending(this.workspaceIds),
|
||||
lastProcessedAt: this.lastProcessedAt?.toISOString() ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
async runBatch(signal: AbortSignal): Promise<BatchResult> {
|
||||
const since = this.cursor;
|
||||
const sync = await this.matrix.sync(since, signal);
|
||||
|
||||
for (const roomId of sync.invited) {
|
||||
if (!this.byRoom.has(roomId)) continue;
|
||||
await this.matrix.joinRoom(roomId, signal);
|
||||
this.logger.info({ roomId }, "archive-bot: joined mapped room");
|
||||
}
|
||||
|
||||
let retryAt: Date | null = null;
|
||||
for (const [roomId, room] of sync.joined) {
|
||||
const mapping = this.byRoom.get(roomId);
|
||||
if (mapping === undefined) continue;
|
||||
let events = room.timeline.events;
|
||||
if (room.timeline.limited && room.timeline.prevBatch !== null && since !== null) {
|
||||
events = [...(await this.matrix.backfill(roomId, room.timeline.prevBatch, since, signal)), ...events];
|
||||
}
|
||||
if (events.length === 0) continue;
|
||||
const encryption = await this.encryptionState(mapping, [...room.stateEvents, ...events], signal);
|
||||
for (const event of events) {
|
||||
const outcome = await this.handleEvent(mapping, event, encryption, signal);
|
||||
if (outcome !== null && (retryAt === null || outcome.blockedUntil < retryAt)) retryAt = outcome.blockedUntil;
|
||||
}
|
||||
}
|
||||
|
||||
if (retryAt !== null) return { advanced: false, retryAt };
|
||||
await this.store.saveCursor(this.workspaceIds, sync.nextBatch);
|
||||
this.cursor = sync.nextBatch;
|
||||
return { advanced: true, retryAt: null };
|
||||
}
|
||||
|
||||
/** Long-running loop; returns when `signal` aborts. */
|
||||
async run(signal: AbortSignal, relogin: (signal: AbortSignal) => Promise<void>): Promise<void> {
|
||||
let consecutiveErrors = 0;
|
||||
while (!signal.aborted) {
|
||||
try {
|
||||
const result = await this.runBatch(signal);
|
||||
consecutiveErrors = 0;
|
||||
if (result.retryAt !== null) {
|
||||
const wait = Math.min(Math.max(result.retryAt.getTime() - this.now().getTime(), 1000), BACKOFF_MAX_MS);
|
||||
this.logger.info({ waitMs: wait }, "archive-bot: batch has pending rows; holding cursor");
|
||||
await abortableSleep(wait, signal);
|
||||
}
|
||||
} catch (err) {
|
||||
if (signal.aborted) break;
|
||||
consecutiveErrors += 1;
|
||||
if (err instanceof MatrixHttpError && err.status === 401) {
|
||||
this.logger.warn("archive-bot: access token rejected; logging in again");
|
||||
await relogin(signal).catch((loginErr: unknown) => {
|
||||
this.logger.error({ err: errorText(loginErr) }, "archive-bot: re-login failed");
|
||||
});
|
||||
} else {
|
||||
this.logger.error({ err: errorText(err) }, "archive-bot: batch failed; cursor held");
|
||||
}
|
||||
await abortableSleep(backoffMs(consecutiveErrors), signal);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Fail closed: an encryption event anywhere in this batch wins; otherwise ask the room state. */
|
||||
private async encryptionState(mapping: RoomMapping, events: MatrixEvent[], signal: AbortSignal): Promise<RoomEncryption> {
|
||||
const sawEncryption = events.some((e) => e.type === "m.room.encryption" && e.state_key === "");
|
||||
const state = sawEncryption ? "encrypted" : await this.matrix.roomEncryption(mapping.roomId, signal);
|
||||
if (state !== "unencrypted") {
|
||||
this.logger.warn({ roomId: mapping.roomId, state }, "archive-bot: room not verifiably unencrypted; not mirroring");
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
private async handleEvent(
|
||||
mapping: RoomMapping,
|
||||
event: MatrixEvent,
|
||||
encryption: RoomEncryption,
|
||||
signal: AbortSignal,
|
||||
): Promise<Outcome> {
|
||||
const classified = classifyEvent(event, mapping, encryption, this.matrix.userId);
|
||||
switch (classified.kind) {
|
||||
case "skip":
|
||||
if (classified.reason !== "own-event" && classified.reason !== "not-an-attachment" && classified.reason !== "not-a-message") {
|
||||
this.logger.info({ roomId: mapping.roomId, eventId: event.event_id, reason: classified.reason }, "archive-bot: skipped");
|
||||
}
|
||||
return null;
|
||||
case "redaction": {
|
||||
const key = claimKey(mapping.workspaceId, mapping.roomId, classified.redacts);
|
||||
const affected = await this.store.markRedacted(mapping.workspaceId, mapping.roomId, classified.redacts, key);
|
||||
this.unsentReplies.delete(key);
|
||||
if (affected > 0) {
|
||||
this.logger.info({ roomId: mapping.roomId, eventId: classified.redacts }, "archive-bot: redacted; index marked unavailable");
|
||||
}
|
||||
this.lastProcessedAt = this.now();
|
||||
return null;
|
||||
}
|
||||
case "attachment":
|
||||
return this.handleAttachment(mapping, classified, signal);
|
||||
}
|
||||
}
|
||||
|
||||
private async handleAttachment(mapping: RoomMapping, work: AttachmentWork, signal: AbortSignal): Promise<Outcome> {
|
||||
const eventId = work.event.event_id;
|
||||
const key = claimKey(mapping.workspaceId, mapping.roomId, eventId);
|
||||
const row = await this.store.claim(mapping.workspaceId, mapping.roomId, eventId, key);
|
||||
const log = { roomId: mapping.roomId, eventId, outboxId: row.id };
|
||||
|
||||
if (row.status === "failed") return null;
|
||||
if (row.status === "done") return this.sendReply(mapping, eventId, key, signal);
|
||||
|
||||
const due = nextAttemptAt(row.attempts, row.updatedAt);
|
||||
if (row.attempts > 0 && due > this.now()) return { blockedUntil: due };
|
||||
|
||||
try {
|
||||
const archived = await this.archiver.archive(mapping, work, row, key, signal);
|
||||
this.logger.info({ ...log, fileId: archived.file.id, sha256: archived.file.checksum.value }, "archive-bot: archived");
|
||||
this.lastProcessedAt = this.now();
|
||||
this.unsentReplies.set(key, archived.safeName);
|
||||
return this.sendReply(mapping, eventId, key, signal);
|
||||
} catch (err) {
|
||||
if (signal.aborted) throw err;
|
||||
const terminal = err instanceof PermanentArchiveError || row.attempts + 1 >= MAX_ATTEMPTS;
|
||||
const updated = await this.store.recordFailure(mapping.workspaceId, row.id, errorText(err), terminal);
|
||||
if (terminal) {
|
||||
this.logger.error({ ...log, err: errorText(err), attempts: updated.attempts }, "archive-bot: archive failed permanently");
|
||||
return null;
|
||||
}
|
||||
const retry = nextAttemptAt(updated.attempts, updated.updatedAt);
|
||||
this.logger.warn({ ...log, err: errorText(err), attempts: updated.attempts, retryAt: retry.toISOString() }, "archive-bot: archive failed; will retry");
|
||||
return { blockedUntil: retry };
|
||||
}
|
||||
}
|
||||
|
||||
/** Exactly one reply per committed claim; the stable txn ID makes resends idempotent server-side. */
|
||||
private async sendReply(mapping: RoomMapping, eventId: string, key: string, signal: AbortSignal): Promise<Outcome> {
|
||||
const safeName = this.unsentReplies.get(key);
|
||||
if (safeName === undefined) return null;
|
||||
try {
|
||||
await this.matrix.sendNotice(mapping.roomId, replyTxnId(key), replyContent(safeName, eventId), signal);
|
||||
this.unsentReplies.delete(key);
|
||||
return null;
|
||||
} catch (err) {
|
||||
if (signal.aborted) throw err;
|
||||
this.logger.warn({ roomId: mapping.roomId, eventId, err: errorText(err) }, "archive-bot: reply failed; will retry");
|
||||
return { blockedUntil: new Date(this.now().getTime() + backoffMs(1)) };
|
||||
}
|
||||
}
|
||||
}
|
||||
223
apps/archive-bot/src/store.ts
Normal file
223
apps/archive-bot/src/store.ts
Normal file
|
|
@ -0,0 +1,223 @@
|
|||
import { Pool, type PoolClient } from "pg";
|
||||
|
||||
export type OutboxStatus = "pending" | "done" | "failed";
|
||||
|
||||
export interface OutboxRow {
|
||||
id: string;
|
||||
status: OutboxStatus;
|
||||
attempts: number;
|
||||
lastError: string | null;
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
export interface ArchiveRecord {
|
||||
workspaceId: string;
|
||||
groupId: string;
|
||||
roomId: string;
|
||||
eventId: string;
|
||||
senderMxid: string;
|
||||
matrixEventJson: unknown;
|
||||
originalName: string;
|
||||
safeName: string;
|
||||
mime: string;
|
||||
mimeClass: string;
|
||||
byteSize: number;
|
||||
sha256: string;
|
||||
ocisResourceId: string;
|
||||
ocisResourceVersion: string;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
/** Durable state the processor depends on; faked in tests. */
|
||||
export interface ArchiveStore {
|
||||
loadCursor(workspaceIds: readonly string[]): Promise<string | null>;
|
||||
saveCursor(workspaceIds: readonly string[], cursor: string): Promise<void>;
|
||||
claim(workspaceId: string, roomId: string, eventId: string, claimKey: string): Promise<OutboxRow>;
|
||||
/** Index row + outbox done in one transaction. Returns the archive row id and indexed time. */
|
||||
commitArchived(record: ArchiveRecord, outboxId: string): Promise<{ id: string; indexedAt: Date }>;
|
||||
recordFailure(workspaceId: string, outboxId: string, error: string, terminal: boolean): Promise<OutboxRow>;
|
||||
/** Mark index rows unavailable and stop any not-yet-archived claim. Returns rows affected. */
|
||||
markRedacted(workspaceId: string, roomId: string, eventId: string, claimKey: string): Promise<number>;
|
||||
countPending(workspaceIds: readonly string[]): Promise<number>;
|
||||
}
|
||||
|
||||
interface OutboxDbRow {
|
||||
id: string;
|
||||
status: OutboxStatus;
|
||||
attempts: number;
|
||||
last_error: string | null;
|
||||
updated_at: Date;
|
||||
}
|
||||
|
||||
function toOutbox(row: OutboxDbRow): OutboxRow {
|
||||
return { id: row.id, status: row.status, attempts: row.attempts, lastError: row.last_error, updatedAt: row.updated_at };
|
||||
}
|
||||
|
||||
const OUTBOX_COLUMNS = "id, status, attempts, last_error, updated_at";
|
||||
const LAST_ERROR_MAX = 1000;
|
||||
|
||||
export class PgArchiveStore implements ArchiveStore {
|
||||
constructor(
|
||||
private readonly pool: Pool,
|
||||
private readonly cursorId: string,
|
||||
) {}
|
||||
|
||||
/** Single service connection; tenant context per transaction, like apps/api withTenant. */
|
||||
static create(databaseUrl: string, cursorId: string): PgArchiveStore {
|
||||
return new PgArchiveStore(new Pool({ connectionString: databaseUrl, max: 1 }), cursorId);
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this.pool.end();
|
||||
}
|
||||
|
||||
/**
|
||||
* The sync stream is global but rows are tenant-scoped, so each mapped
|
||||
* workspace holds a copy. A partial write leaves copies disagreeing: resume
|
||||
* from the least recently written (oldest) one — replay is idempotent.
|
||||
*/
|
||||
async loadCursor(workspaceIds: readonly string[]): Promise<string | null> {
|
||||
let oldest: { cursor: string; at: Date } | null = null;
|
||||
for (const workspaceId of workspaceIds) {
|
||||
const row = await this.withTenant(workspaceId, async (c) => {
|
||||
const res = await c.query<{ sync_cursor: string | null; updated_at: Date }>(
|
||||
"SELECT sync_cursor, updated_at FROM archive_sync_state WHERE id = $1 AND workspace_id = $2",
|
||||
[this.stateId(workspaceId), workspaceId],
|
||||
);
|
||||
return res.rows[0];
|
||||
});
|
||||
if (row?.sync_cursor == null) continue;
|
||||
if (oldest === null || row.updated_at < oldest.at) oldest = { cursor: row.sync_cursor, at: row.updated_at };
|
||||
}
|
||||
return oldest?.cursor ?? null;
|
||||
}
|
||||
|
||||
async saveCursor(workspaceIds: readonly string[], cursor: string): Promise<void> {
|
||||
for (const workspaceId of workspaceIds) {
|
||||
await this.withTenant(workspaceId, (c) => c.query(
|
||||
`INSERT INTO archive_sync_state (id, workspace_id, sync_cursor, updated_at)
|
||||
VALUES ($1, $2, $3, now())
|
||||
ON CONFLICT (id) DO UPDATE SET sync_cursor = EXCLUDED.sync_cursor, updated_at = now()`,
|
||||
[this.stateId(workspaceId), workspaceId, cursor],
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
async claim(workspaceId: string, roomId: string, eventId: string, claimKey: string): Promise<OutboxRow> {
|
||||
return this.withTenant(workspaceId, async (c) => {
|
||||
await c.query(
|
||||
`INSERT INTO archive_sync_outbox (workspace_id, room_id, event_id, claim_key)
|
||||
SELECT $1, $2, $3, $4
|
||||
WHERE NOT EXISTS (SELECT 1 FROM archive_sync_outbox WHERE workspace_id = $1 AND claim_key = $4)
|
||||
ON CONFLICT DO NOTHING`,
|
||||
[workspaceId, roomId, eventId, claimKey],
|
||||
);
|
||||
const res = await c.query<OutboxDbRow>(
|
||||
`SELECT ${OUTBOX_COLUMNS} FROM archive_sync_outbox
|
||||
WHERE workspace_id = $1 AND claim_key = $2 ORDER BY created_at LIMIT 1`,
|
||||
[workspaceId, claimKey],
|
||||
);
|
||||
const row = res.rows[0];
|
||||
if (row === undefined) throw new Error("outbox claim not visible after insert");
|
||||
return toOutbox(row);
|
||||
});
|
||||
}
|
||||
|
||||
async commitArchived(record: ArchiveRecord, outboxId: string): Promise<{ id: string; indexedAt: Date }> {
|
||||
return this.withTenant(record.workspaceId, async (c) => {
|
||||
await c.query(
|
||||
`INSERT INTO chat_media_archive (
|
||||
workspace_id, group_id, room_id, event_id, origin, sender_mxid, matrix_event_json,
|
||||
original_name, safe_name, mime, mime_class, byte_size, sha256,
|
||||
ocis_resource_id, ocis_resource_version, created_at
|
||||
) VALUES ($1, $2, $3, $4, 'matrix', $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
|
||||
ON CONFLICT DO NOTHING`,
|
||||
[
|
||||
record.workspaceId, record.groupId, record.roomId, record.eventId, record.senderMxid,
|
||||
JSON.stringify(record.matrixEventJson), record.originalName, record.safeName, record.mime,
|
||||
record.mimeClass, record.byteSize, record.sha256, record.ocisResourceId,
|
||||
record.ocisResourceVersion, record.createdAt,
|
||||
],
|
||||
);
|
||||
const indexed = await c.query<{ id: string; indexed_at: Date }>(
|
||||
`SELECT id, indexed_at FROM chat_media_archive
|
||||
WHERE workspace_id = $1 AND room_id = $2 AND event_id = $3 LIMIT 1`,
|
||||
[record.workspaceId, record.roomId, record.eventId],
|
||||
);
|
||||
const row = indexed.rows[0];
|
||||
if (row === undefined) throw new Error("archive row not visible after insert");
|
||||
await c.query(
|
||||
`UPDATE archive_sync_outbox SET status = 'done', last_error = NULL, updated_at = now()
|
||||
WHERE id = $1 AND workspace_id = $2`,
|
||||
[outboxId, record.workspaceId],
|
||||
);
|
||||
return { id: row.id, indexedAt: row.indexed_at };
|
||||
});
|
||||
}
|
||||
|
||||
async recordFailure(workspaceId: string, outboxId: string, error: string, terminal: boolean): Promise<OutboxRow> {
|
||||
return this.withTenant(workspaceId, async (c) => {
|
||||
const res = await c.query<OutboxDbRow>(
|
||||
`UPDATE archive_sync_outbox
|
||||
SET attempts = attempts + 1, last_error = $3, updated_at = now(),
|
||||
status = CASE WHEN $4 THEN 'failed' ELSE 'pending' END
|
||||
WHERE id = $1 AND workspace_id = $2 AND status = 'pending'
|
||||
RETURNING ${OUTBOX_COLUMNS}`,
|
||||
[outboxId, workspaceId, error.slice(0, LAST_ERROR_MAX), terminal],
|
||||
);
|
||||
const row = res.rows[0];
|
||||
if (row === undefined) throw new Error("outbox row not pending");
|
||||
return toOutbox(row);
|
||||
});
|
||||
}
|
||||
|
||||
async markRedacted(workspaceId: string, roomId: string, eventId: string, claimKey: string): Promise<number> {
|
||||
return this.withTenant(workspaceId, async (c) => {
|
||||
const res = await c.query(
|
||||
`UPDATE chat_media_archive SET available = false
|
||||
WHERE workspace_id = $1 AND room_id = $2 AND event_id = $3 AND available`,
|
||||
[workspaceId, roomId, eventId],
|
||||
);
|
||||
await c.query(
|
||||
`UPDATE archive_sync_outbox SET status = 'failed', last_error = 'redacted', updated_at = now()
|
||||
WHERE workspace_id = $1 AND claim_key = $2 AND status = 'pending'`,
|
||||
[workspaceId, claimKey],
|
||||
);
|
||||
return res.rowCount ?? 0;
|
||||
});
|
||||
}
|
||||
|
||||
async countPending(workspaceIds: readonly string[]): Promise<number> {
|
||||
let total = 0;
|
||||
for (const workspaceId of workspaceIds) {
|
||||
total += await this.withTenant(workspaceId, async (c) => {
|
||||
const res = await c.query<{ n: string }>(
|
||||
"SELECT count(*) AS n FROM archive_sync_outbox WHERE workspace_id = $1 AND status = 'pending'",
|
||||
[workspaceId],
|
||||
);
|
||||
return Number(res.rows[0]?.n ?? 0);
|
||||
});
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
private stateId(workspaceId: string): string {
|
||||
return `${this.cursorId}:${workspaceId}`;
|
||||
}
|
||||
|
||||
private async withTenant<T>(workspaceId: string, fn: (client: PoolClient) => Promise<T>): Promise<T> {
|
||||
const client = await this.pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]);
|
||||
const result = await fn(client);
|
||||
await client.query("COMMIT");
|
||||
return result;
|
||||
} catch (err) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw err;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
21
apps/archive-bot/src/webdav.test.ts
Normal file
21
apps/archive-bot/src/webdav.test.ts
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { davPath, parsePropfind } from "./webdav.js";
|
||||
|
||||
describe("davPath", () => {
|
||||
it("encodes every segment under the service user's files root", () => {
|
||||
expect(davPath("svc", ["Library", "ws", "grp", "abc-My file #1.pdf"]))
|
||||
.toBe("/remote.php/dav/files/svc/Library/ws/grp/abc-My%20file%20%231.pdf");
|
||||
});
|
||||
});
|
||||
|
||||
describe("parsePropfind", () => {
|
||||
it("reads oc:fileid and an unquoted etag", () => {
|
||||
const xml = `<d:multistatus xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns"><d:response><d:propstat><d:prop>
|
||||
<oc:fileid>space$abc!def</oc:fileid><d:getetag>"9f1c"</d:getetag></d:prop></d:propstat></d:response></d:multistatus>`;
|
||||
expect(parsePropfind(xml)).toEqual({ resourceId: "space$abc!def", version: "9f1c" });
|
||||
});
|
||||
|
||||
it("returns null when properties are missing", () => {
|
||||
expect(parsePropfind("<d:multistatus/>")).toBeNull();
|
||||
});
|
||||
});
|
||||
116
apps/archive-bot/src/webdav.ts
Normal file
116
apps/archive-bot/src/webdav.ts
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
export interface LibraryResource {
|
||||
path: string;
|
||||
url: string;
|
||||
resourceId: string;
|
||||
version: string;
|
||||
}
|
||||
|
||||
/** oCIS library surface; faked in tests. */
|
||||
export interface LibraryPort {
|
||||
/** Stream bytes to a deterministic staging resource (overwritten on retry). */
|
||||
putStaging(dir: string[], stagingName: string, body: ReadableStream<Uint8Array>, signal: AbortSignal): Promise<void>;
|
||||
/** Move staging to its content-addressed final name; an existing final resource is reused. */
|
||||
commit(dir: string[], stagingName: string, finalName: string, signal: AbortSignal): Promise<LibraryResource>;
|
||||
}
|
||||
|
||||
export class WebDavError extends Error {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
const STAGING_DIR = ".archive-staging";
|
||||
|
||||
/** Percent-encode each segment; segments themselves are already normalised safe names/UUIDs. */
|
||||
export function davPath(user: string, segments: readonly string[]): string {
|
||||
return `/remote.php/dav/files/${[user, ...segments].map(encodeURIComponent).join("/")}`;
|
||||
}
|
||||
|
||||
/** Extract `oc:fileid` and `d:getetag` from a Depth:0 PROPFIND multistatus body. */
|
||||
export function parsePropfind(xml: string): { resourceId: string; version: string } | null {
|
||||
const fileId = /<(?:[\w-]+:)?fileid>([^<]+)<\/(?:[\w-]+:)?fileid>/.exec(xml)?.[1];
|
||||
const etag = /<(?:[\w-]+:)?getetag>([^<]+)<\/(?:[\w-]+:)?getetag>/.exec(xml)?.[1];
|
||||
if (fileId === undefined || etag === undefined) return null;
|
||||
return { resourceId: fileId.trim(), version: etag.trim().replace(/^"|"$/g, "").replace(/"/g, "") };
|
||||
}
|
||||
|
||||
const PROPFIND_BODY = `<?xml version="1.0" encoding="utf-8"?>
|
||||
<d:propfind xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns"><d:prop><oc:fileid/><d:getetag/></d:prop></d:propfind>`;
|
||||
|
||||
/** WebDAV client for the scoped oCIS service identity. Credentials never leave the configured origin. */
|
||||
export class OcisWebDav implements LibraryPort {
|
||||
private readonly authorization: string;
|
||||
private readonly knownDirs = new Set<string>();
|
||||
|
||||
constructor(
|
||||
private readonly baseUrl: URL,
|
||||
private readonly user: string,
|
||||
password: string,
|
||||
) {
|
||||
this.authorization = `Basic ${Buffer.from(`${user}:${password}`, "utf8").toString("base64")}`;
|
||||
}
|
||||
|
||||
async putStaging(dir: string[], stagingName: string, body: ReadableStream<Uint8Array>, signal: AbortSignal): Promise<void> {
|
||||
await this.ensureDir([...dir, STAGING_DIR], signal);
|
||||
const res = await this.send("PUT", [...dir, STAGING_DIR, stagingName], signal, {
|
||||
body,
|
||||
headers: { "content-type": "application/octet-stream" },
|
||||
});
|
||||
if (res.status !== 201 && res.status !== 204) throw new WebDavError(res.status, `WebDAV PUT failed: ${res.status}`);
|
||||
}
|
||||
|
||||
async commit(dir: string[], stagingName: string, finalName: string, signal: AbortSignal): Promise<LibraryResource> {
|
||||
const staging = [...dir, STAGING_DIR, stagingName];
|
||||
const final = [...dir, finalName];
|
||||
const destination = new URL(davPath(this.user, final), this.baseUrl).toString();
|
||||
const moved = await this.send("MOVE", staging, signal, { headers: { destination, overwrite: "F" } });
|
||||
if (moved.status === 412) {
|
||||
// Same sha256 + name already archived (crash after MOVE, or a duplicate post): reuse it.
|
||||
const removed = await this.send("DELETE", staging, signal);
|
||||
if (!removed.ok && removed.status !== 404) throw new WebDavError(removed.status, `WebDAV DELETE failed: ${removed.status}`);
|
||||
} else if (moved.status !== 201 && moved.status !== 204) {
|
||||
throw new WebDavError(moved.status, `WebDAV MOVE failed: ${moved.status}`);
|
||||
}
|
||||
const found = await this.send("PROPFIND", final, signal, {
|
||||
body: PROPFIND_BODY,
|
||||
headers: { depth: "0", "content-type": "application/xml" },
|
||||
});
|
||||
if (found.status !== 207) throw new WebDavError(found.status, `WebDAV PROPFIND failed: ${found.status}`);
|
||||
const props = parsePropfind(await found.text());
|
||||
if (props === null) throw new WebDavError(207, "WebDAV PROPFIND missing fileid/etag");
|
||||
return { path: final.join("/"), url: destination, ...props };
|
||||
}
|
||||
|
||||
private async ensureDir(segments: string[], signal: AbortSignal): Promise<void> {
|
||||
for (let depth = 1; depth <= segments.length; depth += 1) {
|
||||
const dir = segments.slice(0, depth);
|
||||
const key = dir.join("/");
|
||||
if (this.knownDirs.has(key)) continue;
|
||||
const res = await this.send("MKCOL", dir, signal);
|
||||
// 405: already exists.
|
||||
if (res.status !== 201 && res.status !== 405) throw new WebDavError(res.status, `WebDAV MKCOL failed: ${res.status}`);
|
||||
this.knownDirs.add(key);
|
||||
}
|
||||
}
|
||||
|
||||
private async send(
|
||||
method: string,
|
||||
segments: string[],
|
||||
signal: AbortSignal,
|
||||
init: { body?: ReadableStream<Uint8Array> | string; headers?: Record<string, string> } = {},
|
||||
): Promise<Response> {
|
||||
const res = await fetch(new URL(davPath(this.user, segments), this.baseUrl), {
|
||||
method,
|
||||
headers: { authorization: this.authorization, ...init.headers },
|
||||
body: init.body,
|
||||
duplex: init.body instanceof ReadableStream ? "half" : undefined,
|
||||
redirect: "error",
|
||||
signal,
|
||||
});
|
||||
if (method !== "PROPFIND") await res.body?.cancel();
|
||||
return res;
|
||||
}
|
||||
}
|
||||
4
apps/archive-bot/tsconfig.build.json
Normal file
4
apps/archive-bot/tsconfig.build.json
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"exclude": ["src/**/*.test.ts"]
|
||||
}
|
||||
11
apps/archive-bot/tsconfig.json
Normal file
11
apps/archive-bot/tsconfig.json
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext"
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
7
apps/archive-bot/vitest.config.ts
Normal file
7
apps/archive-bot/vitest.config.ts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
include: ["src/**/*.test.ts"],
|
||||
},
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue