351 lines
15 KiB
TypeScript
351 lines
15 KiB
TypeScript
import type { Logger } from "pino";
|
|
import type { Archiver } from "./archiver.js";
|
|
import { backoffMs, BACKOFF_MAX_MS, MAX_ATTEMPTS, nextAttemptAt } from "./backoff.js";
|
|
import type { RoomMapping } from "./config.js";
|
|
import { classifyEvent, type AttachmentWork, type MatrixEvent, type RoomEncryption } from "./events.js";
|
|
import { PermanentArchiveError } from "./inspector.js";
|
|
import { claimKey, replyTxnId } from "./keys.js";
|
|
import { MatrixHttpError, type MatrixPort } from "./matrix.js";
|
|
import type { ArchiveStore, RoomCursorTarget } from "./store.js";
|
|
import type { BundleExporter } from "./exporter.js";
|
|
import { exportNotice, isExportCommand } from "./exporter.js";
|
|
import { senderIsRoomAdmin, toEncryptionHealth, type EncryptionLintReport, type MatrixLintPort } from "./encryption-lint.js";
|
|
import type { ExportConfig } from "./config.js";
|
|
|
|
export interface BatchResult {
|
|
/** True when the sync cursor was durably advanced past this batch. */
|
|
advanced: boolean;
|
|
/** Earliest time a blocked row may be retried; null when nothing is blocked. */
|
|
retryAt: Date | null;
|
|
}
|
|
|
|
export interface HealthSnapshot {
|
|
rooms: number;
|
|
cursor: string | null;
|
|
pending: number;
|
|
lastProcessedAt: string | null;
|
|
/** Phase-B: encrypted export bundle state, null when exports are not configured. */
|
|
export: {
|
|
enabled: boolean;
|
|
every: number;
|
|
lastExportAt: string | null;
|
|
lastObjectKey: string | null;
|
|
roomsExported: string[];
|
|
} | null;
|
|
/** Phase-B: encryption-state lint (unmapped unencrypted rooms = violations). */
|
|
encryption: ReturnType<typeof toEncryptionHealth>;
|
|
}
|
|
|
|
type Outcome = { blockedUntil: Date } | null;
|
|
|
|
function lintEmpty(now: Date): EncryptionLintReport {
|
|
return { checked: 0, mappedUnencrypted: [], issues: [], checkedAt: now.toISOString() };
|
|
}
|
|
|
|
export function replyContent(safeName: string, eventId: string): Record<string, unknown> {
|
|
return {
|
|
msgtype: "m.notice",
|
|
body: `Archived to the group library: ${safeName}`,
|
|
"m.relates_to": { "m.in_reply_to": { event_id: eventId } },
|
|
};
|
|
}
|
|
|
|
function errorText(err: unknown): string {
|
|
return err instanceof Error ? err.message : String(err);
|
|
}
|
|
|
|
function abortableSleep(ms: number, signal: AbortSignal): Promise<void> {
|
|
return new Promise((resolve) => {
|
|
if (signal.aborted) return resolve();
|
|
const timer = setTimeout(done, ms);
|
|
function done(): void {
|
|
clearTimeout(timer);
|
|
signal.removeEventListener("abort", done);
|
|
resolve();
|
|
}
|
|
signal.addEventListener("abort", done, { once: true });
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Sync-loop orchestration. The cursor is persisted only after every event in
|
|
* the batch has reached a durable terminal state (done/failed/skipped) and its
|
|
* reply was sent; otherwise the same `since` is re-synced after the earliest
|
|
* per-row backoff, so no attachment is lost to a transient failure.
|
|
*/
|
|
export class ArchiveProcessor {
|
|
private cursor: string | null = null;
|
|
private lastProcessedAt: Date | null = null;
|
|
/** Committed claims whose single reply has not been delivered yet (key → safe name). */
|
|
private readonly unsentReplies = new Map<string, string>();
|
|
private readonly byRoom: Map<string, RoomMapping>;
|
|
private readonly cursorRooms: RoomCursorTarget[];
|
|
private readonly workspaceIds: string[];
|
|
private readonly exporter: BundleExporter | null;
|
|
private readonly exportConfig: ExportConfig | null;
|
|
private lastExportAt: Date | null = null;
|
|
private lastObjectKey: string | null = null;
|
|
private readonly roomsExported = new Set<string>();
|
|
private encryptionReport: EncryptionLintReport | null = null;
|
|
|
|
constructor(
|
|
private readonly matrix: MatrixPort,
|
|
private readonly store: ArchiveStore,
|
|
private readonly archiver: Archiver,
|
|
mappings: readonly RoomMapping[],
|
|
private readonly logger: Logger,
|
|
private readonly now: () => Date = () => new Date(),
|
|
exporter: BundleExporter | null = null,
|
|
exportConfig: ExportConfig | null = null,
|
|
) {
|
|
this.byRoom = new Map(mappings.map((m) => [m.roomId, m]));
|
|
this.cursorRooms = mappings.map((m) => ({ roomId: m.roomId, workspaceId: m.workspaceId }));
|
|
this.workspaceIds = [...new Set(mappings.map((m) => m.workspaceId))];
|
|
this.exporter = exporter;
|
|
this.exportConfig = exportConfig;
|
|
}
|
|
|
|
async init(): Promise<void> {
|
|
this.cursor = await this.store.loadCursor(this.cursorRooms);
|
|
}
|
|
|
|
async health(): Promise<HealthSnapshot> {
|
|
return {
|
|
rooms: this.byRoom.size,
|
|
cursor: this.cursor,
|
|
pending: await this.store.countPending(this.workspaceIds),
|
|
lastProcessedAt: this.lastProcessedAt?.toISOString() ?? null,
|
|
export: this.exportConfig === null ? null : {
|
|
enabled: this.exportConfig.enabled,
|
|
every: this.exportConfig.every,
|
|
lastExportAt: this.lastExportAt?.toISOString() ?? null,
|
|
lastObjectKey: this.lastObjectKey,
|
|
roomsExported: [...this.roomsExported],
|
|
},
|
|
encryption: toEncryptionHealth(this.encryptionReport),
|
|
};
|
|
}
|
|
|
|
/** One encryption-lint pass over every joined room (startup + periodic). */
|
|
async lintEncryption(signal: AbortSignal): Promise<EncryptionLintReport> {
|
|
const lintPort = this.matrix as MatrixLintPort;
|
|
if (typeof lintPort.joinedRooms !== "function") {
|
|
this.logger.warn("archive-bot: matrix port lacks joinedRooms; encryption lint skipped");
|
|
return this.encryptionReport ?? lintEmpty(new Date());
|
|
}
|
|
const { scanEncryption } = await import("./encryption-lint.js");
|
|
this.encryptionReport = await scanEncryption(lintPort, [...this.byRoom.keys()], this.logger, signal, this.now());
|
|
return this.encryptionReport;
|
|
}
|
|
|
|
async runBatch(signal: AbortSignal): Promise<BatchResult> {
|
|
const since = this.cursor;
|
|
const sync = await this.matrix.sync(since, signal);
|
|
|
|
for (const roomId of sync.invited) {
|
|
if (!this.byRoom.has(roomId)) continue;
|
|
await this.matrix.joinRoom(roomId, signal);
|
|
this.logger.info({ roomId }, "archive-bot: joined mapped room");
|
|
}
|
|
|
|
let retryAt: Date | null = null;
|
|
for (const [roomId, room] of sync.joined) {
|
|
const mapping = this.byRoom.get(roomId);
|
|
if (mapping === undefined) continue;
|
|
let events = room.timeline.events;
|
|
if (room.timeline.limited && room.timeline.prevBatch !== null && since !== null) {
|
|
events = [...(await this.matrix.backfill(roomId, room.timeline.prevBatch, since, signal)), ...events];
|
|
}
|
|
if (events.length === 0) continue;
|
|
const encryption = await this.encryptionState(mapping, [...room.stateEvents, ...events], signal);
|
|
for (const event of events) {
|
|
const outcome = await this.handleEvent(mapping, event, encryption, signal);
|
|
if (outcome !== null && (retryAt === null || outcome.blockedUntil < retryAt)) retryAt = outcome.blockedUntil;
|
|
}
|
|
}
|
|
|
|
if (retryAt !== null) return { advanced: false, retryAt };
|
|
await this.store.saveCursor(this.cursorRooms, sync.nextBatch);
|
|
this.cursor = sync.nextBatch;
|
|
return { advanced: true, retryAt: null };
|
|
}
|
|
|
|
/** Long-running loop; returns when `signal` aborts. */
|
|
async run(signal: AbortSignal, relogin: (signal: AbortSignal) => Promise<void>): Promise<void> {
|
|
let consecutiveErrors = 0;
|
|
while (!signal.aborted) {
|
|
try {
|
|
const result = await this.runBatch(signal);
|
|
consecutiveErrors = 0;
|
|
if (result.retryAt !== null) {
|
|
const wait = Math.min(Math.max(result.retryAt.getTime() - this.now().getTime(), 1000), BACKOFF_MAX_MS);
|
|
this.logger.info({ waitMs: wait }, "archive-bot: batch has pending rows; holding cursor");
|
|
await abortableSleep(wait, signal);
|
|
}
|
|
} catch (err) {
|
|
if (signal.aborted) break;
|
|
consecutiveErrors += 1;
|
|
if (err instanceof MatrixHttpError && err.status === 401) {
|
|
this.logger.warn("archive-bot: access token rejected; logging in again");
|
|
await relogin(signal).catch((loginErr: unknown) => {
|
|
this.logger.error({ err: errorText(loginErr) }, "archive-bot: re-login failed");
|
|
});
|
|
} else {
|
|
this.logger.error({ err: errorText(err) }, "archive-bot: batch failed; cursor held");
|
|
}
|
|
await abortableSleep(backoffMs(consecutiveErrors), signal);
|
|
}
|
|
}
|
|
}
|
|
|
|
/** Fail closed: an encryption event anywhere in this batch wins; otherwise ask the room state. */
|
|
private async encryptionState(mapping: RoomMapping, events: MatrixEvent[], signal: AbortSignal): Promise<RoomEncryption> {
|
|
const sawEncryption = events.some((e) => e.type === "m.room.encryption" && e.state_key === "");
|
|
const state = sawEncryption ? "encrypted" : await this.matrix.roomEncryption(mapping.roomId, signal);
|
|
if (state !== "unencrypted") {
|
|
this.logger.warn({ roomId: mapping.roomId, state }, "archive-bot: room not verifiably unencrypted; not mirroring");
|
|
}
|
|
return state;
|
|
}
|
|
|
|
private async handleEvent(
|
|
mapping: RoomMapping,
|
|
event: MatrixEvent,
|
|
encryption: RoomEncryption,
|
|
signal: AbortSignal,
|
|
): Promise<Outcome> {
|
|
// Admin export command: an m.text message, so it is never an attachment;
|
|
// check it before classification (which would skip it as not-an-attachment).
|
|
if (isExportCommand(event.content)) {
|
|
return this.handleExportCommand(mapping, event, signal);
|
|
}
|
|
const classified = classifyEvent(event, mapping, encryption, this.matrix.userId);
|
|
switch (classified.kind) {
|
|
case "skip":
|
|
if (classified.reason !== "own-event" && classified.reason !== "not-an-attachment" && classified.reason !== "not-a-message") {
|
|
this.logger.info({ roomId: mapping.roomId, eventId: event.event_id, reason: classified.reason }, "archive-bot: skipped");
|
|
}
|
|
return null;
|
|
case "redaction": {
|
|
const key = claimKey(mapping.workspaceId, mapping.roomId, classified.redacts);
|
|
const affected = await this.store.markRedacted(mapping.workspaceId, mapping.roomId, classified.redacts, key);
|
|
this.unsentReplies.delete(key);
|
|
if (affected > 0) {
|
|
this.logger.info({ roomId: mapping.roomId, eventId: classified.redacts }, "archive-bot: redacted; index marked unavailable");
|
|
}
|
|
this.lastProcessedAt = this.now();
|
|
return null;
|
|
}
|
|
case "attachment":
|
|
return this.handleAttachment(mapping, classified, signal);
|
|
}
|
|
}
|
|
|
|
private async handleAttachment(mapping: RoomMapping, work: AttachmentWork, signal: AbortSignal): Promise<Outcome> {
|
|
const eventId = work.event.event_id;
|
|
const key = claimKey(mapping.workspaceId, mapping.roomId, eventId);
|
|
const row = await this.store.claim(mapping.workspaceId, mapping.roomId, eventId, key);
|
|
const log = { roomId: mapping.roomId, eventId, outboxId: row.id };
|
|
|
|
if (row.status === "failed") return null;
|
|
if (row.status === "done") return this.sendReply(mapping, eventId, key, signal);
|
|
|
|
const due = nextAttemptAt(row.attempts, row.updatedAt);
|
|
if (row.attempts > 0 && due > this.now()) return { blockedUntil: due };
|
|
|
|
try {
|
|
const archived = await this.archiver.archive(mapping, work, row, key, signal);
|
|
this.logger.info({ ...log, fileId: archived.file.id, sha256: archived.file.checksum.value }, "archive-bot: archived");
|
|
this.lastProcessedAt = this.now();
|
|
this.unsentReplies.set(key, archived.safeName);
|
|
await this.maybeAutoExport(mapping, signal);
|
|
return this.sendReply(mapping, eventId, key, signal);
|
|
} catch (err) {
|
|
if (signal.aborted) throw err;
|
|
const terminal = err instanceof PermanentArchiveError || row.attempts + 1 >= MAX_ATTEMPTS;
|
|
const updated = await this.store.recordFailure(mapping.workspaceId, row.id, errorText(err), terminal);
|
|
if (terminal) {
|
|
this.logger.error({ ...log, err: errorText(err), attempts: updated.attempts }, "archive-bot: archive failed permanently");
|
|
return null;
|
|
}
|
|
const retry = nextAttemptAt(updated.attempts, updated.updatedAt);
|
|
this.logger.warn({ ...log, err: errorText(err), attempts: updated.attempts, retryAt: retry.toISOString() }, "archive-bot: archive failed; will retry");
|
|
return { blockedUntil: retry };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* `!archive export` from a mapped room. Power level >= 50 required; the
|
|
* bundle is built, uploaded to R2 and the ciphertext object key is posted
|
|
* back as a notice (the key material lives only in the E2EE admin room).
|
|
*/
|
|
private async handleExportCommand(
|
|
mapping: RoomMapping,
|
|
event: MatrixEvent,
|
|
signal: AbortSignal,
|
|
): Promise<Outcome> {
|
|
if (this.exporter === null || this.exportConfig === null || !this.exportConfig.enabled) {
|
|
this.logger.info({ roomId: mapping.roomId }, "archive-export: not configured; command ignored");
|
|
return null;
|
|
}
|
|
try {
|
|
const powerLevels = await this.matrix.roomPowerLevels(mapping.roomId, signal);
|
|
if (!senderIsRoomAdmin(powerLevels, event.sender)) {
|
|
this.logger.warn({ roomId: mapping.roomId, sender: event.sender }, "archive-export: sender is not a room admin");
|
|
return null;
|
|
}
|
|
} catch (err) {
|
|
this.logger.warn({ roomId: mapping.roomId, err: errorText(err) }, "archive-export: power-level check failed");
|
|
return null;
|
|
}
|
|
const result = await this.exporter.exportRoom(mapping, signal, "manual");
|
|
this.recordExport(mapping.roomId, result.objectKey);
|
|
await this.matrix.sendNotice(
|
|
mapping.roomId,
|
|
replyTxnId(`export:${result.objectKey}`),
|
|
exportNotice(result.objectKey, this.exportConfig.r2.bucket, this.exportConfig.ageRecipients),
|
|
signal,
|
|
);
|
|
return null;
|
|
}
|
|
|
|
private recordExport(roomId: string, objectKey: string): void {
|
|
this.lastExportAt = this.now();
|
|
this.lastObjectKey = objectKey;
|
|
this.roomsExported.add(roomId);
|
|
}
|
|
|
|
/** Rolling auto-export after ARCHIVE_EXPORT_EVERY archived attachments in a room. */
|
|
private async maybeAutoExport(mapping: RoomMapping, signal: AbortSignal): Promise<void> {
|
|
if (this.exporter === null || this.exportConfig === null || !this.exportConfig.enabled) return;
|
|
this.exporter.countAttachment(mapping.roomId);
|
|
if (!this.exporter.shouldAutoExport(mapping.roomId, this.exportConfig.every)) return;
|
|
try {
|
|
const result = await this.exporter.exportRoom(mapping.roomId === mapping.roomId ? mapping : mapping, signal, "auto");
|
|
this.recordExport(mapping.roomId, result.objectKey);
|
|
await this.matrix.sendNotice(
|
|
mapping.roomId,
|
|
replyTxnId(`export:${result.objectKey}`),
|
|
exportNotice(result.objectKey, this.exportConfig.r2.bucket, this.exportConfig.ageRecipients),
|
|
signal,
|
|
);
|
|
} catch (err) {
|
|
this.logger.error({ roomId: mapping.roomId, err: errorText(err) }, "archive-export: automatic export failed");
|
|
}
|
|
}
|
|
|
|
/** Exactly one reply per committed claim; the stable txn ID makes resends idempotent server-side. */
|
|
private async sendReply(mapping: RoomMapping, eventId: string, key: string, signal: AbortSignal): Promise<Outcome> {
|
|
const safeName = this.unsentReplies.get(key);
|
|
if (safeName === undefined) return null;
|
|
try {
|
|
await this.matrix.sendNotice(mapping.roomId, replyTxnId(key), replyContent(safeName, eventId), signal);
|
|
this.unsentReplies.delete(key);
|
|
return null;
|
|
} catch (err) {
|
|
if (signal.aborted) throw err;
|
|
this.logger.warn({ roomId: mapping.roomId, eventId, err: errorText(err) }, "archive-bot: reply failed; will retry");
|
|
return { blockedUntil: new Date(this.now().getTime() + backoffMs(1)) };
|
|
}
|
|
}
|
|
}
|