import type { Logger } from "pino"; import type { Archiver } from "./archiver.js"; import { backoffMs, BACKOFF_MAX_MS, MAX_ATTEMPTS, nextAttemptAt } from "./backoff.js"; import type { RoomMapping } from "./config.js"; import { classifyEvent, type AttachmentWork, type MatrixEvent, type RoomEncryption } from "./events.js"; import { PermanentArchiveError } from "./inspector.js"; import { claimKey, replyTxnId } from "./keys.js"; import { MatrixHttpError, type MatrixPort } from "./matrix.js"; import type { ArchiveStore, RoomCursorTarget } from "./store.js"; import type { BundleExporter } from "./exporter.js"; import { exportNotice, isExportCommand } from "./exporter.js"; import { senderIsRoomAdmin, toEncryptionHealth, type EncryptionLintReport, type MatrixLintPort } from "./encryption-lint.js"; import type { ExportConfig } from "./config.js"; export interface BatchResult { /** True when the sync cursor was durably advanced past this batch. */ advanced: boolean; /** Earliest time a blocked row may be retried; null when nothing is blocked. */ retryAt: Date | null; } export interface HealthSnapshot { rooms: number; cursor: string | null; pending: number; lastProcessedAt: string | null; /** Phase-B: encrypted export bundle state, null when exports are not configured. */ export: { enabled: boolean; every: number; lastExportAt: string | null; lastObjectKey: string | null; roomsExported: string[]; } | null; /** Phase-B: encryption-state lint (unmapped unencrypted rooms = violations). */ encryption: ReturnType; } type Outcome = { blockedUntil: Date } | null; function lintEmpty(now: Date): EncryptionLintReport { return { checked: 0, mappedUnencrypted: [], issues: [], checkedAt: now.toISOString() }; } export function replyContent(safeName: string, eventId: string): Record { return { msgtype: "m.notice", body: `Archived to the group library: ${safeName}`, "m.relates_to": { "m.in_reply_to": { event_id: eventId } }, }; } function errorText(err: unknown): string { return err instanceof Error ? err.message : String(err); } function abortableSleep(ms: number, signal: AbortSignal): Promise { return new Promise((resolve) => { if (signal.aborted) return resolve(); const timer = setTimeout(done, ms); function done(): void { clearTimeout(timer); signal.removeEventListener("abort", done); resolve(); } signal.addEventListener("abort", done, { once: true }); }); } /** * Sync-loop orchestration. The cursor is persisted only after every event in * the batch has reached a durable terminal state (done/failed/skipped) and its * reply was sent; otherwise the same `since` is re-synced after the earliest * per-row backoff, so no attachment is lost to a transient failure. */ export class ArchiveProcessor { private cursor: string | null = null; private lastProcessedAt: Date | null = null; /** Committed claims whose single reply has not been delivered yet (key → safe name). */ private readonly unsentReplies = new Map(); private readonly byRoom: Map; private readonly cursorRooms: RoomCursorTarget[]; private readonly workspaceIds: string[]; private readonly exporter: BundleExporter | null; private readonly exportConfig: ExportConfig | null; private lastExportAt: Date | null = null; private lastObjectKey: string | null = null; private readonly roomsExported = new Set(); private encryptionReport: EncryptionLintReport | null = null; constructor( private readonly matrix: MatrixPort, private readonly store: ArchiveStore, private readonly archiver: Archiver, mappings: readonly RoomMapping[], private readonly logger: Logger, private readonly now: () => Date = () => new Date(), exporter: BundleExporter | null = null, exportConfig: ExportConfig | null = null, ) { this.byRoom = new Map(mappings.map((m) => [m.roomId, m])); this.cursorRooms = mappings.map((m) => ({ roomId: m.roomId, workspaceId: m.workspaceId })); this.workspaceIds = [...new Set(mappings.map((m) => m.workspaceId))]; this.exporter = exporter; this.exportConfig = exportConfig; } async init(): Promise { this.cursor = await this.store.loadCursor(this.cursorRooms); } async health(): Promise { return { rooms: this.byRoom.size, cursor: this.cursor, pending: await this.store.countPending(this.workspaceIds), lastProcessedAt: this.lastProcessedAt?.toISOString() ?? null, export: this.exportConfig === null ? null : { enabled: this.exportConfig.enabled, every: this.exportConfig.every, lastExportAt: this.lastExportAt?.toISOString() ?? null, lastObjectKey: this.lastObjectKey, roomsExported: [...this.roomsExported], }, encryption: toEncryptionHealth(this.encryptionReport), }; } /** One encryption-lint pass over every joined room (startup + periodic). */ async lintEncryption(signal: AbortSignal): Promise { const lintPort = this.matrix as MatrixLintPort; if (typeof lintPort.joinedRooms !== "function") { this.logger.warn("archive-bot: matrix port lacks joinedRooms; encryption lint skipped"); return this.encryptionReport ?? lintEmpty(new Date()); } const { scanEncryption } = await import("./encryption-lint.js"); this.encryptionReport = await scanEncryption(lintPort, [...this.byRoom.keys()], this.logger, signal, this.now()); return this.encryptionReport; } async runBatch(signal: AbortSignal): Promise { const since = this.cursor; const sync = await this.matrix.sync(since, signal); for (const roomId of sync.invited) { if (!this.byRoom.has(roomId)) continue; await this.matrix.joinRoom(roomId, signal); this.logger.info({ roomId }, "archive-bot: joined mapped room"); } let retryAt: Date | null = null; for (const [roomId, room] of sync.joined) { const mapping = this.byRoom.get(roomId); if (mapping === undefined) continue; let events = room.timeline.events; if (room.timeline.limited && room.timeline.prevBatch !== null && since !== null) { events = [...(await this.matrix.backfill(roomId, room.timeline.prevBatch, since, signal)), ...events]; } if (events.length === 0) continue; const encryption = await this.encryptionState(mapping, [...room.stateEvents, ...events], signal); for (const event of events) { const outcome = await this.handleEvent(mapping, event, encryption, signal); if (outcome !== null && (retryAt === null || outcome.blockedUntil < retryAt)) retryAt = outcome.blockedUntil; } } if (retryAt !== null) return { advanced: false, retryAt }; await this.store.saveCursor(this.cursorRooms, sync.nextBatch); this.cursor = sync.nextBatch; return { advanced: true, retryAt: null }; } /** Long-running loop; returns when `signal` aborts. */ async run(signal: AbortSignal, relogin: (signal: AbortSignal) => Promise): Promise { let consecutiveErrors = 0; while (!signal.aborted) { try { const result = await this.runBatch(signal); consecutiveErrors = 0; if (result.retryAt !== null) { const wait = Math.min(Math.max(result.retryAt.getTime() - this.now().getTime(), 1000), BACKOFF_MAX_MS); this.logger.info({ waitMs: wait }, "archive-bot: batch has pending rows; holding cursor"); await abortableSleep(wait, signal); } } catch (err) { if (signal.aborted) break; consecutiveErrors += 1; if (err instanceof MatrixHttpError && err.status === 401) { this.logger.warn("archive-bot: access token rejected; logging in again"); await relogin(signal).catch((loginErr: unknown) => { this.logger.error({ err: errorText(loginErr) }, "archive-bot: re-login failed"); }); } else { this.logger.error({ err: errorText(err) }, "archive-bot: batch failed; cursor held"); } await abortableSleep(backoffMs(consecutiveErrors), signal); } } } /** Fail closed: an encryption event anywhere in this batch wins; otherwise ask the room state. */ private async encryptionState(mapping: RoomMapping, events: MatrixEvent[], signal: AbortSignal): Promise { const sawEncryption = events.some((e) => e.type === "m.room.encryption" && e.state_key === ""); const state = sawEncryption ? "encrypted" : await this.matrix.roomEncryption(mapping.roomId, signal); if (state !== "unencrypted") { this.logger.warn({ roomId: mapping.roomId, state }, "archive-bot: room not verifiably unencrypted; not mirroring"); } return state; } private async handleEvent( mapping: RoomMapping, event: MatrixEvent, encryption: RoomEncryption, signal: AbortSignal, ): Promise { // Admin export command: an m.text message, so it is never an attachment; // check it before classification (which would skip it as not-an-attachment). if (isExportCommand(event.content)) { return this.handleExportCommand(mapping, event, signal); } const classified = classifyEvent(event, mapping, encryption, this.matrix.userId); switch (classified.kind) { case "skip": if (classified.reason !== "own-event" && classified.reason !== "not-an-attachment" && classified.reason !== "not-a-message") { this.logger.info({ roomId: mapping.roomId, eventId: event.event_id, reason: classified.reason }, "archive-bot: skipped"); } return null; case "redaction": { const key = claimKey(mapping.workspaceId, mapping.roomId, classified.redacts); const affected = await this.store.markRedacted(mapping.workspaceId, mapping.roomId, classified.redacts, key); this.unsentReplies.delete(key); if (affected > 0) { this.logger.info({ roomId: mapping.roomId, eventId: classified.redacts }, "archive-bot: redacted; index marked unavailable"); } this.lastProcessedAt = this.now(); return null; } case "attachment": return this.handleAttachment(mapping, classified, signal); } } private async handleAttachment(mapping: RoomMapping, work: AttachmentWork, signal: AbortSignal): Promise { const eventId = work.event.event_id; const key = claimKey(mapping.workspaceId, mapping.roomId, eventId); const row = await this.store.claim(mapping.workspaceId, mapping.roomId, eventId, key); const log = { roomId: mapping.roomId, eventId, outboxId: row.id }; if (row.status === "failed") return null; if (row.status === "done") return this.sendReply(mapping, eventId, key, signal); const due = nextAttemptAt(row.attempts, row.updatedAt); if (row.attempts > 0 && due > this.now()) return { blockedUntil: due }; try { const archived = await this.archiver.archive(mapping, work, row, key, signal); this.logger.info({ ...log, fileId: archived.file.id, sha256: archived.file.checksum.value }, "archive-bot: archived"); this.lastProcessedAt = this.now(); this.unsentReplies.set(key, archived.safeName); await this.maybeAutoExport(mapping, signal); return this.sendReply(mapping, eventId, key, signal); } catch (err) { if (signal.aborted) throw err; const terminal = err instanceof PermanentArchiveError || row.attempts + 1 >= MAX_ATTEMPTS; const updated = await this.store.recordFailure(mapping.workspaceId, row.id, errorText(err), terminal); if (terminal) { this.logger.error({ ...log, err: errorText(err), attempts: updated.attempts }, "archive-bot: archive failed permanently"); return null; } const retry = nextAttemptAt(updated.attempts, updated.updatedAt); this.logger.warn({ ...log, err: errorText(err), attempts: updated.attempts, retryAt: retry.toISOString() }, "archive-bot: archive failed; will retry"); return { blockedUntil: retry }; } } /** * `!archive export` from a mapped room. Power level >= 50 required; the * bundle is built, uploaded to R2 and the ciphertext object key is posted * back as a notice (the key material lives only in the E2EE admin room). */ private async handleExportCommand( mapping: RoomMapping, event: MatrixEvent, signal: AbortSignal, ): Promise { if (this.exporter === null || this.exportConfig === null || !this.exportConfig.enabled) { this.logger.info({ roomId: mapping.roomId }, "archive-export: not configured; command ignored"); return null; } try { const powerLevels = await this.matrix.roomPowerLevels(mapping.roomId, signal); if (!senderIsRoomAdmin(powerLevels, event.sender)) { this.logger.warn({ roomId: mapping.roomId, sender: event.sender }, "archive-export: sender is not a room admin"); return null; } } catch (err) { this.logger.warn({ roomId: mapping.roomId, err: errorText(err) }, "archive-export: power-level check failed"); return null; } const result = await this.exporter.exportRoom(mapping, signal, "manual"); this.recordExport(mapping.roomId, result.objectKey); await this.matrix.sendNotice( mapping.roomId, replyTxnId(`export:${result.objectKey}`), exportNotice(result.objectKey, this.exportConfig.r2.bucket, this.exportConfig.ageRecipients), signal, ); return null; } private recordExport(roomId: string, objectKey: string): void { this.lastExportAt = this.now(); this.lastObjectKey = objectKey; this.roomsExported.add(roomId); } /** Rolling auto-export after ARCHIVE_EXPORT_EVERY archived attachments in a room. */ private async maybeAutoExport(mapping: RoomMapping, signal: AbortSignal): Promise { if (this.exporter === null || this.exportConfig === null || !this.exportConfig.enabled) return; this.exporter.countAttachment(mapping.roomId); if (!this.exporter.shouldAutoExport(mapping.roomId, this.exportConfig.every)) return; try { const result = await this.exporter.exportRoom(mapping.roomId === mapping.roomId ? mapping : mapping, signal, "auto"); this.recordExport(mapping.roomId, result.objectKey); await this.matrix.sendNotice( mapping.roomId, replyTxnId(`export:${result.objectKey}`), exportNotice(result.objectKey, this.exportConfig.r2.bucket, this.exportConfig.ageRecipients), signal, ); } catch (err) { this.logger.error({ roomId: mapping.roomId, err: errorText(err) }, "archive-export: automatic export failed"); } } /** Exactly one reply per committed claim; the stable txn ID makes resends idempotent server-side. */ private async sendReply(mapping: RoomMapping, eventId: string, key: string, signal: AbortSignal): Promise { const safeName = this.unsentReplies.get(key); if (safeName === undefined) return null; try { await this.matrix.sendNotice(mapping.roomId, replyTxnId(key), replyContent(safeName, eventId), signal); this.unsentReplies.delete(key); return null; } catch (err) { if (signal.aborted) throw err; this.logger.warn({ roomId: mapping.roomId, eventId, err: errorText(err) }, "archive-bot: reply failed; will retry"); return { blockedUntil: new Date(this.now().getTime() + backoffMs(1)) }; } } }