Add group cloud: files index api, web UI, archive sync bot
The Telegram-like group file contract (packages/core shared-files.ts) becomes a working subsystem, built in three parallel tracks: API (NestJS): migration 029 group_files (chat_media_archive metadata-only index with partial replay-unique + RLS, group_file_shares with sha256 token persistence, archive_sync_state/outbox durable-claim seam incl. a unique claim_key index), group-files module (list/search with cursor pagination, direct upload ≤20 MiB streamed+hashed to oCIS WebDAV, download with ACL + retention rechecks, share links with expiry, retention modes incl. legalHold) — RBAC group-membership authz + audit chain events. Web: Files section per group — group picker, drag-drop upload (20 MiB cap), search + MIME/origin filters, cursor pagination, download via authenticated blob fetch, share dialog with expiry + shown-once token, admin retention dialog; bilingual EN/ES (wiki i18n pattern); nav sidebar/palette/bottom-nav wiring; MIME classes aligned to the DB CHECK (document/other). Archive bot (apps/archive-bot): plain sync-loop Matrix bot mirroring attachments from explicitly-mapped unencrypted archive rooms — raw /sync with durable since-cursor, outbox-claimed ingestion (hash-verified stream, no temp files), magic-byte MIME sniffing, safe-name normalization, WebDAV upload, indexed + replied with link, redaction→unavailable; health endpoint; Dockerfile. Tests 74/74, typecheck clean across api/web/bot.
This commit is contained in:
parent
eb1b60f67d
commit
d58db63fe6
58 changed files with 5543 additions and 4 deletions
235
apps/archive-bot/src/processor.ts
Normal file
235
apps/archive-bot/src/processor.ts
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
import type { Logger } from "pino";
|
||||
import type { Archiver } from "./archiver.js";
|
||||
import { backoffMs, BACKOFF_MAX_MS, MAX_ATTEMPTS, nextAttemptAt } from "./backoff.js";
|
||||
import type { RoomMapping } from "./config.js";
|
||||
import { classifyEvent, type AttachmentWork, type MatrixEvent, type RoomEncryption } from "./events.js";
|
||||
import { PermanentArchiveError } from "./inspector.js";
|
||||
import { claimKey, replyTxnId } from "./keys.js";
|
||||
import { MatrixHttpError, type MatrixPort } from "./matrix.js";
|
||||
import type { ArchiveStore } from "./store.js";
|
||||
|
||||
export interface BatchResult {
|
||||
/** True when the sync cursor was durably advanced past this batch. */
|
||||
advanced: boolean;
|
||||
/** Earliest time a blocked row may be retried; null when nothing is blocked. */
|
||||
retryAt: Date | null;
|
||||
}
|
||||
|
||||
export interface HealthSnapshot {
|
||||
rooms: number;
|
||||
cursor: string | null;
|
||||
pending: number;
|
||||
lastProcessedAt: string | null;
|
||||
}
|
||||
|
||||
type Outcome = { blockedUntil: Date } | null;
|
||||
|
||||
export function replyContent(safeName: string, eventId: string): Record<string, unknown> {
|
||||
return {
|
||||
msgtype: "m.notice",
|
||||
body: `Archived to the group library: ${safeName}`,
|
||||
"m.relates_to": { "m.in_reply_to": { event_id: eventId } },
|
||||
};
|
||||
}
|
||||
|
||||
function errorText(err: unknown): string {
|
||||
return err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
|
||||
function abortableSleep(ms: number, signal: AbortSignal): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
if (signal.aborted) return resolve();
|
||||
const timer = setTimeout(done, ms);
|
||||
function done(): void {
|
||||
clearTimeout(timer);
|
||||
signal.removeEventListener("abort", done);
|
||||
resolve();
|
||||
}
|
||||
signal.addEventListener("abort", done, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync-loop orchestration. The cursor is persisted only after every event in
|
||||
* the batch has reached a durable terminal state (done/failed/skipped) and its
|
||||
* reply was sent; otherwise the same `since` is re-synced after the earliest
|
||||
* per-row backoff, so no attachment is lost to a transient failure.
|
||||
*/
|
||||
export class ArchiveProcessor {
|
||||
private cursor: string | null = null;
|
||||
private lastProcessedAt: Date | null = null;
|
||||
/** Committed claims whose single reply has not been delivered yet (key → safe name). */
|
||||
private readonly unsentReplies = new Map<string, string>();
|
||||
private readonly byRoom: Map<string, RoomMapping>;
|
||||
private readonly workspaceIds: string[];
|
||||
|
||||
constructor(
|
||||
private readonly matrix: MatrixPort,
|
||||
private readonly store: ArchiveStore,
|
||||
private readonly archiver: Archiver,
|
||||
mappings: readonly RoomMapping[],
|
||||
private readonly logger: Logger,
|
||||
private readonly now: () => Date = () => new Date(),
|
||||
) {
|
||||
this.byRoom = new Map(mappings.map((m) => [m.roomId, m]));
|
||||
this.workspaceIds = [...new Set(mappings.map((m) => m.workspaceId))];
|
||||
}
|
||||
|
||||
async init(): Promise<void> {
|
||||
this.cursor = await this.store.loadCursor(this.workspaceIds);
|
||||
}
|
||||
|
||||
async health(): Promise<HealthSnapshot> {
|
||||
return {
|
||||
rooms: this.byRoom.size,
|
||||
cursor: this.cursor,
|
||||
pending: await this.store.countPending(this.workspaceIds),
|
||||
lastProcessedAt: this.lastProcessedAt?.toISOString() ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
async runBatch(signal: AbortSignal): Promise<BatchResult> {
|
||||
const since = this.cursor;
|
||||
const sync = await this.matrix.sync(since, signal);
|
||||
|
||||
for (const roomId of sync.invited) {
|
||||
if (!this.byRoom.has(roomId)) continue;
|
||||
await this.matrix.joinRoom(roomId, signal);
|
||||
this.logger.info({ roomId }, "archive-bot: joined mapped room");
|
||||
}
|
||||
|
||||
let retryAt: Date | null = null;
|
||||
for (const [roomId, room] of sync.joined) {
|
||||
const mapping = this.byRoom.get(roomId);
|
||||
if (mapping === undefined) continue;
|
||||
let events = room.timeline.events;
|
||||
if (room.timeline.limited && room.timeline.prevBatch !== null && since !== null) {
|
||||
events = [...(await this.matrix.backfill(roomId, room.timeline.prevBatch, since, signal)), ...events];
|
||||
}
|
||||
if (events.length === 0) continue;
|
||||
const encryption = await this.encryptionState(mapping, [...room.stateEvents, ...events], signal);
|
||||
for (const event of events) {
|
||||
const outcome = await this.handleEvent(mapping, event, encryption, signal);
|
||||
if (outcome !== null && (retryAt === null || outcome.blockedUntil < retryAt)) retryAt = outcome.blockedUntil;
|
||||
}
|
||||
}
|
||||
|
||||
if (retryAt !== null) return { advanced: false, retryAt };
|
||||
await this.store.saveCursor(this.workspaceIds, sync.nextBatch);
|
||||
this.cursor = sync.nextBatch;
|
||||
return { advanced: true, retryAt: null };
|
||||
}
|
||||
|
||||
/** Long-running loop; returns when `signal` aborts. */
|
||||
async run(signal: AbortSignal, relogin: (signal: AbortSignal) => Promise<void>): Promise<void> {
|
||||
let consecutiveErrors = 0;
|
||||
while (!signal.aborted) {
|
||||
try {
|
||||
const result = await this.runBatch(signal);
|
||||
consecutiveErrors = 0;
|
||||
if (result.retryAt !== null) {
|
||||
const wait = Math.min(Math.max(result.retryAt.getTime() - this.now().getTime(), 1000), BACKOFF_MAX_MS);
|
||||
this.logger.info({ waitMs: wait }, "archive-bot: batch has pending rows; holding cursor");
|
||||
await abortableSleep(wait, signal);
|
||||
}
|
||||
} catch (err) {
|
||||
if (signal.aborted) break;
|
||||
consecutiveErrors += 1;
|
||||
if (err instanceof MatrixHttpError && err.status === 401) {
|
||||
this.logger.warn("archive-bot: access token rejected; logging in again");
|
||||
await relogin(signal).catch((loginErr: unknown) => {
|
||||
this.logger.error({ err: errorText(loginErr) }, "archive-bot: re-login failed");
|
||||
});
|
||||
} else {
|
||||
this.logger.error({ err: errorText(err) }, "archive-bot: batch failed; cursor held");
|
||||
}
|
||||
await abortableSleep(backoffMs(consecutiveErrors), signal);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Fail closed: an encryption event anywhere in this batch wins; otherwise ask the room state. */
|
||||
private async encryptionState(mapping: RoomMapping, events: MatrixEvent[], signal: AbortSignal): Promise<RoomEncryption> {
|
||||
const sawEncryption = events.some((e) => e.type === "m.room.encryption" && e.state_key === "");
|
||||
const state = sawEncryption ? "encrypted" : await this.matrix.roomEncryption(mapping.roomId, signal);
|
||||
if (state !== "unencrypted") {
|
||||
this.logger.warn({ roomId: mapping.roomId, state }, "archive-bot: room not verifiably unencrypted; not mirroring");
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
private async handleEvent(
|
||||
mapping: RoomMapping,
|
||||
event: MatrixEvent,
|
||||
encryption: RoomEncryption,
|
||||
signal: AbortSignal,
|
||||
): Promise<Outcome> {
|
||||
const classified = classifyEvent(event, mapping, encryption, this.matrix.userId);
|
||||
switch (classified.kind) {
|
||||
case "skip":
|
||||
if (classified.reason !== "own-event" && classified.reason !== "not-an-attachment" && classified.reason !== "not-a-message") {
|
||||
this.logger.info({ roomId: mapping.roomId, eventId: event.event_id, reason: classified.reason }, "archive-bot: skipped");
|
||||
}
|
||||
return null;
|
||||
case "redaction": {
|
||||
const key = claimKey(mapping.workspaceId, mapping.roomId, classified.redacts);
|
||||
const affected = await this.store.markRedacted(mapping.workspaceId, mapping.roomId, classified.redacts, key);
|
||||
this.unsentReplies.delete(key);
|
||||
if (affected > 0) {
|
||||
this.logger.info({ roomId: mapping.roomId, eventId: classified.redacts }, "archive-bot: redacted; index marked unavailable");
|
||||
}
|
||||
this.lastProcessedAt = this.now();
|
||||
return null;
|
||||
}
|
||||
case "attachment":
|
||||
return this.handleAttachment(mapping, classified, signal);
|
||||
}
|
||||
}
|
||||
|
||||
private async handleAttachment(mapping: RoomMapping, work: AttachmentWork, signal: AbortSignal): Promise<Outcome> {
|
||||
const eventId = work.event.event_id;
|
||||
const key = claimKey(mapping.workspaceId, mapping.roomId, eventId);
|
||||
const row = await this.store.claim(mapping.workspaceId, mapping.roomId, eventId, key);
|
||||
const log = { roomId: mapping.roomId, eventId, outboxId: row.id };
|
||||
|
||||
if (row.status === "failed") return null;
|
||||
if (row.status === "done") return this.sendReply(mapping, eventId, key, signal);
|
||||
|
||||
const due = nextAttemptAt(row.attempts, row.updatedAt);
|
||||
if (row.attempts > 0 && due > this.now()) return { blockedUntil: due };
|
||||
|
||||
try {
|
||||
const archived = await this.archiver.archive(mapping, work, row, key, signal);
|
||||
this.logger.info({ ...log, fileId: archived.file.id, sha256: archived.file.checksum.value }, "archive-bot: archived");
|
||||
this.lastProcessedAt = this.now();
|
||||
this.unsentReplies.set(key, archived.safeName);
|
||||
return this.sendReply(mapping, eventId, key, signal);
|
||||
} catch (err) {
|
||||
if (signal.aborted) throw err;
|
||||
const terminal = err instanceof PermanentArchiveError || row.attempts + 1 >= MAX_ATTEMPTS;
|
||||
const updated = await this.store.recordFailure(mapping.workspaceId, row.id, errorText(err), terminal);
|
||||
if (terminal) {
|
||||
this.logger.error({ ...log, err: errorText(err), attempts: updated.attempts }, "archive-bot: archive failed permanently");
|
||||
return null;
|
||||
}
|
||||
const retry = nextAttemptAt(updated.attempts, updated.updatedAt);
|
||||
this.logger.warn({ ...log, err: errorText(err), attempts: updated.attempts, retryAt: retry.toISOString() }, "archive-bot: archive failed; will retry");
|
||||
return { blockedUntil: retry };
|
||||
}
|
||||
}
|
||||
|
||||
/** Exactly one reply per committed claim; the stable txn ID makes resends idempotent server-side. */
|
||||
private async sendReply(mapping: RoomMapping, eventId: string, key: string, signal: AbortSignal): Promise<Outcome> {
|
||||
const safeName = this.unsentReplies.get(key);
|
||||
if (safeName === undefined) return null;
|
||||
try {
|
||||
await this.matrix.sendNotice(mapping.roomId, replyTxnId(key), replyContent(safeName, eventId), signal);
|
||||
this.unsentReplies.delete(key);
|
||||
return null;
|
||||
} catch (err) {
|
||||
if (signal.aborted) throw err;
|
||||
this.logger.warn({ roomId: mapping.roomId, eventId, err: errorText(err) }, "archive-bot: reply failed; will retry");
|
||||
return { blockedUntil: new Date(this.now().getTime() + backoffMs(1)) };
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue