Add group cloud: files index api, web UI, archive sync bot
The Telegram-like group file contract (packages/core shared-files.ts) becomes a working subsystem, built in three parallel tracks: API (NestJS): migration 029 group_files (chat_media_archive metadata-only index with partial replay-unique + RLS, group_file_shares with sha256 token persistence, archive_sync_state/outbox durable-claim seam incl. a unique claim_key index), group-files module (list/search with cursor pagination, direct upload ≤20 MiB streamed+hashed to oCIS WebDAV, download with ACL + retention rechecks, share links with expiry, retention modes incl. legalHold) — RBAC group-membership authz + audit chain events. Web: Files section per group — group picker, drag-drop upload (20 MiB cap), search + MIME/origin filters, cursor pagination, download via authenticated blob fetch, share dialog with expiry + shown-once token, admin retention dialog; bilingual EN/ES (wiki i18n pattern); nav sidebar/palette/bottom-nav wiring; MIME classes aligned to the DB CHECK (document/other). Archive bot (apps/archive-bot): plain sync-loop Matrix bot mirroring attachments from explicitly-mapped unencrypted archive rooms — raw /sync with durable since-cursor, outbox-claimed ingestion (hash-verified stream, no temp files), magic-byte MIME sniffing, safe-name normalization, WebDAV upload, indexed + replied with link, redaction→unavailable; health endpoint; Dockerfile. Tests 74/74, typecheck clean across api/web/bot.
This commit is contained in:
parent
eb1b60f67d
commit
d58db63fe6
58 changed files with 5543 additions and 4 deletions
267
apps/archive-bot/src/processor.test.ts
Normal file
267
apps/archive-bot/src/processor.test.ts
Normal file
|
|
@ -0,0 +1,267 @@
|
|||
import pino from "pino";
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import { Archiver } from "./archiver.js";
|
||||
import { BACKOFF_BASE_MS } from "./backoff.js";
|
||||
import type { RoomMapping } from "./config.js";
|
||||
import type { MatrixEvent, RoomEncryption } from "./events.js";
|
||||
import { claimKey, replyTxnId } from "./keys.js";
|
||||
import type { MatrixPort, MediaDownload, SyncResult } from "./matrix.js";
|
||||
import { ArchiveProcessor } from "./processor.js";
|
||||
import type { ArchiveRecord, ArchiveStore, OutboxRow } from "./store.js";
|
||||
import type { LibraryPort, LibraryResource } from "./webdav.js";
|
||||
|
||||
const mapping: RoomMapping = {
|
||||
roomId: "!room:example.org",
|
||||
workspaceId: "11111111-1111-4111-8111-111111111111",
|
||||
groupId: "22222222-2222-4222-8222-222222222222",
|
||||
};
|
||||
const BOT = "@archive:example.org";
|
||||
const PNG = Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3]);
|
||||
|
||||
function imageEvent(eventId: string, extra: Partial<MatrixEvent> = {}): MatrixEvent {
|
||||
return {
|
||||
type: "m.room.message",
|
||||
event_id: eventId,
|
||||
sender: "@alice:example.org",
|
||||
origin_server_ts: 1_700_000_000_000,
|
||||
content: { msgtype: "m.image", body: "../cat", url: `mxc://example.org/${eventId.slice(1)}` },
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
class FakeMatrix implements MatrixPort {
|
||||
readonly userId = BOT;
|
||||
batches: SyncResult[] = [];
|
||||
syncedWith: (string | null)[] = [];
|
||||
encryption: RoomEncryption = "unencrypted";
|
||||
notices: { roomId: string; txnId: string; content: Record<string, unknown> }[] = [];
|
||||
failNotices = 0;
|
||||
joined: string[] = [];
|
||||
|
||||
async sync(since: string | null): Promise<SyncResult> {
|
||||
this.syncedWith.push(since);
|
||||
const next = this.batches.shift();
|
||||
if (next === undefined) throw new Error("no batch queued");
|
||||
return next;
|
||||
}
|
||||
async backfill(): Promise<MatrixEvent[]> {
|
||||
return [];
|
||||
}
|
||||
async roomEncryption(): Promise<RoomEncryption> {
|
||||
return this.encryption;
|
||||
}
|
||||
async joinRoom(roomId: string): Promise<void> {
|
||||
this.joined.push(roomId);
|
||||
}
|
||||
async downloadMedia(): Promise<MediaDownload> {
|
||||
return {
|
||||
body: new ReadableStream({ start: (c) => { c.enqueue(PNG); c.close(); } }),
|
||||
contentLength: PNG.byteLength,
|
||||
};
|
||||
}
|
||||
async sendNotice(roomId: string, txnId: string, content: Record<string, unknown>): Promise<void> {
|
||||
if (this.failNotices > 0) {
|
||||
this.failNotices -= 1;
|
||||
throw new Error("notice failed");
|
||||
}
|
||||
this.notices.push({ roomId, txnId, content });
|
||||
}
|
||||
}
|
||||
|
||||
class FakeLibrary implements LibraryPort {
|
||||
failPuts = 0;
|
||||
committed: string[] = [];
|
||||
async putStaging(_dir: string[], _name: string, body: ReadableStream<Uint8Array>): Promise<void> {
|
||||
for await (const chunk of body) void chunk;
|
||||
if (this.failPuts > 0) {
|
||||
this.failPuts -= 1;
|
||||
throw new Error("WebDAV PUT failed: 507");
|
||||
}
|
||||
}
|
||||
async commit(dir: string[], _staging: string, finalName: string): Promise<LibraryResource> {
|
||||
const path = [...dir, finalName].join("/");
|
||||
this.committed.push(path);
|
||||
return { path, url: `https://files/${path}`, resourceId: "rid", version: "v1" };
|
||||
}
|
||||
}
|
||||
|
||||
class FakeStore implements ArchiveStore {
|
||||
cursor: string | null = null;
|
||||
outbox = new Map<string, OutboxRow & { claimKey: string }>();
|
||||
archive: (ArchiveRecord & { available: boolean })[] = [];
|
||||
async loadCursor(): Promise<string | null> {
|
||||
return this.cursor;
|
||||
}
|
||||
async saveCursor(_ws: readonly string[], cursor: string): Promise<void> {
|
||||
this.cursor = cursor;
|
||||
}
|
||||
async claim(_ws: string, _room: string, _event: string, key: string): Promise<OutboxRow> {
|
||||
const existing = this.outbox.get(key);
|
||||
if (existing !== undefined) return { ...existing };
|
||||
const row = { id: `ob${this.outbox.size + 1}`, status: "pending" as const, attempts: 0, lastError: null, updatedAt: new Date(0), claimKey: key };
|
||||
this.outbox.set(key, row);
|
||||
return { ...row };
|
||||
}
|
||||
async commitArchived(record: ArchiveRecord, outboxId: string): Promise<{ id: string; indexedAt: Date }> {
|
||||
this.archive.push({ ...record, available: true });
|
||||
this.byId(outboxId).status = "done";
|
||||
return { id: `file-${this.archive.length}`, indexedAt: new Date(0) };
|
||||
}
|
||||
async recordFailure(_ws: string, outboxId: string, error: string, terminal: boolean): Promise<OutboxRow> {
|
||||
const row = this.byId(outboxId);
|
||||
row.attempts += 1;
|
||||
row.lastError = error;
|
||||
row.status = terminal ? "failed" : "pending";
|
||||
row.updatedAt = clock.now;
|
||||
return { ...row };
|
||||
}
|
||||
async markRedacted(_ws: string, _room: string, eventId: string, key: string): Promise<number> {
|
||||
const rows = this.archive.filter((r) => r.eventId === eventId && r.available);
|
||||
for (const r of rows) r.available = false;
|
||||
const claim = this.outbox.get(key);
|
||||
if (claim?.status === "pending") claim.status = "failed";
|
||||
return rows.length;
|
||||
}
|
||||
async countPending(): Promise<number> {
|
||||
return [...this.outbox.values()].filter((r) => r.status === "pending").length;
|
||||
}
|
||||
private byId(id: string): OutboxRow {
|
||||
const row = [...this.outbox.values()].find((r) => r.id === id);
|
||||
if (row === undefined) throw new Error("no row");
|
||||
return row;
|
||||
}
|
||||
}
|
||||
|
||||
const clock = { now: new Date(1_000_000) };
|
||||
|
||||
function batch(nextBatch: string, events: MatrixEvent[], extra: { stateEvents?: MatrixEvent[]; invited?: string[] } = {}): SyncResult {
|
||||
return {
|
||||
nextBatch,
|
||||
invited: extra.invited ?? [],
|
||||
joined: new Map([[mapping.roomId, { timeline: { events, limited: false, prevBatch: null }, stateEvents: extra.stateEvents ?? [] }]]),
|
||||
};
|
||||
}
|
||||
|
||||
describe("ArchiveProcessor", () => {
|
||||
let matrix: FakeMatrix;
|
||||
let library: FakeLibrary;
|
||||
let store: FakeStore;
|
||||
let processor: ArchiveProcessor;
|
||||
const signal = new AbortController().signal;
|
||||
|
||||
beforeEach(async () => {
|
||||
clock.now = new Date(1_000_000);
|
||||
matrix = new FakeMatrix();
|
||||
library = new FakeLibrary();
|
||||
store = new FakeStore();
|
||||
processor = new ArchiveProcessor(
|
||||
matrix, store, new Archiver(matrix, library, store), [mapping], pino({ level: "silent" }), () => clock.now,
|
||||
);
|
||||
await processor.init();
|
||||
});
|
||||
|
||||
it("archives, indexes, replies once with the stable txn id, then advances the cursor", async () => {
|
||||
matrix.batches.push(batch("s1", [imageEvent("$a")]));
|
||||
expect(await processor.runBatch(signal)).toEqual({ advanced: true, retryAt: null });
|
||||
|
||||
expect(store.cursor).toBe("s1");
|
||||
expect(store.archive).toHaveLength(1);
|
||||
const row = store.archive[0];
|
||||
expect(row).toMatchObject({ mime: "image/png", mimeClass: "image", safeName: "cat.png", originalName: "../cat", byteSize: PNG.byteLength });
|
||||
expect(library.committed[0]).toBe(`Library/${mapping.workspaceId}/${mapping.groupId}/${row?.sha256}-cat.png`);
|
||||
const key = claimKey(mapping.workspaceId, mapping.roomId, "$a");
|
||||
expect(matrix.notices).toEqual([{
|
||||
roomId: mapping.roomId,
|
||||
txnId: replyTxnId(key),
|
||||
content: expect.objectContaining({ body: "Archived to the group library: cat.png" }) as unknown,
|
||||
}]);
|
||||
|
||||
// Replaying the same event (e.g. after a crash before cursor save) neither re-archives nor re-replies.
|
||||
matrix.batches.push(batch("s2", [imageEvent("$a")]));
|
||||
await processor.runBatch(signal);
|
||||
expect(store.archive).toHaveLength(1);
|
||||
expect(matrix.notices).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("holds the cursor on a WebDAV failure, records attempts, and retries after backoff", async () => {
|
||||
library.failPuts = 1;
|
||||
matrix.batches.push(batch("s1", [imageEvent("$a")]));
|
||||
const first = await processor.runBatch(signal);
|
||||
expect(first.advanced).toBe(false);
|
||||
expect(first.retryAt?.getTime()).toBe(clock.now.getTime() + BACKOFF_BASE_MS);
|
||||
expect(store.cursor).toBeNull();
|
||||
const row = [...store.outbox.values()][0];
|
||||
expect(row).toMatchObject({ status: "pending", attempts: 1, lastError: "WebDAV PUT failed: 507" });
|
||||
|
||||
// Re-sync before the row is due: still blocked, nothing uploaded.
|
||||
matrix.batches.push(batch("s1b", [imageEvent("$a")]));
|
||||
expect((await processor.runBatch(signal)).advanced).toBe(false);
|
||||
expect(store.archive).toHaveLength(0);
|
||||
expect(matrix.syncedWith).toEqual([null, null]);
|
||||
|
||||
clock.now = new Date(clock.now.getTime() + BACKOFF_BASE_MS);
|
||||
matrix.batches.push(batch("s1c", [imageEvent("$a")]));
|
||||
expect((await processor.runBatch(signal)).advanced).toBe(true);
|
||||
expect(store.cursor).toBe("s1c");
|
||||
expect(store.archive).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("retries a failed reply without re-archiving", async () => {
|
||||
matrix.failNotices = 1;
|
||||
matrix.batches.push(batch("s1", [imageEvent("$a")]), batch("s2", [imageEvent("$a")]));
|
||||
expect((await processor.runBatch(signal)).advanced).toBe(false);
|
||||
expect(store.archive).toHaveLength(1);
|
||||
expect((await processor.runBatch(signal)).advanced).toBe(true);
|
||||
expect(matrix.notices).toHaveLength(1);
|
||||
expect(store.archive).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("marks permanent payload errors failed and lets the cursor advance", async () => {
|
||||
matrix.downloadMedia = async () => ({
|
||||
body: new ReadableStream({ start: (c) => { c.enqueue(PNG); c.close(); } }),
|
||||
contentLength: PNG.byteLength + 10,
|
||||
});
|
||||
matrix.batches.push(batch("s1", [imageEvent("$a")]));
|
||||
expect((await processor.runBatch(signal)).advanced).toBe(true);
|
||||
expect([...store.outbox.values()][0]).toMatchObject({ status: "failed", attempts: 1 });
|
||||
expect(matrix.notices).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("never mirrors when the room is encrypted or its state is unknown", async () => {
|
||||
matrix.encryption = "unknown";
|
||||
matrix.batches.push(batch("s1", [imageEvent("$a")]));
|
||||
await processor.runBatch(signal);
|
||||
matrix.encryption = "unencrypted";
|
||||
const encryptionEvent: MatrixEvent = {
|
||||
type: "m.room.encryption", event_id: "$enc", sender: "@admin:x", state_key: "", content: { algorithm: "m.megolm.v1.aes-sha2" },
|
||||
};
|
||||
matrix.batches.push(batch("s2", [imageEvent("$b"), encryptionEvent]));
|
||||
await processor.runBatch(signal);
|
||||
expect(store.outbox.size).toBe(0);
|
||||
expect(store.archive).toHaveLength(0);
|
||||
expect(store.cursor).toBe("s2");
|
||||
});
|
||||
|
||||
it("ignores its own events", async () => {
|
||||
matrix.batches.push(batch("s1", [imageEvent("$a", { sender: BOT })]));
|
||||
await processor.runBatch(signal);
|
||||
expect(store.outbox.size).toBe(0);
|
||||
});
|
||||
|
||||
it("marks redacted attachments unavailable without touching the library", async () => {
|
||||
matrix.batches.push(batch("s1", [imageEvent("$a")]));
|
||||
await processor.runBatch(signal);
|
||||
const redaction: MatrixEvent = { type: "m.room.redaction", event_id: "$r", sender: "@mod:x", content: { redacts: "$a" } };
|
||||
matrix.batches.push(batch("s2", [redaction]));
|
||||
await processor.runBatch(signal);
|
||||
expect(store.archive[0]?.available).toBe(false);
|
||||
expect(library.committed).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("joins invites only for mapped rooms and reports health", async () => {
|
||||
matrix.batches.push({ nextBatch: "s1", joined: new Map(), invited: [mapping.roomId, "!other:x"] });
|
||||
await processor.runBatch(signal);
|
||||
expect(matrix.joined).toEqual([mapping.roomId]);
|
||||
expect(await processor.health()).toEqual({ rooms: 1, cursor: "s1", pending: 0, lastProcessedAt: null });
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue