Phase B: encrypted archive export bundles (age), power-level-gated export command, encryption-state linter
This commit is contained in:
parent
a7ad1f059f
commit
699b9412c1
22 changed files with 2325 additions and 9 deletions
244
apps/archive-bot/src/exporter.test.ts
Normal file
244
apps/archive-bot/src/exporter.test.ts
Normal file
|
|
@ -0,0 +1,244 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import pino from "pino";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { RoomMapping } from "./config.js";
|
||||
import { BundleExporter, isExportCommand, exportNotice, ExportError } from "./exporter.js";
|
||||
import { senderIsRoomAdmin } from "./encryption-lint.js";
|
||||
import { readTarZstd, drainToBuffer } from "./bundle.js";
|
||||
import { R2ExportStore } from "./r2-export.js";
|
||||
import type { ArchiveIndexPort, IndexedFile, BundleManifest } from "./archive-index.js";
|
||||
import * as age from "age-encryption";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import { afterEach } from "vitest";
|
||||
|
||||
const mapping: RoomMapping = {
|
||||
roomId: "!room:example.org",
|
||||
workspaceId: "11111111-1111-4111-8111-111111111111",
|
||||
groupId: "22222222-2222-4222-8222-222222222222",
|
||||
};
|
||||
|
||||
const PNG = Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3]);
|
||||
const PDF = new TextEncoder().encode("%PDF-1.7\narchive body");
|
||||
|
||||
function indexed(rows: Partial<IndexedFile>[]): IndexedFile[] {
|
||||
return rows.map((row, i) => ({
|
||||
id: `id-${i}`,
|
||||
eventId: `$e${i}`,
|
||||
mxcUri: `mxc://example.org/m${i}`,
|
||||
originalName: `original ${i}.png`,
|
||||
safeName: `safe_${i}.png`,
|
||||
mime: "image/png",
|
||||
mimeClass: "image",
|
||||
byteSize: PNG.byteLength,
|
||||
sha256: createHash("sha256").update(PNG).digest("hex"),
|
||||
sender: "@alice:example.org",
|
||||
createdAt: `2026-01-0${i + 1}T00:00:00.000Z`,
|
||||
ocisResourceId: `ocis-${i}`,
|
||||
available: true,
|
||||
...row,
|
||||
}));
|
||||
}
|
||||
|
||||
class FakeIndex implements ArchiveIndexPort {
|
||||
rows: IndexedFile[] = [];
|
||||
async listRoom(_ws: string, _room: string): Promise<IndexedFile[]> {
|
||||
return this.rows;
|
||||
}
|
||||
}
|
||||
|
||||
class FakeLibrary {
|
||||
files = new Map<string, Uint8Array>();
|
||||
failGet = 0;
|
||||
async getResource(_dir: string[], name: string): Promise<ReadableStream<Uint8Array>> {
|
||||
if (this.failGet > 0) {
|
||||
this.failGet -= 1;
|
||||
throw new Error("WebDAV GET failed");
|
||||
}
|
||||
const bytes = this.files.get(name);
|
||||
if (bytes === undefined) throw new Error(`missing ${name}`);
|
||||
return new Blob([bytes]).stream() as ReadableStream<Uint8Array>;
|
||||
}
|
||||
}
|
||||
|
||||
function s3Capture(): { server: Server; puts: { key: string; length: number; body: Buffer }[]; port: () => number } {
|
||||
const puts: { key: string; length: number; body: Buffer }[] = [];
|
||||
const server = createServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (c: Buffer) => chunks.push(c));
|
||||
req.on("end", () => {
|
||||
puts.push({ key: req.url ?? "", length: Number(req.headers["content-length"] ?? -1), body: Buffer.concat(chunks) });
|
||||
res.writeHead(200).end("{}");
|
||||
});
|
||||
});
|
||||
server.listen(0, "127.0.0.1");
|
||||
return { server, puts, port: () => (server.address() as { port: number }).port };
|
||||
}
|
||||
|
||||
let cleanup: (() => Promise<void>)[] = [];
|
||||
afterEach(async () => {
|
||||
await Promise.all(cleanup.map((fn) => fn().catch(() => undefined)));
|
||||
cleanup = [];
|
||||
});
|
||||
|
||||
describe("isExportCommand", () => {
|
||||
it("matches only the exact admin command (optionally with arguments)", () => {
|
||||
expect(isExportCommand({ body: "!archive export" })).toBe(true);
|
||||
expect(isExportCommand({ body: " !archive export " })).toBe(true);
|
||||
expect(isExportCommand({ body: "!archive export now" })).toBe(true);
|
||||
expect(isExportCommand({ body: "!archive exportall" })).toBe(false);
|
||||
expect(isExportCommand({ body: "!archive" })).toBe(false);
|
||||
expect(isExportCommand({})).toBe(false);
|
||||
expect(isExportCommand({ body: 123 })).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("senderIsRoomAdmin", () => {
|
||||
const levels = { users: { "@admin:x": 100, "@mod:x": 50 }, users_default: 0 };
|
||||
it("requires power level 50+", () => {
|
||||
expect(senderIsRoomAdmin(levels, "@admin:x")).toBe(true);
|
||||
expect(senderIsRoomAdmin(levels, "@mod:x")).toBe(true);
|
||||
expect(senderIsRoomAdmin(levels, "@peasant:x")).toBe(false);
|
||||
expect(senderIsRoomAdmin({}, "@anyone:x")).toBe(false);
|
||||
});
|
||||
it("honours a custom minimum", () => {
|
||||
expect(senderIsRoomAdmin(levels, "@mod:x", 51)).toBe(false);
|
||||
expect(senderIsRoomAdmin(levels, "@admin:x", 51)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("BundleExporter", () => {
|
||||
it("builds an encrypted bundle with a verified manifest and uploads to R2", async () => {
|
||||
const index = new FakeIndex();
|
||||
const library = new FakeLibrary();
|
||||
const rows = indexed([{ sha256: createHash("sha256").update(PNG).digest("hex") }, {
|
||||
eventId: "$e1",
|
||||
safeName: "doc.pdf",
|
||||
sha256: createHash("sha256").update(PDF).digest("hex"),
|
||||
byteSize: PDF.byteLength,
|
||||
}]);
|
||||
index.rows = rows;
|
||||
for (const row of rows) library.files.set(`${row.sha256}-${row.safeName}`, row.eventId === "$e0" ? PNG : PDF);
|
||||
const { server, puts, port } = s3Capture();
|
||||
cleanup.push(async () => await new Promise<void>((r) => server.close(() => r())));
|
||||
await new Promise<void>((r) => server.once("listening", r));
|
||||
const dir = await mkdtemp(join(tmpdir(), "exporter-"));
|
||||
cleanup.push(() => rm(dir, { recursive: true, force: true }));
|
||||
|
||||
const identity = await age.generateIdentity();
|
||||
const recipient = await age.identityToRecipient(identity);
|
||||
const store = new R2ExportStore({
|
||||
endpoint: `http://127.0.0.1:${port()}`, region: "auto", accessKeyId: "k", secretAccessKey: "s", spillDir: dir,
|
||||
allowInsecureEndpoint: true,
|
||||
});
|
||||
const exporter = new BundleExporter(index, library as never, store, [recipient], "hub-matrix-media", pino({ level: "silent" }));
|
||||
const signal = new AbortController().signal;
|
||||
const result = await exporter.exportRoom(mapping, signal, "manual");
|
||||
|
||||
expect(result.fileCount).toBe(2);
|
||||
expect(result.totalBytes).toBe(PNG.byteLength + PDF.byteLength);
|
||||
expect(puts).toHaveLength(1);
|
||||
expect(puts[0]?.key).toContain("/hub-matrix-media/exports/");
|
||||
expect(puts[0]?.key).toContain(encodeURIComponent(mapping.roomId).replace(/!/g, "%21"));
|
||||
expect(puts[0]?.length).toBe(puts[0]?.body.byteLength);
|
||||
// what was uploaded is genuine age ciphertext (binary format, no tar magic)
|
||||
expect(puts[0]?.body.subarray(257, 263).toString("latin1")).not.toBe("ustar\0");
|
||||
expect(puts[0]?.body.subarray(257, 262).toString("latin1")).not.toBe("ustar");
|
||||
expect(puts[0]?.body.length).toBeGreaterThan(PNG.byteLength + PDF.byteLength);
|
||||
|
||||
// end-to-end verify: decrypt the exact uploaded bytes and check manifest + payloads
|
||||
const dec = new age.Decrypter();
|
||||
dec.addIdentity(identity);
|
||||
const plain = await dec.decrypt(new Blob([puts[0]?.body ?? Buffer.alloc(0)]).stream() as ReadableStream<Uint8Array>);
|
||||
const entries = new Map<string, Uint8Array>();
|
||||
for await (const entry of readTarZstd(plain)) entries.set(entry.name, await drainToBuffer(entry.stream));
|
||||
const manifestBytes = entries.get("manifest.json");
|
||||
if (manifestBytes === undefined) throw new Error("manifest missing");
|
||||
const manifest = JSON.parse(new TextDecoder().decode(manifestBytes)) as BundleManifest;
|
||||
expect(manifest.roomId).toBe(mapping.roomId);
|
||||
expect(manifest.recipients).toEqual([recipient]);
|
||||
expect(manifest.files.map((f) => f.safeName)).toEqual(["safe_0.png", "doc.pdf"]);
|
||||
expect(entries.get("files/$e0/safe_0.png")).toEqual(PNG);
|
||||
expect(entries.get("files/$e1/doc.pdf")).toEqual(PDF);
|
||||
});
|
||||
|
||||
it("excludes unavailable (redacted) rows from payloads but records them in the manifest", async () => {
|
||||
const index = new FakeIndex();
|
||||
const library = new FakeLibrary();
|
||||
const rows = indexed([{ available: false }]);
|
||||
index.rows = rows;
|
||||
const { server, port } = s3Capture();
|
||||
cleanup.push(async () => await new Promise<void>((r) => server.close(() => r())));
|
||||
await new Promise<void>((r) => server.once("listening", r));
|
||||
const dir = await mkdtemp(join(tmpdir(), "exporter-"));
|
||||
cleanup.push(() => rm(dir, { recursive: true, force: true }));
|
||||
const identity = await age.generateIdentity();
|
||||
const recipient = await age.identityToRecipient(identity);
|
||||
const store = new R2ExportStore({
|
||||
endpoint: `http://127.0.0.1:${port()}`, region: "auto", accessKeyId: "k", secretAccessKey: "s", spillDir: dir,
|
||||
allowInsecureEndpoint: true,
|
||||
});
|
||||
const exporter = new BundleExporter(index, library as never, store, [recipient], "hub-matrix-media", pino({ level: "silent" }));
|
||||
const result = await exporter.exportRoom(mapping, new AbortController().signal, "auto");
|
||||
expect(result.fileCount).toBe(0);
|
||||
expect(result.totalBytes).toBe(0);
|
||||
});
|
||||
|
||||
it("fails when the library payload is missing or the wrong size", async () => {
|
||||
const index = new FakeIndex();
|
||||
const library = new FakeLibrary();
|
||||
index.rows = indexed([{ byteSize: PNG.byteLength + 5 }]); // declared size ≠ actual
|
||||
for (const row of rows0(index)) library.files.set(`${row.sha256}-${row.safeName}`, PNG);
|
||||
const { server, port } = s3Capture();
|
||||
cleanup.push(async () => await new Promise<void>((r) => server.close(() => r())));
|
||||
await new Promise<void>((r) => server.once("listening", r));
|
||||
const dir = await mkdtemp(join(tmpdir(), "exporter-"));
|
||||
cleanup.push(() => rm(dir, { recursive: true, force: true }));
|
||||
const identity = await age.generateIdentity();
|
||||
const recipient = await age.identityToRecipient(identity);
|
||||
const store = new R2ExportStore({
|
||||
endpoint: `http://127.0.0.1:${port()}`, region: "auto", accessKeyId: "k", secretAccessKey: "s", spillDir: dir,
|
||||
allowInsecureEndpoint: true,
|
||||
});
|
||||
const exporter = new BundleExporter(index, library as never, store, [recipient], "hub-matrix-media", pino({ level: "silent" }));
|
||||
await expect(exporter.exportRoom(mapping, new AbortController().signal, "manual")).rejects.toThrow(ExportError);
|
||||
});
|
||||
|
||||
it("counts attachments and auto-exports on the rolling threshold", () => {
|
||||
const exporter = new BundleExporter(new FakeIndex(), new FakeLibrary() as never, fakeStore(), ["age1" + "q".repeat(58)], "b", pino({ level: "silent" }));
|
||||
expect(exporter.shouldAutoExport(mapping.roomId, 2)).toBe(false);
|
||||
exporter.countAttachment(mapping.roomId);
|
||||
expect(exporter.shouldAutoExport(mapping.roomId, 2)).toBe(false);
|
||||
exporter.countAttachment(mapping.roomId);
|
||||
expect(exporter.shouldAutoExport(mapping.roomId, 2)).toBe(true);
|
||||
// counter reset after firing
|
||||
expect(exporter.shouldAutoExport(mapping.roomId, 2)).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects malformed recipients at construction", () => {
|
||||
expect(() => new BundleExporter(new FakeIndex(), new FakeLibrary() as never, fakeStore(), ["not-a-key"], "b", pino({ level: "silent" })))
|
||||
.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
function rows0(index: FakeIndex): IndexedFile[] {
|
||||
return index.rows;
|
||||
}
|
||||
|
||||
function fakeStore(): R2ExportStore {
|
||||
return new R2ExportStore({
|
||||
endpoint: "https://example.invalid", region: "auto", accessKeyId: "k", secretAccessKey: "s",
|
||||
});
|
||||
}
|
||||
|
||||
describe("exportNotice", () => {
|
||||
it("announces the object key and recipient count without secrets", () => {
|
||||
const notice = exportNotice("exports/!r:x/2026.tar.zst.age", "hub-matrix-media", ["age1a", "age1b"]);
|
||||
expect(notice.msgtype).toBe("m.notice");
|
||||
expect(String(notice.body)).toContain("exports/!r:x/2026.tar.zst.age");
|
||||
expect(String(notice.body)).toContain("2 age recipient");
|
||||
expect(String(notice.body)).not.toContain("AGE-SECRET-KEY");
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue