Add group cloud: files index api, web UI, archive sync bot
The Telegram-like group file contract (packages/core shared-files.ts) becomes a working subsystem, built in three parallel tracks: API (NestJS): migration 029 group_files (chat_media_archive metadata-only index with partial replay-unique + RLS, group_file_shares with sha256 token persistence, archive_sync_state/outbox durable-claim seam incl. a unique claim_key index), group-files module (list/search with cursor pagination, direct upload ≤20 MiB streamed+hashed to oCIS WebDAV, download with ACL + retention rechecks, share links with expiry, retention modes incl. legalHold) — RBAC group-membership authz + audit chain events. Web: Files section per group — group picker, drag-drop upload (20 MiB cap), search + MIME/origin filters, cursor pagination, download via authenticated blob fetch, share dialog with expiry + shown-once token, admin retention dialog; bilingual EN/ES (wiki i18n pattern); nav sidebar/palette/bottom-nav wiring; MIME classes aligned to the DB CHECK (document/other). Archive bot (apps/archive-bot): plain sync-loop Matrix bot mirroring attachments from explicitly-mapped unencrypted archive rooms — raw /sync with durable since-cursor, outbox-claimed ingestion (hash-verified stream, no temp files), magic-byte MIME sniffing, safe-name normalization, WebDAV upload, indexed + replied with link, redaction→unavailable; health endpoint; Dockerfile. Tests 74/74, typecheck clean across api/web/bot.
This commit is contained in:
parent
eb1b60f67d
commit
d58db63fe6
58 changed files with 5543 additions and 4 deletions
242
apps/api/src/group-files/group-files.test.ts
Normal file
242
apps/api/src/group-files/group-files.test.ts
Normal file
|
|
@ -0,0 +1,242 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { BadRequestException, ServiceUnavailableException } from "@nestjs/common";
|
||||
import { Readable } from "node:stream";
|
||||
import {
|
||||
decodeFileListCursor,
|
||||
encodeFileListCursor,
|
||||
} from "./file-cursor.js";
|
||||
import {
|
||||
parseGroupFile,
|
||||
parseMatrixAttachmentCandidate,
|
||||
parseRetention,
|
||||
parseRetentionInput,
|
||||
parseSharedFileOrigin,
|
||||
requireMxid,
|
||||
requireMxcUri,
|
||||
requireRoomId,
|
||||
requireSha256,
|
||||
} from "./group-files.validation.js";
|
||||
import { OcisVerificationError, ocisConfigFromEnv, OcisWebdavClient, verifyWhileStreaming } from "./ocis-webdav.js";
|
||||
import { hashShareToken, isPlausibleShareToken, mintShareToken, sameToken } from "./share-token.js";
|
||||
|
||||
/** Deterministic 64-hex helper: repeat a pattern to exactly 64 chars. */
|
||||
function hex64(seed: string): string {
|
||||
let out = "";
|
||||
while (out.length < 64) out += Buffer.from(seed).toString("hex");
|
||||
return out.slice(0, 64);
|
||||
}
|
||||
|
||||
describe("group-files runtime validation (shared contracts)", () => {
|
||||
it("accepts the four SharedFileOrigin variants and rejects anything else", () => {
|
||||
expect(parseSharedFileOrigin({ kind: "matrix", mxcUri: "mxc://example.org/abc" })).toEqual({
|
||||
kind: "matrix",
|
||||
mxcUri: "mxc://example.org/abc",
|
||||
});
|
||||
expect(
|
||||
parseSharedFileOrigin({ kind: "bridge", mxcUri: "mxc://example.org/abc", providerId: "telegram" }),
|
||||
).toEqual({ kind: "bridge", mxcUri: "mxc://example.org/abc", providerId: "telegram" });
|
||||
expect(parseSharedFileOrigin({ kind: "bridge", mxcUri: "mxc://example.org/abc", providerId: "t", remoteEventId: "42" })).toEqual({
|
||||
kind: "bridge",
|
||||
mxcUri: "mxc://example.org/abc",
|
||||
providerId: "t",
|
||||
remoteEventId: "42",
|
||||
});
|
||||
expect(parseSharedFileOrigin({ kind: "upload" })).toEqual({ kind: "upload" });
|
||||
expect(parseSharedFileOrigin({ kind: "pad-export", padId: "p", revision: "7" })).toEqual({
|
||||
kind: "pad-export",
|
||||
padId: "p",
|
||||
revision: "7",
|
||||
});
|
||||
for (const bad of [null, {}, { kind: "nope" }, { kind: "matrix" }, { kind: "matrix", mxcUri: "https://evil/x" }]) {
|
||||
expect(() => parseSharedFileOrigin(bad)).toThrow(BadRequestException);
|
||||
}
|
||||
});
|
||||
|
||||
it("validates MXIDs, room ids, mxc URIs and sha256 shape", () => {
|
||||
expect(requireMxid("@alice:example.org")).toBe("@alice:example.org");
|
||||
expect(() => requireMxid("Alice (display)")).toThrow(BadRequestException);
|
||||
expect(() => requireMxid("#alias:example.org")).toThrow(BadRequestException);
|
||||
expect(requireRoomId("!archive:example.org")).toBe("!archive:example.org");
|
||||
expect(() => requireRoomId("#alias:example.org")).toThrow(BadRequestException); // alias, never an identity
|
||||
expect(() => requireRoomId("")).toThrow(BadRequestException);
|
||||
expect(requireMxcUri("mxc://example.org/AbC123")).toBe("mxc://example.org/AbC123");
|
||||
expect(() => requireMxcUri("mxc://example.org/")).toThrow(BadRequestException);
|
||||
expect(requireSha256(hex64("a"))).toBe(hex64("a"));
|
||||
expect(() => requireSha256("nope")).toThrow(BadRequestException);
|
||||
// hex digits include letters: an uppercase digest must be rejected
|
||||
const upperHex = "A".repeat(64);
|
||||
expect(/[A-F]/.test(upperHex)).toBe(true);
|
||||
expect(() => requireSha256(upperHex)).toThrow(BadRequestException);
|
||||
expect(() => requireSha256("0".repeat(63))).toThrow(BadRequestException); // wrong length
|
||||
});
|
||||
|
||||
it("keeps retention separate from link expiry and enforces shape coupling", () => {
|
||||
expect(parseRetention({ mode: "keep", legalHold: false })).toEqual({ mode: "keep", legalHold: false });
|
||||
// legalHold is a FLAG alongside the mode, not a third mode (FileRetention contract)
|
||||
expect(parseRetention({ mode: "keep", legalHold: true })).toEqual({ mode: "keep", legalHold: true });
|
||||
expect(parseRetention({ mode: "until", legalHold: true, expiresAt: "2030-01-01T00:00:00Z" })).toEqual({
|
||||
mode: "until",
|
||||
legalHold: true,
|
||||
expiresAt: "2030-01-01T00:00:00Z",
|
||||
});
|
||||
expect(() => parseRetention({ mode: "burn", legalHold: false })).toThrow(BadRequestException);
|
||||
expect(() => parseRetention({ mode: "until", legalHold: false })).toThrow(BadRequestException);
|
||||
expect(() => parseRetention({ mode: "keep", legalHold: false, expiresAt: "2030-01-01T00:00:00Z" })).toThrow(BadRequestException);
|
||||
expect(() => parseRetention({ mode: "until", legalHold: false, expiresAt: "next tuesday" })).toThrow(BadRequestException);
|
||||
// input variant
|
||||
expect(parseRetentionInput({ mode: "until", until: "2030-01-01T00:00:00Z" })).toEqual({
|
||||
mode: "until",
|
||||
legalHold: false,
|
||||
expiresAt: "2030-01-01T00:00:00Z",
|
||||
});
|
||||
expect(() => parseRetentionInput({ mode: "until" })).toThrow(BadRequestException);
|
||||
expect(() => parseRetentionInput({ mode: "keep", until: "2030-01-01T00:00:00Z" })).toThrow(BadRequestException);
|
||||
expect(parseRetentionInput({ mode: "keep", legalHold: true })).toEqual({ mode: "keep", legalHold: true });
|
||||
expect(() => parseRetentionInput({ mode: "legalHold" })).toThrow(BadRequestException); // not a mode
|
||||
});
|
||||
|
||||
it("fails closed on Matrix attachment candidates with unknown/bad state", () => {
|
||||
const good = {
|
||||
roomId: "!a:example.org",
|
||||
eventId: "$1",
|
||||
senderId: "@s:example.org",
|
||||
eventType: "m.room.message",
|
||||
msgtype: "m.image",
|
||||
roomEncryption: "unencrypted",
|
||||
encryptedFile: false,
|
||||
redacted: false,
|
||||
mxcUri: "mxc://example.org/x",
|
||||
};
|
||||
expect(parseMatrixAttachmentCandidate(good).roomEncryption).toBe("unencrypted");
|
||||
// shape-level failures
|
||||
for (const patch of [
|
||||
{ roomId: "#alias:example.org" },
|
||||
{ senderId: "Bob" },
|
||||
{ mxcUri: "https://attacker/x" },
|
||||
{ encryptedFile: "yes" },
|
||||
]) {
|
||||
expect(() => parseMatrixAttachmentCandidate({ ...good, ...patch })).toThrow(BadRequestException);
|
||||
}
|
||||
// state-level eligibility is the CORE helper's call, not the parser's:
|
||||
// parse accepts encrypted/unknown/redacted states; canArchiveAttachment
|
||||
// is what fails closed on them (core test shared-files.test.ts).
|
||||
const parsed = parseMatrixAttachmentCandidate({ ...good, roomEncryption: "unknown" });
|
||||
expect(parsed.roomEncryption).toBe("unknown");
|
||||
});
|
||||
|
||||
it("round-trips a full GroupFile wire shape", () => {
|
||||
const value = {
|
||||
id: "b3a1c2d4-1111-4222-8333-444455556666",
|
||||
workspaceId: "c3a1c2d4-1111-4222-8333-444455556666",
|
||||
groupId: "d3a1c2d4-1111-4222-8333-444455556666",
|
||||
roomId: "!r:example.org",
|
||||
senderId: "@s:example.org",
|
||||
eventId: "$evt",
|
||||
origin: { kind: "matrix", mxcUri: "mxc://example.org/x" },
|
||||
name: "clip.mp4",
|
||||
mimeType: "video/mp4",
|
||||
sizeBytes: 12,
|
||||
checksum: { algorithm: "sha256", value: hex64("z") },
|
||||
createdAt: "2026-01-02T03:04:05Z",
|
||||
indexedAt: "2026-01-02T03:04:05Z",
|
||||
file: { spaceId: "sp", resourceId: "res", version: "v1" },
|
||||
shareLink: { url: "https://files.example.org/f/res", access: "members", expiresAt: null },
|
||||
retention: { mode: "keep", legalHold: false },
|
||||
status: "available",
|
||||
};
|
||||
expect(parseGroupFile(value)).toEqual(value);
|
||||
expect(() => parseGroupFile({ ...value, sizeBytes: -1 })).toThrow(BadRequestException);
|
||||
expect(() => parseGroupFile({ ...value, senderId: "Bob" })).toThrow(BadRequestException);
|
||||
expect(() => parseGroupFile({ ...value, checksum: { algorithm: "sha256", value: "zz" } })).toThrow(BadRequestException);
|
||||
});
|
||||
});
|
||||
|
||||
describe("file-cursor", () => {
|
||||
it("encodes/decodes (createdAt,id) and rejects malformed scopes", () => {
|
||||
const cursor = { createdAt: "2026-01-02T03:04:05Z", id: "b3a1c2d4-1111-4222-8333-444455556666" };
|
||||
expect(decodeFileListCursor(encodeFileListCursor(cursor))).toEqual(cursor);
|
||||
for (const bad of ["", "!!!", Buffer.from(JSON.stringify({ createdAt: "nope", id: "x" })).toString("base64url")]) {
|
||||
expect(() => decodeFileListCursor(bad)).toThrow(BadRequestException);
|
||||
}
|
||||
// a cursor from another tenant carries no scope: it is just an empty page,
|
||||
// never a leak — the scope is re-bound server-side on every request.
|
||||
expect(decodeFileListCursor(encodeFileListCursor({ createdAt: "2026-01-02T03:04:05Z", id: "00000000-0000-4000-8000-000000000000" })).id).toBe(
|
||||
"00000000-0000-4000-8000-000000000000",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("share tokens", () => {
|
||||
it("mints 256-bit capabilities and stores only sha256(token)", () => {
|
||||
const token = mintShareToken();
|
||||
expect(token.length).toBeGreaterThanOrEqual(40);
|
||||
expect(isPlausibleShareToken(token)).toBe(true);
|
||||
expect(isPlausibleShareToken("short")).toBe(false);
|
||||
expect(isPlausibleShareToken("not base64url!!")).toBe(false);
|
||||
const hash = hashShareToken(token);
|
||||
expect(hash).toMatch(/^[0-9a-f]{64}$/);
|
||||
expect(hashShareToken(token)).toBe(hash); // deterministic
|
||||
expect(hashShareToken(mintShareToken())).not.toBe(hash); // unique per mint
|
||||
expect(sameToken(token, token)).toBe(true);
|
||||
expect(sameToken(token, mintShareToken())).toBe(false);
|
||||
// the plaintext can never be derived from the hash
|
||||
expect(hash).not.toContain(token);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ocis-webdav config + streaming verification", () => {
|
||||
it("reads env lazily and throws 503 when unconfigured", () => {
|
||||
expect(() => ocisConfigFromEnv({})).toThrow(ServiceUnavailableException);
|
||||
expect(() => ocisConfigFromEnv({ FILES_OCID_WEBDAV_URL: "https://files.example.org" })).toThrow(ServiceUnavailableException);
|
||||
const config = ocisConfigFromEnv({
|
||||
FILES_OCID_WEBDAV_URL: "https://files.example.org/",
|
||||
FILES_OCID_USER: "svc",
|
||||
FILES_OCID_PASSWORD: "pw",
|
||||
FILES_PUBLIC_BASE: "https://files.example.org/",
|
||||
});
|
||||
expect(config.webdavUrl).toBe("https://files.example.org");
|
||||
expect(config.publicBase).toBe("https://files.example.org");
|
||||
});
|
||||
|
||||
it("uploads hash the actual streamed bytes and surface oc-fileid/oc-etag", async () => {
|
||||
const bytes = Buffer.from("group file payload");
|
||||
const fetcher = (async () =>
|
||||
new Response(null, {
|
||||
status: 201,
|
||||
headers: { "oc-fileid": "res-1", "oc-etag": "v-9" },
|
||||
})) as unknown as typeof fetch;
|
||||
const client = new OcisWebdavClient(
|
||||
{ webdavUrl: "https://files.example.org", user: "svc", password: "pw", publicBase: "https://files.example.org" },
|
||||
fetcher,
|
||||
);
|
||||
const result = await client.upload("group-1", "clip.mp4", Readable.from([bytes]), "video/mp4");
|
||||
const { createHash } = await import("node:crypto");
|
||||
expect(result.sha256).toBe(createHash("sha256").update(bytes).digest("hex"));
|
||||
expect(result.sizeBytes).toBe(bytes.byteLength);
|
||||
expect(result.resourceId).toBe("res-1");
|
||||
expect(result.version).toBe("v-9");
|
||||
expect(client.uploadUrl("g", "n.mp4")).toContain("/remote.php/dav/files/svc/hub-groups/g/n.mp4");
|
||||
expect(client.libraryUrl("res-1")).toBe("https://files.example.org/f/res-1");
|
||||
});
|
||||
|
||||
it("verifyWhileStreaming passes on match and aborts on tamper", async () => {
|
||||
const { createHash } = await import("node:crypto");
|
||||
const payload = Buffer.from("0123456789");
|
||||
const good = createHash("sha256").update(payload).digest("hex");
|
||||
const sinkChunks: Buffer[] = [];
|
||||
const out = await verifyWhileStreaming(
|
||||
{ stream: Readable.from([payload.subarray(0, 4), payload.subarray(4)]), sizeBytes: payload.byteLength, contentType: "application/octet-stream" },
|
||||
good,
|
||||
(chunk) => sinkChunks.push(chunk),
|
||||
);
|
||||
expect(out.sizeBytes).toBe(10);
|
||||
expect(Buffer.concat(sinkChunks).toString()).toBe("0123456789");
|
||||
await expect(
|
||||
verifyWhileStreaming(
|
||||
{ stream: Readable.from([Buffer.from("tampered")]), sizeBytes: 8, contentType: null },
|
||||
good,
|
||||
() => undefined,
|
||||
),
|
||||
).rejects.toBeInstanceOf(OcisVerificationError);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue