Add group cloud: files index api, web UI, archive sync bot

The Telegram-like group file contract (packages/core shared-files.ts) becomes
a working subsystem, built in three parallel tracks:

API (NestJS): migration 029 group_files (chat_media_archive metadata-only
index with partial replay-unique + RLS, group_file_shares with sha256 token
persistence, archive_sync_state/outbox durable-claim seam incl. a unique
claim_key index), group-files module (list/search with cursor pagination,
direct upload ≤20 MiB streamed+hashed to oCIS WebDAV, download with ACL +
retention rechecks, share links with expiry, retention modes incl.
legalHold) — RBAC group-membership authz + audit chain events.

Web: Files section per group — group picker, drag-drop upload (20 MiB cap),
search + MIME/origin filters, cursor pagination, download via authenticated
blob fetch, share dialog with expiry + shown-once token, admin retention
dialog; bilingual EN/ES (wiki i18n pattern); nav sidebar/palette/bottom-nav
wiring; MIME classes aligned to the DB CHECK (document/other).

Archive bot (apps/archive-bot): plain sync-loop Matrix bot mirroring
attachments from explicitly-mapped unencrypted archive rooms — raw /sync
with durable since-cursor, outbox-claimed ingestion (hash-verified stream,
no temp files), magic-byte MIME sniffing, safe-name normalization, WebDAV
upload, indexed + replied with link, redaction→unavailable; health endpoint;
Dockerfile. Tests 74/74, typecheck clean across api/web/bot.
This commit is contained in:
glass beadian 2026-10-11 08:21:11 +00:00
parent eb1b60f67d
commit d58db63fe6
58 changed files with 5543 additions and 4 deletions

View file

@ -0,0 +1,573 @@
/**
* Group files service — the metadata index + share/link lifecycle over
* chat_media_archive (029) and the oCIS WebDAV port.
*
* Invariants this service owns (docs/occupymedia-storage.md):
* * List/search return ONLY available, unexpired rows visible through
* current group/file ACLs — expired/retention-past/revoked rows never
* surface, and non-Owner/Admin callers only see their groups' files.
* * Attribution by sender MXID; origin is one of the four SharedFileOrigin
* kinds, reconstructed from origin + matrix_event_json.
* * Retention is SEPARATE from link expiry: an expired share never deletes
* or hides a file; 'legalHold' blocks deletion outright.
* * Share tokens: plaintext returned ONCE, sha256(token) persisted, public
* access requires an explicit grant + bounded expiry, revoke stops
* serving immediately.
* * Downloads stream through the API with the SHA-256 re-verified; no
* presigned object URL is ever minted or stored.
*
* Every mutation lands its audit row on the SAME tenant transaction
* (audit_append), the house pattern.
*/
import { BadRequestException, ConflictException, ForbiddenException, GoneException, Injectable, NotFoundException, Optional, Inject, ServiceUnavailableException } from "@nestjs/common";
import { createHash } from "node:crypto";
import type { PoolClient } from "pg";
import { attachmentArchiveKey, type FileRetention, type GroupFile } from "@tactical-media-broadcast-hub/core";
import { appendTenantAudit, type AuditEvent } from "../audit/audit.service.js";
import { withSystem, withTenant } from "../db/client.js";
import { isUuid, parseLimit } from "../http/validation.js";
import { decodeFileListCursor, encodeFileListCursor } from "./file-cursor.js";
import { ocisConfigFromEnv, OcisWebdavClient, type FetchLike, type OcisDownload } from "./ocis-webdav.js";
import { hashShareToken, mintShareToken } from "./share-token.js";
import { MIME_CLASSES, parseRetentionInput, type MimeClass } from "./group-files.validation.js";
export const GROUP_FILES_DEPENDENCIES: unique symbol = Symbol("GROUP_FILES_DEPENDENCIES");
/** Wraps the raw payload handle the controller streams out. */
export interface FileDownloadHandle {
item: GroupFile;
download: OcisDownload;
}
export interface ShareCreated {
id: string;
itemId: string;
access: "members" | "public";
url: string;
expiresAt: string;
revokedAt: string | null;
/** Plaintext capability — returned ONCE here, never persisted anywhere. */
token: string;
}
export interface FileListFilters {
groupId: string | null;
senderMxid: string | null;
mimeClass: MimeClass | null;
name: string | null;
from: string | null;
to: string | null;
}
export interface FilePage {
items: GroupFile[];
nextCursor: string | null;
}
const MAX_SHARE_TTL_HOURS = 720;
const WORKSPACE_WIDE_ROLES: readonly string[] = ["Owner", "Admin"];
interface ArchiveRow {
id: string;
workspace_id: string;
group_id: string;
room_id: string;
event_id: string | null;
origin: string;
sender_mxid: string;
matrix_event_json: Record<string, unknown> | null;
original_name: string;
safe_name: string;
mime: string;
mime_class: string;
byte_size: string | number;
sha256: string;
ocis_resource_id: string;
ocis_resource_version: string | null;
created_at: Date;
indexed_at: Date;
available: boolean;
retention_mode: string;
legal_hold: boolean;
retention_until: Date | null;
share_access: string | null;
share_expires_at: Date | null;
share_revoked_at: Date | null;
}
interface ShareRow {
id: string;
workspace_id: string;
item_id: string;
access: string;
expires_at: Date;
revoked_at: Date | null;
}
const ARCHIVE_SELECT = `SELECT a.*, s.access AS share_access, s.expires_at AS share_expires_at, s.revoked_at AS share_revoked_at
FROM chat_media_archive a
LEFT JOIN LATERAL (
SELECT access, expires_at, revoked_at FROM group_file_shares
WHERE item_id = a.id AND revoked_at IS NULL AND expires_at > now()
ORDER BY created_at DESC LIMIT 1
) s ON true`;
function iso(value: Date): string {
return new Date(value).toISOString();
}
function safeNumber(value: string | number): number {
const n = typeof value === "number" ? value : Number(value);
if (!Number.isSafeInteger(n)) throw new Error(`unsafe numeric column: ${String(value)}`);
return n;
}
/** Row → GroupFile, reconstructing origin + retention + share link. */
function rowToGroupFile(row: ArchiveRow, libraryUrl: (resourceId: string) => string): GroupFile {
const payload = row.matrix_event_json ?? {};
const origin: GroupFile["origin"] = (() => {
if (row.origin === "matrix") return { kind: "matrix", mxcUri: String(payload.mxcUri ?? "") };
if (row.origin === "bridge") {
return {
kind: "bridge",
mxcUri: String(payload.mxcUri ?? ""),
providerId: String(payload.providerId ?? ""),
...(payload.remoteEventId === undefined ? {} : { remoteEventId: String(payload.remoteEventId) }),
};
}
if (row.origin === "pad-export") {
return { kind: "pad-export", padId: String(payload.padId ?? ""), revision: String(payload.revision ?? "") };
}
return { kind: "upload" };
})();
const retention: FileRetention =
row.retention_mode === "until" && row.retention_until !== null
? { mode: "until", legalHold: row.legal_hold, expiresAt: iso(row.retention_until) }
: { mode: "keep", legalHold: row.legal_hold };
const status: GroupFile["status"] = !row.available
? "revoked"
: retention.mode === "until" && new Date(retention.expiresAt).getTime() <= Date.now()
? "expired"
: "available";
return {
id: row.id,
workspaceId: row.workspace_id,
groupId: row.group_id,
roomId: row.room_id,
senderId: row.sender_mxid,
eventId: row.event_id,
origin,
name: row.original_name,
mimeType: row.mime,
sizeBytes: safeNumber(row.byte_size),
checksum: { algorithm: "sha256", value: row.sha256 },
createdAt: iso(row.created_at),
indexedAt: iso(row.indexed_at),
file: {
spaceId: row.group_id,
resourceId: row.ocis_resource_id,
version: row.ocis_resource_version ?? row.ocis_resource_id,
},
shareLink: {
url: libraryUrl(row.ocis_resource_id),
access: row.share_access === "public" ? "public" : "members",
expiresAt: row.share_expires_at === null ? null : iso(row.share_expires_at),
},
retention,
status,
};
}
@Injectable()
export class GroupFilesService {
private client: OcisWebdavClient | null = null;
constructor(@Optional() @Inject(GROUP_FILES_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike }) {}
/** Lazily bound: the oCIS env may be absent in dev/test contexts. */
private webdav(): OcisWebdavClient {
if (this.client === null) {
const fetcher: FetchLike = this.deps?.fetcher ?? ((url, init) => fetch(url, init));
this.client = new OcisWebdavClient(ocisConfigFromEnv(), fetcher);
}
return this.client;
}
// -------------------------------------------------------------------------
// Read paths
// -------------------------------------------------------------------------
/**
* Keyset page over the visible slice: available, unexpired, and — for
* non-Owner/Admin callers — restricted to the caller's groups (the ACL
* rule "List/search return only available, unexpired rows visible through
* current group/file ACLs").
*/
async listFiles(
workspaceId: string,
userId: string,
role: string,
query: Record<string, unknown>,
): Promise<FilePage> {
const filters = parseFilters(query);
const limit = parseLimit(query.limit, 50, 100);
const cursor = typeof query.cursor === "string" && query.cursor !== "" ? decodeFileListCursor(query.cursor) : undefined;
return withTenant(workspaceId, async (client) => {
// values[1] is always the caller's id: the membership subquery needs it
// for non-Owner/Admin callers; Owner/Admin leave $2 bound but unused so
// every placeholder stays positional and no SQL text carries a value.
const values: unknown[] = [workspaceId, userId];
const conds: string[] = ["a.available = true", "(a.retention_mode <> 'until' OR a.retention_until > now())"];
if (!WORKSPACE_WIDE_ROLES.includes(role)) {
conds.push("a.group_id IN (SELECT group_id FROM group_members WHERE user_id = $2 AND status = 'active')");
}
if (filters.groupId !== null) {
values.push(filters.groupId);
conds.push(`a.group_id = $${values.length}::uuid`);
}
if (filters.senderMxid !== null) {
values.push(filters.senderMxid);
conds.push(`a.sender_mxid = $${values.length}`);
}
if (filters.mimeClass !== null) {
values.push(filters.mimeClass);
conds.push(`a.mime_class = $${values.length}`);
}
if (filters.name !== null) {
values.push(`%${filters.name}%`);
conds.push(`a.safe_name ILIKE $${values.length}`);
}
if (filters.from !== null) {
values.push(filters.from);
conds.push(`a.created_at >= $${values.length}::timestamptz`);
}
if (filters.to !== null) {
values.push(filters.to);
conds.push(`a.created_at < $${values.length}::timestamptz`);
}
if (cursor !== undefined) {
values.push(cursor.createdAt, cursor.id);
conds.push(`(a.created_at, a.id) < ($${values.length - 1}::timestamptz, $${values.length}::uuid)`);
}
values.push(limit + 1);
const sql = `${ARCHIVE_SELECT}
WHERE ${conds.join(" AND ")}
ORDER BY a.created_at DESC, a.id DESC
LIMIT $${values.length}`;
const result = await client.query<ArchiveRow>(sql, values);
const page = result.rows.slice(0, limit);
const last = page.at(-1);
const lib = (resourceId: string): string => this.webdav().libraryUrl(resourceId);
return {
items: page.map((row) => rowToGroupFile(row, lib)),
nextCursor: result.rows.length > limit && last !== undefined
? encodeFileListCursor({ createdAt: iso(last.created_at), id: last.id })
: null,
};
});
}
async getFile(workspaceId: string, userId: string, role: string, itemId: string): Promise<GroupFile> {
return withTenant(workspaceId, async (client) => {
const row = await this.requireRow(client, itemId);
await this.requireAccess(client, userId, role, row.group_id);
return rowToGroupFile(row, (resourceId) => this.webdav().libraryUrl(resourceId));
});
}
/** Open the payload for streaming; membership/ACL + retention rechecked here. */
async openDownload(workspaceId: string, userId: string, role: string, itemId: string): Promise<FileDownloadHandle> {
const item = await this.getFile(workspaceId, userId, role, itemId);
if (item.status !== "available") throw new GoneException(`file is ${item.status}`);
const download = await this.webdav().download(item.file.resourceId).catch(() => {
throw new ServiceUnavailableException("file storage is unavailable");
});
return { item, download };
}
// -------------------------------------------------------------------------
// Retention / availability
// -------------------------------------------------------------------------
async setRetention(workspaceId: string, itemId: string, retentionInput: unknown, actor: ActorRef): Promise<GroupFile> {
const retention = parseRetentionInput(retentionInput);
return withTenant(workspaceId, async (client) => {
const before = await this.requireRow(client, itemId);
await client.query(
`UPDATE chat_media_archive
SET retention_mode = $2, legal_hold = $3, retention_until = $4
WHERE id = $1`,
[itemId, retention.mode, retention.legalHold, retention.mode === "until" ? new Date(retention.expiresAt) : null],
);
await this.audit(client, workspaceId, actor, {
action: "file.retention_changed",
targetId: itemId,
diff: {
from: { mode: before.retention_mode, legalHold: before.legal_hold, until: before.retention_until === null ? null : iso(before.retention_until) },
to: { mode: retention.mode, legalHold: retention.legalHold, until: retention.mode === "until" ? retention.expiresAt : null },
name: before.safe_name,
},
});
const after = await this.requireRow(client, itemId);
return rowToGroupFile(after, (resourceId) => this.webdav().libraryUrl(resourceId));
});
}
/** Redaction/membership-removal path: revoke links, stop serving. No deletion. */
async markRevoked(workspaceId: string, itemId: string, actor: ActorRef): Promise<GroupFile> {
return withTenant(workspaceId, async (client) => {
const before = await this.requireRow(client, itemId);
if (!before.available) throw new ConflictException("file is already revoked");
await client.query("UPDATE chat_media_archive SET available = false WHERE id = $1", [itemId]);
await client.query("UPDATE group_file_shares SET revoked_at = now() WHERE item_id = $1 AND revoked_at IS NULL", [itemId]);
await this.audit(client, workspaceId, actor, {
action: "file.revoked",
targetId: itemId,
diff: { name: before.safe_name, roomId: before.room_id, eventId: before.event_id },
});
const after = await this.requireRow(client, itemId);
return rowToGroupFile(after, (resourceId) => this.webdav().libraryUrl(resourceId));
});
}
// -------------------------------------------------------------------------
// Shares / public links
// -------------------------------------------------------------------------
async createShare(
workspaceId: string,
itemId: string,
input: { access?: unknown; expiresInHours?: unknown },
actor: ActorRef,
): Promise<ShareCreated> {
const access = input.access === undefined ? "members" : input.access;
if (access !== "members" && access !== "public") throw new BadRequestException("access must be 'members' or 'public'");
const hours = typeof input.expiresInHours === "number" ? input.expiresInHours : NaN;
if (!Number.isInteger(hours) || hours < 1 || hours > MAX_SHARE_TTL_HOURS) {
throw new BadRequestException(`expiresInHours must be an integer between 1 and ${MAX_SHARE_TTL_HOURS}`);
}
const token = mintShareToken();
const tokenHash = hashShareToken(token);
const expiresAt = new Date(Date.now() + hours * 3_600_000);
return withTenant(workspaceId, async (client) => {
const row = await this.requireRow(client, itemId);
const inserted = await client.query<ShareRow>(
`INSERT INTO group_file_shares (workspace_id, item_id, access, token_hash, created_by, expires_at)
VALUES ($1, $2, $3, $4, $5, $6) RETURNING id, workspace_id, item_id, access, expires_at, revoked_at`,
[workspaceId, itemId, access, tokenHash, actor.userId, expiresAt],
);
const share = inserted.rows[0];
if (share === undefined) throw new Error("share insert returned no row");
await this.audit(client, workspaceId, actor, {
action: "file.share_created",
targetId: share.id,
diff: { itemId, access, expiresAt: iso(expiresAt), name: row.safe_name }, // never the token
});
return {
id: share.id,
itemId,
access,
url: `${this.publicWebBase()}/files/s/${token}`,
expiresAt: iso(expiresAt),
revokedAt: null,
token,
};
});
}
async revokeShare(workspaceId: string, shareId: string, actor: ActorRef): Promise<void> {
await withTenant(workspaceId, async (client) => {
if (!isUuid(shareId)) throw new BadRequestException("shareId must be a UUID");
const updated = await client.query(
"UPDATE group_file_shares SET revoked_at = now() WHERE id = $1 AND workspace_id = $2 AND revoked_at IS NULL RETURNING item_id",
[shareId, workspaceId],
);
if ((updated.rowCount ?? 0) === 0) {
const existing = await client.query("SELECT item_id FROM group_file_shares WHERE id = $1 AND workspace_id = $2", [shareId, workspaceId]);
if (existing.rows[0] === undefined) throw new NotFoundException(`share ${shareId} not found`);
throw new ConflictException("share is already revoked");
}
await this.audit(client, workspaceId, actor, {
action: "file.share_revoked",
targetId: shareId,
diff: { itemId: String(updated.rows[0]?.item_id) },
});
});
}
/**
* Public claim path (no session): resolve the token through the narrow
* SECURITY DEFINER lookup — no tenant context exists yet, exactly like the
* invite-link claim — then enter the row's tenant to recheck availability
* and retention before opening the payload. Serving stops immediately on
* revoke/expiry (Gone).
*/
async claimPublicDownload(token: string): Promise<FileDownloadHandle> {
if (token === "") throw new NotFoundException("share token is invalid");
const tokenHash = createHash("sha256").update(token, "utf8").digest("hex");
const share = await withSystem(async (client) => {
const result = await client.query<ShareRow>(
"SELECT id, workspace_id, item_id, access, expires_at, revoked_at FROM share_for_public_claim($1)",
[tokenHash],
);
return result.rows[0];
});
if (share === undefined) throw new NotFoundException("share token is invalid or has expired");
if (share.access !== "public") throw new ForbiddenException("share token is not a public link");
return withTenant(share.workspace_id, async (client) => {
const row = await this.requireRow(client, share.item_id);
if (!row.available) throw new GoneException("file is revoked");
if (row.retention_mode === "until" && row.retention_until !== null && row.retention_until.getTime() <= Date.now()) {
throw new GoneException("file retention has expired");
}
const item = rowToGroupFile(row, (resourceId) => this.webdav().libraryUrl(resourceId));
const download = await this.webdav().download(item.file.resourceId).catch(() => {
throw new ServiceUnavailableException("file storage is unavailable");
});
return { item, download };
});
}
// -------------------------------------------------------------------------
// Ingest seam (hub-archive adapter / direct upload), same idempotency rule
// -------------------------------------------------------------------------
/**
* Register a committed library reference. Replay-safe: the partial unique
* (workspace,room,event) makes a second attempt for the same attachment
* event a conflict, and the claim key is attachmentArchiveKey().
*/
async registerArchiveEntry(
workspaceId: string,
entry: ArchiveEntryInput,
): Promise<{ id: string; claimKey: string | null }> {
const claimKey = entry.eventId === null ? null : attachmentArchiveKey(workspaceId, entry.roomId, entry.eventId);
return withTenant(workspaceId, async (client) => {
let inserted: { rows: Array<{ id: string }> };
try {
inserted = await client.query<{ id: string }>(
`INSERT INTO chat_media_archive
(workspace_id, group_id, room_id, event_id, origin, sender_mxid, matrix_event_json,
original_name, safe_name, mime, mime_class, byte_size, sha256,
ocis_resource_id, ocis_resource_version, created_at)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)
RETURNING id`,
[
workspaceId, entry.groupId, entry.roomId, entry.eventId, entry.originKind, entry.senderMxid,
JSON.stringify(entry.origin), entry.originalName, entry.safeName, entry.mime, entry.mimeClass,
entry.byteSize, entry.sha256, entry.ocisResourceId, entry.ocisResourceVersion, entry.createdAt,
],
);
} catch (err) {
if (typeof err === "object" && err !== null && (err as { code?: string }).code === "23505") {
throw new ConflictException("attachment event is already archived");
}
throw err;
}
const id = inserted.rows[0]?.id;
if (id === undefined) throw new Error("archive insert returned no row");
return { id, claimKey };
});
}
// -------------------------------------------------------------------------
// internals
// -------------------------------------------------------------------------
private publicWebBase(): string {
const base = process.env.FILES_PUBLIC_BASE ?? "";
return base === "" ? "" : base.replace(/\/+$/, "");
}
private async requireRow(client: PoolClient, itemId: string): Promise<ArchiveRow> {
if (!isUuid(itemId)) throw new BadRequestException("itemId must be a UUID");
const result = await client.query<ArchiveRow>(`${ARCHIVE_SELECT} WHERE a.id = $1`, [itemId]);
const row = result.rows[0];
if (row === undefined) throw new NotFoundException(`file ${itemId} not found`);
return row;
}
/** Default links require current group membership; Owner/Admin pass. */
private async requireAccess(client: PoolClient, userId: string, role: string, groupId: string): Promise<void> {
if (WORKSPACE_WIDE_ROLES.includes(role)) return;
const result = await client.query(
"SELECT 1 FROM group_members WHERE group_id = $1 AND user_id = $2 AND status = 'active'",
[groupId, userId],
);
if ((result.rowCount ?? 0) === 0) throw new ForbiddenException("file belongs to a group you are not a member of");
}
private async audit(
client: PoolClient,
workspaceId: string,
actor: ActorRef,
event: Omit<AuditEvent, "actorUserId" | "actorRole" | "targetType">,
): Promise<void> {
await client.query("SELECT audit_append($1, $2, $3, $4, $5, $6, $7)", [
workspaceId,
actor.userId,
actor.role,
event.action,
"group_file",
event.targetId,
event.diff ?? null,
]);
}
}
export interface ActorRef {
userId: string;
role: string;
}
export interface ArchiveEntryInput {
groupId: string;
roomId: string;
eventId: string | null;
origin: Record<string, unknown>;
originKind: string;
senderMxid: string;
originalName: string;
safeName: string;
mime: string;
mimeClass: MimeClass;
byteSize: number;
sha256: string;
ocisResourceId: string;
ocisResourceVersion: string | null;
createdAt: Date;
}
/** Kept for the audit import contract; appendTenantAudit is used by callers that prefer a fresh tx. */
export { appendTenantAudit };
function parseFilters(query: Record<string, unknown>): FileListFilters {
const groupId = optionalUuid(query.groupId);
const senderMxid = typeof query.senderMxid === "string" && query.senderMxid.trim() !== "" ? query.senderMxid.trim() : null;
if (senderMxid !== null && senderMxid.length > 320) throw new BadRequestException("senderMxid is too long");
const rawClass = typeof query.mimeClass === "string" && query.mimeClass !== "" ? query.mimeClass : null;
if (rawClass !== null && !(MIME_CLASSES as readonly string[]).includes(rawClass)) {
throw new BadRequestException(`mimeClass must be one of: ${MIME_CLASSES.join(", ")}`);
}
const name = typeof query.name === "string" && query.name.trim() !== "" ? query.name.trim().slice(0, 255) : null;
const from = parseBound(query.from, "from");
const to = parseBound(query.to, "to");
if (from !== null && to !== null && Date.parse(from) > Date.parse(to)) throw new BadRequestException("from must not be after to");
return { groupId, senderMxid, mimeClass: rawClass as MimeClass | null, name, from, to };
}
function optionalUuid(value: unknown): string | null {
if (value === undefined || value === null || value === "") return null;
if (typeof value !== "string" || !isUuid(value)) throw new BadRequestException("groupId must be a UUID");
return value;
}
const ISO_BOUND = /^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?$/;
function parseBound(value: unknown, field: string): string | null {
if (value === undefined || value === null || value === "") return null;
if (typeof value !== "string" || value.length > 32 || !ISO_BOUND.test(value) || !Number.isFinite(Date.parse(value))) {
throw new BadRequestException(`${field} must be an ISO-8601 date or timestamp`);
}
return value;
}