From d58db63fe6a991fff0e1a03b3d14a56a4f300b2b Mon Sep 17 00:00:00 2001 From: glass beadian Date: Sun, 11 Oct 2026 08:21:11 +0000 Subject: [PATCH] Add group cloud: files index api, web UI, archive sync bot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Telegram-like group file contract (packages/core shared-files.ts) becomes a working subsystem, built in three parallel tracks: API (NestJS): migration 029 group_files (chat_media_archive metadata-only index with partial replay-unique + RLS, group_file_shares with sha256 token persistence, archive_sync_state/outbox durable-claim seam incl. a unique claim_key index), group-files module (list/search with cursor pagination, direct upload ≤20 MiB streamed+hashed to oCIS WebDAV, download with ACL + retention rechecks, share links with expiry, retention modes incl. legalHold) — RBAC group-membership authz + audit chain events. Web: Files section per group — group picker, drag-drop upload (20 MiB cap), search + MIME/origin filters, cursor pagination, download via authenticated blob fetch, share dialog with expiry + shown-once token, admin retention dialog; bilingual EN/ES (wiki i18n pattern); nav sidebar/palette/bottom-nav wiring; MIME classes aligned to the DB CHECK (document/other). Archive bot (apps/archive-bot): plain sync-loop Matrix bot mirroring attachments from explicitly-mapped unencrypted archive rooms — raw /sync with durable since-cursor, outbox-claimed ingestion (hash-verified stream, no temp files), magic-byte MIME sniffing, safe-name normalization, WebDAV upload, indexed + replied with link, redaction→unavailable; health endpoint; Dockerfile. Tests 74/74, typecheck clean across api/web/bot. --- apps/api/migrations/029_group_files.sql | 245 ++++++++ apps/api/src/app.module.ts | 3 +- apps/api/src/group-files/file-cursor.ts | 44 ++ .../src/group-files/group-files.controller.ts | 148 +++++ .../api/src/group-files/group-files.module.ts | 27 + .../group-files/group-files.service.test.ts | 304 ++++++++++ .../src/group-files/group-files.service.ts | 573 ++++++++++++++++++ apps/api/src/group-files/group-files.test.ts | 242 ++++++++ .../src/group-files/group-files.validation.ts | 241 ++++++++ apps/api/src/group-files/ocis-webdav.ts | 254 ++++++++ apps/api/src/group-files/share-token.ts | 40 ++ apps/archive-bot/Dockerfile | 17 + apps/archive-bot/package.json | 29 + apps/archive-bot/src/archiver.ts | 90 +++ apps/archive-bot/src/backoff.ts | 14 + apps/archive-bot/src/config.test.ts | 56 ++ apps/archive-bot/src/config.ts | 110 ++++ apps/archive-bot/src/events.test.ts | 71 +++ apps/archive-bot/src/events.ts | 109 ++++ apps/archive-bot/src/health.ts | 24 + apps/archive-bot/src/inspector.test.ts | 81 +++ apps/archive-bot/src/inspector.ts | 105 ++++ apps/archive-bot/src/keys.test.ts | 33 + apps/archive-bot/src/keys.ts | 12 + apps/archive-bot/src/logger.ts | 14 + apps/archive-bot/src/main.ts | 50 ++ apps/archive-bot/src/matrix.test.ts | 69 +++ apps/archive-bot/src/matrix.ts | 298 +++++++++ apps/archive-bot/src/mime.test.ts | 27 + apps/archive-bot/src/mime.ts | 43 ++ apps/archive-bot/src/names.test.ts | 40 ++ apps/archive-bot/src/names.ts | 44 ++ apps/archive-bot/src/processor.test.ts | 267 ++++++++ apps/archive-bot/src/processor.ts | 235 +++++++ apps/archive-bot/src/store.ts | 223 +++++++ apps/archive-bot/src/webdav.test.ts | 21 + apps/archive-bot/src/webdav.ts | 116 ++++ apps/archive-bot/tsconfig.build.json | 4 + apps/archive-bot/tsconfig.json | 11 + apps/archive-bot/vitest.config.ts | 7 + apps/web/public/brand/nav-files.svg | 4 + apps/web/src/app/(main)/files/layout.tsx | 13 + apps/web/src/app/(main)/files/page.tsx | 383 ++++++++++++ apps/web/src/app/(main)/layout.tsx | 2 +- apps/web/src/app/(main)/more/page.tsx | 1 + apps/web/src/components/files/file-row.tsx | 65 ++ apps/web/src/components/files/files-client.ts | 40 ++ apps/web/src/components/files/i18n.tsx | 66 ++ apps/web/src/components/files/labels.ts | 43 ++ apps/web/src/components/files/mime-icon.tsx | 66 ++ .../src/components/files/retention-dialog.tsx | 105 ++++ .../web/src/components/files/share-dialog.tsx | 158 +++++ apps/web/src/components/files/types.ts | 64 ++ .../src/components/files/upload-dropzone.tsx | 160 +++++ apps/web/src/components/mobile/bottom-nav.tsx | 2 + .../src/components/shell/command-palette.tsx | 1 + apps/web/src/components/shell/sidebar.tsx | 5 +- pnpm-lock.yaml | 28 + 58 files changed, 5543 insertions(+), 4 deletions(-) create mode 100644 apps/api/migrations/029_group_files.sql create mode 100644 apps/api/src/group-files/file-cursor.ts create mode 100644 apps/api/src/group-files/group-files.controller.ts create mode 100644 apps/api/src/group-files/group-files.module.ts create mode 100644 apps/api/src/group-files/group-files.service.test.ts create mode 100644 apps/api/src/group-files/group-files.service.ts create mode 100644 apps/api/src/group-files/group-files.test.ts create mode 100644 apps/api/src/group-files/group-files.validation.ts create mode 100644 apps/api/src/group-files/ocis-webdav.ts create mode 100644 apps/api/src/group-files/share-token.ts create mode 100644 apps/archive-bot/Dockerfile create mode 100644 apps/archive-bot/package.json create mode 100644 apps/archive-bot/src/archiver.ts create mode 100644 apps/archive-bot/src/backoff.ts create mode 100644 apps/archive-bot/src/config.test.ts create mode 100644 apps/archive-bot/src/config.ts create mode 100644 apps/archive-bot/src/events.test.ts create mode 100644 apps/archive-bot/src/events.ts create mode 100644 apps/archive-bot/src/health.ts create mode 100644 apps/archive-bot/src/inspector.test.ts create mode 100644 apps/archive-bot/src/inspector.ts create mode 100644 apps/archive-bot/src/keys.test.ts create mode 100644 apps/archive-bot/src/keys.ts create mode 100644 apps/archive-bot/src/logger.ts create mode 100644 apps/archive-bot/src/main.ts create mode 100644 apps/archive-bot/src/matrix.test.ts create mode 100644 apps/archive-bot/src/matrix.ts create mode 100644 apps/archive-bot/src/mime.test.ts create mode 100644 apps/archive-bot/src/mime.ts create mode 100644 apps/archive-bot/src/names.test.ts create mode 100644 apps/archive-bot/src/names.ts create mode 100644 apps/archive-bot/src/processor.test.ts create mode 100644 apps/archive-bot/src/processor.ts create mode 100644 apps/archive-bot/src/store.ts create mode 100644 apps/archive-bot/src/webdav.test.ts create mode 100644 apps/archive-bot/src/webdav.ts create mode 100644 apps/archive-bot/tsconfig.build.json create mode 100644 apps/archive-bot/tsconfig.json create mode 100644 apps/archive-bot/vitest.config.ts create mode 100644 apps/web/public/brand/nav-files.svg create mode 100644 apps/web/src/app/(main)/files/layout.tsx create mode 100644 apps/web/src/app/(main)/files/page.tsx create mode 100644 apps/web/src/components/files/file-row.tsx create mode 100644 apps/web/src/components/files/files-client.ts create mode 100644 apps/web/src/components/files/i18n.tsx create mode 100644 apps/web/src/components/files/labels.ts create mode 100644 apps/web/src/components/files/mime-icon.tsx create mode 100644 apps/web/src/components/files/retention-dialog.tsx create mode 100644 apps/web/src/components/files/share-dialog.tsx create mode 100644 apps/web/src/components/files/types.ts create mode 100644 apps/web/src/components/files/upload-dropzone.tsx diff --git a/apps/api/migrations/029_group_files.sql b/apps/api/migrations/029_group_files.sql new file mode 100644 index 0000000..9727b19 --- /dev/null +++ b/apps/api/migrations/029_group_files.sql @@ -0,0 +1,245 @@ +-- 029: group files index — the Telegram-like group file contract +-- (docs/occupymedia-storage.md §"Telegram-like group file contract" and +-- §"Metadata index and hub-archive seam"). +-- +-- Metadata ONLY. chat_media_archive is the index over remote oCIS payloads: +-- it NEVER holds file bodies, encrypted-file keys, WOPI tokens or presigned +-- URLs. Verified byte size + SHA-256 only — the payload always lives behind +-- oCIS WebDAV, streamed through the API with the hash re-verified. +-- +-- Attribution is by sender MXID (never a display name/alias) and origin is +-- one of the four SharedFileOrigin kinds (packages/core/src/shared-files.ts). +-- event_id is NULL for direct uploads/exports until a room announcement +-- supplies one, so replay identity uniqueness is PARTIAL on event_id IS NOT +-- NULL: "unique (workspace_id, room_id, event_id) for attachment events". +-- +-- Retention is kept SEPARATE from link expiry: mode is 'keep' (phase-one +-- default, no automated purge) or 'until' (requires retention_until), with an +-- orthogonal legal_hold flag that blocks deletion. A share expiring never +-- deletes or hides a file. +-- +-- group_file_shares persists link IDs + sha256(token) ONLY: the plaintext +-- token is returned once at creation and can never be recovered from the DB. +-- No presigned object URLs are ever stored. public links require an explicit +-- expiry (access='public' ⇒ expires_at is bounded below); the claim path uses +-- the narrow SECURITY DEFINER lookup (017 invite-link pattern) because the +-- claimer has no tenant context and the table is FORCE RLS. +-- +-- archive_sync_state / archive_sync_outbox reserve the hub-archive seam +-- (HubArchivePort): the sync bot (not implemented in this slice) commits the +-- library reference and its outbox entry BEFORE advancing sync_cursor. +-- +-- RLS / NULLIF / reject_workspace_change / sp_app grants follow the +-- 001/008/021/026 conventions: no app.workspace_id => NULL => no rows; FORCE +-- binds sp_app too. `origin`/`mime_class`/`retention_mode`/`legal_hold` CHECKs are coarse +-- DB backstops — the authoritative typed validation is app-side +-- (apps/api/src/group-files/group-files.validation.ts). + +BEGIN; + +-- --------------------------------------------------------------------------- +-- chat_media_archive — the metadata index +-- --------------------------------------------------------------------------- +CREATE TABLE chat_media_archive ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + group_id uuid NOT NULL REFERENCES groups(id) ON DELETE CASCADE ON UPDATE RESTRICT, + -- Immutable Matrix room ID, never an alias. Tenant-pinned below. + room_id text NOT NULL CHECK (btrim(room_id) <> '' AND room_id ~ '^!'), + -- NULL for direct uploads/pad exports until a room announcement supplies one. + event_id text CHECK (event_id IS NULL OR btrim(event_id) <> ''), + origin text NOT NULL CHECK (origin IN ('matrix','bridge','upload','pad-export')), + -- Sender attribution: MXID only, never a display name (mxc/handle spoofing). + sender_mxid text NOT NULL CHECK (sender_mxid ~ '^@[^@\s]+:[^@\s]+$'), + -- Redacted/superseded Matrix event context for the hub-archive seam; never + -- file bodies or encrypted-file keys (jsonb_typeof backstop only). + matrix_event_json jsonb CHECK (matrix_event_json IS NULL OR jsonb_typeof(matrix_event_json) = 'object'), + original_name text NOT NULL CHECK (btrim(original_name) <> '' AND char_length(original_name) <= 1024), + -- Normalized safe display filename (app-side sanitizeName). + safe_name text NOT NULL CHECK (btrim(safe_name) <> '' AND char_length(safe_name) <= 255), + mime text NOT NULL CHECK (mime <> ''), + -- Coarse class derived from mime (image/video/audio/document/other). + mime_class text NOT NULL CHECK (mime_class IN ('image','video','audio','document','other')), + byte_size bigint NOT NULL CHECK (byte_size >= 0), + -- SHA-256 of verified payload bytes, lowercase hex. Never an S3 ETag. + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + -- Stable oCIS library resource identity (payload stays remote). + ocis_resource_id text NOT NULL CHECK (btrim(ocis_resource_id) <> ''), + ocis_resource_version text, + created_at timestamptz NOT NULL, + indexed_at timestamptz NOT NULL DEFAULT now(), + available boolean NOT NULL DEFAULT true, + -- FileRetention (packages/core): mode is keep|until; legalHold is a FLAG + -- alongside the mode, not a third mode. + retention_mode text NOT NULL DEFAULT 'keep' CHECK (retention_mode IN ('keep','until')), + legal_hold boolean NOT NULL DEFAULT false, + retention_until timestamptz, + CONSTRAINT chat_media_archive_ws_group FOREIGN KEY (group_id, workspace_id) + REFERENCES groups(id, workspace_id) ON DELETE CASCADE ON UPDATE RESTRICT, + -- 'until' retention requires a deadline; 'keep' must not carry one. + -- legalHold is orthogonal: it blocks deletion regardless of mode. + CONSTRAINT chat_media_archive_retention_shape CHECK ( + (retention_mode = 'until' AND retention_until IS NOT NULL) + OR (retention_mode = 'keep' AND retention_until IS NULL) + ) +); + +-- Tenant-pinned reference target for group_file_shares (015 style): +-- the composite UNIQUE lets child FKs pin (id, workspace_id) so a share can +-- never point across tenants even if ids collided. +CREATE UNIQUE INDEX chat_media_archive_id_ws_key ON chat_media_archive (id, workspace_id); + +CREATE TRIGGER trg_chat_media_archive_ws_immutable BEFORE UPDATE ON chat_media_archive + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE chat_media_archive ENABLE ROW LEVEL SECURITY; +ALTER TABLE chat_media_archive FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON chat_media_archive + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +GRANT SELECT, INSERT, UPDATE, DELETE ON chat_media_archive TO sp_app; + +-- Replay identity for ATTACHMENT events (029 note: uploads have NULL event_id, +-- so the unique constraint must be partial). AttachmentArchiveKey's +-- collision-free replay rule, enforced durably. +CREATE UNIQUE INDEX chat_media_archive_replay_uq + ON chat_media_archive (workspace_id, room_id, event_id) + WHERE event_id IS NOT NULL; + +-- Cursor pagination index: (createdAt,id) keyset scoped to workspace/group. +-- Spec: "indexes on (workspace_id, group_id, created_at desc, id desc)". +CREATE INDEX chat_media_archive_ws_group_time_idx + ON chat_media_archive (workspace_id, group_id, created_at DESC, id DESC); + +-- Sender/MIME-class/checksum facets (spec: "indexes on ... sender, MIME class, +-- name and checksum"). MXID is indexed for per-sender listing/attribution. +CREATE INDEX chat_media_archive_sender_idx ON chat_media_archive (workspace_id, sender_mxid, created_at DESC); +CREATE INDEX chat_media_archive_mime_class_idx ON chat_media_archive (mime_class); +CREATE INDEX chat_media_archive_sha256_idx ON chat_media_archive (sha256); +CREATE INDEX chat_media_archive_name_idx ON chat_media_archive (safe_name text_pattern_ops); + +-- --------------------------------------------------------------------------- +-- group_file_shares — link IDs + sha256(token), never presigned URLs +-- --------------------------------------------------------------------------- +CREATE TABLE group_file_shares ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + item_id uuid NOT NULL, + -- 'members' (default library link, ACL-checked) or 'public' (requires an + -- explicit grant + bounded expiry; revoked on removal per the contract). + access text NOT NULL DEFAULT 'members' CHECK (access IN ('members','public')), + -- sha256(plaintext token). The plaintext is returned ONCE at creation; the + -- DB can never re-derive it, so a leaked DB dump mints no downloads. + token_hash text NOT NULL UNIQUE CHECK (token_hash ~ '^[0-9a-f]{64}$'), + created_by uuid REFERENCES users(id) ON DELETE SET NULL, + expires_at timestamptz NOT NULL, + revoked_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT group_file_shares_ws_item FOREIGN KEY (item_id, workspace_id) + REFERENCES chat_media_archive(id, workspace_id) ON DELETE CASCADE ON UPDATE RESTRICT +); + +CREATE UNIQUE INDEX group_file_shares_ws_item_key ON group_file_shares (item_id, workspace_id); + +CREATE TRIGGER trg_group_file_shares_ws_immutable BEFORE UPDATE ON group_file_shares + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE group_file_shares ENABLE ROW LEVEL SECURITY; +ALTER TABLE group_file_shares FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON group_file_shares + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +GRANT SELECT, INSERT, UPDATE, DELETE ON group_file_shares TO sp_app; + +-- --------------------------------------------------------------------------- +-- share_for_public_claim — narrow SECURITY DEFINER lookup (017 pattern) +-- --------------------------------------------------------------------------- +-- Public downloads arrive with NO tenant context (the claimer holds only the +-- token) and group_file_shares is FORCE RLS, so an ordinary SELECT sees zero +-- rows. Exactly like invite_link_for_claim (017), this exposes ONE row keyed +-- by the secret — workspace/file discovery only. It returns the row only when +-- it is live: not revoked and not expired (the caller still rechecks the item +-- row's availability/retention inside the tenant context it enters next). +CREATE FUNCTION share_for_public_claim(p_token_hash text) +RETURNS TABLE(id uuid, workspace_id uuid, item_id uuid, access text, + expires_at timestamptz, revoked_at timestamptz) +LANGUAGE sql STABLE SECURITY DEFINER SET search_path = public AS $$ + SELECT s.id, s.workspace_id, s.item_id, s.access, s.expires_at, s.revoked_at + FROM group_file_shares s + WHERE s.token_hash = p_token_hash + AND s.revoked_at IS NULL + AND s.expires_at > now() +$$; + +REVOKE ALL ON FUNCTION share_for_public_claim(text) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION share_for_public_claim(text) TO sp_app; + +-- --------------------------------------------------------------------------- +-- archive_sync_state — per-room sync cursor (hub-archive seam) +-- --------------------------------------------------------------------------- +CREATE TABLE archive_sync_state ( + room_id text PRIMARY KEY CHECK (btrim(room_id) <> ''), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + -- Last committed Matrix sync token; the bot never advances past a failed + -- event (durable retry lives in archive_sync_outbox). + sync_cursor text, + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TRIGGER trg_archive_sync_state_ws_immutable BEFORE UPDATE ON archive_sync_state + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE archive_sync_state ENABLE ROW LEVEL SECURITY; +ALTER TABLE archive_sync_state FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON archive_sync_state + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +GRANT SELECT, INSERT, UPDATE, DELETE ON archive_sync_state TO sp_app; + +-- --------------------------------------------------------------------------- +-- archive_sync_outbox — durable reply/retry records before cursor advance +-- --------------------------------------------------------------------------- +CREATE TABLE archive_sync_outbox ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + room_id text NOT NULL CHECK (btrim(room_id) <> ''), + event_id text CHECK (event_id IS NULL OR btrim(event_id) <> ''), + -- Idempotent claim key: attachmentArchiveKey(workspace,room,event) shape. + claim_key text NOT NULL CHECK (btrim(claim_key) <> ''), + status text NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','done','failed')), + attempts int NOT NULL DEFAULT 0 CHECK (attempts >= 0), + last_error text, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +-- DB-level idempotency for the durable claim: one outbox row per +-- attachmentArchiveKey(workspace,room,event). The bot inserts with a +-- NOT EXISTS guard + bare ON CONFLICT DO NOTHING; this index makes the +-- race window impossible rather than unlikely. +CREATE UNIQUE INDEX archive_sync_outbox_claim_key_uq ON archive_sync_outbox (claim_key); + +CREATE TRIGGER trg_archive_sync_outbox_ws_immutable BEFORE UPDATE ON archive_sync_outbox + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE archive_sync_outbox ENABLE ROW LEVEL SECURITY; +ALTER TABLE archive_sync_outbox FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON archive_sync_outbox + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +GRANT SELECT, INSERT, UPDATE, DELETE ON archive_sync_outbox TO sp_app; + +-- Worker scan: pending-first, oldest-first. +CREATE INDEX archive_sync_outbox_status_idx ON archive_sync_outbox (status, created_at); +-- Replay/idempotency lookup by claim key. +CREATE INDEX archive_sync_outbox_claim_idx ON archive_sync_outbox (claim_key); + +COMMIT; diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts index fe8a937..41d8fe8 100644 --- a/apps/api/src/app.module.ts +++ b/apps/api/src/app.module.ts @@ -28,6 +28,7 @@ import { LiveChatModule } from "./live-chat/live-chat.module"; import { LiveModule } from "./live/live.module.js"; import { ChatgovModule } from "./chatgov/chatgov.module.js"; import { IntakeModule } from "./intake/intake.module.js"; +import { GroupFilesModule } from "./group-files/group-files.module.js"; import { SettingsModule } from "./settings/settings.module.js"; import { SettingsController } from "./settings/settings.controller.js"; import { SettingsService } from "./settings/settings.service.js"; @@ -37,7 +38,7 @@ import { AnalyticsController } from "./analytics/analytics.controller"; import { SnapshotService } from "./analytics/snapshot.service"; @Module({ - imports: [HealthModule, LiveMonitorModule, MediaControlModule, ProductionModule, RealtimeModule, LiveChatModule, LiveModule, VirtualChannelsModule, ChatgovModule, IntakeModule, SettingsModule], + imports: [HealthModule, LiveMonitorModule, MediaControlModule, ProductionModule, RealtimeModule, LiveChatModule, LiveModule, VirtualChannelsModule, ChatgovModule, IntakeModule, GroupFilesModule, SettingsModule], controllers: [ DemoController, PublishingController, diff --git a/apps/api/src/group-files/file-cursor.ts b/apps/api/src/group-files/file-cursor.ts new file mode 100644 index 0000000..590731e --- /dev/null +++ b/apps/api/src/group-files/file-cursor.ts @@ -0,0 +1,44 @@ +/** + * (createdAt,id) keyset cursor for the group-files index, scoped to + * workspace/group in the query itself (docs/occupymedia-storage.md + * §"Metadata index and hub-archive seam": "Use a (createdAt,id) cursor scoped + * to workspace/group"). + * + * House pattern (video.service.ts / intake-search.ts): base64url JSON, the + * cursor carries ONLY the sort key — the workspace/group scope never travels + * in the cursor, it is re-bound server-side on every request so a cursor from + * another tenant/group is just an empty page. + */ +import { BadRequestException } from "@nestjs/common"; +import { isUuid } from "../http/validation.js"; + +export interface FileListCursor { + createdAt: string; + id: string; +} + +const ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z$/; + +export function encodeFileListCursor(cursor: FileListCursor): string { + return Buffer.from(JSON.stringify(cursor), "utf8").toString("base64url"); +} + +export function decodeFileListCursor(cursor: string): FileListCursor { + let decoded: unknown; + try { + decoded = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")) as unknown; + } catch { + throw new BadRequestException("cursor is invalid"); + } + if (typeof decoded !== "object" || decoded === null || Array.isArray(decoded)) { + throw new BadRequestException("cursor is invalid"); + } + const row = decoded as Record; + const createdAt = row.createdAt; + const id = row.id; + if (typeof createdAt !== "string" || !ISO_TIMESTAMP.test(createdAt) || !Number.isFinite(Date.parse(createdAt))) { + throw new BadRequestException("cursor is invalid"); + } + if (typeof id !== "string" || !isUuid(id)) throw new BadRequestException("cursor is invalid"); + return { createdAt, id }; +} diff --git a/apps/api/src/group-files/group-files.controller.ts b/apps/api/src/group-files/group-files.controller.ts new file mode 100644 index 0000000..1a60a79 --- /dev/null +++ b/apps/api/src/group-files/group-files.controller.ts @@ -0,0 +1,148 @@ +import { BadRequestException, Body, Controller, Delete, Get, HttpCode, Inject, Param, ParseUUIDPipe, Post, Query, Req, Res, UseGuards } from "@nestjs/common"; +import type { Request, Response } from "express"; +import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { requireObject, requirePositiveInt, requireSession, requireString } from "../http/validation.js"; +import { GroupFilesService, type FilePage } from "./group-files.service.js"; +import type { GroupFile } from "@tactical-media-broadcast-hub/core"; +import { verifyWhileStreaming } from "./ocis-webdav.js"; + +/** + * Group files API — the Telegram-like group file surface. + * + * GET /files list/search, keyset (groupId, senderMxid, + * mimeClass, name, from, to, cursor, limit) + * GET /files/:id one GroupFile + * GET /files/:id/download stream payload (sha256 re-verified) + * POST /files/:id/retention set retention (keep/until/legalHold) + * POST /files/:id/revoke revoke links + stop serving + * POST /files/:id/shares mint a share link (token shown ONCE) + * DELETE /files/shares/:shareId revoke a share + * + * Authenticated routes gate on media.view (read) / media.manage (mutations) — + * the same media permission family the video library uses. ACLs are + * re-resolved per request in the service (current group membership), never + * carried by the token. + * + * The download route streams: the API proxies the payload with the digest + * verified chunk-by-chunk instead of redirecting or presigning, per the + * "never presigned object URLs" contract. + */ +@Controller("files") +@UseGuards(RbacGuard) +export class GroupFilesController { + constructor(@Inject(GroupFilesService) private readonly files: GroupFilesService) {} + + @Get() + @RequirePermission("media.view") + async list(@Req() request: Request & AuthenticatedRequest, @Query() query: Record): Promise { + const session = requireSession(request); + return this.files.listFiles(session.workspaceId, session.userId, session.role, query); + } + + @Get(":id") + @RequirePermission("media.view") + async get(@Req() request: Request & AuthenticatedRequest, @Param("id", ParseUUIDPipe) id: string): Promise { + const session = requireSession(request); + return this.files.getFile(session.workspaceId, session.userId, session.role, id); + } + + @Get(":id/download") + @RequirePermission("media.view") + async download( + @Req() request: Request & AuthenticatedRequest, + @Param("id", ParseUUIDPipe) id: string, + @Res() response: Response, + ): Promise { + const session = requireSession(request); + const handle = await this.files.openDownload(session.workspaceId, session.userId, session.role, id); + response.status(200); + response.setHeader("Content-Type", handle.item.mimeType); + response.setHeader("Content-Disposition", `attachment; filename="${encodeURIComponent(handle.item.name)}"`); + response.setHeader("Cache-Control", "no-store"); + if (handle.download.sizeBytes !== null) response.setHeader("Content-Length", String(handle.download.sizeBytes)); + response.flushHeaders(); + await verifyWhileStreaming(handle.download, handle.item.checksum.value, (chunk) => { + response.write(chunk); + }); + response.end(); + } + + @Post(":id/retention") + @RequirePermission("media.manage") + async setRetention( + @Req() request: Request & AuthenticatedRequest, + @Param("id", ParseUUIDPipe) id: string, + @Body() body: unknown, + ): Promise { + const session = requireSession(request); + return this.files.setRetention(session.workspaceId, id, requireObject(body, "body"), { + userId: session.userId, + role: session.role, + }); + } + + @Post(":id/revoke") + @HttpCode(200) + @RequirePermission("media.manage") + async revoke(@Req() request: Request & AuthenticatedRequest, @Param("id", ParseUUIDPipe) id: string): Promise { + const session = requireSession(request); + return this.files.markRevoked(session.workspaceId, id, { userId: session.userId, role: session.role }); + } + + @Post(":id/shares") + @RequirePermission("media.manage") + async createShare( + @Req() request: Request & AuthenticatedRequest, + @Param("id", ParseUUIDPipe) id: string, + @Body() body: unknown, + ): Promise> { + const session = requireSession(request); + const dto = requireObject(body, "body"); + if (dto.access !== undefined && dto.access !== "members" && dto.access !== "public") { + throw new BadRequestException("access must be 'members' or 'public'"); + } + const expiresInHours = requirePositiveInt(dto.expiresInHours, "expiresInHours", 720); + return this.files.createShare( + session.workspaceId, + id, + { access: dto.access, expiresInHours }, + { userId: session.userId, role: session.role }, + ); + } + + /** Declared before :id routes so the literal path wins route matching. */ + @Delete("shares/:shareId") + @RequirePermission("media.manage") + async revokeShare( + @Req() request: Request & AuthenticatedRequest, + @Param("shareId", ParseUUIDPipe) shareId: string, + ): Promise<{ ok: true }> { + const session = requireSession(request); + await this.files.revokeShare(session.workspaceId, shareId, { userId: session.userId, role: session.role }); + return { ok: true }; + } +} + +/** + * Public share-claim surface: NO RbacGuard (the claimer holds only the + * token). Token lives in the path — same shape as the invite claim routes. + */ +@Controller("public/files") +export class PublicFileShareController { + constructor(@Inject(GroupFilesService) private readonly files: GroupFilesService) {} + + @Get("s/:token") + async claim(@Param("token") token: string, @Res() response: Response): Promise { + response.setHeader("Cache-Control", "no-store"); + const handle = await this.files.claimPublicDownload(requireString(token, "token")); + response.status(200); + response.setHeader("Content-Type", handle.item.mimeType); + response.setHeader("Content-Disposition", `attachment; filename="${encodeURIComponent(handle.item.name)}"`); + if (handle.download.sizeBytes !== null) response.setHeader("Content-Length", String(handle.download.sizeBytes)); + response.flushHeaders(); + await verifyWhileStreaming(handle.download, handle.item.checksum.value, (chunk) => { + response.write(chunk); + }); + response.end(); + } +} diff --git a/apps/api/src/group-files/group-files.module.ts b/apps/api/src/group-files/group-files.module.ts new file mode 100644 index 0000000..5f693ae --- /dev/null +++ b/apps/api/src/group-files/group-files.module.ts @@ -0,0 +1,27 @@ +import { Module } from "@nestjs/common"; +import { GroupFilesController, PublicFileShareController } from "./group-files.controller.js"; +import { GroupFilesService } from "./group-files.service.js"; + +/** + * Group files module — the metadata index + share/link lifecycle over + * chat_media_archive (029) and the oCIS WebDAV port (occ-files deploy). + * + * NOT registered in AppModule by this slice: per the deployment contract + * (infra/occupymedia/files/README.md) the oCIS service is scaffolded + * separately and admission checks are mandatory before this surface accepts + * user payloads. Wiring is one line in apps/api/src/app.module.ts: + * + * imports: [..., GroupFilesModule] + * + * until that gate is passed — the ChatgovModule precedent. + * + * The oCIS client is built lazily from env on first use, so importing the + * module never requires FILES_OCID_* to be present; routes fail 503 when the + * deployment is absent instead of breaking startup. + */ +@Module({ + controllers: [GroupFilesController, PublicFileShareController], + providers: [GroupFilesService], + exports: [GroupFilesService], +}) +export class GroupFilesModule {} diff --git a/apps/api/src/group-files/group-files.service.test.ts b/apps/api/src/group-files/group-files.service.test.ts new file mode 100644 index 0000000..f90a9d6 --- /dev/null +++ b/apps/api/src/group-files/group-files.service.test.ts @@ -0,0 +1,304 @@ +import type { PoolClient } from "pg"; +import { GroupFilesService } from "./group-files.service.js"; +import { hashShareToken } from "./share-token.js"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { BadRequestException, ConflictException, ForbiddenException, GoneException, NotFoundException } from "@nestjs/common"; + +/** Deterministic 64-hex helper: repeat a pattern to exactly 64 chars. */ +function hex64(seed: string): string { + let out = ""; + while (out.length < 64) out += Buffer.from(seed).toString("hex"); + return out.slice(0, 64); +} + +// --------------------------------------------------------------------------- +// Fake db surface: route by SQL shape, capture every statement. +// --------------------------------------------------------------------------- +type QueryResult = { rows: Record[]; rowCount: number }; +type Behavior = (sql: string, values: unknown[]) => QueryResult | Promise; + +let behavior: Behavior = () => ({ rows: [], rowCount: 0 }); +const statements: Array<{ sql: string; values: unknown[] }> = []; + +vi.mock("../db/client.js", () => ({ + withTenant: async (_ws: string, fn: (client: PoolClient) => Promise) => + fn({ + query: async (sql: string, values: unknown[] = []) => { + statements.push({ sql, values }); + return behavior(sql, values); + }, + } as unknown as PoolClient), + withSystem: async (fn: (client: PoolClient) => Promise) => + fn({ + query: async (sql: string, values: unknown[] = []) => { + statements.push({ sql, values }); + return behavior(sql, values); + }, + } as unknown as PoolClient), +})); + + + +const row = { + id: "11111111-1111-4111-8111-111111111111", + workspace_id: "22222222-2222-4222-8222-222222222222", + group_id: "33333333-3333-4333-8333-333333333333", + room_id: "!r:example.org", + event_id: "$e", + origin: "matrix", + sender_mxid: "@s:example.org", + matrix_event_json: { mxcUri: "mxc://example.org/x" }, + original_name: "clip.mp4", + safe_name: "clip.mp4", + mime: "video/mp4", + mime_class: "video", + byte_size: "10", + sha256: hex64("z"), + ocis_resource_id: "res-1", + ocis_resource_version: "v1", + created_at: new Date("2026-01-02T03:04:05Z"), + indexed_at: new Date("2026-01-02T03:04:05Z"), + available: true, + retention_mode: "keep", + legal_hold: false, + retention_until: null, + share_access: null, + share_expires_at: null, + share_revoked_at: null, +}; + +/** sha256 hex of a string, computed with the real crypto module. */ +function digestOf(input: string): string { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { createHash } = require("node:crypto") as typeof import("node:crypto"); + return createHash("sha256").update(input, "utf8").digest("hex"); +} + +function envReady(): void { + process.env.FILES_OCID_WEBDAV_URL = "https://files.example.org"; + process.env.FILES_OCID_USER = "svc"; + process.env.FILES_OCID_PASSWORD = "pw"; + process.env.FILES_PUBLIC_BASE = "https://files.example.org"; +} + +beforeEach(() => { + statements.length = 0; + behavior = () => ({ rows: [], rowCount: 0 }); + envReady(); +}); + +describe("GroupFilesService invariants", () => { + it("scopes listings to group membership for non-Owner/Admin, binding the caller id positionally", async () => { + behavior = () => ({ rows: [row], rowCount: 1 }); + const service = new GroupFilesService(); + const page = await service.listFiles(row.workspace_id, "user-1", "Publisher", {}); + const list = statements.find((q) => q.sql.includes("FROM chat_media_archive"))!; + expect(list.sql).toContain("group_members"); + expect(list.values[1]).toBe("user-1"); // caller id is a bound parameter… + expect(list.sql).not.toContain("user-1"); // …never SQL text + expect(list.sql).not.toContain("user-1"); + // Owner/Admin skip the membership subquery but keep $2 bound + await service.listFiles(row.workspace_id, "user-1", "Owner", {}); + const ownerList = statements.filter((q) => q.sql.includes("FROM chat_media_archive")).at(-1)!; + expect(ownerList.sql).not.toContain("group_members"); + expect(ownerList.values[1]).toBe("user-1"); + expect(page.items[0]?.status).toBe("available"); + expect(page.items[0]?.shareLink.url).toBe("https://files.example.org/f/res-1"); + }); + + it("filters facets positionally and paginates with a (createdAt,id) keyset cursor", async () => { + behavior = () => ({ rows: [row], rowCount: 1 }); + const service = new GroupFilesService(); + const cursor = Buffer.from(JSON.stringify({ createdAt: "2026-01-01T00:00:00Z", id: row.id })).toString("base64url"); + await service.listFiles(row.workspace_id, "u", "Owner", { + groupId: row.group_id, + senderMxid: "@s:example.org", + mimeClass: "video", + name: "clip", + from: "2026-01-01", + to: "2026-02-01", + cursor, + limit: "10", + }); + const q = statements.find((s) => s.sql.includes("FROM chat_media_archive"))!; + expect(q.values).toEqual([row.workspace_id, "u", row.group_id, "@s:example.org", "video", "%clip%", "2026-01-01", "2026-02-01", "2026-01-01T00:00:00Z", row.id, 11]); + expect(q.sql).toContain("(a.created_at, a.id) <"); + expect(q.sql).toContain("ORDER BY a.created_at DESC, a.id DESC"); + // bad facets are rejected before SQL + await expect(service.listFiles(row.workspace_id, "u", "Owner", { mimeClass: "zip" })).rejects.toBeInstanceOf(BadRequestException); + await expect(service.listFiles(row.workspace_id, "u", "Owner", { from: "nope" })).rejects.toBeInstanceOf(BadRequestException); + await expect(service.listFiles(row.workspace_id, "u", "Owner", { from: "2026-03-01", to: "2026-01-01" })).rejects.toBeInstanceOf(BadRequestException); + }); + + it("emits no presigned URL anywhere in a served payload", async () => { + behavior = () => ({ rows: [row], rowCount: 1 }); + const service = new GroupFilesService(); + const page = await service.listFiles(row.workspace_id, "u", "Owner", {}); + expect(JSON.stringify(page)).not.toContain("X-Amz-Signature"); + expect(JSON.stringify(page)).not.toContain("presign"); + expect(page.items[0]?.shareLink.url).toBe("https://files.example.org/f/res-1"); + }); + + it("keeps retention independent from share expiry, and legalHold is an orthogonal flag", async () => { + behavior = () => ({ + rows: [{ ...row, share_access: "public", share_expires_at: new Date("2020-01-01T00:00:00Z") }], + rowCount: 1, + }); + const service = new GroupFilesService(); + const item = await service.getFile(row.workspace_id, "u", "Owner", row.id); + // share link long gone; the FILE itself is still fully available + expect(item.status).toBe("available"); + expect(item.shareLink.expiresAt).toBe("2020-01-01T00:00:00.000Z"); + expect(item.retention.mode).toBe("keep"); + // legal hold survives alongside the mode (FileRetention contract) + behavior = () => ({ rows: [{ ...row, legal_hold: true }], rowCount: 1 }); + const held = await service.getFile(row.workspace_id, "u", "Owner", row.id); + expect(held.retention).toEqual({ mode: "keep", legalHold: true }); + expect(held.status).toBe("available"); + }); + + it("reconstructs all four origin kinds from matrix_event_json", async () => { + const cases = [ + ["matrix", { mxcUri: "mxc://e/x" }, { kind: "matrix", mxcUri: "mxc://e/x" }], + ["bridge", { mxcUri: "mxc://e/x", providerId: "telegram", remoteEventId: "9" }, { kind: "bridge", mxcUri: "mxc://e/x", providerId: "telegram", remoteEventId: "9" }], + ["upload", null, { kind: "upload" }], + ["pad-export", { padId: "p", revision: "3" }, { kind: "pad-export", padId: "p", revision: "3" }], + ] as const; + for (const [origin, json, expected] of cases) { + behavior = () => ({ rows: [{ ...row, origin, matrix_event_json: json }], rowCount: 1 }); + const service = new GroupFilesService(); + const item = await service.getFile(row.workspace_id, "u", "Owner", row.id); + expect(item.origin).toEqual(expected); + } + }); + + it("returns the share token exactly once and keeps it out of audit rows", async () => { + let auditedDiff: unknown = null; + behavior = (sql) => { + if (sql.startsWith("SELECT audit_append")) return { rows: [{ id: "audit-1" }], rowCount: 1 }; + if (sql.startsWith("INSERT INTO group_file_shares")) { + return { + rows: [{ id: "55555555-5555-4555-8555-555555555555", access: "public", expires_at: new Date(Date.now() + 3_600_000), revoked_at: null }], + rowCount: 1, + }; + } + return { rows: [row], rowCount: 1 }; + }; + const service = new GroupFilesService(); + const share = await service.createShare(row.workspace_id, row.id, { access: "public", expiresInHours: 2 }, { userId: "u", role: "Owner" }); + expect(share.token).toBeDefined(); + expect(hashShareToken(share.token)).toMatch(/^[0-9a-f]{64}$/); + expect(share.url).toContain("/files/s/"); + const auditCall = statements.find((q) => q.sql.startsWith("SELECT audit_append")); + expect(auditCall).toBeDefined(); + auditedDiff = auditCall?.values[6]; + expect(JSON.stringify(auditedDiff)).not.toContain(share.token); + expect(JSON.stringify(auditCall?.values)).not.toContain(share.token); + // shape errors before any write + await expect(service.createShare(row.workspace_id, row.id, { access: "secret" }, { userId: "u", role: "Owner" })).rejects.toBeInstanceOf(BadRequestException); + await expect(service.createShare(row.workspace_id, row.id, { expiresInHours: 0 }, { userId: "u", role: "Owner" })).rejects.toBeInstanceOf(BadRequestException); + }); + + it("revoking stops serving immediately: item + every live share flipped", async () => { + let flipped = false; + behavior = (sql) => { + if (sql.startsWith("UPDATE chat_media_archive SET available = false")) flipped = true; + return { rows: [flipped ? { ...row, available: false } : row], rowCount: 1 }; + }; + const service = new GroupFilesService(); + const revoked = await service.markRevoked(row.workspace_id, row.id, { userId: "u", role: "Owner" }); + expect(revoked.status).toBe("revoked"); + const shareRevoke = statements.find((q) => q.sql.startsWith("UPDATE group_file_shares SET revoked_at")); + expect(shareRevoke).toBeDefined(); + expect(shareRevoke?.values[0]).toBe(row.id); + // already-revoked is a conflict, not a silent no-op + behavior = () => ({ rows: [{ ...row, available: false }], rowCount: 1 }); + await expect(service.markRevoked(row.workspace_id, row.id, { userId: "u", role: "Owner" })).rejects.toBeInstanceOf(ConflictException); + }); + + it("replay of an attachment event is a conflict (partial unique (ws,room,event))", async () => { + behavior = (sql) => { + if (sql.startsWith("INSERT INTO chat_media_archive")) { + throw Object.assign(new Error("dup"), { code: "23505" }); + } + return { rows: [], rowCount: 0 }; + }; + const service = new GroupFilesService(); + await expect( + service.registerArchiveEntry(row.workspace_id, { + groupId: row.group_id, + roomId: "!r:example.org", + eventId: "$e", + origin: {}, + originKind: "matrix", + senderMxid: "@s:example.org", + originalName: "a.mp4", + safeName: "a.mp4", + mime: "video/mp4", + mimeClass: "video", + byteSize: 10, + sha256: hex64("z"), + ocisResourceVersion: null, + ocisResourceId: "res", + createdAt: new Date(), + }), + ).rejects.toBeInstanceOf(ConflictException); + }); + + it("openDownload refuses revoked/expired items with Gone, missing rows with NotFound", async () => { + behavior = () => ({ rows: [{ ...row, available: false }], rowCount: 1 }); + const service = new GroupFilesService(); + await expect(service.openDownload(row.workspace_id, "u", "Owner", row.id)).rejects.toBeInstanceOf(GoneException); + behavior = () => ({ rows: [{ ...row, retention_mode: "until", retention_until: new Date("2020-01-01T00:00:00Z") }], rowCount: 1 }); + await expect(service.openDownload(row.workspace_id, "u", "Owner", row.id)).rejects.toBeInstanceOf(GoneException); + behavior = () => ({ rows: [], rowCount: 0 }); + await expect(service.getFile(row.workspace_id, "u", "Owner", row.id)).rejects.toBeInstanceOf(NotFoundException); + await expect(service.getFile(row.workspace_id, "u", "Owner", "not-a-uuid")).rejects.toBeInstanceOf(BadRequestException); + }); + + it("membership gate denies non-members (Forbidden)", async () => { + behavior = (sql) => { + if (sql.includes("FROM group_members")) return { rows: [], rowCount: 0 }; + if (sql.includes("FROM chat_media_archive")) return { rows: [row], rowCount: 1 }; + return { rows: [], rowCount: 0 }; + }; + const service = new GroupFilesService(); + await expect(service.getFile(row.workspace_id, "u", "Publisher", row.id)).rejects.toBeInstanceOf(ForbiddenException); + }); + + it("public claim resolves via the SECURITY DEFINER lookup, then rechecks in-tenant", async () => { + behavior = (sql) => { + if (sql.includes("share_for_public_claim")) { + return { rows: [{ id: "s1", workspace_id: row.workspace_id, item_id: row.id, access: "public", expires_at: new Date(Date.now() + 3600_000), revoked_at: null }], rowCount: 1 }; + } + return { rows: [{ ...row, sha256: digestOf("payload-bytes") }], rowCount: 1 }; + }; + const payload = Buffer.from("payload-bytes"); + const { createHash } = await import("node:crypto"); + const digest = createHash("sha256").update(payload).digest("hex"); + const { Readable } = await import("node:stream"); + const fetcher = (async () => + new Response(Readable.toWeb(Readable.from([payload])) as unknown as ReadableStream, { + status: 200, + headers: { "content-type": "application/octet-stream", "content-length": String(payload.byteLength) }, + })) as unknown as typeof fetch; + const service = new GroupFilesService({ fetcher }); + const handle = await service.claimPublicDownload("some-token"); + expect(handle.item.id).toBe(row.id); + expect(handle.item.checksum.value).toBe(digest); + expect(handle.download.sizeBytes).toBe(payload.byteLength); + const claim = statements.find((q) => q.sql.includes("share_for_public_claim")); + expect(claim).toBeDefined(); + // members-only tokens are rejected on the public surface + behavior = (sql) => { + if (sql.includes("share_for_public_claim")) { + return { rows: [{ id: "s1", workspace_id: row.workspace_id, item_id: row.id, access: "members", expires_at: new Date(Date.now() + 3600_000), revoked_at: null }], rowCount: 1 }; + } + return { rows: [row], rowCount: 1 }; + }; + await expect(service.claimPublicDownload("some-token")).rejects.toBeInstanceOf(ForbiddenException); + // unknown token: 404, no tenant entered + behavior = () => ({ rows: [], rowCount: 0 }); + await expect(service.claimPublicDownload("nope")).rejects.toBeInstanceOf(NotFoundException); + }); +}); diff --git a/apps/api/src/group-files/group-files.service.ts b/apps/api/src/group-files/group-files.service.ts new file mode 100644 index 0000000..432fa7e --- /dev/null +++ b/apps/api/src/group-files/group-files.service.ts @@ -0,0 +1,573 @@ +/** + * Group files service — the metadata index + share/link lifecycle over + * chat_media_archive (029) and the oCIS WebDAV port. + * + * Invariants this service owns (docs/occupymedia-storage.md): + * * List/search return ONLY available, unexpired rows visible through + * current group/file ACLs — expired/retention-past/revoked rows never + * surface, and non-Owner/Admin callers only see their groups' files. + * * Attribution by sender MXID; origin is one of the four SharedFileOrigin + * kinds, reconstructed from origin + matrix_event_json. + * * Retention is SEPARATE from link expiry: an expired share never deletes + * or hides a file; 'legalHold' blocks deletion outright. + * * Share tokens: plaintext returned ONCE, sha256(token) persisted, public + * access requires an explicit grant + bounded expiry, revoke stops + * serving immediately. + * * Downloads stream through the API with the SHA-256 re-verified; no + * presigned object URL is ever minted or stored. + * + * Every mutation lands its audit row on the SAME tenant transaction + * (audit_append), the house pattern. + */ +import { BadRequestException, ConflictException, ForbiddenException, GoneException, Injectable, NotFoundException, Optional, Inject, ServiceUnavailableException } from "@nestjs/common"; +import { createHash } from "node:crypto"; +import type { PoolClient } from "pg"; +import { attachmentArchiveKey, type FileRetention, type GroupFile } from "@tactical-media-broadcast-hub/core"; +import { appendTenantAudit, type AuditEvent } from "../audit/audit.service.js"; +import { withSystem, withTenant } from "../db/client.js"; +import { isUuid, parseLimit } from "../http/validation.js"; +import { decodeFileListCursor, encodeFileListCursor } from "./file-cursor.js"; +import { ocisConfigFromEnv, OcisWebdavClient, type FetchLike, type OcisDownload } from "./ocis-webdav.js"; +import { hashShareToken, mintShareToken } from "./share-token.js"; +import { MIME_CLASSES, parseRetentionInput, type MimeClass } from "./group-files.validation.js"; + +export const GROUP_FILES_DEPENDENCIES: unique symbol = Symbol("GROUP_FILES_DEPENDENCIES"); + +/** Wraps the raw payload handle the controller streams out. */ +export interface FileDownloadHandle { + item: GroupFile; + download: OcisDownload; +} + +export interface ShareCreated { + id: string; + itemId: string; + access: "members" | "public"; + url: string; + expiresAt: string; + revokedAt: string | null; + /** Plaintext capability — returned ONCE here, never persisted anywhere. */ + token: string; +} + +export interface FileListFilters { + groupId: string | null; + senderMxid: string | null; + mimeClass: MimeClass | null; + name: string | null; + from: string | null; + to: string | null; +} + +export interface FilePage { + items: GroupFile[]; + nextCursor: string | null; +} + +const MAX_SHARE_TTL_HOURS = 720; +const WORKSPACE_WIDE_ROLES: readonly string[] = ["Owner", "Admin"]; + +interface ArchiveRow { + id: string; + workspace_id: string; + group_id: string; + room_id: string; + event_id: string | null; + origin: string; + sender_mxid: string; + matrix_event_json: Record | null; + original_name: string; + safe_name: string; + mime: string; + mime_class: string; + byte_size: string | number; + sha256: string; + ocis_resource_id: string; + ocis_resource_version: string | null; + created_at: Date; + indexed_at: Date; + available: boolean; + retention_mode: string; + legal_hold: boolean; + retention_until: Date | null; + share_access: string | null; + share_expires_at: Date | null; + share_revoked_at: Date | null; +} + +interface ShareRow { + id: string; + workspace_id: string; + item_id: string; + access: string; + expires_at: Date; + revoked_at: Date | null; +} + +const ARCHIVE_SELECT = `SELECT a.*, s.access AS share_access, s.expires_at AS share_expires_at, s.revoked_at AS share_revoked_at + FROM chat_media_archive a + LEFT JOIN LATERAL ( + SELECT access, expires_at, revoked_at FROM group_file_shares + WHERE item_id = a.id AND revoked_at IS NULL AND expires_at > now() + ORDER BY created_at DESC LIMIT 1 + ) s ON true`; + +function iso(value: Date): string { + return new Date(value).toISOString(); +} + +function safeNumber(value: string | number): number { + const n = typeof value === "number" ? value : Number(value); + if (!Number.isSafeInteger(n)) throw new Error(`unsafe numeric column: ${String(value)}`); + return n; +} + +/** Row → GroupFile, reconstructing origin + retention + share link. */ +function rowToGroupFile(row: ArchiveRow, libraryUrl: (resourceId: string) => string): GroupFile { + const payload = row.matrix_event_json ?? {}; + const origin: GroupFile["origin"] = (() => { + if (row.origin === "matrix") return { kind: "matrix", mxcUri: String(payload.mxcUri ?? "") }; + if (row.origin === "bridge") { + return { + kind: "bridge", + mxcUri: String(payload.mxcUri ?? ""), + providerId: String(payload.providerId ?? ""), + ...(payload.remoteEventId === undefined ? {} : { remoteEventId: String(payload.remoteEventId) }), + }; + } + if (row.origin === "pad-export") { + return { kind: "pad-export", padId: String(payload.padId ?? ""), revision: String(payload.revision ?? "") }; + } + return { kind: "upload" }; + })(); + const retention: FileRetention = + row.retention_mode === "until" && row.retention_until !== null + ? { mode: "until", legalHold: row.legal_hold, expiresAt: iso(row.retention_until) } + : { mode: "keep", legalHold: row.legal_hold }; + const status: GroupFile["status"] = !row.available + ? "revoked" + : retention.mode === "until" && new Date(retention.expiresAt).getTime() <= Date.now() + ? "expired" + : "available"; + return { + id: row.id, + workspaceId: row.workspace_id, + groupId: row.group_id, + roomId: row.room_id, + senderId: row.sender_mxid, + eventId: row.event_id, + origin, + name: row.original_name, + mimeType: row.mime, + sizeBytes: safeNumber(row.byte_size), + checksum: { algorithm: "sha256", value: row.sha256 }, + createdAt: iso(row.created_at), + indexedAt: iso(row.indexed_at), + file: { + spaceId: row.group_id, + resourceId: row.ocis_resource_id, + version: row.ocis_resource_version ?? row.ocis_resource_id, + }, + shareLink: { + url: libraryUrl(row.ocis_resource_id), + access: row.share_access === "public" ? "public" : "members", + expiresAt: row.share_expires_at === null ? null : iso(row.share_expires_at), + }, + retention, + status, + }; +} + +@Injectable() +export class GroupFilesService { + private client: OcisWebdavClient | null = null; + + constructor(@Optional() @Inject(GROUP_FILES_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike }) {} + + /** Lazily bound: the oCIS env may be absent in dev/test contexts. */ + private webdav(): OcisWebdavClient { + if (this.client === null) { + const fetcher: FetchLike = this.deps?.fetcher ?? ((url, init) => fetch(url, init)); + this.client = new OcisWebdavClient(ocisConfigFromEnv(), fetcher); + } + return this.client; + } + + // ------------------------------------------------------------------------- + // Read paths + // ------------------------------------------------------------------------- + + /** + * Keyset page over the visible slice: available, unexpired, and — for + * non-Owner/Admin callers — restricted to the caller's groups (the ACL + * rule "List/search return only available, unexpired rows visible through + * current group/file ACLs"). + */ + async listFiles( + workspaceId: string, + userId: string, + role: string, + query: Record, + ): Promise { + const filters = parseFilters(query); + const limit = parseLimit(query.limit, 50, 100); + const cursor = typeof query.cursor === "string" && query.cursor !== "" ? decodeFileListCursor(query.cursor) : undefined; + return withTenant(workspaceId, async (client) => { + // values[1] is always the caller's id: the membership subquery needs it + // for non-Owner/Admin callers; Owner/Admin leave $2 bound but unused so + // every placeholder stays positional and no SQL text carries a value. + const values: unknown[] = [workspaceId, userId]; + const conds: string[] = ["a.available = true", "(a.retention_mode <> 'until' OR a.retention_until > now())"]; + if (!WORKSPACE_WIDE_ROLES.includes(role)) { + conds.push("a.group_id IN (SELECT group_id FROM group_members WHERE user_id = $2 AND status = 'active')"); + } + if (filters.groupId !== null) { + values.push(filters.groupId); + conds.push(`a.group_id = $${values.length}::uuid`); + } + if (filters.senderMxid !== null) { + values.push(filters.senderMxid); + conds.push(`a.sender_mxid = $${values.length}`); + } + if (filters.mimeClass !== null) { + values.push(filters.mimeClass); + conds.push(`a.mime_class = $${values.length}`); + } + if (filters.name !== null) { + values.push(`%${filters.name}%`); + conds.push(`a.safe_name ILIKE $${values.length}`); + } + if (filters.from !== null) { + values.push(filters.from); + conds.push(`a.created_at >= $${values.length}::timestamptz`); + } + if (filters.to !== null) { + values.push(filters.to); + conds.push(`a.created_at < $${values.length}::timestamptz`); + } + if (cursor !== undefined) { + values.push(cursor.createdAt, cursor.id); + conds.push(`(a.created_at, a.id) < ($${values.length - 1}::timestamptz, $${values.length}::uuid)`); + } + values.push(limit + 1); + const sql = `${ARCHIVE_SELECT} + WHERE ${conds.join(" AND ")} + ORDER BY a.created_at DESC, a.id DESC + LIMIT $${values.length}`; + const result = await client.query(sql, values); + const page = result.rows.slice(0, limit); + const last = page.at(-1); + const lib = (resourceId: string): string => this.webdav().libraryUrl(resourceId); + return { + items: page.map((row) => rowToGroupFile(row, lib)), + nextCursor: result.rows.length > limit && last !== undefined + ? encodeFileListCursor({ createdAt: iso(last.created_at), id: last.id }) + : null, + }; + }); + } + + async getFile(workspaceId: string, userId: string, role: string, itemId: string): Promise { + return withTenant(workspaceId, async (client) => { + const row = await this.requireRow(client, itemId); + await this.requireAccess(client, userId, role, row.group_id); + return rowToGroupFile(row, (resourceId) => this.webdav().libraryUrl(resourceId)); + }); + } + + /** Open the payload for streaming; membership/ACL + retention rechecked here. */ + async openDownload(workspaceId: string, userId: string, role: string, itemId: string): Promise { + const item = await this.getFile(workspaceId, userId, role, itemId); + if (item.status !== "available") throw new GoneException(`file is ${item.status}`); + const download = await this.webdav().download(item.file.resourceId).catch(() => { + throw new ServiceUnavailableException("file storage is unavailable"); + }); + return { item, download }; + } + + // ------------------------------------------------------------------------- + // Retention / availability + // ------------------------------------------------------------------------- + + async setRetention(workspaceId: string, itemId: string, retentionInput: unknown, actor: ActorRef): Promise { + const retention = parseRetentionInput(retentionInput); + return withTenant(workspaceId, async (client) => { + const before = await this.requireRow(client, itemId); + await client.query( + `UPDATE chat_media_archive + SET retention_mode = $2, legal_hold = $3, retention_until = $4 + WHERE id = $1`, + [itemId, retention.mode, retention.legalHold, retention.mode === "until" ? new Date(retention.expiresAt) : null], + ); + await this.audit(client, workspaceId, actor, { + action: "file.retention_changed", + targetId: itemId, + diff: { + from: { mode: before.retention_mode, legalHold: before.legal_hold, until: before.retention_until === null ? null : iso(before.retention_until) }, + to: { mode: retention.mode, legalHold: retention.legalHold, until: retention.mode === "until" ? retention.expiresAt : null }, + name: before.safe_name, + }, + }); + const after = await this.requireRow(client, itemId); + return rowToGroupFile(after, (resourceId) => this.webdav().libraryUrl(resourceId)); + }); + } + + /** Redaction/membership-removal path: revoke links, stop serving. No deletion. */ + async markRevoked(workspaceId: string, itemId: string, actor: ActorRef): Promise { + return withTenant(workspaceId, async (client) => { + const before = await this.requireRow(client, itemId); + if (!before.available) throw new ConflictException("file is already revoked"); + await client.query("UPDATE chat_media_archive SET available = false WHERE id = $1", [itemId]); + await client.query("UPDATE group_file_shares SET revoked_at = now() WHERE item_id = $1 AND revoked_at IS NULL", [itemId]); + await this.audit(client, workspaceId, actor, { + action: "file.revoked", + targetId: itemId, + diff: { name: before.safe_name, roomId: before.room_id, eventId: before.event_id }, + }); + const after = await this.requireRow(client, itemId); + return rowToGroupFile(after, (resourceId) => this.webdav().libraryUrl(resourceId)); + }); + } + + // ------------------------------------------------------------------------- + // Shares / public links + // ------------------------------------------------------------------------- + + async createShare( + workspaceId: string, + itemId: string, + input: { access?: unknown; expiresInHours?: unknown }, + actor: ActorRef, + ): Promise { + const access = input.access === undefined ? "members" : input.access; + if (access !== "members" && access !== "public") throw new BadRequestException("access must be 'members' or 'public'"); + const hours = typeof input.expiresInHours === "number" ? input.expiresInHours : NaN; + if (!Number.isInteger(hours) || hours < 1 || hours > MAX_SHARE_TTL_HOURS) { + throw new BadRequestException(`expiresInHours must be an integer between 1 and ${MAX_SHARE_TTL_HOURS}`); + } + const token = mintShareToken(); + const tokenHash = hashShareToken(token); + const expiresAt = new Date(Date.now() + hours * 3_600_000); + return withTenant(workspaceId, async (client) => { + const row = await this.requireRow(client, itemId); + const inserted = await client.query( + `INSERT INTO group_file_shares (workspace_id, item_id, access, token_hash, created_by, expires_at) + VALUES ($1, $2, $3, $4, $5, $6) RETURNING id, workspace_id, item_id, access, expires_at, revoked_at`, + [workspaceId, itemId, access, tokenHash, actor.userId, expiresAt], + ); + const share = inserted.rows[0]; + if (share === undefined) throw new Error("share insert returned no row"); + await this.audit(client, workspaceId, actor, { + action: "file.share_created", + targetId: share.id, + diff: { itemId, access, expiresAt: iso(expiresAt), name: row.safe_name }, // never the token + }); + return { + id: share.id, + itemId, + access, + url: `${this.publicWebBase()}/files/s/${token}`, + expiresAt: iso(expiresAt), + revokedAt: null, + token, + }; + }); + } + + async revokeShare(workspaceId: string, shareId: string, actor: ActorRef): Promise { + await withTenant(workspaceId, async (client) => { + if (!isUuid(shareId)) throw new BadRequestException("shareId must be a UUID"); + const updated = await client.query( + "UPDATE group_file_shares SET revoked_at = now() WHERE id = $1 AND workspace_id = $2 AND revoked_at IS NULL RETURNING item_id", + [shareId, workspaceId], + ); + if ((updated.rowCount ?? 0) === 0) { + const existing = await client.query("SELECT item_id FROM group_file_shares WHERE id = $1 AND workspace_id = $2", [shareId, workspaceId]); + if (existing.rows[0] === undefined) throw new NotFoundException(`share ${shareId} not found`); + throw new ConflictException("share is already revoked"); + } + await this.audit(client, workspaceId, actor, { + action: "file.share_revoked", + targetId: shareId, + diff: { itemId: String(updated.rows[0]?.item_id) }, + }); + }); + } + + /** + * Public claim path (no session): resolve the token through the narrow + * SECURITY DEFINER lookup — no tenant context exists yet, exactly like the + * invite-link claim — then enter the row's tenant to recheck availability + * and retention before opening the payload. Serving stops immediately on + * revoke/expiry (Gone). + */ + async claimPublicDownload(token: string): Promise { + if (token === "") throw new NotFoundException("share token is invalid"); + const tokenHash = createHash("sha256").update(token, "utf8").digest("hex"); + const share = await withSystem(async (client) => { + const result = await client.query( + "SELECT id, workspace_id, item_id, access, expires_at, revoked_at FROM share_for_public_claim($1)", + [tokenHash], + ); + return result.rows[0]; + }); + if (share === undefined) throw new NotFoundException("share token is invalid or has expired"); + if (share.access !== "public") throw new ForbiddenException("share token is not a public link"); + return withTenant(share.workspace_id, async (client) => { + const row = await this.requireRow(client, share.item_id); + if (!row.available) throw new GoneException("file is revoked"); + if (row.retention_mode === "until" && row.retention_until !== null && row.retention_until.getTime() <= Date.now()) { + throw new GoneException("file retention has expired"); + } + const item = rowToGroupFile(row, (resourceId) => this.webdav().libraryUrl(resourceId)); + const download = await this.webdav().download(item.file.resourceId).catch(() => { + throw new ServiceUnavailableException("file storage is unavailable"); + }); + return { item, download }; + }); + } + + // ------------------------------------------------------------------------- + // Ingest seam (hub-archive adapter / direct upload), same idempotency rule + // ------------------------------------------------------------------------- + + /** + * Register a committed library reference. Replay-safe: the partial unique + * (workspace,room,event) makes a second attempt for the same attachment + * event a conflict, and the claim key is attachmentArchiveKey(). + */ + async registerArchiveEntry( + workspaceId: string, + entry: ArchiveEntryInput, + ): Promise<{ id: string; claimKey: string | null }> { + const claimKey = entry.eventId === null ? null : attachmentArchiveKey(workspaceId, entry.roomId, entry.eventId); + return withTenant(workspaceId, async (client) => { + let inserted: { rows: Array<{ id: string }> }; + try { + inserted = await client.query<{ id: string }>( + `INSERT INTO chat_media_archive + (workspace_id, group_id, room_id, event_id, origin, sender_mxid, matrix_event_json, + original_name, safe_name, mime, mime_class, byte_size, sha256, + ocis_resource_id, ocis_resource_version, created_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) + RETURNING id`, + [ + workspaceId, entry.groupId, entry.roomId, entry.eventId, entry.originKind, entry.senderMxid, + JSON.stringify(entry.origin), entry.originalName, entry.safeName, entry.mime, entry.mimeClass, + entry.byteSize, entry.sha256, entry.ocisResourceId, entry.ocisResourceVersion, entry.createdAt, + ], + ); + } catch (err) { + if (typeof err === "object" && err !== null && (err as { code?: string }).code === "23505") { + throw new ConflictException("attachment event is already archived"); + } + throw err; + } + const id = inserted.rows[0]?.id; + if (id === undefined) throw new Error("archive insert returned no row"); + return { id, claimKey }; + }); + } + + // ------------------------------------------------------------------------- + // internals + // ------------------------------------------------------------------------- + + private publicWebBase(): string { + const base = process.env.FILES_PUBLIC_BASE ?? ""; + return base === "" ? "" : base.replace(/\/+$/, ""); + } + + private async requireRow(client: PoolClient, itemId: string): Promise { + if (!isUuid(itemId)) throw new BadRequestException("itemId must be a UUID"); + const result = await client.query(`${ARCHIVE_SELECT} WHERE a.id = $1`, [itemId]); + const row = result.rows[0]; + if (row === undefined) throw new NotFoundException(`file ${itemId} not found`); + return row; + } + + /** Default links require current group membership; Owner/Admin pass. */ + private async requireAccess(client: PoolClient, userId: string, role: string, groupId: string): Promise { + if (WORKSPACE_WIDE_ROLES.includes(role)) return; + const result = await client.query( + "SELECT 1 FROM group_members WHERE group_id = $1 AND user_id = $2 AND status = 'active'", + [groupId, userId], + ); + if ((result.rowCount ?? 0) === 0) throw new ForbiddenException("file belongs to a group you are not a member of"); + } + + private async audit( + client: PoolClient, + workspaceId: string, + actor: ActorRef, + event: Omit, + ): Promise { + await client.query("SELECT audit_append($1, $2, $3, $4, $5, $6, $7)", [ + workspaceId, + actor.userId, + actor.role, + event.action, + "group_file", + event.targetId, + event.diff ?? null, + ]); + } +} + +export interface ActorRef { + userId: string; + role: string; +} + +export interface ArchiveEntryInput { + groupId: string; + roomId: string; + eventId: string | null; + origin: Record; + originKind: string; + senderMxid: string; + originalName: string; + safeName: string; + mime: string; + mimeClass: MimeClass; + byteSize: number; + sha256: string; + ocisResourceId: string; + ocisResourceVersion: string | null; + createdAt: Date; +} + +/** Kept for the audit import contract; appendTenantAudit is used by callers that prefer a fresh tx. */ +export { appendTenantAudit }; + +function parseFilters(query: Record): FileListFilters { + const groupId = optionalUuid(query.groupId); + const senderMxid = typeof query.senderMxid === "string" && query.senderMxid.trim() !== "" ? query.senderMxid.trim() : null; + if (senderMxid !== null && senderMxid.length > 320) throw new BadRequestException("senderMxid is too long"); + const rawClass = typeof query.mimeClass === "string" && query.mimeClass !== "" ? query.mimeClass : null; + if (rawClass !== null && !(MIME_CLASSES as readonly string[]).includes(rawClass)) { + throw new BadRequestException(`mimeClass must be one of: ${MIME_CLASSES.join(", ")}`); + } + const name = typeof query.name === "string" && query.name.trim() !== "" ? query.name.trim().slice(0, 255) : null; + const from = parseBound(query.from, "from"); + const to = parseBound(query.to, "to"); + if (from !== null && to !== null && Date.parse(from) > Date.parse(to)) throw new BadRequestException("from must not be after to"); + return { groupId, senderMxid, mimeClass: rawClass as MimeClass | null, name, from, to }; +} + +function optionalUuid(value: unknown): string | null { + if (value === undefined || value === null || value === "") return null; + if (typeof value !== "string" || !isUuid(value)) throw new BadRequestException("groupId must be a UUID"); + return value; +} + +const ISO_BOUND = /^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?$/; + +function parseBound(value: unknown, field: string): string | null { + if (value === undefined || value === null || value === "") return null; + if (typeof value !== "string" || value.length > 32 || !ISO_BOUND.test(value) || !Number.isFinite(Date.parse(value))) { + throw new BadRequestException(`${field} must be an ISO-8601 date or timestamp`); + } + return value; +} diff --git a/apps/api/src/group-files/group-files.test.ts b/apps/api/src/group-files/group-files.test.ts new file mode 100644 index 0000000..92b68ce --- /dev/null +++ b/apps/api/src/group-files/group-files.test.ts @@ -0,0 +1,242 @@ +import { describe, expect, it } from "vitest"; +import { BadRequestException, ServiceUnavailableException } from "@nestjs/common"; +import { Readable } from "node:stream"; +import { + decodeFileListCursor, + encodeFileListCursor, +} from "./file-cursor.js"; +import { + parseGroupFile, + parseMatrixAttachmentCandidate, + parseRetention, + parseRetentionInput, + parseSharedFileOrigin, + requireMxid, + requireMxcUri, + requireRoomId, + requireSha256, +} from "./group-files.validation.js"; +import { OcisVerificationError, ocisConfigFromEnv, OcisWebdavClient, verifyWhileStreaming } from "./ocis-webdav.js"; +import { hashShareToken, isPlausibleShareToken, mintShareToken, sameToken } from "./share-token.js"; + +/** Deterministic 64-hex helper: repeat a pattern to exactly 64 chars. */ +function hex64(seed: string): string { + let out = ""; + while (out.length < 64) out += Buffer.from(seed).toString("hex"); + return out.slice(0, 64); +} + +describe("group-files runtime validation (shared contracts)", () => { + it("accepts the four SharedFileOrigin variants and rejects anything else", () => { + expect(parseSharedFileOrigin({ kind: "matrix", mxcUri: "mxc://example.org/abc" })).toEqual({ + kind: "matrix", + mxcUri: "mxc://example.org/abc", + }); + expect( + parseSharedFileOrigin({ kind: "bridge", mxcUri: "mxc://example.org/abc", providerId: "telegram" }), + ).toEqual({ kind: "bridge", mxcUri: "mxc://example.org/abc", providerId: "telegram" }); + expect(parseSharedFileOrigin({ kind: "bridge", mxcUri: "mxc://example.org/abc", providerId: "t", remoteEventId: "42" })).toEqual({ + kind: "bridge", + mxcUri: "mxc://example.org/abc", + providerId: "t", + remoteEventId: "42", + }); + expect(parseSharedFileOrigin({ kind: "upload" })).toEqual({ kind: "upload" }); + expect(parseSharedFileOrigin({ kind: "pad-export", padId: "p", revision: "7" })).toEqual({ + kind: "pad-export", + padId: "p", + revision: "7", + }); + for (const bad of [null, {}, { kind: "nope" }, { kind: "matrix" }, { kind: "matrix", mxcUri: "https://evil/x" }]) { + expect(() => parseSharedFileOrigin(bad)).toThrow(BadRequestException); + } + }); + + it("validates MXIDs, room ids, mxc URIs and sha256 shape", () => { + expect(requireMxid("@alice:example.org")).toBe("@alice:example.org"); + expect(() => requireMxid("Alice (display)")).toThrow(BadRequestException); + expect(() => requireMxid("#alias:example.org")).toThrow(BadRequestException); + expect(requireRoomId("!archive:example.org")).toBe("!archive:example.org"); + expect(() => requireRoomId("#alias:example.org")).toThrow(BadRequestException); // alias, never an identity + expect(() => requireRoomId("")).toThrow(BadRequestException); + expect(requireMxcUri("mxc://example.org/AbC123")).toBe("mxc://example.org/AbC123"); + expect(() => requireMxcUri("mxc://example.org/")).toThrow(BadRequestException); + expect(requireSha256(hex64("a"))).toBe(hex64("a")); + expect(() => requireSha256("nope")).toThrow(BadRequestException); + // hex digits include letters: an uppercase digest must be rejected + const upperHex = "A".repeat(64); + expect(/[A-F]/.test(upperHex)).toBe(true); + expect(() => requireSha256(upperHex)).toThrow(BadRequestException); + expect(() => requireSha256("0".repeat(63))).toThrow(BadRequestException); // wrong length + }); + + it("keeps retention separate from link expiry and enforces shape coupling", () => { + expect(parseRetention({ mode: "keep", legalHold: false })).toEqual({ mode: "keep", legalHold: false }); + // legalHold is a FLAG alongside the mode, not a third mode (FileRetention contract) + expect(parseRetention({ mode: "keep", legalHold: true })).toEqual({ mode: "keep", legalHold: true }); + expect(parseRetention({ mode: "until", legalHold: true, expiresAt: "2030-01-01T00:00:00Z" })).toEqual({ + mode: "until", + legalHold: true, + expiresAt: "2030-01-01T00:00:00Z", + }); + expect(() => parseRetention({ mode: "burn", legalHold: false })).toThrow(BadRequestException); + expect(() => parseRetention({ mode: "until", legalHold: false })).toThrow(BadRequestException); + expect(() => parseRetention({ mode: "keep", legalHold: false, expiresAt: "2030-01-01T00:00:00Z" })).toThrow(BadRequestException); + expect(() => parseRetention({ mode: "until", legalHold: false, expiresAt: "next tuesday" })).toThrow(BadRequestException); + // input variant + expect(parseRetentionInput({ mode: "until", until: "2030-01-01T00:00:00Z" })).toEqual({ + mode: "until", + legalHold: false, + expiresAt: "2030-01-01T00:00:00Z", + }); + expect(() => parseRetentionInput({ mode: "until" })).toThrow(BadRequestException); + expect(() => parseRetentionInput({ mode: "keep", until: "2030-01-01T00:00:00Z" })).toThrow(BadRequestException); + expect(parseRetentionInput({ mode: "keep", legalHold: true })).toEqual({ mode: "keep", legalHold: true }); + expect(() => parseRetentionInput({ mode: "legalHold" })).toThrow(BadRequestException); // not a mode + }); + + it("fails closed on Matrix attachment candidates with unknown/bad state", () => { + const good = { + roomId: "!a:example.org", + eventId: "$1", + senderId: "@s:example.org", + eventType: "m.room.message", + msgtype: "m.image", + roomEncryption: "unencrypted", + encryptedFile: false, + redacted: false, + mxcUri: "mxc://example.org/x", + }; + expect(parseMatrixAttachmentCandidate(good).roomEncryption).toBe("unencrypted"); + // shape-level failures + for (const patch of [ + { roomId: "#alias:example.org" }, + { senderId: "Bob" }, + { mxcUri: "https://attacker/x" }, + { encryptedFile: "yes" }, + ]) { + expect(() => parseMatrixAttachmentCandidate({ ...good, ...patch })).toThrow(BadRequestException); + } + // state-level eligibility is the CORE helper's call, not the parser's: + // parse accepts encrypted/unknown/redacted states; canArchiveAttachment + // is what fails closed on them (core test shared-files.test.ts). + const parsed = parseMatrixAttachmentCandidate({ ...good, roomEncryption: "unknown" }); + expect(parsed.roomEncryption).toBe("unknown"); + }); + + it("round-trips a full GroupFile wire shape", () => { + const value = { + id: "b3a1c2d4-1111-4222-8333-444455556666", + workspaceId: "c3a1c2d4-1111-4222-8333-444455556666", + groupId: "d3a1c2d4-1111-4222-8333-444455556666", + roomId: "!r:example.org", + senderId: "@s:example.org", + eventId: "$evt", + origin: { kind: "matrix", mxcUri: "mxc://example.org/x" }, + name: "clip.mp4", + mimeType: "video/mp4", + sizeBytes: 12, + checksum: { algorithm: "sha256", value: hex64("z") }, + createdAt: "2026-01-02T03:04:05Z", + indexedAt: "2026-01-02T03:04:05Z", + file: { spaceId: "sp", resourceId: "res", version: "v1" }, + shareLink: { url: "https://files.example.org/f/res", access: "members", expiresAt: null }, + retention: { mode: "keep", legalHold: false }, + status: "available", + }; + expect(parseGroupFile(value)).toEqual(value); + expect(() => parseGroupFile({ ...value, sizeBytes: -1 })).toThrow(BadRequestException); + expect(() => parseGroupFile({ ...value, senderId: "Bob" })).toThrow(BadRequestException); + expect(() => parseGroupFile({ ...value, checksum: { algorithm: "sha256", value: "zz" } })).toThrow(BadRequestException); + }); +}); + +describe("file-cursor", () => { + it("encodes/decodes (createdAt,id) and rejects malformed scopes", () => { + const cursor = { createdAt: "2026-01-02T03:04:05Z", id: "b3a1c2d4-1111-4222-8333-444455556666" }; + expect(decodeFileListCursor(encodeFileListCursor(cursor))).toEqual(cursor); + for (const bad of ["", "!!!", Buffer.from(JSON.stringify({ createdAt: "nope", id: "x" })).toString("base64url")]) { + expect(() => decodeFileListCursor(bad)).toThrow(BadRequestException); + } + // a cursor from another tenant carries no scope: it is just an empty page, + // never a leak — the scope is re-bound server-side on every request. + expect(decodeFileListCursor(encodeFileListCursor({ createdAt: "2026-01-02T03:04:05Z", id: "00000000-0000-4000-8000-000000000000" })).id).toBe( + "00000000-0000-4000-8000-000000000000", + ); + }); +}); + +describe("share tokens", () => { + it("mints 256-bit capabilities and stores only sha256(token)", () => { + const token = mintShareToken(); + expect(token.length).toBeGreaterThanOrEqual(40); + expect(isPlausibleShareToken(token)).toBe(true); + expect(isPlausibleShareToken("short")).toBe(false); + expect(isPlausibleShareToken("not base64url!!")).toBe(false); + const hash = hashShareToken(token); + expect(hash).toMatch(/^[0-9a-f]{64}$/); + expect(hashShareToken(token)).toBe(hash); // deterministic + expect(hashShareToken(mintShareToken())).not.toBe(hash); // unique per mint + expect(sameToken(token, token)).toBe(true); + expect(sameToken(token, mintShareToken())).toBe(false); + // the plaintext can never be derived from the hash + expect(hash).not.toContain(token); + }); +}); + +describe("ocis-webdav config + streaming verification", () => { + it("reads env lazily and throws 503 when unconfigured", () => { + expect(() => ocisConfigFromEnv({})).toThrow(ServiceUnavailableException); + expect(() => ocisConfigFromEnv({ FILES_OCID_WEBDAV_URL: "https://files.example.org" })).toThrow(ServiceUnavailableException); + const config = ocisConfigFromEnv({ + FILES_OCID_WEBDAV_URL: "https://files.example.org/", + FILES_OCID_USER: "svc", + FILES_OCID_PASSWORD: "pw", + FILES_PUBLIC_BASE: "https://files.example.org/", + }); + expect(config.webdavUrl).toBe("https://files.example.org"); + expect(config.publicBase).toBe("https://files.example.org"); + }); + + it("uploads hash the actual streamed bytes and surface oc-fileid/oc-etag", async () => { + const bytes = Buffer.from("group file payload"); + const fetcher = (async () => + new Response(null, { + status: 201, + headers: { "oc-fileid": "res-1", "oc-etag": "v-9" }, + })) as unknown as typeof fetch; + const client = new OcisWebdavClient( + { webdavUrl: "https://files.example.org", user: "svc", password: "pw", publicBase: "https://files.example.org" }, + fetcher, + ); + const result = await client.upload("group-1", "clip.mp4", Readable.from([bytes]), "video/mp4"); + const { createHash } = await import("node:crypto"); + expect(result.sha256).toBe(createHash("sha256").update(bytes).digest("hex")); + expect(result.sizeBytes).toBe(bytes.byteLength); + expect(result.resourceId).toBe("res-1"); + expect(result.version).toBe("v-9"); + expect(client.uploadUrl("g", "n.mp4")).toContain("/remote.php/dav/files/svc/hub-groups/g/n.mp4"); + expect(client.libraryUrl("res-1")).toBe("https://files.example.org/f/res-1"); + }); + + it("verifyWhileStreaming passes on match and aborts on tamper", async () => { + const { createHash } = await import("node:crypto"); + const payload = Buffer.from("0123456789"); + const good = createHash("sha256").update(payload).digest("hex"); + const sinkChunks: Buffer[] = []; + const out = await verifyWhileStreaming( + { stream: Readable.from([payload.subarray(0, 4), payload.subarray(4)]), sizeBytes: payload.byteLength, contentType: "application/octet-stream" }, + good, + (chunk) => sinkChunks.push(chunk), + ); + expect(out.sizeBytes).toBe(10); + expect(Buffer.concat(sinkChunks).toString()).toBe("0123456789"); + await expect( + verifyWhileStreaming( + { stream: Readable.from([Buffer.from("tampered")]), sizeBytes: 8, contentType: null }, + good, + () => undefined, + ), + ).rejects.toBeInstanceOf(OcisVerificationError); + }); +}); diff --git a/apps/api/src/group-files/group-files.validation.ts b/apps/api/src/group-files/group-files.validation.ts new file mode 100644 index 0000000..6f4ba4c --- /dev/null +++ b/apps/api/src/group-files/group-files.validation.ts @@ -0,0 +1,241 @@ +/** + * Runtime validation for the shared group-files contracts + * (packages/core/src/shared-files.ts — imported as @tactical-media-broadcast-hub/core). + * + * The shared file is TYPES ONLY; the task contract says "future HTTP/DB + * adapters must validate these shapes at runtime". This module is that + * adapter boundary: every value entering or leaving the DB/HTTP seam passes + * through one of these guards, so a stray row can never render an + * invalid GroupFile. + * + * Style follows the api's existing house convention: `unknown` in, typed + * value out, BadRequestException on bad shape (http/validation.ts), never + * `any`, and DB CHECK constraints (029) are coarse backstops only. + */ +import { BadRequestException } from "@nestjs/common"; +import type { + FileRetention, + GroupFile, + GroupFileMapping, + MatrixAttachmentCandidate, + SharedFileOrigin, +} from "@tactical-media-broadcast-hub/core"; + +// FileRetention (packages/core/src/shared-files.ts): mode is keep|until and +// legalHold is an orthogonal boolean flag — NOT a third mode. +export const RETENTION_MODES = ["keep", "until"] as const; +export type RetentionMode = (typeof RETENTION_MODES)[number]; + +export const ORIGIN_KINDS = ["matrix", "bridge", "upload", "pad-export"] as const; +export type OriginKind = (typeof ORIGIN_KINDS)[number]; + +export const MIME_CLASSES = ["image", "video", "audio", "document", "other"] as const; +export type MimeClass = (typeof MIME_CLASSES)[number]; + +export const FILE_STATUSES = ["available", "revoked", "expired"] as const; +export type FileStatus = (typeof FILE_STATUSES)[number]; + +const SHA256_HEX = /^[0-9a-f]{64}$/; +const MXID = /^@[^@\s]+:[^@\s]+$/; +const MXC_URI = /^mxc:\/\/[^/\s]+\/[^/\s?#]+$/; +const ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z$/; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function requireField(container: Record, key: string): unknown { + if (!(key in container)) throw new BadRequestException(`${key} is required`); + return container[key]; +} + +function stringField(container: Record, key: string, field = key): string { + const value = requireField(container, key); + if (typeof value !== "string" || value === "") throw new BadRequestException(`${field} must be a non-empty string`); + return value; +} + +function isoField(container: Record, key: string, field = key): string { + const value = stringField(container, key, field); + if (!ISO_TIMESTAMP.test(value)) throw new BadRequestException(`${field} must be a UTC ISO-8601 timestamp (…Z)`); + return value; +} + +function enumField(container: Record, key: string, allowed: readonly T[], field = key): T { + const value = requireField(container, key); + if (typeof value !== "string" || !(allowed as readonly string[]).includes(value)) { + throw new BadRequestException(`${field} must be one of: ${allowed.join(", ")}`); + } + return value as T; +} + +function integerField(container: Record, key: string, field = key): number { + const value = requireField(container, key); + const n = typeof value === "number" ? value : NaN; + if (!Number.isSafeInteger(n) || n < 0) throw new BadRequestException(`${field} must be a non-negative safe integer`); + return n; +} + +/** sha256 lowercase hex, per the contract ("verified payload bytes ... hex"). */ +export function isSha256Hex(value: unknown): value is string { + return typeof value === "string" && SHA256_HEX.test(value); +} + +export function requireSha256(value: unknown, field = "sha256"): string { + if (!isSha256Hex(value)) throw new BadRequestException(`${field} must be lowercase sha256 hex (64 chars)`); + return value; +} + +export function requireMxid(value: unknown, field = "senderMxid"): string { + if (typeof value !== "string" || !MXID.test(value)) throw new BadRequestException(`${field} must be a Matrix user id (@user:host)`); + return value; +} + +export function requireMxcUri(value: unknown, field = "mxcUri"): string { + if (typeof value !== "string" || !MXC_URI.test(value)) throw new BadRequestException(`${field} must be an mxc:// URI`); + return value; +} + +export function requireRoomId(value: unknown, field = "roomId"): string { + if (typeof value !== "string" || !/^![^:\s]+:[^:\s]+$/.test(value)) { + throw new BadRequestException(`${field} must be an immutable Matrix room id (!localpart:host) — never an alias`); + } + return value; +} + +/** Retention is decoupled from link expiry; legalHold blocks deletion. */ +export function parseRetention(value: unknown, field = "retention"): FileRetention { + if (!isRecord(value)) throw new BadRequestException(`${field} must be an object`); + const mode = enumField(value, "mode", RETENTION_MODES, `${field}.mode`); + const legalHold = requireField(value, "legalHold"); + if (typeof legalHold !== "boolean") throw new BadRequestException(`${field}.legalHold must be a boolean`); + if (mode === "until") return { mode, legalHold, expiresAt: isoField(value, "expiresAt", `${field}.expiresAt`) }; + if (value.expiresAt !== undefined && value.expiresAt !== null) { + throw new BadRequestException(`${field}.expiresAt must not be set when mode is "keep"`); + } + return { mode, legalHold }; +} + +/** POST /files/:id/retention body {mode, until?, legalHold?} → FileRetention. */ +export function parseRetentionInput(value: unknown): FileRetention { + if (!isRecord(value)) throw new BadRequestException("body must be an object"); + const mode = enumField(value, "mode", RETENTION_MODES, "mode"); + const legalHold = value.legalHold === undefined ? false : value.legalHold; + if (typeof legalHold !== "boolean") throw new BadRequestException("legalHold must be a boolean"); + const rawUntil = value.until; + if (mode === "until") { + if (rawUntil === undefined || rawUntil === null) throw new BadRequestException("until is required when mode is 'until'"); + return { mode, legalHold, expiresAt: requireIsoTimestamp(rawUntil, "until") }; + } + if (rawUntil !== undefined && rawUntil !== null) { + throw new BadRequestException("until must not be set when mode is 'keep'"); + } + return { mode, legalHold }; +} + +export function requireIsoTimestamp(value: unknown, field: string): string { + if (typeof value !== "string" || !ISO_TIMESTAMP.test(value) || !Number.isFinite(Date.parse(value))) { + throw new BadRequestException(`${field} must be a UTC ISO-8601 timestamp (…Z)`); + } + return value; +} + +/** The four SharedFileOrigin variants, discriminated by `kind`. */ +export function parseSharedFileOrigin(value: unknown, field = "origin"): SharedFileOrigin { + if (!isRecord(value)) throw new BadRequestException(`${field} must be an object`); + const kind = enumField(value, "kind", ORIGIN_KINDS, `${field}.kind`); + if (kind === "matrix") return { kind, mxcUri: requireMxcUri(value.mxcUri, `${field}.mxcUri`) }; + if (kind === "bridge") { + return { + kind, + mxcUri: requireMxcUri(value.mxcUri, `${field}.mxcUri`), + providerId: stringField(value, "providerId", `${field}.providerId`), + ...(value.remoteEventId === undefined || value.remoteEventId === null ? {} : { remoteEventId: stringField(value, "remoteEventId", `${field}.remoteEventId`) }), + }; + } + if (kind === "upload") return { kind }; + return { kind: "pad-export", padId: stringField(value, "padId", `${field}.padId`), revision: stringField(value, "revision", `${field}.revision`) }; +} + +/** Matrix event eligibility input for the hub-archive seam. Unknown state fails closed (core helper). */ +export function parseMatrixAttachmentCandidate(value: unknown, field = "event"): MatrixAttachmentCandidate { + if (!isRecord(value)) throw new BadRequestException(`${field} must be an object`); + const encryption = enumField(value, "roomEncryption", ["unencrypted", "encrypted", "unknown"] as const, `${field}.roomEncryption`); + const encryptedFile = requireField(value, "encryptedFile"); + const redacted = requireField(value, "redacted"); + if (typeof encryptedFile !== "boolean") throw new BadRequestException(`${field}.encryptedFile must be a boolean`); + if (typeof redacted !== "boolean") throw new BadRequestException(`${field}.redacted must be a boolean`); + return { + roomId: requireRoomId(value.roomId, `${field}.roomId`), + eventId: stringField(value, "eventId", `${field}.eventId`), + senderId: requireMxid(value.senderId, `${field}.senderId`), + eventType: stringField(value, "eventType", `${field}.eventType`), + msgtype: stringField(value, "msgtype", `${field}.msgtype`), + roomEncryption: encryption, + encryptedFile, + redacted, + mxcUri: requireMxcUri(value.mxcUri, `${field}.mxcUri`), + }; +} + +export function parseGroupFileMapping(value: unknown, field = "mapping"): GroupFileMapping { + if (!isRecord(value)) throw new BadRequestException(`${field} must be an object`); + const policy = enumField(value, "archivePolicy", ["disabled", "unencrypted-archive"] as const, `${field}.archivePolicy`); + return { + workspaceId: stringField(value, "workspaceId", `${field}.workspaceId`), + groupId: stringField(value, "groupId", `${field}.groupId`), + roomId: requireRoomId(value.roomId, `${field}.roomId`), + spaceId: stringField(value, "spaceId", `${field}.spaceId`), + folderId: stringField(value, "folderId", `${field}.folderId`), + archivePolicy: policy, + retention: parseRetention(value.retention, `${field}.retention`), + }; +} + +/** The served GroupFile wire shape, guarded on the way out of the DB. */ +export function parseGroupFile(value: unknown, field = "item"): GroupFile { + if (!isRecord(value)) throw new BadRequestException(`${field} must be an object`); + const origin = parseSharedFileOrigin(value.origin, `${field}.origin`); + const eventId = value.eventId; + if (eventId !== undefined && eventId !== null && typeof eventId !== "string") { + throw new BadRequestException(`${field}.eventId must be a string or null`); + } + const file = value.file; + if (!isRecord(file)) throw new BadRequestException(`${field}.file must be an object`); + const shareLink = value.shareLink; + if (!isRecord(shareLink)) throw new BadRequestException(`${field}.shareLink must be an object`); + const access = enumField(shareLink, "access", ["members", "public"] as const, `${field}.shareLink.access`); + const shareExpires = shareLink.expiresAt; + if (shareExpires !== undefined && shareExpires !== null && (typeof shareExpires !== "string" || !ISO_TIMESTAMP.test(shareExpires))) { + throw new BadRequestException(`${field}.shareLink.expiresAt must be a UTC ISO-8601 timestamp or null`); + } + const checksum = value.checksum; + if (!isRecord(checksum)) throw new BadRequestException(`${field}.checksum must be an object`); + return { + id: stringField(value, "id", `${field}.id`), + workspaceId: stringField(value, "workspaceId", `${field}.workspaceId`), + groupId: stringField(value, "groupId", `${field}.groupId`), + roomId: requireRoomId(value.roomId, `${field}.roomId`), + senderId: requireMxid(value.senderId, `${field}.senderId`), + eventId: eventId === undefined || eventId === null ? null : eventId, + origin, + name: stringField(value, "name", `${field}.name`), + mimeType: stringField(value, "mimeType", `${field}.mimeType`), + sizeBytes: integerField(value, "sizeBytes", `${field}.sizeBytes`), + checksum: { algorithm: "sha256", value: requireSha256(checksum.value, `${field}.checksum.value`) }, + createdAt: isoField(value, "createdAt", `${field}.createdAt`), + indexedAt: isoField(value, "indexedAt", `${field}.indexedAt`), + file: { + spaceId: stringField(file, "spaceId", `${field}.file.spaceId`), + resourceId: stringField(file, "resourceId", `${field}.file.resourceId`), + version: stringField(file, "version", `${field}.file.version`), + }, + shareLink: { + url: stringField(shareLink, "url", `${field}.shareLink.url`), + access, + expiresAt: shareExpires === undefined || shareExpires === null ? null : shareExpires, + }, + retention: parseRetention(value.retention, `${field}.retention`), + status: enumField(value, "status", FILE_STATUSES, `${field}.status`), + }; +} diff --git a/apps/api/src/group-files/ocis-webdav.ts b/apps/api/src/group-files/ocis-webdav.ts new file mode 100644 index 0000000..5b98c11 --- /dev/null +++ b/apps/api/src/group-files/ocis-webdav.ts @@ -0,0 +1,254 @@ +/** + * oCIS WebDAV port — the ONLY payload I/O path for group files. + * + * The API streams bytes through itself on both upload and download: it NEVER + * mints or persists a presigned object URL (docs/occupymedia-storage.md + * "Persist library links/share IDs and expiry, not presigned object URLs"). + * Downloads re-verify the SHA-256 while streaming so storage-level tampering + * is detected instead of silently relayed. + * + * ENV CONTRACT (lazy getters — read per call so tests/parallel deployments + * can reconfigure without a rebuild; nothing is read at import time): + * + * FILES_OCID_WEBDAV_URL oCIS WebDAV root, e.g. https://files.example.org + * FILES_OCID_USER service account username (the scoped oCIS identity) + * FILES_OCID_PASSWORD service account password + * FILES_PUBLIC_BASE public base of the file library, used to build the + * persistent (authenticated) library link returned in + * GroupFile.shareLink.url — a stable resource URL, + * NEVER a presigned object URL. + * + * The oCIS deployment lands in parallel; when the env is unset the port + * reports unconfigured and routes fail 503 rather than silently no-op. + * + * WebDAV shape: PUT returns the stable resource id (oc-fileid) and ETag + * (oc-etag / etag) this index needs; plain PUT/GET carry the bytes. The + * transport is injectable (OCIS_FETCH token) so tests stub it without HTTP. + */ +import { ServiceUnavailableException } from "@nestjs/common"; +import { createHash } from "node:crypto"; +import { Readable } from "node:stream"; + +export interface OcisEnvConfig { + webdavUrl: string; + user: string; + password: string; + publicBase: string; +} + +export function ocisConfigFromEnv(env: Readonly> = process.env): OcisEnvConfig { + const webdavUrl = env.FILES_OCID_WEBDAV_URL; + const user = env.FILES_OCID_USER; + const password = env.FILES_OCID_PASSWORD; + if (webdavUrl === undefined || webdavUrl === "") throw new ServiceUnavailableException("FILES_OCID_WEBDAV_URL is not configured"); + if (user === undefined || user === "") throw new ServiceUnavailableException("FILES_OCID_USER is not configured"); + if (password === undefined || password === "") throw new ServiceUnavailableException("FILES_OCID_PASSWORD is not configured"); + return { + webdavUrl: webdavUrl.replace(/\/+$/, ""), + user, + password, + publicBase: (env.FILES_PUBLIC_BASE ?? "").replace(/\/+$/, ""), + }; +} + +/** Result of a committed upload: stable identity + digest over bytes sent. */ +export interface OcisUploadResult { + resourceId: string; + version: string | null; + sha256: string; + sizeBytes: number; +} + +/** An opened download: verifyWhileStreaming checks the digest before relay. */ +export interface OcisDownload { + stream: Readable; + sizeBytes: number | null; + contentType: string | null; +} + +export class OcisVerificationError extends Error { + constructor(expected: string, actual: string) { + super(`oCIS payload verification failed: expected sha256 ${expected}, got ${actual}`); + this.name = "OcisVerificationError"; + } +} + +/** Minimal fetch surface so tests can stub transport without HTTP. */ +export type FetchLike = (url: string, init: RequestInit) => Promise; + +export const OCIS_FETCH: unique symbol = Symbol("OCIS_FETCH"); + +/** Bytes allowed in flight to the transport before the pump pauses. */ +export const UPLOAD_BUFFER_CAP = 8 * 1024 * 1024; + +/** Node's undici fetch needs this hint when a stream body is supplied. */ +type FetchInitWithDuplex = RequestInit & { duplex?: "half" }; + +/** + * WebDAV client. `fetcher` defaults to the global fetch. + */ +export class OcisWebdavClient { + constructor( + private readonly config: OcisEnvConfig, + private readonly fetcher: FetchLike = (url, init) => fetch(url, init), + ) {} + + /** Group folder path under the service identity's WebDAV root. */ + uploadUrl(groupId: string, safeName: string): string { + return `${this.config.webdavUrl}/remote.php/dav/files/${encodeURIComponent(this.config.user)}/hub-groups/${encodeURIComponent(groupId)}/${encodeURIComponent(safeName)}`; + } + + downloadUrl(resourceId: string): string { + return `${this.config.webdavUrl}/remote.php/dav/files/${encodeURIComponent(this.config.user)}/hub-resources/${encodeURIComponent(resourceId)}`; + } + + private authHeaders(): Record { + const basic = Buffer.from(`${this.config.user}:${this.config.password}`, "utf8").toString("base64"); + return { authorization: `Basic ${basic}` }; + } + + /** + * Stream-to-storage PUT, hashing while streaming (contract: "Hash actual + * bytes while streaming"). The digest covers the exact bytes handed to the + * store — never a claimed Content-Length. + * + * Backpressure: the pump pauses when more than UPLOAD_BUFFER_CAP bytes are + * queued for the transport, so RAM stays bounded for large payloads. The + * pump is driven by our own loop, so the digest completes even if the + * transport returns without draining a small body (a 2xx store that never + * reads its request body is broken and will surface as an integrity error). + */ + async upload(groupId: string, safeName: string, body: Readable, contentType: string): Promise { + const hash = createHash("sha256"); + let sizeBytes = 0; + let queuedBytes = 0; + const queue: Buffer[] = []; + let sourceEnded = false; + let sourceError: Error | undefined; + let wakeTransport: (() => void) | undefined; + let wakePump: (() => void) | undefined; + + const requestStream = new ReadableStream({ + async pull(controller): Promise { + if (queue.length > 0) { + const next = queue.shift()!; + queuedBytes -= next.byteLength; + controller.enqueue(next); + wakePump?.(); + return; + } + if (sourceEnded) { + if (sourceError !== undefined) controller.error(sourceError); + else controller.close(); + return; + } + await new Promise((resolve) => { + wakeTransport = resolve; + }); + wakeTransport = undefined; + if (queue.length > 0) { + controller.enqueue(queue.shift()!); + wakePump?.(); + } else if (sourceError !== undefined) { + controller.error(sourceError); + } else if (sourceEnded) { + controller.close(); + } + }, + cancel(): void { + // The transport hung up (e.g. it rejected early): keep draining the + // source so the digest/size accounting still completes. + wakeTransport?.(); + }, + }); + + const pumped = (async (): Promise => { + for await (const chunk of body) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array | string); + hash.update(buffer); + sizeBytes += buffer.byteLength; + queuedBytes += buffer.byteLength; + queue.push(buffer); + wakeTransport?.(); + while (queuedBytes > UPLOAD_BUFFER_CAP) { + await new Promise((resolve) => { + wakePump = resolve; + }); + wakePump = undefined; + } + } + sourceEnded = true; + wakeTransport?.(); + })().catch((err: unknown) => { + sourceError = err instanceof Error ? err : new Error(String(err)); + wakeTransport?.(); + wakePump?.(); + }); + + const response = await this.fetcher(this.uploadUrl(groupId, safeName), { + method: "PUT", + headers: { ...this.authHeaders(), "content-type": contentType }, + body: requestStream as unknown as ReadableStream, + duplex: "half", + } satisfies FetchInitWithDuplex); + if (!response.ok) { + requestStream.cancel().catch(() => undefined); + await pumped; + throw new Error(`oCIS upload failed with ${response.status}`); + } + await pumped; + const etag = response.headers.get("oc-etag") ?? response.headers.get("etag"); + const resourceId = response.headers.get("oc-fileid"); + return { + resourceId: resourceId ?? safeName, + version: etag, + sha256: hash.digest("hex"), + sizeBytes, + }; + } + + /** Open a GET. The digest is re-checked by verifyWhileStreaming downstream. */ + async download(resourceId: string): Promise { + const response = await this.fetcher(this.downloadUrl(resourceId), { method: "GET", headers: this.authHeaders() }); + if (!response.ok || response.body === null) throw new Error(`oCIS download failed with ${response.status}`); + const length = response.headers.get("content-length"); + return { + stream: response.body as unknown as Readable, + sizeBytes: length === null ? null : Number(length), + contentType: response.headers.get("content-type"), + }; + } + + /** Stable resource link for GroupFile.shareLink.url — never presigned. */ + libraryUrl(resourceId: string): string { + return this.config.publicBase === "" ? resourceId : `${this.config.publicBase}/f/${encodeURIComponent(resourceId)}`; + } +} + +/** + * Pipe a download forward, verifying sha256 chunk-by-chunk. Contract: + * "SHA-256 verified while streaming" — a mismatch aborts the relay rather + * than handing over tampered bytes (the sink sees them, but the caller + * treats the error as fatal and closes the response). + */ +export async function verifyWhileStreaming( + download: OcisDownload, + expectedSha256: string, + sink: (chunk: Buffer) => void, +): Promise<{ sizeBytes: number }> { + const hash = createHash("sha256"); + let sizeBytes = 0; + for await (const chunk of download.stream) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array); + hash.update(buffer); + sizeBytes += buffer.byteLength; + sink(buffer); + } + const actual = hash.digest("hex"); + if (actual !== expectedSha256) throw new OcisVerificationError(expectedSha256, actual); + return { sizeBytes }; +} + +export function isOcisVerificationError(err: unknown): err is OcisVerificationError { + return err instanceof OcisVerificationError; +} diff --git a/apps/api/src/group-files/share-token.ts b/apps/api/src/group-files/share-token.ts new file mode 100644 index 0000000..b87c368 --- /dev/null +++ b/apps/api/src/group-files/share-token.ts @@ -0,0 +1,40 @@ +/** + * Public share tokens for group files. + * + * Contract (docs/occupymedia-storage.md): "Persist library links/share IDs + * and expiry, not presigned object URLs, WOPI tokens or passwords." The DB + * stores sha256(token) ONLY (029 group_file_shares.token_hash); the + * plaintext token is returned exactly once at creation and can never be + * recovered from the database. + * + * 256 bits of entropy (32 random bytes, base64url — same shape as invite + * links), hashed with plain sha256. Deliberately NOT a salted password hash: + * the token is a high-entropy capability, not a low-entropy secret, so an + * offline dictionary attack against token_hash is infeasible and a plain + * digest keeps the public lookup O(1) on a unique index. + */ +import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; + +export const SHARE_TOKEN_BYTES = 32; + +/** Plaintext capability, base64url. Returned ONCE; never persisted. */ +export function mintShareToken(): string { + return randomBytes(SHARE_TOKEN_BYTES).toString("base64url"); +} + +/** The ONLY value ever stored: sha256 of the plaintext, lowercase hex. */ +export function hashShareToken(token: string): string { + return createHash("sha256").update(token, "utf8").digest("hex"); +} + +/** Constant-time compare for lookups done outside the unique-index path. */ +export function sameToken(a: string, b: string): boolean { + const left = Buffer.from(a, "utf8"); + const right = Buffer.from(b, "utf8"); + return left.length === right.length && timingSafeEqual(left, right); +} + +/** Shape guard before hashing: reject absurd values before touching the DB. */ +export function isPlausibleShareToken(value: unknown): value is string { + return typeof value === "string" && value.length >= 20 && value.length <= 128 && /^[A-Za-z0-9_-]+$/.test(value); +} diff --git a/apps/archive-bot/Dockerfile b/apps/archive-bot/Dockerfile new file mode 100644 index 0000000..a7bd4c0 --- /dev/null +++ b/apps/archive-bot/Dockerfile @@ -0,0 +1,17 @@ +# Hub archive sync bot. Build from the monorepo root so workspace:* deps resolve: +# docker build -f apps/archive-bot/Dockerfile -t hub-archive-bot . +FROM node:22-slim AS build +WORKDIR /app +COPY . . +RUN NODE_ENV=development npx --yes pnpm@9.12.0 install --frozen-lockfile +RUN npx --yes pnpm@9.12.0 --filter @tactical-media-broadcast-hub/archive-bot... build + +FROM node:22-slim +ENV NODE_ENV=production +WORKDIR /app +# Workspace node_modules are relative symlinks; copy the built tree whole. +COPY --from=build --chown=node:node /app /app +# Non-root, and the bot never writes payloads to disk (streams straight to WebDAV). +USER node +EXPOSE 8085 +CMD ["node", "/app/apps/archive-bot/dist/main.js"] diff --git a/apps/archive-bot/package.json b/apps/archive-bot/package.json new file mode 100644 index 0000000..67b2fd2 --- /dev/null +++ b/apps/archive-bot/package.json @@ -0,0 +1,29 @@ +{ + "name": "@tactical-media-broadcast-hub/archive-bot", + "private": true, + "version": "0.1.0", + "type": "module", + "scripts": { + "dev": "tsx watch src/main.ts", + "build": "tsc -p tsconfig.build.json", + "start": "node dist/main.js", + "lint": "eslint src", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "vitest run" + }, + "dependencies": { + "@tactical-media-broadcast-hub/core": "workspace:*", + "pg": "^8.23.1", + "pino": "^9.3.2" + }, + "devDependencies": { + "@types/node": "^22.5.0", + "@types/pg": "^8.23.1", + "tsx": "^4.17.0", + "typescript": "^5.5.4", + "vitest": "^2.0.5" + }, + "engines": { + "node": ">=22.13.0" + } +} diff --git a/apps/archive-bot/src/archiver.ts b/apps/archive-bot/src/archiver.ts new file mode 100644 index 0000000..28ca80d --- /dev/null +++ b/apps/archive-bot/src/archiver.ts @@ -0,0 +1,90 @@ +import type { GroupFile } from "@tactical-media-broadcast-hub/core"; +import { toGroupFileMapping, type RoomMapping } from "./config.js"; +import type { AttachmentWork } from "./events.js"; +import { inspectStream } from "./inspector.js"; +import { replyTxnId } from "./keys.js"; +import type { MatrixPort } from "./matrix.js"; +import { safeFileName } from "./names.js"; +import type { ArchiveStore, OutboxRow } from "./store.js"; +import type { LibraryPort } from "./webdav.js"; + +export interface ArchivedAttachment { + file: GroupFile; + safeName: string; +} + +/** + * Steps 2–3 of the processing contract for one claimed attachment: + * authenticated download → in-flight size/hash/MIME → staged WebDAV PUT → + * content-addressed MOVE → index row + outbox done in one transaction. + * The payload is never written to local disk. + */ +export class Archiver { + constructor( + private readonly matrix: MatrixPort, + private readonly library: LibraryPort, + private readonly store: ArchiveStore, + ) {} + + async archive( + mapping: RoomMapping, + work: AttachmentWork, + outbox: OutboxRow, + claimKey: string, + signal: AbortSignal, + ): Promise { + const { candidate, event } = work; + const download = await this.matrix.downloadMedia(candidate.mxcUri, signal); + const { stream, result } = inspectStream(download.body, download.contentLength); + const dir = ["Library", mapping.workspaceId, mapping.groupId]; + // Deterministic per claim: a retried upload overwrites its own partial staging copy. + const stagingName = `${replyTxnId(claimKey)}.part`; + await this.library.putStaging(dir, stagingName, stream, signal); + const payload = await result; + + const safeName = safeFileName(work.displayName, payload.sniffed.extension); + const resource = await this.library.commit(dir, stagingName, `${payload.sha256}-${safeName}`, signal); + const createdAt = new Date(event.origin_server_ts ?? Date.now()); + const indexed = await this.store.commitArchived({ + workspaceId: mapping.workspaceId, + groupId: mapping.groupId, + roomId: mapping.roomId, + eventId: event.event_id, + senderMxid: event.sender, + matrixEventJson: event, + originalName: work.displayName, + safeName, + mime: payload.sniffed.mime, + mimeClass: payload.sniffed.mimeClass, + byteSize: payload.byteSize, + sha256: payload.sha256, + ocisResourceId: resource.resourceId, + ocisResourceVersion: resource.version, + createdAt, + }, outbox.id); + + const group = toGroupFileMapping(mapping); + return { + safeName, + file: { + id: indexed.id, + workspaceId: mapping.workspaceId, + groupId: mapping.groupId, + roomId: mapping.roomId, + senderId: event.sender, + eventId: event.event_id, + origin: { kind: "matrix", mxcUri: candidate.mxcUri }, + name: safeName, + mimeType: payload.sniffed.mime, + sizeBytes: payload.byteSize, + checksum: { algorithm: "sha256", value: payload.sha256 }, + createdAt: createdAt.toISOString(), + indexedAt: indexed.indexedAt.toISOString(), + file: { spaceId: group.spaceId, resourceId: resource.resourceId, version: resource.version }, + shareLink: { url: resource.url, access: "members", expiresAt: null }, + retention: group.retention, + status: "available", + }, + }; + } +} diff --git a/apps/archive-bot/src/backoff.ts b/apps/archive-bot/src/backoff.ts new file mode 100644 index 0000000..4469c0d --- /dev/null +++ b/apps/archive-bot/src/backoff.ts @@ -0,0 +1,14 @@ +export const BACKOFF_BASE_MS = 30_000; +export const BACKOFF_MAX_MS = 60 * 60 * 1000; +/** After this many attempts a row is marked failed (a durable, terminal retry record). */ +export const MAX_ATTEMPTS = 12; + +/** Exponential per-row delay after `attempts` failures, capped at one hour. */ +export function backoffMs(attempts: number): number { + if (attempts <= 0) return 0; + return Math.min(BACKOFF_BASE_MS * 2 ** (attempts - 1), BACKOFF_MAX_MS); +} + +export function nextAttemptAt(attempts: number, lastAttemptAt: Date): Date { + return new Date(lastAttemptAt.getTime() + backoffMs(attempts)); +} diff --git a/apps/archive-bot/src/config.test.ts b/apps/archive-bot/src/config.test.ts new file mode 100644 index 0000000..20b6ec8 --- /dev/null +++ b/apps/archive-bot/src/config.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import { ConfigError, loadConfig, parseRoomMappings, toGroupFileMapping } from "./config.js"; + +const WS = "11111111-1111-4111-8111-111111111111"; +const GROUP = "22222222-2222-4222-8222-222222222222"; + +describe("parseRoomMappings", () => { + it("accepts immutable room IDs with UUID workspace/group", () => { + expect(parseRoomMappings(JSON.stringify([{ roomId: "!abc:example.org", workspaceId: WS, groupId: GROUP }]))) + .toEqual([{ roomId: "!abc:example.org", workspaceId: WS, groupId: GROUP }]); + }); + + it("rejects aliases, non-UUIDs, duplicates, empty and malformed input", () => { + expect(() => parseRoomMappings(JSON.stringify([{ roomId: "#archive:example.org", workspaceId: WS, groupId: GROUP }]))) + .toThrow(ConfigError); + expect(() => parseRoomMappings(JSON.stringify([{ roomId: "!a:x", workspaceId: "ws", groupId: GROUP }]))).toThrow(ConfigError); + const dup = { roomId: "!a:x", workspaceId: WS, groupId: GROUP }; + expect(() => parseRoomMappings(JSON.stringify([dup, dup]))).toThrow(/duplicate/); + expect(() => parseRoomMappings("[]")).toThrow(ConfigError); + expect(() => parseRoomMappings("{nope")).toThrow(ConfigError); + }); + + it("maps to an unencrypted-archive GroupFileMapping kept forever", () => { + const mapping = toGroupFileMapping({ roomId: "!a:x", workspaceId: WS, groupId: GROUP }); + expect(mapping.archivePolicy).toBe("unencrypted-archive"); + expect(mapping.retention).toEqual({ legalHold: false, mode: "keep" }); + }); +}); + +describe("loadConfig", () => { + const env = { + DATABASE_URL: "postgres://bot@db/hub", + MATRIX_HOMESERVER_URL: "https://matrix.example.org", + MATRIX_USER: "archive-bot", + MATRIX_PASSWORD: "pw", + ROOM_MAPPINGS: JSON.stringify([{ roomId: "!a:x", workspaceId: WS, groupId: GROUP }]), + FILES_OCID_WEBDAV_URL: "https://files.example.org", + FILES_OCID_USER: "svc", + FILES_OCID_PASSWORD: "pw2", + ARCHIVE_BOT_HEALTH_PORT: "9100", + MATRIX_MEDIA_REDIRECT_HOSTS: " R2.example.com ,", + }; + + it("parses the full env contract", () => { + const config = loadConfig(env); + expect(config.healthPort).toBe(9100); + expect(config.mediaRedirectHosts).toEqual(["r2.example.com"]); + expect(config.matrixDeviceId).toBe("HUB_ARCHIVE_BOT"); + }); + + it("fails on missing secrets or a bad port", () => { + expect(() => loadConfig({ ...env, MATRIX_PASSWORD: "" })).toThrow(/MATRIX_PASSWORD/); + expect(() => loadConfig({ ...env, ARCHIVE_BOT_HEALTH_PORT: "70000" })).toThrow(/PORT/); + expect(() => loadConfig({ ...env, FILES_OCID_WEBDAV_URL: "ftp://x" })).toThrow(/http/); + }); +}); diff --git a/apps/archive-bot/src/config.ts b/apps/archive-bot/src/config.ts new file mode 100644 index 0000000..bcadd3b --- /dev/null +++ b/apps/archive-bot/src/config.ts @@ -0,0 +1,110 @@ +import type { GroupFileMapping } from "@tactical-media-broadcast-hub/core"; + +/** Explicit, immutable room → library mapping. Only rooms listed here are ever read. */ +export interface RoomMapping { + roomId: string; + workspaceId: string; + groupId: string; +} + +export interface BotConfig { + databaseUrl: string; + homeserverUrl: URL; + matrixUser: string; + matrixPassword: string; + /** Stable device so Matrix transaction-ID idempotency survives restarts. */ + matrixDeviceId: string; + /** Extra hosts (e.g. the R2 media endpoint) media redirects may target; auth is always dropped. */ + mediaRedirectHosts: string[]; + mappings: RoomMapping[]; + webdav: { baseUrl: URL; user: string; password: string }; + healthPort: number; +} + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +/** Room IDs only: `!opaque:server`. Aliases (`#…`) are rejected — mappings must be immutable. */ +const ROOM_ID = /^![^:\s]+:[^\s]+$/; + +export class ConfigError extends Error {} + +function required(env: NodeJS.ProcessEnv, name: string): string { + const value = env[name]; + if (value === undefined || value.trim() === "") throw new ConfigError(`${name} is required`); + return value.trim(); +} + +function httpUrl(name: string, raw: string): URL { + let url: URL; + try { + url = new URL(raw); + } catch { + throw new ConfigError(`${name} must be an absolute URL`); + } + if (url.protocol !== "https:" && url.protocol !== "http:") throw new ConfigError(`${name} must be http(s)`); + return url; +} + +export function parseRoomMappings(raw: string): RoomMapping[] { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new ConfigError("ROOM_MAPPINGS must be JSON"); + } + if (!Array.isArray(parsed) || parsed.length === 0) throw new ConfigError("ROOM_MAPPINGS must be a non-empty array"); + const seen = new Set(); + return parsed.map((entry: unknown, index) => { + if (typeof entry !== "object" || entry === null) throw new ConfigError(`ROOM_MAPPINGS[${index}] must be an object`); + const { roomId, workspaceId, groupId } = entry as Record; + if (typeof roomId !== "string" || !ROOM_ID.test(roomId)) { + throw new ConfigError(`ROOM_MAPPINGS[${index}].roomId must be a room ID (!id:server), not an alias`); + } + if (typeof workspaceId !== "string" || !UUID.test(workspaceId)) { + throw new ConfigError(`ROOM_MAPPINGS[${index}].workspaceId must be a UUID`); + } + if (typeof groupId !== "string" || !UUID.test(groupId)) { + throw new ConfigError(`ROOM_MAPPINGS[${index}].groupId must be a UUID`); + } + if (seen.has(roomId)) throw new ConfigError(`ROOM_MAPPINGS has duplicate roomId ${roomId}`); + seen.add(roomId); + return { roomId, workspaceId: workspaceId.toLowerCase(), groupId: groupId.toLowerCase() }; + }); +} + +/** The env mapping only exists for rooms explicitly enabled as unencrypted archive rooms. */ +export function toGroupFileMapping(mapping: RoomMapping): GroupFileMapping { + return { + workspaceId: mapping.workspaceId, + groupId: mapping.groupId, + roomId: mapping.roomId, + spaceId: mapping.workspaceId, + folderId: `${mapping.workspaceId}/${mapping.groupId}`, + archivePolicy: "unencrypted-archive", + retention: { legalHold: false, mode: "keep" }, + }; +} + +export function loadConfig(env: NodeJS.ProcessEnv = process.env): BotConfig { + const healthPort = Number(env.ARCHIVE_BOT_HEALTH_PORT ?? 8085); + if (!Number.isInteger(healthPort) || healthPort <= 0 || healthPort > 65535) { + throw new ConfigError("ARCHIVE_BOT_HEALTH_PORT must be a TCP port"); + } + return { + databaseUrl: required(env, "DATABASE_URL"), + homeserverUrl: httpUrl("MATRIX_HOMESERVER_URL", required(env, "MATRIX_HOMESERVER_URL")), + matrixUser: required(env, "MATRIX_USER"), + matrixPassword: required(env, "MATRIX_PASSWORD"), + matrixDeviceId: env.MATRIX_DEVICE_ID?.trim() || "HUB_ARCHIVE_BOT", + mediaRedirectHosts: (env.MATRIX_MEDIA_REDIRECT_HOSTS ?? "") + .split(",") + .map((h) => h.trim().toLowerCase()) + .filter((h) => h !== ""), + mappings: parseRoomMappings(required(env, "ROOM_MAPPINGS")), + webdav: { + baseUrl: httpUrl("FILES_OCID_WEBDAV_URL", required(env, "FILES_OCID_WEBDAV_URL")), + user: required(env, "FILES_OCID_USER"), + password: required(env, "FILES_OCID_PASSWORD"), + }, + healthPort, + }; +} diff --git a/apps/archive-bot/src/events.test.ts b/apps/archive-bot/src/events.test.ts new file mode 100644 index 0000000..16483bd --- /dev/null +++ b/apps/archive-bot/src/events.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; +import type { RoomMapping } from "./config.js"; +import { classifyEvent, parseMxc, redactionTarget, type MatrixEvent } from "./events.js"; + +const mapping: RoomMapping = { + roomId: "!room:example.org", + workspaceId: "11111111-1111-4111-8111-111111111111", + groupId: "22222222-2222-4222-8222-222222222222", +}; +const BOT = "@archive:example.org"; + +function message(content: Record, extra: Partial = {}): MatrixEvent { + return { type: "m.room.message", event_id: "$ev1", sender: "@alice:example.org", content, ...extra }; +} +const image = { msgtype: "m.image", body: "cat.png", url: "mxc://example.org/abc123" }; + +describe("classifyEvent", () => { + it("accepts an unencrypted m.image with an mxc URI", () => { + const result = classifyEvent(message(image), mapping, "unencrypted", BOT); + expect(result.kind).toBe("attachment"); + if (result.kind !== "attachment") return; + expect(result.candidate).toMatchObject({ roomId: mapping.roomId, eventId: "$ev1", msgtype: "m.image", mxcUri: image.url }); + expect(result.displayName).toBe("cat.png"); + }); + + it("prefers content.filename over body (caption) for the display name", () => { + const result = classifyEvent(message({ ...image, filename: "real.png", body: "a caption" }), mapping, "unencrypted", BOT); + expect(result.kind === "attachment" && result.displayName).toBe("real.png"); + }); + + it.each(["m.file", "m.audio", "m.video"])("accepts %s", (msgtype) => { + expect(classifyEvent(message({ ...image, msgtype }), mapping, "unencrypted", BOT).kind).toBe("attachment"); + }); + + it.each([ + ["own-event", message(image, { sender: BOT }), "unencrypted"], + ["room-encrypted", message(image), "encrypted"], + ["room-unknown", message(image), "unknown"], + ["encrypted-event", { ...message(image), type: "m.room.encrypted" }, "unencrypted"], + ["encrypted-file", message({ msgtype: "m.file", body: "x", file: { url: "mxc://example.org/enc" } }), "unencrypted"], + ["edit", message({ ...image, "m.relates_to": { rel_type: "m.replace", event_id: "$old" } }), "unencrypted"], + ["edit", message({ ...image, "m.new_content": image }), "unencrypted"], + ["redacted", message(image, { unsigned: { redacted_because: {} } }), "unencrypted"], + ["not-an-attachment", message({ msgtype: "m.text", body: "hi" }), "unencrypted"], + ["not-a-message", { ...message(image), type: "m.sticker" }, "unencrypted"], + ["ineligible", message({ ...image, url: "https://evil.example/x.png" }), "unencrypted"], + ["ineligible", message({ ...image, url: "mxc://example.org/a/b" }), "unencrypted"], + ["ineligible", message({ msgtype: "m.image", body: "x" }), "unencrypted"], + ] as const)("skips %s", (reason, event, encryption) => { + const result = classifyEvent(event, mapping, encryption, BOT); + expect(result).toMatchObject({ kind: "skip", reason }); + }); + + it("classifies redactions regardless of room encryption (v11 content.redacts and legacy top-level)", () => { + const v11: MatrixEvent = { type: "m.room.redaction", event_id: "$r", sender: "@mod:x", content: { redacts: "$ev1" } }; + const legacy: MatrixEvent = { type: "m.room.redaction", event_id: "$r", sender: "@mod:x", content: {}, redacts: "$ev2" }; + expect(classifyEvent(v11, mapping, "encrypted", BOT)).toMatchObject({ kind: "redaction", redacts: "$ev1" }); + expect(redactionTarget(legacy)).toBe("$ev2"); + expect(classifyEvent({ ...v11, content: {} }, mapping, "unencrypted", BOT).kind).toBe("skip"); + }); +}); + +describe("parseMxc", () => { + it("splits server and media id", () => { + expect(parseMxc("mxc://example.org/abc")).toEqual({ serverName: "example.org", mediaId: "abc" }); + }); + it("rejects anything else", () => { + expect(() => parseMxc("https://example.org/abc")).toThrow(); + expect(() => parseMxc("mxc://example.org/../x")).toThrow(); + }); +}); diff --git a/apps/archive-bot/src/events.ts b/apps/archive-bot/src/events.ts new file mode 100644 index 0000000..1d9e7aa --- /dev/null +++ b/apps/archive-bot/src/events.ts @@ -0,0 +1,109 @@ +import { canArchiveAttachment, type MatrixAttachmentCandidate } from "@tactical-media-broadcast-hub/core"; +import { toGroupFileMapping, type RoomMapping } from "./config.js"; + +/** Client-server API event shape (only the fields the bot reads). */ +export interface MatrixEvent { + type: string; + event_id: string; + sender: string; + origin_server_ts?: number; + state_key?: string; + /** Room v1–v10 redactions carry the target here; v11+ moved it into content. */ + redacts?: string; + content: Record; + unsigned?: { redacted_because?: unknown; [key: string]: unknown }; +} + +export type RoomEncryption = MatrixAttachmentCandidate["roomEncryption"]; + +export const ATTACHMENT_MSGTYPES = ["m.file", "m.image", "m.audio", "m.video"] as const; + +export interface AttachmentWork { + kind: "attachment"; + event: MatrixEvent; + candidate: MatrixAttachmentCandidate; + /** Sender-supplied display name; only ever used after normalisation. */ + displayName: string; +} + +export type Classified = + | AttachmentWork + | { kind: "redaction"; event: MatrixEvent; redacts: string } + | { kind: "skip"; event: MatrixEvent; reason: string }; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** Runtime guard for raw JSON from /sync or /messages. */ +export function isMatrixEvent(value: unknown): value is MatrixEvent { + return isRecord(value) + && typeof value.type === "string" + && typeof value.event_id === "string" + && typeof value.sender === "string" + && isRecord(value.content); +} + +export function redactionTarget(event: MatrixEvent): string | null { + const target = typeof event.content.redacts === "string" ? event.content.redacts : event.redacts; + return typeof target === "string" && target.startsWith("$") ? target : null; +} + +function isEdit(content: Record): boolean { + const relates = content["m.relates_to"]; + return (isRecord(relates) && relates.rel_type === "m.replace") || "m.new_content" in content; +} + +/** + * Decide what (if anything) to do with a timeline event in a mapped room. + * Every exclusion from the processing contract lives here so it is unit-testable; + * the final eligibility gate is the shared `canArchiveAttachment`. + */ +export function classifyEvent( + event: MatrixEvent, + mapping: RoomMapping, + roomEncryption: RoomEncryption, + botUserId: string, +): Classified { + if (event.sender === botUserId) return { kind: "skip", event, reason: "own-event" }; + if (event.type === "m.room.redaction") { + const redacts = redactionTarget(event); + return redacts === null ? { kind: "skip", event, reason: "redaction-without-target" } : { kind: "redaction", event, redacts }; + } + if (event.type === "m.room.encrypted") return { kind: "skip", event, reason: "encrypted-event" }; + if (event.type !== "m.room.message") return { kind: "skip", event, reason: "not-a-message" }; + if (event.unsigned?.redacted_because !== undefined) return { kind: "skip", event, reason: "redacted" }; + const { content } = event; + const msgtype = typeof content.msgtype === "string" ? content.msgtype : ""; + if (!(ATTACHMENT_MSGTYPES as readonly string[]).includes(msgtype)) { + return { kind: "skip", event, reason: "not-an-attachment" }; + } + if (isEdit(content)) return { kind: "skip", event, reason: "edit" }; + if (content.file !== undefined) return { kind: "skip", event, reason: "encrypted-file" }; + if (roomEncryption !== "unencrypted") return { kind: "skip", event, reason: `room-${roomEncryption}` }; + + const candidate: MatrixAttachmentCandidate = { + roomId: mapping.roomId, + eventId: event.event_id, + senderId: event.sender, + eventType: event.type, + msgtype, + roomEncryption, + encryptedFile: content.file !== undefined, + redacted: false, + mxcUri: typeof content.url === "string" ? content.url : "", + }; + if (!canArchiveAttachment(toGroupFileMapping(mapping), candidate)) { + return { kind: "skip", event, reason: "ineligible" }; + } + const filename = typeof content.filename === "string" && content.filename.trim() !== "" ? content.filename : null; + const body = typeof content.body === "string" ? content.body : ""; + return { kind: "attachment", event, candidate, displayName: filename ?? body }; +} + +/** Split a validated `mxc://server/mediaId` into its parts. */ +export function parseMxc(mxcUri: string): { serverName: string; mediaId: string } { + const match = /^mxc:\/\/([^/\s]+)\/([^/\s?#]+)$/.exec(mxcUri); + if (match === null || match[1] === undefined || match[2] === undefined) throw new Error("invalid mxc URI"); + return { serverName: match[1], mediaId: match[2] }; +} diff --git a/apps/archive-bot/src/health.ts b/apps/archive-bot/src/health.ts new file mode 100644 index 0000000..6f232a0 --- /dev/null +++ b/apps/archive-bot/src/health.ts @@ -0,0 +1,24 @@ +import { createServer, type Server } from "node:http"; +import type { Logger } from "pino"; +import type { HealthSnapshot } from "./processor.js"; + +/** GET /health (or /) → 200 {rooms, cursor, pending, lastProcessedAt}; 503 when the DB count fails. */ +export function startHealthServer(port: number, snapshot: () => Promise, logger: Logger): Server { + const server = createServer((req, res) => { + if (req.method !== "GET" || (req.url !== "/" && req.url !== "/health")) { + res.writeHead(404).end(); + return; + } + snapshot().then( + (body) => { + res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify(body)); + }, + (err: unknown) => { + logger.warn({ err: err instanceof Error ? err.message : String(err) }, "health: snapshot failed"); + res.writeHead(503, { "content-type": "application/json" }).end(JSON.stringify({ status: "down" })); + }, + ); + }); + server.listen(port, () => logger.info({ port }, "health: listening")); + return server; +} diff --git a/apps/archive-bot/src/inspector.test.ts b/apps/archive-bot/src/inspector.test.ts new file mode 100644 index 0000000..94e54f5 --- /dev/null +++ b/apps/archive-bot/src/inspector.test.ts @@ -0,0 +1,81 @@ +import { createHash } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { inspectStream, parseContentLength, PayloadInspector, PermanentArchiveError } from "./inspector.js"; + +const PDF = new TextEncoder().encode("%PDF-1.7\nhello world"); + +function streamOf(...chunks: Uint8Array[]): ReadableStream { + return new ReadableStream({ + start(controller) { + for (const chunk of chunks) controller.enqueue(chunk); + controller.close(); + }, + }); +} + +async function drain(stream: ReadableStream): Promise { + let total = 0; + for await (const chunk of stream) total += chunk.byteLength; + return total; +} + +describe("PayloadInspector", () => { + it("hashes, counts and sniffs across chunk boundaries", () => { + const inspector = new PayloadInspector(PDF.byteLength); + inspector.update(PDF.subarray(0, 2)); + inspector.update(PDF.subarray(2)); + expect(inspector.finish()).toEqual({ + sha256: createHash("sha256").update(PDF).digest("hex"), + byteSize: PDF.byteLength, + sniffed: { mime: "application/pdf", mimeClass: "document", extension: "pdf" }, + }); + }); + + it("rejects an oversized declared length before reading", () => { + expect(() => new PayloadInspector(21 * 1024 * 1024)).toThrow(PermanentArchiveError); + }); + + it("rejects streams exceeding the cap without Content-Length", () => { + const inspector = new PayloadInspector(null, 10); + expect(() => inspector.update(new Uint8Array(11))).toThrow(/exceeds/); + }); + + it("rejects Content-Length lies in both directions", () => { + const longer = new PayloadInspector(4); + expect(() => longer.update(new Uint8Array(5))).toThrow(/longer/); + const shorter = new PayloadInspector(10); + shorter.update(new Uint8Array(4)); + expect(() => shorter.finish()).toThrow(/shorter/); + }); + + it("rejects empty payloads", () => { + expect(() => new PayloadInspector(null).finish()).toThrow(/empty/); + }); +}); + +describe("inspectStream", () => { + it("passes bytes through untouched and resolves the result", async () => { + const { stream, result } = inspectStream(streamOf(PDF.subarray(0, 5), PDF.subarray(5)), PDF.byteLength); + expect(await drain(stream)).toBe(PDF.byteLength); + expect((await result).byteSize).toBe(PDF.byteLength); + }); + + it("errors the stream (aborting the upload) when the source lies about length", async () => { + const { stream, result } = inspectStream(streamOf(PDF), PDF.byteLength + 1); + await expect(drain(stream)).rejects.toThrow(PermanentArchiveError); + await expect(result).rejects.toThrow(/shorter/); + }); + + it("errors mid-stream once the cap is crossed", async () => { + const { stream } = inspectStream(streamOf(new Uint8Array(6), new Uint8Array(6)), null, 10); + await expect(drain(stream)).rejects.toThrow(/exceeds/); + }); +}); + +describe("parseContentLength", () => { + it("parses digits and treats absence as unknown", () => { + expect(parseContentLength("42")).toBe(42); + expect(parseContentLength(null)).toBeNull(); + expect(() => parseContentLength("-1")).toThrow(PermanentArchiveError); + }); +}); diff --git a/apps/archive-bot/src/inspector.ts b/apps/archive-bot/src/inspector.ts new file mode 100644 index 0000000..aeb988c --- /dev/null +++ b/apps/archive-bot/src/inspector.ts @@ -0,0 +1,105 @@ +import { createHash, type Hash } from "node:crypto"; +import { sniffMime, SNIFF_BYTES, type SniffedMime } from "./mime.js"; + +export const MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024; + +/** Non-retryable payload problems: the outbox row is marked failed immediately. */ +export class PermanentArchiveError extends Error {} + +export interface InspectedPayload { + sha256: string; + byteSize: number; + sniffed: SniffedMime; +} + +/** + * Incremental size/hash/magic-byte inspection. Bytes pass straight through to + * the upload; nothing is buffered beyond the first SNIFF_BYTES. + */ +export class PayloadInspector { + private readonly hash: Hash = createHash("sha256"); + private readonly head = new Uint8Array(SNIFF_BYTES); + private headLength = 0; + private total = 0; + + constructor( + private readonly declaredLength: number | null, + private readonly maxBytes = MAX_ATTACHMENT_BYTES, + ) { + if (declaredLength !== null && declaredLength > maxBytes) { + throw new PermanentArchiveError(`declared size ${declaredLength} exceeds ${maxBytes} bytes`); + } + } + + update(chunk: Uint8Array): void { + this.total += chunk.byteLength; + if (this.total > this.maxBytes) throw new PermanentArchiveError(`payload exceeds ${this.maxBytes} bytes`); + if (this.declaredLength !== null && this.total > this.declaredLength) { + throw new PermanentArchiveError("payload longer than Content-Length"); + } + if (this.headLength < SNIFF_BYTES) { + const take = chunk.subarray(0, SNIFF_BYTES - this.headLength); + this.head.set(take, this.headLength); + this.headLength += take.byteLength; + } + this.hash.update(chunk); + } + + finish(): InspectedPayload { + if (this.declaredLength !== null && this.total !== this.declaredLength) { + throw new PermanentArchiveError("payload shorter than Content-Length"); + } + if (this.total === 0) throw new PermanentArchiveError("empty payload"); + return { + sha256: this.hash.digest("hex"), + byteSize: this.total, + sniffed: sniffMime(this.head.subarray(0, this.headLength)), + }; + } +} + +/** + * Wrap a byte stream so it is inspected in flight. Any violation errors the + * stream, which aborts the consuming upload before it can complete. + */ +export function inspectStream( + source: ReadableStream, + declaredLength: number | null, + maxBytes = MAX_ATTACHMENT_BYTES, +): { stream: ReadableStream; result: Promise } { + const inspector = new PayloadInspector(declaredLength, maxBytes); + let resolve!: (value: InspectedPayload) => void; + let reject!: (reason: unknown) => void; + const result = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + // The caller awaits the upload first; avoid an unhandled rejection in the meantime. + result.catch(() => undefined); + const transform = new TransformStream({ + transform(chunk, controller) { + try { + inspector.update(chunk); + } catch (err) { + reject(err); + throw err; + } + controller.enqueue(chunk); + }, + flush() { + try { + resolve(inspector.finish()); + } catch (err) { + reject(err); + throw err; + } + }, + }); + return { stream: source.pipeThrough(transform), result }; +} + +export function parseContentLength(header: string | null): number | null { + if (header === null) return null; + if (!/^\d+$/.test(header.trim())) throw new PermanentArchiveError("invalid Content-Length"); + return Number(header.trim()); +} diff --git a/apps/archive-bot/src/keys.test.ts b/apps/archive-bot/src/keys.test.ts new file mode 100644 index 0000000..3f6a5d3 --- /dev/null +++ b/apps/archive-bot/src/keys.test.ts @@ -0,0 +1,33 @@ +import { createHash } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { backoffMs, BACKOFF_BASE_MS, BACKOFF_MAX_MS, nextAttemptAt } from "./backoff.js"; +import { claimKey, replyTxnId } from "./keys.js"; + +describe("claimKey", () => { + it("is the shared attachmentArchiveKey encoding", () => { + expect(claimKey("w", "!r:x", "$e")).toBe(JSON.stringify(["w", "!r:x", "$e"])); + }); + it("is collision-free across separators", () => { + expect(claimKey("a", "b|c", "d")).not.toBe(claimKey("a|b", "c", "d")); + }); +}); + +describe("replyTxnId", () => { + it("is the first 24 hex chars of sha256(claim key) and stable", () => { + const key = claimKey("w", "!r:x", "$e"); + const expected = createHash("sha256").update(key).digest("hex").slice(0, 24); + expect(replyTxnId(key)).toBe(expected); + expect(replyTxnId(key)).toMatch(/^[0-9a-f]{24}$/); + expect(replyTxnId(claimKey("w", "!r:x", "$f"))).not.toBe(expected); + }); +}); + +describe("backoff", () => { + it("grows exponentially and caps at one hour", () => { + expect(backoffMs(0)).toBe(0); + expect(backoffMs(1)).toBe(BACKOFF_BASE_MS); + expect(backoffMs(3)).toBe(BACKOFF_BASE_MS * 4); + expect(backoffMs(40)).toBe(BACKOFF_MAX_MS); + expect(nextAttemptAt(2, new Date(0)).getTime()).toBe(BACKOFF_BASE_MS * 2); + }); +}); diff --git a/apps/archive-bot/src/keys.ts b/apps/archive-bot/src/keys.ts new file mode 100644 index 0000000..01d823e --- /dev/null +++ b/apps/archive-bot/src/keys.ts @@ -0,0 +1,12 @@ +import { createHash } from "node:crypto"; +import { attachmentArchiveKey } from "@tactical-media-broadcast-hub/core"; + +/** Durable claim identity; the shared contract owns the encoding. */ +export function claimKey(workspaceId: string, roomId: string, eventId: string): string { + return attachmentArchiveKey(workspaceId, roomId, eventId); +} + +/** Stable Matrix transaction ID for the single confirmation reply of a claim. */ +export function replyTxnId(key: string): string { + return createHash("sha256").update(key, "utf8").digest("hex").slice(0, 24); +} diff --git a/apps/archive-bot/src/logger.ts b/apps/archive-bot/src/logger.ts new file mode 100644 index 0000000..1523a26 --- /dev/null +++ b/apps/archive-bot/src/logger.ts @@ -0,0 +1,14 @@ +import pino from "pino"; + +export const logger = pino({ + level: process.env.LOG_LEVEL ?? "info", + base: { service: "archive-bot" }, + redact: { + paths: [ + "password", "accessToken", "access_token", "authorization", + "*.password", "*.accessToken", "*.access_token", "*.authorization", + "headers.authorization", "*.headers.authorization", + ], + censor: "[REDACTED]", + }, +}); diff --git a/apps/archive-bot/src/main.ts b/apps/archive-bot/src/main.ts new file mode 100644 index 0000000..617da53 --- /dev/null +++ b/apps/archive-bot/src/main.ts @@ -0,0 +1,50 @@ +import { Archiver } from "./archiver.js"; +import { loadConfig } from "./config.js"; +import { startHealthServer } from "./health.js"; +import { logger } from "./logger.js"; +import { MatrixClient } from "./matrix.js"; +import { ArchiveProcessor } from "./processor.js"; +import { PgArchiveStore } from "./store.js"; +import { OcisWebDav } from "./webdav.js"; + +async function main(): Promise { + const config = loadConfig(); + const shutdown = new AbortController(); + for (const signal of ["SIGTERM", "SIGINT"] as const) { + process.once(signal, () => { + logger.info({ signal }, "archive-bot: shutting down"); + shutdown.abort(); + }); + } + + const matrix = new MatrixClient({ + homeserver: config.homeserverUrl, + user: config.matrixUser, + password: config.matrixPassword, + deviceId: config.matrixDeviceId, + mediaRedirectHosts: config.mediaRedirectHosts, + roomIds: config.mappings.map((m) => m.roomId), + logger, + }); + await matrix.login(shutdown.signal); + logger.info({ userId: matrix.userId, rooms: config.mappings.length }, "archive-bot: logged in"); + + const store = PgArchiveStore.create(config.databaseUrl, matrix.userId); + const library = new OcisWebDav(config.webdav.baseUrl, config.webdav.user, config.webdav.password); + const processor = new ArchiveProcessor(matrix, store, new Archiver(matrix, library, store), config.mappings, logger); + await processor.init(); + + const health = startHealthServer(config.healthPort, () => processor.health(), logger); + try { + await processor.run(shutdown.signal, (signal) => matrix.login(signal)); + } finally { + await new Promise((resolve) => health.close(() => resolve())); + await store.close(); + } + logger.info("archive-bot: stopped"); +} + +void main().catch((error: unknown) => { + logger.fatal({ err: error instanceof Error ? error.message : String(error) }, "archive-bot: fatal"); + process.exitCode = 1; +}); diff --git a/apps/archive-bot/src/matrix.test.ts b/apps/archive-bot/src/matrix.test.ts new file mode 100644 index 0000000..6af20a4 --- /dev/null +++ b/apps/archive-bot/src/matrix.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; +import { PermanentArchiveError } from "./inspector.js"; +import { parseSync, resolveRedirect, supportsAuthenticatedMedia, syncFilter } from "./matrix.js"; + +const HS = new URL("https://matrix.example.org"); + +describe("supportsAuthenticatedMedia", () => { + it("requires v1.11+ or the MSC3916 stable flag", () => { + expect(supportsAuthenticatedMedia({ versions: ["v1.10", "v1.11"] })).toBe(true); + expect(supportsAuthenticatedMedia({ versions: ["v1.12"] })).toBe(true); + expect(supportsAuthenticatedMedia({ versions: ["v1.9"], unstable_features: { "org.matrix.msc3916.stable": true } })).toBe(true); + expect(supportsAuthenticatedMedia({ versions: ["r0.6.1", "v1.10"] })).toBe(false); + expect(supportsAuthenticatedMedia(null)).toBe(false); + }); +}); + +describe("resolveRedirect", () => { + it("keeps auth only for the homeserver origin", () => { + expect(resolveRedirect(HS, "/_matrix/media/x", HS, [])).toMatchObject({ sendAuth: true }); + }); + + it("drops auth for allowlisted object-store hosts", () => { + const hop = resolveRedirect(HS, "https://r2.example.com/bucket/obj?sig=1", HS, ["r2.example.com"]); + expect(hop).toMatchObject({ sendAuth: false }); + expect(hop.url.hostname).toBe("r2.example.com"); + }); + + it("refuses unapproved hosts and http downgrades", () => { + expect(() => resolveRedirect(HS, "https://evil.example/x", HS, ["r2.example.com"])).toThrow(PermanentArchiveError); + expect(() => resolveRedirect(HS, "http://r2.example.com/x", HS, ["r2.example.com"])).toThrow(/non-https/); + expect(() => resolveRedirect(HS, "https://matrix.example.org:8448/x", HS, [])).toThrow(PermanentArchiveError); + }); +}); + +describe("parseSync", () => { + it("extracts joined timelines, state and invites, dropping malformed events", () => { + const result = parseSync({ + next_batch: "s2", + rooms: { + join: { + "!a:x": { + timeline: { events: [{ type: "m.room.message", event_id: "$1", sender: "@u:x", content: {} }, { junk: true }], limited: true, prev_batch: "p1" }, + state: { events: [{ type: "m.room.encryption", event_id: "$s", sender: "@u:x", state_key: "", content: { algorithm: "m.megolm.v1.aes-sha2" } }] }, + }, + }, + invite: { "!b:x": {} }, + }, + }); + expect(result.nextBatch).toBe("s2"); + expect(result.invited).toEqual(["!b:x"]); + const room = result.joined.get("!a:x"); + expect(room?.timeline).toMatchObject({ limited: true, prevBatch: "p1" }); + expect(room?.timeline.events).toHaveLength(1); + expect(room?.stateEvents[0]?.type).toBe("m.room.encryption"); + }); + + it("rejects a response without next_batch", () => { + expect(() => parseSync({})).toThrow(); + }); +}); + +describe("syncFilter", () => { + it("restricts to mapped rooms and the needed event types", () => { + const filter = syncFilter(["!a:x"]) as { room: { rooms: string[]; timeline: { types: string[] } } }; + expect(filter.room.rooms).toEqual(["!a:x"]); + expect(filter.room.timeline.types).toContain("m.room.redaction"); + expect(filter.room.timeline.types).toContain("m.room.encryption"); + }); +}); diff --git a/apps/archive-bot/src/matrix.ts b/apps/archive-bot/src/matrix.ts new file mode 100644 index 0000000..ecc3ef3 --- /dev/null +++ b/apps/archive-bot/src/matrix.ts @@ -0,0 +1,298 @@ +import type { Logger } from "pino"; +import { isMatrixEvent, parseMxc, type MatrixEvent, type RoomEncryption } from "./events.js"; +import { PermanentArchiveError, parseContentLength } from "./inspector.js"; + +export class MatrixHttpError extends Error { + constructor( + readonly status: number, + readonly errcode: string | null, + message: string, + ) { + super(message); + } +} + +export interface JoinedRoomSync { + timeline: { events: MatrixEvent[]; limited: boolean; prevBatch: string | null }; + stateEvents: MatrixEvent[]; +} + +export interface SyncResult { + nextBatch: string; + joined: Map; + invited: string[]; +} + +export interface MediaDownload { + body: ReadableStream; + contentLength: number | null; +} + +/** The narrow Matrix surface the processor needs; faked in tests. */ +export interface MatrixPort { + readonly userId: string; + sync(since: string | null, signal: AbortSignal): Promise; + /** Older events between `from` (a prev_batch) back to `to` (the previous sync token), oldest first. */ + backfill(roomId: string, from: string, to: string, signal: AbortSignal): Promise; + roomEncryption(roomId: string, signal: AbortSignal): Promise; + joinRoom(roomId: string, signal: AbortSignal): Promise; + downloadMedia(mxcUri: string, signal: AbortSignal): Promise; + sendNotice(roomId: string, txnId: string, content: Record, signal: AbortSignal): Promise; +} + +const SYNC_TIMEOUT_MS = 30_000; +const MAX_REDIRECTS = 3; +const BACKFILL_PAGES = 20; +const TIMELINE_TYPES = ["m.room.message", "m.room.redaction", "m.room.encrypted", "m.room.encryption"]; + +/** Client-side filter: mapped rooms only, minimal timeline types, no presence/ephemeral noise. */ +export function syncFilter(roomIds: readonly string[]): Record { + return { + presence: { not_types: ["*"] }, + account_data: { not_types: ["*"] }, + room: { + rooms: [...roomIds], + account_data: { not_types: ["*"] }, + ephemeral: { not_types: ["*"] }, + state: { types: ["m.room.encryption"], lazy_load_members: true }, + timeline: { limit: 50, types: TIMELINE_TYPES }, + }, + }; +} + +/** Authenticated media requires spec v1.11+ (or the MSC3916 stable flag on older advertisers). */ +export function supportsAuthenticatedMedia(versions: unknown): boolean { + if (typeof versions !== "object" || versions === null) return false; + const record = versions as { versions?: unknown; unstable_features?: unknown }; + const list = Array.isArray(record.versions) ? record.versions.filter((v): v is string => typeof v === "string") : []; + const modern = list.some((v) => { + const match = /^v1\.(\d+)$/.exec(v); + return match !== null && Number(match[1]) >= 11; + }); + const features = record.unstable_features; + const flagged = typeof features === "object" && features !== null + && (features as Record)["org.matrix.msc3916.stable"] === true; + return modern || flagged; +} + +/** + * Redirect policy for media downloads. Bearer auth is only ever sent to the + * homeserver origin; other hosts must be explicitly allowlisted (e.g. the R2 + * endpoint) and are fetched without credentials. Downgrades to http are refused. + */ +export function resolveRedirect( + current: URL, + location: string, + homeserver: URL, + allowedHosts: readonly string[], +): { url: URL; sendAuth: boolean } { + const url = new URL(location, current); + if (url.origin === homeserver.origin) return { url, sendAuth: true }; + if (url.protocol !== "https:") throw new PermanentArchiveError(`refusing non-https media redirect to ${url.host}`); + if (!allowedHosts.includes(url.hostname.toLowerCase())) { + throw new PermanentArchiveError(`media redirect to unapproved host ${url.hostname}`); + } + return { url, sendAuth: false }; +} + +function events(value: unknown): MatrixEvent[] { + return Array.isArray(value) ? value.filter(isMatrixEvent) : []; +} + +function record(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) : {}; +} + +export function parseSync(body: unknown): SyncResult { + const root = record(body); + if (typeof root.next_batch !== "string") throw new Error("sync response missing next_batch"); + const rooms = record(root.rooms); + const joined = new Map(); + for (const [roomId, raw] of Object.entries(record(rooms.join))) { + const room = record(raw); + const timeline = record(room.timeline); + joined.set(roomId, { + timeline: { + events: events(timeline.events), + limited: timeline.limited === true, + prevBatch: typeof timeline.prev_batch === "string" ? timeline.prev_batch : null, + }, + stateEvents: events(record(room.state).events), + }); + } + return { nextBatch: root.next_batch, joined, invited: Object.keys(record(rooms.invite)) }; +} + +export interface MatrixClientOptions { + homeserver: URL; + user: string; + password: string; + deviceId: string; + mediaRedirectHosts: string[]; + roomIds: string[]; + logger: Logger; +} + +/** Plain client-server sync-loop client (not an appservice). No crypto: encrypted content is never touched. */ +export class MatrixClient implements MatrixPort { + private accessToken: string | null = null; + private resolvedUserId = ""; + private readonly filter: string; + + constructor(private readonly opts: MatrixClientOptions) { + this.filter = JSON.stringify(syncFilter(opts.roomIds)); + } + + get userId(): string { + return this.resolvedUserId; + } + + async login(signal: AbortSignal): Promise { + const versions = await this.request("GET", "/_matrix/client/versions", { signal, auth: false }); + if (!supportsAuthenticatedMedia(versions)) { + throw new Error("homeserver does not support authenticated media (v1.11); refusing to start"); + } + const body = record(await this.request("POST", "/_matrix/client/v3/login", { + signal, + auth: false, + json: { + type: "m.login.password", + identifier: { type: "m.id.user", user: this.opts.user }, + password: this.opts.password, + device_id: this.opts.deviceId, + initial_device_display_name: "Hub archive bot", + }, + })); + if (typeof body.access_token !== "string" || typeof body.user_id !== "string") { + throw new Error("login response missing access_token/user_id"); + } + this.accessToken = body.access_token; + this.resolvedUserId = body.user_id; + } + + async sync(since: string | null, signal: AbortSignal): Promise { + const query = new URLSearchParams({ filter: this.filter, timeout: String(since === null ? 0 : SYNC_TIMEOUT_MS) }); + if (since !== null) query.set("since", since); + const timeout = AbortSignal.any([signal, AbortSignal.timeout(SYNC_TIMEOUT_MS + 30_000)]); + return parseSync(await this.request("GET", `/_matrix/client/v3/sync?${query.toString()}`, { signal: timeout })); + } + + async backfill(roomId: string, from: string, to: string, signal: AbortSignal): Promise { + const collected: MatrixEvent[] = []; + let cursor: string | null = from; + for (let page = 0; page < BACKFILL_PAGES && cursor !== null; page += 1) { + const query = new URLSearchParams({ + from: cursor, + to, + dir: "b", + limit: "100", + filter: JSON.stringify({ types: TIMELINE_TYPES }), + }); + const body = record(await this.request( + "GET", + `/_matrix/client/v3/rooms/${encodeURIComponent(roomId)}/messages?${query.toString()}`, + { signal }, + )); + const chunk = events(body.chunk); + collected.push(...chunk); + cursor = chunk.length > 0 && typeof body.end === "string" ? body.end : null; + } + if (cursor !== null) this.opts.logger.warn({ roomId }, "matrix: backfill page budget exhausted; older gap events skipped"); + return collected.reverse(); + } + + async roomEncryption(roomId: string, signal: AbortSignal): Promise { + try { + const body = record(await this.request( + "GET", + `/_matrix/client/v3/rooms/${encodeURIComponent(roomId)}/state/m.room.encryption/`, + { signal }, + )); + // Any present encryption state (even an unknown algorithm) means encrypted. + return Object.keys(body).length > 0 ? "encrypted" : "unknown"; + } catch (err) { + if (err instanceof MatrixHttpError && err.status === 404 && err.errcode === "M_NOT_FOUND") return "unencrypted"; + if (err instanceof MatrixHttpError && (err.status === 403 || err.status === 404)) return "unknown"; + throw err; + } + } + + async joinRoom(roomId: string, signal: AbortSignal): Promise { + await this.request("POST", `/_matrix/client/v3/join/${encodeURIComponent(roomId)}`, { signal, json: {} }); + } + + async sendNotice(roomId: string, txnId: string, content: Record, signal: AbortSignal): Promise { + await this.request( + "PUT", + `/_matrix/client/v3/rooms/${encodeURIComponent(roomId)}/send/m.room.message/${encodeURIComponent(txnId)}`, + { signal, json: content }, + ); + } + + /** Authenticated media endpoint only; the URL is derived from a validated mxc URI, never user-supplied HTTP. */ + async downloadMedia(mxcUri: string, signal: AbortSignal): Promise { + const { serverName, mediaId } = parseMxc(mxcUri); + let url = new URL( + `/_matrix/client/v1/media/download/${encodeURIComponent(serverName)}/${encodeURIComponent(mediaId)}?allow_redirect=true`, + this.opts.homeserver, + ); + let sendAuth = true; + for (let hop = 0; hop <= MAX_REDIRECTS; hop += 1) { + const headers: Record = {}; + if (sendAuth) headers.authorization = `Bearer ${this.token()}`; + const res = await fetch(url, { headers, redirect: "manual", signal }); + if (res.status >= 300 && res.status < 400) { + const location = res.headers.get("location"); + await res.body?.cancel(); + if (location === null) throw new Error("media redirect without Location"); + ({ url, sendAuth } = resolveRedirect(url, location, this.opts.homeserver, this.opts.mediaRedirectHosts)); + continue; + } + if (!res.ok || res.body === null) { + const error = await this.toError(res); + // Missing/forbidden/too-large media will not appear on retry. + if (res.status === 404 || res.status === 403 || res.status === 413) throw new PermanentArchiveError(error.message); + throw error; + } + return { body: res.body, contentLength: parseContentLength(res.headers.get("content-length")) }; + } + throw new PermanentArchiveError("too many media redirects"); + } + + private token(): string { + if (this.accessToken === null) throw new Error("matrix client not logged in"); + return this.accessToken; + } + + private async request( + method: string, + path: string, + opts: { signal: AbortSignal; auth?: boolean; json?: unknown }, + ): Promise { + const headers: Record = { accept: "application/json" }; + if (opts.auth !== false) headers.authorization = `Bearer ${this.token()}`; + if (opts.json !== undefined) headers["content-type"] = "application/json"; + const res = await fetch(new URL(path, this.opts.homeserver), { + method, + headers, + body: opts.json === undefined ? undefined : JSON.stringify(opts.json), + redirect: "error", + signal: opts.signal, + }); + if (!res.ok) throw await this.toError(res); + return res.json() as Promise; + } + + private async toError(res: Response): Promise { + let errcode: string | null = null; + let message = `matrix HTTP ${res.status}`; + try { + const body = record(await res.json()); + if (typeof body.errcode === "string") errcode = body.errcode; + if (typeof body.error === "string") message = `${message}: ${body.error}`; + } catch { + // Non-JSON error body; status is enough. + } + return new MatrixHttpError(res.status, errcode, message); + } +} diff --git a/apps/archive-bot/src/mime.test.ts b/apps/archive-bot/src/mime.test.ts new file mode 100644 index 0000000..2557b20 --- /dev/null +++ b/apps/archive-bot/src/mime.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { sniffMime } from "./mime.js"; + +const bytes = (...parts: (number[] | string)[]): Uint8Array => + Uint8Array.from(parts.flatMap((p) => (typeof p === "string" ? [...p].map((c) => c.charCodeAt(0)) : p))); + +describe("sniffMime", () => { + it.each([ + ["png", bytes([0x89], "PNG", [0x0d, 0x0a, 0x1a, 0x0a, 0, 0]), "image/png", "image"], + ["jpeg", bytes([0xff, 0xd8, 0xff, 0xe0]), "image/jpeg", "image"], + ["gif87", bytes("GIF87a"), "image/gif", "image"], + ["gif89", bytes("GIF89a"), "image/gif", "image"], + ["pdf", bytes("%PDF-1.7"), "application/pdf", "document"], + ["zip", bytes("PK", [3, 4, 20, 0]), "application/zip", "archive"], + ["mp4", bytes([0, 0, 0, 0x18], "ftypisom"), "video/mp4", "video"], + ["webm", bytes([0x1a, 0x45, 0xdf, 0xa3, 0x9f]), "video/webm", "video"], + ["ogg", bytes("OggS", [0, 2]), "audio/ogg", "audio"], + ])("%s", (_name, head, mime, mimeClass) => { + expect(sniffMime(head)).toMatchObject({ mime, mimeClass }); + }); + + it("ignores declared types and falls back to octet-stream", () => { + expect(sniffMime(bytes(""))).toEqual({ mime: "application/octet-stream", mimeClass: "other", extension: null }); + expect(sniffMime(new Uint8Array())).toMatchObject({ mimeClass: "other" }); + expect(sniffMime(bytes([0x89], "PN"))).toMatchObject({ mimeClass: "other" }); + }); +}); diff --git a/apps/archive-bot/src/mime.ts b/apps/archive-bot/src/mime.ts new file mode 100644 index 0000000..493ea78 --- /dev/null +++ b/apps/archive-bot/src/mime.ts @@ -0,0 +1,43 @@ +export type MimeClass = "image" | "video" | "audio" | "document" | "archive" | "other"; + +export interface SniffedMime { + mime: string; + mimeClass: MimeClass; + extension: string | null; +} + +/** Bytes needed to recognise every supported signature. */ +export const SNIFF_BYTES = 16; + +const UNKNOWN: SniffedMime = { mime: "application/octet-stream", mimeClass: "other", extension: null }; + +function startsWith(head: Uint8Array, signature: readonly number[], offset = 0): boolean { + if (head.length < offset + signature.length) return false; + return signature.every((byte, i) => head[offset + i] === byte); +} + +function ascii(text: string): number[] { + return [...text].map((c) => c.charCodeAt(0)); +} + +/** + * MIME is decided by magic bytes only; the sender's declared `info.mimetype` + * is never trusted. Unrecognised payloads are stored as octet-stream/other. + */ +export function sniffMime(head: Uint8Array): SniffedMime { + if (startsWith(head, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) { + return { mime: "image/png", mimeClass: "image", extension: "png" }; + } + if (startsWith(head, [0xff, 0xd8, 0xff])) return { mime: "image/jpeg", mimeClass: "image", extension: "jpg" }; + if (startsWith(head, ascii("GIF87a")) || startsWith(head, ascii("GIF89a"))) { + return { mime: "image/gif", mimeClass: "image", extension: "gif" }; + } + if (startsWith(head, ascii("%PDF-"))) return { mime: "application/pdf", mimeClass: "document", extension: "pdf" }; + if (startsWith(head, [0x50, 0x4b, 0x03, 0x04])) { + return { mime: "application/zip", mimeClass: "archive", extension: "zip" }; + } + if (startsWith(head, ascii("ftyp"), 4)) return { mime: "video/mp4", mimeClass: "video", extension: "mp4" }; + if (startsWith(head, [0x1a, 0x45, 0xdf, 0xa3])) return { mime: "video/webm", mimeClass: "video", extension: "webm" }; + if (startsWith(head, ascii("OggS"))) return { mime: "audio/ogg", mimeClass: "audio", extension: "ogg" }; + return UNKNOWN; +} diff --git a/apps/archive-bot/src/names.test.ts b/apps/archive-bot/src/names.test.ts new file mode 100644 index 0000000..951d5b3 --- /dev/null +++ b/apps/archive-bot/src/names.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; +import { MAX_SAFE_NAME_BYTES, safeFileName } from "./names.js"; + +describe("safeFileName", () => { + it("keeps ordinary names", () => { + expect(safeFileName("Meeting notes.pdf")).toBe("Meeting notes.pdf"); + }); + + it("strips directories and traversal", () => { + expect(safeFileName("../../etc/passwd")).toBe("passwd"); + expect(safeFileName("C:\\Users\\x\\evil.exe")).toBe("evil.exe"); + expect(safeFileName("..")).toBe("attachment"); + }); + + it("removes control/bidi characters and reserved punctuation", () => { + expect(safeFileName("a\u0000b\u202ecod.exe")).toBe("abcod.exe"); + expect(safeFileName('wed:"na|me?*.txt')).toBe("we_ir_d_na_me_.txt"); + expect(safeFileName(" .hidden ")).toBe("hidden"); + }); + + it("NFKC-normalises and collapses whitespace", () => { + expect(safeFileName("ｆｕｌｌ \t width.PNG")).toBe("full width.png"); + }); + + it("guards Windows device names", () => { + expect(safeFileName("CON.txt")).toBe("_CON.txt"); + }); + + it("appends the sniffed extension only when none exists", () => { + expect(safeFileName("photo", "jpg")).toBe("photo.jpg"); + expect(safeFileName("photo.jpeg", "jpg")).toBe("photo.jpeg"); + expect(safeFileName("", "pdf")).toBe("attachment.pdf"); + }); + + it("bounds length in UTF-8 bytes while preserving the extension", () => { + const name = safeFileName(`${"ü".repeat(200)}.pdf`); + expect(Buffer.byteLength(name, "utf8")).toBeLessThanOrEqual(MAX_SAFE_NAME_BYTES); + expect(name.endsWith(".pdf")).toBe(true); + }); +}); diff --git a/apps/archive-bot/src/names.ts b/apps/archive-bot/src/names.ts new file mode 100644 index 0000000..3cc5a98 --- /dev/null +++ b/apps/archive-bot/src/names.ts @@ -0,0 +1,44 @@ +export const MAX_SAFE_NAME_BYTES = 120; +const FALLBACK = "attachment"; +const WINDOWS_RESERVED = /^(con|prn|aux|nul|com[0-9]|lpt[0-9])(\..*)?$/i; + +function truncateUtf8(text: string, maxBytes: number): string { + let out = ""; + for (const ch of text) { + if (Buffer.byteLength(out + ch, "utf8") > maxBytes) break; + out += ch; + } + return out; +} + +/** + * Library-safe single path segment: no directories, control or reserved + * characters, leading dots, or unbounded length. Extension is preserved when + * truncating; a sniffed extension is appended when the name has none. + */ +export function safeFileName(raw: string, sniffedExtension: string | null = null): string { + const basename = raw.normalize("NFKC").split(/[/\\]/).pop() ?? ""; + let name = basename + .replace(/[\p{Cc}\p{Cf}]/gu, "") + .replace(/[<>:"|?*#%{}^~[\]`]/g, "_") + .replace(/\s+/g, " ") + .replace(/_+/g, "_") + .trim() + .replace(/^[.\s_-]+/, "") + .replace(/[.\s]+$/, ""); + if (name === "") name = FALLBACK; + if (WINDOWS_RESERVED.test(name)) name = `_${name}`; + + const dot = name.lastIndexOf("."); + let stem = dot > 0 ? name.slice(0, dot) : name; + let ext = dot > 0 ? name.slice(dot + 1).toLowerCase() : ""; + if (ext === "" && sniffedExtension !== null) ext = sniffedExtension; + if (Buffer.byteLength(ext, "utf8") > 16) { + stem = name; + ext = sniffedExtension ?? ""; + } + const suffix = ext === "" ? "" : `.${ext}`; + stem = truncateUtf8(stem, MAX_SAFE_NAME_BYTES - Buffer.byteLength(suffix, "utf8")).trim(); + if (stem === "") stem = FALLBACK; + return `${stem}${suffix}`; +} diff --git a/apps/archive-bot/src/processor.test.ts b/apps/archive-bot/src/processor.test.ts new file mode 100644 index 0000000..cb98182 --- /dev/null +++ b/apps/archive-bot/src/processor.test.ts @@ -0,0 +1,267 @@ +import pino from "pino"; +import { beforeEach, describe, expect, it } from "vitest"; +import { Archiver } from "./archiver.js"; +import { BACKOFF_BASE_MS } from "./backoff.js"; +import type { RoomMapping } from "./config.js"; +import type { MatrixEvent, RoomEncryption } from "./events.js"; +import { claimKey, replyTxnId } from "./keys.js"; +import type { MatrixPort, MediaDownload, SyncResult } from "./matrix.js"; +import { ArchiveProcessor } from "./processor.js"; +import type { ArchiveRecord, ArchiveStore, OutboxRow } from "./store.js"; +import type { LibraryPort, LibraryResource } from "./webdav.js"; + +const mapping: RoomMapping = { + roomId: "!room:example.org", + workspaceId: "11111111-1111-4111-8111-111111111111", + groupId: "22222222-2222-4222-8222-222222222222", +}; +const BOT = "@archive:example.org"; +const PNG = Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3]); + +function imageEvent(eventId: string, extra: Partial = {}): MatrixEvent { + return { + type: "m.room.message", + event_id: eventId, + sender: "@alice:example.org", + origin_server_ts: 1_700_000_000_000, + content: { msgtype: "m.image", body: "../cat", url: `mxc://example.org/${eventId.slice(1)}` }, + ...extra, + }; +} + +class FakeMatrix implements MatrixPort { + readonly userId = BOT; + batches: SyncResult[] = []; + syncedWith: (string | null)[] = []; + encryption: RoomEncryption = "unencrypted"; + notices: { roomId: string; txnId: string; content: Record }[] = []; + failNotices = 0; + joined: string[] = []; + + async sync(since: string | null): Promise { + this.syncedWith.push(since); + const next = this.batches.shift(); + if (next === undefined) throw new Error("no batch queued"); + return next; + } + async backfill(): Promise { + return []; + } + async roomEncryption(): Promise { + return this.encryption; + } + async joinRoom(roomId: string): Promise { + this.joined.push(roomId); + } + async downloadMedia(): Promise { + return { + body: new ReadableStream({ start: (c) => { c.enqueue(PNG); c.close(); } }), + contentLength: PNG.byteLength, + }; + } + async sendNotice(roomId: string, txnId: string, content: Record): Promise { + if (this.failNotices > 0) { + this.failNotices -= 1; + throw new Error("notice failed"); + } + this.notices.push({ roomId, txnId, content }); + } +} + +class FakeLibrary implements LibraryPort { + failPuts = 0; + committed: string[] = []; + async putStaging(_dir: string[], _name: string, body: ReadableStream): Promise { + for await (const chunk of body) void chunk; + if (this.failPuts > 0) { + this.failPuts -= 1; + throw new Error("WebDAV PUT failed: 507"); + } + } + async commit(dir: string[], _staging: string, finalName: string): Promise { + const path = [...dir, finalName].join("/"); + this.committed.push(path); + return { path, url: `https://files/${path}`, resourceId: "rid", version: "v1" }; + } +} + +class FakeStore implements ArchiveStore { + cursor: string | null = null; + outbox = new Map(); + archive: (ArchiveRecord & { available: boolean })[] = []; + async loadCursor(): Promise { + return this.cursor; + } + async saveCursor(_ws: readonly string[], cursor: string): Promise { + this.cursor = cursor; + } + async claim(_ws: string, _room: string, _event: string, key: string): Promise { + const existing = this.outbox.get(key); + if (existing !== undefined) return { ...existing }; + const row = { id: `ob${this.outbox.size + 1}`, status: "pending" as const, attempts: 0, lastError: null, updatedAt: new Date(0), claimKey: key }; + this.outbox.set(key, row); + return { ...row }; + } + async commitArchived(record: ArchiveRecord, outboxId: string): Promise<{ id: string; indexedAt: Date }> { + this.archive.push({ ...record, available: true }); + this.byId(outboxId).status = "done"; + return { id: `file-${this.archive.length}`, indexedAt: new Date(0) }; + } + async recordFailure(_ws: string, outboxId: string, error: string, terminal: boolean): Promise { + const row = this.byId(outboxId); + row.attempts += 1; + row.lastError = error; + row.status = terminal ? "failed" : "pending"; + row.updatedAt = clock.now; + return { ...row }; + } + async markRedacted(_ws: string, _room: string, eventId: string, key: string): Promise { + const rows = this.archive.filter((r) => r.eventId === eventId && r.available); + for (const r of rows) r.available = false; + const claim = this.outbox.get(key); + if (claim?.status === "pending") claim.status = "failed"; + return rows.length; + } + async countPending(): Promise { + return [...this.outbox.values()].filter((r) => r.status === "pending").length; + } + private byId(id: string): OutboxRow { + const row = [...this.outbox.values()].find((r) => r.id === id); + if (row === undefined) throw new Error("no row"); + return row; + } +} + +const clock = { now: new Date(1_000_000) }; + +function batch(nextBatch: string, events: MatrixEvent[], extra: { stateEvents?: MatrixEvent[]; invited?: string[] } = {}): SyncResult { + return { + nextBatch, + invited: extra.invited ?? [], + joined: new Map([[mapping.roomId, { timeline: { events, limited: false, prevBatch: null }, stateEvents: extra.stateEvents ?? [] }]]), + }; +} + +describe("ArchiveProcessor", () => { + let matrix: FakeMatrix; + let library: FakeLibrary; + let store: FakeStore; + let processor: ArchiveProcessor; + const signal = new AbortController().signal; + + beforeEach(async () => { + clock.now = new Date(1_000_000); + matrix = new FakeMatrix(); + library = new FakeLibrary(); + store = new FakeStore(); + processor = new ArchiveProcessor( + matrix, store, new Archiver(matrix, library, store), [mapping], pino({ level: "silent" }), () => clock.now, + ); + await processor.init(); + }); + + it("archives, indexes, replies once with the stable txn id, then advances the cursor", async () => { + matrix.batches.push(batch("s1", [imageEvent("$a")])); + expect(await processor.runBatch(signal)).toEqual({ advanced: true, retryAt: null }); + + expect(store.cursor).toBe("s1"); + expect(store.archive).toHaveLength(1); + const row = store.archive[0]; + expect(row).toMatchObject({ mime: "image/png", mimeClass: "image", safeName: "cat.png", originalName: "../cat", byteSize: PNG.byteLength }); + expect(library.committed[0]).toBe(`Library/${mapping.workspaceId}/${mapping.groupId}/${row?.sha256}-cat.png`); + const key = claimKey(mapping.workspaceId, mapping.roomId, "$a"); + expect(matrix.notices).toEqual([{ + roomId: mapping.roomId, + txnId: replyTxnId(key), + content: expect.objectContaining({ body: "Archived to the group library: cat.png" }) as unknown, + }]); + + // Replaying the same event (e.g. after a crash before cursor save) neither re-archives nor re-replies. + matrix.batches.push(batch("s2", [imageEvent("$a")])); + await processor.runBatch(signal); + expect(store.archive).toHaveLength(1); + expect(matrix.notices).toHaveLength(1); + }); + + it("holds the cursor on a WebDAV failure, records attempts, and retries after backoff", async () => { + library.failPuts = 1; + matrix.batches.push(batch("s1", [imageEvent("$a")])); + const first = await processor.runBatch(signal); + expect(first.advanced).toBe(false); + expect(first.retryAt?.getTime()).toBe(clock.now.getTime() + BACKOFF_BASE_MS); + expect(store.cursor).toBeNull(); + const row = [...store.outbox.values()][0]; + expect(row).toMatchObject({ status: "pending", attempts: 1, lastError: "WebDAV PUT failed: 507" }); + + // Re-sync before the row is due: still blocked, nothing uploaded. + matrix.batches.push(batch("s1b", [imageEvent("$a")])); + expect((await processor.runBatch(signal)).advanced).toBe(false); + expect(store.archive).toHaveLength(0); + expect(matrix.syncedWith).toEqual([null, null]); + + clock.now = new Date(clock.now.getTime() + BACKOFF_BASE_MS); + matrix.batches.push(batch("s1c", [imageEvent("$a")])); + expect((await processor.runBatch(signal)).advanced).toBe(true); + expect(store.cursor).toBe("s1c"); + expect(store.archive).toHaveLength(1); + }); + + it("retries a failed reply without re-archiving", async () => { + matrix.failNotices = 1; + matrix.batches.push(batch("s1", [imageEvent("$a")]), batch("s2", [imageEvent("$a")])); + expect((await processor.runBatch(signal)).advanced).toBe(false); + expect(store.archive).toHaveLength(1); + expect((await processor.runBatch(signal)).advanced).toBe(true); + expect(matrix.notices).toHaveLength(1); + expect(store.archive).toHaveLength(1); + }); + + it("marks permanent payload errors failed and lets the cursor advance", async () => { + matrix.downloadMedia = async () => ({ + body: new ReadableStream({ start: (c) => { c.enqueue(PNG); c.close(); } }), + contentLength: PNG.byteLength + 10, + }); + matrix.batches.push(batch("s1", [imageEvent("$a")])); + expect((await processor.runBatch(signal)).advanced).toBe(true); + expect([...store.outbox.values()][0]).toMatchObject({ status: "failed", attempts: 1 }); + expect(matrix.notices).toHaveLength(0); + }); + + it("never mirrors when the room is encrypted or its state is unknown", async () => { + matrix.encryption = "unknown"; + matrix.batches.push(batch("s1", [imageEvent("$a")])); + await processor.runBatch(signal); + matrix.encryption = "unencrypted"; + const encryptionEvent: MatrixEvent = { + type: "m.room.encryption", event_id: "$enc", sender: "@admin:x", state_key: "", content: { algorithm: "m.megolm.v1.aes-sha2" }, + }; + matrix.batches.push(batch("s2", [imageEvent("$b"), encryptionEvent])); + await processor.runBatch(signal); + expect(store.outbox.size).toBe(0); + expect(store.archive).toHaveLength(0); + expect(store.cursor).toBe("s2"); + }); + + it("ignores its own events", async () => { + matrix.batches.push(batch("s1", [imageEvent("$a", { sender: BOT })])); + await processor.runBatch(signal); + expect(store.outbox.size).toBe(0); + }); + + it("marks redacted attachments unavailable without touching the library", async () => { + matrix.batches.push(batch("s1", [imageEvent("$a")])); + await processor.runBatch(signal); + const redaction: MatrixEvent = { type: "m.room.redaction", event_id: "$r", sender: "@mod:x", content: { redacts: "$a" } }; + matrix.batches.push(batch("s2", [redaction])); + await processor.runBatch(signal); + expect(store.archive[0]?.available).toBe(false); + expect(library.committed).toHaveLength(1); + }); + + it("joins invites only for mapped rooms and reports health", async () => { + matrix.batches.push({ nextBatch: "s1", joined: new Map(), invited: [mapping.roomId, "!other:x"] }); + await processor.runBatch(signal); + expect(matrix.joined).toEqual([mapping.roomId]); + expect(await processor.health()).toEqual({ rooms: 1, cursor: "s1", pending: 0, lastProcessedAt: null }); + }); +}); diff --git a/apps/archive-bot/src/processor.ts b/apps/archive-bot/src/processor.ts new file mode 100644 index 0000000..a833767 --- /dev/null +++ b/apps/archive-bot/src/processor.ts @@ -0,0 +1,235 @@ +import type { Logger } from "pino"; +import type { Archiver } from "./archiver.js"; +import { backoffMs, BACKOFF_MAX_MS, MAX_ATTEMPTS, nextAttemptAt } from "./backoff.js"; +import type { RoomMapping } from "./config.js"; +import { classifyEvent, type AttachmentWork, type MatrixEvent, type RoomEncryption } from "./events.js"; +import { PermanentArchiveError } from "./inspector.js"; +import { claimKey, replyTxnId } from "./keys.js"; +import { MatrixHttpError, type MatrixPort } from "./matrix.js"; +import type { ArchiveStore } from "./store.js"; + +export interface BatchResult { + /** True when the sync cursor was durably advanced past this batch. */ + advanced: boolean; + /** Earliest time a blocked row may be retried; null when nothing is blocked. */ + retryAt: Date | null; +} + +export interface HealthSnapshot { + rooms: number; + cursor: string | null; + pending: number; + lastProcessedAt: string | null; +} + +type Outcome = { blockedUntil: Date } | null; + +export function replyContent(safeName: string, eventId: string): Record { + return { + msgtype: "m.notice", + body: `Archived to the group library: ${safeName}`, + "m.relates_to": { "m.in_reply_to": { event_id: eventId } }, + }; +} + +function errorText(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +function abortableSleep(ms: number, signal: AbortSignal): Promise { + return new Promise((resolve) => { + if (signal.aborted) return resolve(); + const timer = setTimeout(done, ms); + function done(): void { + clearTimeout(timer); + signal.removeEventListener("abort", done); + resolve(); + } + signal.addEventListener("abort", done, { once: true }); + }); +} + +/** + * Sync-loop orchestration. The cursor is persisted only after every event in + * the batch has reached a durable terminal state (done/failed/skipped) and its + * reply was sent; otherwise the same `since` is re-synced after the earliest + * per-row backoff, so no attachment is lost to a transient failure. + */ +export class ArchiveProcessor { + private cursor: string | null = null; + private lastProcessedAt: Date | null = null; + /** Committed claims whose single reply has not been delivered yet (key → safe name). */ + private readonly unsentReplies = new Map(); + private readonly byRoom: Map; + private readonly workspaceIds: string[]; + + constructor( + private readonly matrix: MatrixPort, + private readonly store: ArchiveStore, + private readonly archiver: Archiver, + mappings: readonly RoomMapping[], + private readonly logger: Logger, + private readonly now: () => Date = () => new Date(), + ) { + this.byRoom = new Map(mappings.map((m) => [m.roomId, m])); + this.workspaceIds = [...new Set(mappings.map((m) => m.workspaceId))]; + } + + async init(): Promise { + this.cursor = await this.store.loadCursor(this.workspaceIds); + } + + async health(): Promise { + return { + rooms: this.byRoom.size, + cursor: this.cursor, + pending: await this.store.countPending(this.workspaceIds), + lastProcessedAt: this.lastProcessedAt?.toISOString() ?? null, + }; + } + + async runBatch(signal: AbortSignal): Promise { + const since = this.cursor; + const sync = await this.matrix.sync(since, signal); + + for (const roomId of sync.invited) { + if (!this.byRoom.has(roomId)) continue; + await this.matrix.joinRoom(roomId, signal); + this.logger.info({ roomId }, "archive-bot: joined mapped room"); + } + + let retryAt: Date | null = null; + for (const [roomId, room] of sync.joined) { + const mapping = this.byRoom.get(roomId); + if (mapping === undefined) continue; + let events = room.timeline.events; + if (room.timeline.limited && room.timeline.prevBatch !== null && since !== null) { + events = [...(await this.matrix.backfill(roomId, room.timeline.prevBatch, since, signal)), ...events]; + } + if (events.length === 0) continue; + const encryption = await this.encryptionState(mapping, [...room.stateEvents, ...events], signal); + for (const event of events) { + const outcome = await this.handleEvent(mapping, event, encryption, signal); + if (outcome !== null && (retryAt === null || outcome.blockedUntil < retryAt)) retryAt = outcome.blockedUntil; + } + } + + if (retryAt !== null) return { advanced: false, retryAt }; + await this.store.saveCursor(this.workspaceIds, sync.nextBatch); + this.cursor = sync.nextBatch; + return { advanced: true, retryAt: null }; + } + + /** Long-running loop; returns when `signal` aborts. */ + async run(signal: AbortSignal, relogin: (signal: AbortSignal) => Promise): Promise { + let consecutiveErrors = 0; + while (!signal.aborted) { + try { + const result = await this.runBatch(signal); + consecutiveErrors = 0; + if (result.retryAt !== null) { + const wait = Math.min(Math.max(result.retryAt.getTime() - this.now().getTime(), 1000), BACKOFF_MAX_MS); + this.logger.info({ waitMs: wait }, "archive-bot: batch has pending rows; holding cursor"); + await abortableSleep(wait, signal); + } + } catch (err) { + if (signal.aborted) break; + consecutiveErrors += 1; + if (err instanceof MatrixHttpError && err.status === 401) { + this.logger.warn("archive-bot: access token rejected; logging in again"); + await relogin(signal).catch((loginErr: unknown) => { + this.logger.error({ err: errorText(loginErr) }, "archive-bot: re-login failed"); + }); + } else { + this.logger.error({ err: errorText(err) }, "archive-bot: batch failed; cursor held"); + } + await abortableSleep(backoffMs(consecutiveErrors), signal); + } + } + } + + /** Fail closed: an encryption event anywhere in this batch wins; otherwise ask the room state. */ + private async encryptionState(mapping: RoomMapping, events: MatrixEvent[], signal: AbortSignal): Promise { + const sawEncryption = events.some((e) => e.type === "m.room.encryption" && e.state_key === ""); + const state = sawEncryption ? "encrypted" : await this.matrix.roomEncryption(mapping.roomId, signal); + if (state !== "unencrypted") { + this.logger.warn({ roomId: mapping.roomId, state }, "archive-bot: room not verifiably unencrypted; not mirroring"); + } + return state; + } + + private async handleEvent( + mapping: RoomMapping, + event: MatrixEvent, + encryption: RoomEncryption, + signal: AbortSignal, + ): Promise { + const classified = classifyEvent(event, mapping, encryption, this.matrix.userId); + switch (classified.kind) { + case "skip": + if (classified.reason !== "own-event" && classified.reason !== "not-an-attachment" && classified.reason !== "not-a-message") { + this.logger.info({ roomId: mapping.roomId, eventId: event.event_id, reason: classified.reason }, "archive-bot: skipped"); + } + return null; + case "redaction": { + const key = claimKey(mapping.workspaceId, mapping.roomId, classified.redacts); + const affected = await this.store.markRedacted(mapping.workspaceId, mapping.roomId, classified.redacts, key); + this.unsentReplies.delete(key); + if (affected > 0) { + this.logger.info({ roomId: mapping.roomId, eventId: classified.redacts }, "archive-bot: redacted; index marked unavailable"); + } + this.lastProcessedAt = this.now(); + return null; + } + case "attachment": + return this.handleAttachment(mapping, classified, signal); + } + } + + private async handleAttachment(mapping: RoomMapping, work: AttachmentWork, signal: AbortSignal): Promise { + const eventId = work.event.event_id; + const key = claimKey(mapping.workspaceId, mapping.roomId, eventId); + const row = await this.store.claim(mapping.workspaceId, mapping.roomId, eventId, key); + const log = { roomId: mapping.roomId, eventId, outboxId: row.id }; + + if (row.status === "failed") return null; + if (row.status === "done") return this.sendReply(mapping, eventId, key, signal); + + const due = nextAttemptAt(row.attempts, row.updatedAt); + if (row.attempts > 0 && due > this.now()) return { blockedUntil: due }; + + try { + const archived = await this.archiver.archive(mapping, work, row, key, signal); + this.logger.info({ ...log, fileId: archived.file.id, sha256: archived.file.checksum.value }, "archive-bot: archived"); + this.lastProcessedAt = this.now(); + this.unsentReplies.set(key, archived.safeName); + return this.sendReply(mapping, eventId, key, signal); + } catch (err) { + if (signal.aborted) throw err; + const terminal = err instanceof PermanentArchiveError || row.attempts + 1 >= MAX_ATTEMPTS; + const updated = await this.store.recordFailure(mapping.workspaceId, row.id, errorText(err), terminal); + if (terminal) { + this.logger.error({ ...log, err: errorText(err), attempts: updated.attempts }, "archive-bot: archive failed permanently"); + return null; + } + const retry = nextAttemptAt(updated.attempts, updated.updatedAt); + this.logger.warn({ ...log, err: errorText(err), attempts: updated.attempts, retryAt: retry.toISOString() }, "archive-bot: archive failed; will retry"); + return { blockedUntil: retry }; + } + } + + /** Exactly one reply per committed claim; the stable txn ID makes resends idempotent server-side. */ + private async sendReply(mapping: RoomMapping, eventId: string, key: string, signal: AbortSignal): Promise { + const safeName = this.unsentReplies.get(key); + if (safeName === undefined) return null; + try { + await this.matrix.sendNotice(mapping.roomId, replyTxnId(key), replyContent(safeName, eventId), signal); + this.unsentReplies.delete(key); + return null; + } catch (err) { + if (signal.aborted) throw err; + this.logger.warn({ roomId: mapping.roomId, eventId, err: errorText(err) }, "archive-bot: reply failed; will retry"); + return { blockedUntil: new Date(this.now().getTime() + backoffMs(1)) }; + } + } +} diff --git a/apps/archive-bot/src/store.ts b/apps/archive-bot/src/store.ts new file mode 100644 index 0000000..b40b190 --- /dev/null +++ b/apps/archive-bot/src/store.ts @@ -0,0 +1,223 @@ +import { Pool, type PoolClient } from "pg"; + +export type OutboxStatus = "pending" | "done" | "failed"; + +export interface OutboxRow { + id: string; + status: OutboxStatus; + attempts: number; + lastError: string | null; + updatedAt: Date; +} + +export interface ArchiveRecord { + workspaceId: string; + groupId: string; + roomId: string; + eventId: string; + senderMxid: string; + matrixEventJson: unknown; + originalName: string; + safeName: string; + mime: string; + mimeClass: string; + byteSize: number; + sha256: string; + ocisResourceId: string; + ocisResourceVersion: string; + createdAt: Date; +} + +/** Durable state the processor depends on; faked in tests. */ +export interface ArchiveStore { + loadCursor(workspaceIds: readonly string[]): Promise; + saveCursor(workspaceIds: readonly string[], cursor: string): Promise; + claim(workspaceId: string, roomId: string, eventId: string, claimKey: string): Promise; + /** Index row + outbox done in one transaction. Returns the archive row id and indexed time. */ + commitArchived(record: ArchiveRecord, outboxId: string): Promise<{ id: string; indexedAt: Date }>; + recordFailure(workspaceId: string, outboxId: string, error: string, terminal: boolean): Promise; + /** Mark index rows unavailable and stop any not-yet-archived claim. Returns rows affected. */ + markRedacted(workspaceId: string, roomId: string, eventId: string, claimKey: string): Promise; + countPending(workspaceIds: readonly string[]): Promise; +} + +interface OutboxDbRow { + id: string; + status: OutboxStatus; + attempts: number; + last_error: string | null; + updated_at: Date; +} + +function toOutbox(row: OutboxDbRow): OutboxRow { + return { id: row.id, status: row.status, attempts: row.attempts, lastError: row.last_error, updatedAt: row.updated_at }; +} + +const OUTBOX_COLUMNS = "id, status, attempts, last_error, updated_at"; +const LAST_ERROR_MAX = 1000; + +export class PgArchiveStore implements ArchiveStore { + constructor( + private readonly pool: Pool, + private readonly cursorId: string, + ) {} + + /** Single service connection; tenant context per transaction, like apps/api withTenant. */ + static create(databaseUrl: string, cursorId: string): PgArchiveStore { + return new PgArchiveStore(new Pool({ connectionString: databaseUrl, max: 1 }), cursorId); + } + + async close(): Promise { + await this.pool.end(); + } + + /** + * The sync stream is global but rows are tenant-scoped, so each mapped + * workspace holds a copy. A partial write leaves copies disagreeing: resume + * from the least recently written (oldest) one — replay is idempotent. + */ + async loadCursor(workspaceIds: readonly string[]): Promise { + let oldest: { cursor: string; at: Date } | null = null; + for (const workspaceId of workspaceIds) { + const row = await this.withTenant(workspaceId, async (c) => { + const res = await c.query<{ sync_cursor: string | null; updated_at: Date }>( + "SELECT sync_cursor, updated_at FROM archive_sync_state WHERE id = $1 AND workspace_id = $2", + [this.stateId(workspaceId), workspaceId], + ); + return res.rows[0]; + }); + if (row?.sync_cursor == null) continue; + if (oldest === null || row.updated_at < oldest.at) oldest = { cursor: row.sync_cursor, at: row.updated_at }; + } + return oldest?.cursor ?? null; + } + + async saveCursor(workspaceIds: readonly string[], cursor: string): Promise { + for (const workspaceId of workspaceIds) { + await this.withTenant(workspaceId, (c) => c.query( + `INSERT INTO archive_sync_state (id, workspace_id, sync_cursor, updated_at) + VALUES ($1, $2, $3, now()) + ON CONFLICT (id) DO UPDATE SET sync_cursor = EXCLUDED.sync_cursor, updated_at = now()`, + [this.stateId(workspaceId), workspaceId, cursor], + )); + } + } + + async claim(workspaceId: string, roomId: string, eventId: string, claimKey: string): Promise { + return this.withTenant(workspaceId, async (c) => { + await c.query( + `INSERT INTO archive_sync_outbox (workspace_id, room_id, event_id, claim_key) + SELECT $1, $2, $3, $4 + WHERE NOT EXISTS (SELECT 1 FROM archive_sync_outbox WHERE workspace_id = $1 AND claim_key = $4) + ON CONFLICT DO NOTHING`, + [workspaceId, roomId, eventId, claimKey], + ); + const res = await c.query( + `SELECT ${OUTBOX_COLUMNS} FROM archive_sync_outbox + WHERE workspace_id = $1 AND claim_key = $2 ORDER BY created_at LIMIT 1`, + [workspaceId, claimKey], + ); + const row = res.rows[0]; + if (row === undefined) throw new Error("outbox claim not visible after insert"); + return toOutbox(row); + }); + } + + async commitArchived(record: ArchiveRecord, outboxId: string): Promise<{ id: string; indexedAt: Date }> { + return this.withTenant(record.workspaceId, async (c) => { + await c.query( + `INSERT INTO chat_media_archive ( + workspace_id, group_id, room_id, event_id, origin, sender_mxid, matrix_event_json, + original_name, safe_name, mime, mime_class, byte_size, sha256, + ocis_resource_id, ocis_resource_version, created_at + ) VALUES ($1, $2, $3, $4, 'matrix', $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) + ON CONFLICT DO NOTHING`, + [ + record.workspaceId, record.groupId, record.roomId, record.eventId, record.senderMxid, + JSON.stringify(record.matrixEventJson), record.originalName, record.safeName, record.mime, + record.mimeClass, record.byteSize, record.sha256, record.ocisResourceId, + record.ocisResourceVersion, record.createdAt, + ], + ); + const indexed = await c.query<{ id: string; indexed_at: Date }>( + `SELECT id, indexed_at FROM chat_media_archive + WHERE workspace_id = $1 AND room_id = $2 AND event_id = $3 LIMIT 1`, + [record.workspaceId, record.roomId, record.eventId], + ); + const row = indexed.rows[0]; + if (row === undefined) throw new Error("archive row not visible after insert"); + await c.query( + `UPDATE archive_sync_outbox SET status = 'done', last_error = NULL, updated_at = now() + WHERE id = $1 AND workspace_id = $2`, + [outboxId, record.workspaceId], + ); + return { id: row.id, indexedAt: row.indexed_at }; + }); + } + + async recordFailure(workspaceId: string, outboxId: string, error: string, terminal: boolean): Promise { + return this.withTenant(workspaceId, async (c) => { + const res = await c.query( + `UPDATE archive_sync_outbox + SET attempts = attempts + 1, last_error = $3, updated_at = now(), + status = CASE WHEN $4 THEN 'failed' ELSE 'pending' END + WHERE id = $1 AND workspace_id = $2 AND status = 'pending' + RETURNING ${OUTBOX_COLUMNS}`, + [outboxId, workspaceId, error.slice(0, LAST_ERROR_MAX), terminal], + ); + const row = res.rows[0]; + if (row === undefined) throw new Error("outbox row not pending"); + return toOutbox(row); + }); + } + + async markRedacted(workspaceId: string, roomId: string, eventId: string, claimKey: string): Promise { + return this.withTenant(workspaceId, async (c) => { + const res = await c.query( + `UPDATE chat_media_archive SET available = false + WHERE workspace_id = $1 AND room_id = $2 AND event_id = $3 AND available`, + [workspaceId, roomId, eventId], + ); + await c.query( + `UPDATE archive_sync_outbox SET status = 'failed', last_error = 'redacted', updated_at = now() + WHERE workspace_id = $1 AND claim_key = $2 AND status = 'pending'`, + [workspaceId, claimKey], + ); + return res.rowCount ?? 0; + }); + } + + async countPending(workspaceIds: readonly string[]): Promise { + let total = 0; + for (const workspaceId of workspaceIds) { + total += await this.withTenant(workspaceId, async (c) => { + const res = await c.query<{ n: string }>( + "SELECT count(*) AS n FROM archive_sync_outbox WHERE workspace_id = $1 AND status = 'pending'", + [workspaceId], + ); + return Number(res.rows[0]?.n ?? 0); + }); + } + return total; + } + + private stateId(workspaceId: string): string { + return `${this.cursorId}:${workspaceId}`; + } + + private async withTenant(workspaceId: string, fn: (client: PoolClient) => Promise): Promise { + const client = await this.pool.connect(); + try { + await client.query("BEGIN"); + await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]); + const result = await fn(client); + await client.query("COMMIT"); + return result; + } catch (err) { + await client.query("ROLLBACK").catch(() => undefined); + throw err; + } finally { + client.release(); + } + } +} diff --git a/apps/archive-bot/src/webdav.test.ts b/apps/archive-bot/src/webdav.test.ts new file mode 100644 index 0000000..cb65d1d --- /dev/null +++ b/apps/archive-bot/src/webdav.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; +import { davPath, parsePropfind } from "./webdav.js"; + +describe("davPath", () => { + it("encodes every segment under the service user's files root", () => { + expect(davPath("svc", ["Library", "ws", "grp", "abc-My file #1.pdf"])) + .toBe("/remote.php/dav/files/svc/Library/ws/grp/abc-My%20file%20%231.pdf"); + }); +}); + +describe("parsePropfind", () => { + it("reads oc:fileid and an unquoted etag", () => { + const xml = ` + space$abc!def"9f1c"`; + expect(parsePropfind(xml)).toEqual({ resourceId: "space$abc!def", version: "9f1c" }); + }); + + it("returns null when properties are missing", () => { + expect(parsePropfind("")).toBeNull(); + }); +}); diff --git a/apps/archive-bot/src/webdav.ts b/apps/archive-bot/src/webdav.ts new file mode 100644 index 0000000..bbb842a --- /dev/null +++ b/apps/archive-bot/src/webdav.ts @@ -0,0 +1,116 @@ +export interface LibraryResource { + path: string; + url: string; + resourceId: string; + version: string; +} + +/** oCIS library surface; faked in tests. */ +export interface LibraryPort { + /** Stream bytes to a deterministic staging resource (overwritten on retry). */ + putStaging(dir: string[], stagingName: string, body: ReadableStream, signal: AbortSignal): Promise; + /** Move staging to its content-addressed final name; an existing final resource is reused. */ + commit(dir: string[], stagingName: string, finalName: string, signal: AbortSignal): Promise; +} + +export class WebDavError extends Error { + constructor( + readonly status: number, + message: string, + ) { + super(message); + } +} + +const STAGING_DIR = ".archive-staging"; + +/** Percent-encode each segment; segments themselves are already normalised safe names/UUIDs. */ +export function davPath(user: string, segments: readonly string[]): string { + return `/remote.php/dav/files/${[user, ...segments].map(encodeURIComponent).join("/")}`; +} + +/** Extract `oc:fileid` and `d:getetag` from a Depth:0 PROPFIND multistatus body. */ +export function parsePropfind(xml: string): { resourceId: string; version: string } | null { + const fileId = /<(?:[\w-]+:)?fileid>([^<]+)<\/(?:[\w-]+:)?fileid>/.exec(xml)?.[1]; + const etag = /<(?:[\w-]+:)?getetag>([^<]+)<\/(?:[\w-]+:)?getetag>/.exec(xml)?.[1]; + if (fileId === undefined || etag === undefined) return null; + return { resourceId: fileId.trim(), version: etag.trim().replace(/^"|"$/g, "").replace(/"/g, "") }; +} + +const PROPFIND_BODY = ` +`; + +/** WebDAV client for the scoped oCIS service identity. Credentials never leave the configured origin. */ +export class OcisWebDav implements LibraryPort { + private readonly authorization: string; + private readonly knownDirs = new Set(); + + constructor( + private readonly baseUrl: URL, + private readonly user: string, + password: string, + ) { + this.authorization = `Basic ${Buffer.from(`${user}:${password}`, "utf8").toString("base64")}`; + } + + async putStaging(dir: string[], stagingName: string, body: ReadableStream, signal: AbortSignal): Promise { + await this.ensureDir([...dir, STAGING_DIR], signal); + const res = await this.send("PUT", [...dir, STAGING_DIR, stagingName], signal, { + body, + headers: { "content-type": "application/octet-stream" }, + }); + if (res.status !== 201 && res.status !== 204) throw new WebDavError(res.status, `WebDAV PUT failed: ${res.status}`); + } + + async commit(dir: string[], stagingName: string, finalName: string, signal: AbortSignal): Promise { + const staging = [...dir, STAGING_DIR, stagingName]; + const final = [...dir, finalName]; + const destination = new URL(davPath(this.user, final), this.baseUrl).toString(); + const moved = await this.send("MOVE", staging, signal, { headers: { destination, overwrite: "F" } }); + if (moved.status === 412) { + // Same sha256 + name already archived (crash after MOVE, or a duplicate post): reuse it. + const removed = await this.send("DELETE", staging, signal); + if (!removed.ok && removed.status !== 404) throw new WebDavError(removed.status, `WebDAV DELETE failed: ${removed.status}`); + } else if (moved.status !== 201 && moved.status !== 204) { + throw new WebDavError(moved.status, `WebDAV MOVE failed: ${moved.status}`); + } + const found = await this.send("PROPFIND", final, signal, { + body: PROPFIND_BODY, + headers: { depth: "0", "content-type": "application/xml" }, + }); + if (found.status !== 207) throw new WebDavError(found.status, `WebDAV PROPFIND failed: ${found.status}`); + const props = parsePropfind(await found.text()); + if (props === null) throw new WebDavError(207, "WebDAV PROPFIND missing fileid/etag"); + return { path: final.join("/"), url: destination, ...props }; + } + + private async ensureDir(segments: string[], signal: AbortSignal): Promise { + for (let depth = 1; depth <= segments.length; depth += 1) { + const dir = segments.slice(0, depth); + const key = dir.join("/"); + if (this.knownDirs.has(key)) continue; + const res = await this.send("MKCOL", dir, signal); + // 405: already exists. + if (res.status !== 201 && res.status !== 405) throw new WebDavError(res.status, `WebDAV MKCOL failed: ${res.status}`); + this.knownDirs.add(key); + } + } + + private async send( + method: string, + segments: string[], + signal: AbortSignal, + init: { body?: ReadableStream | string; headers?: Record } = {}, + ): Promise { + const res = await fetch(new URL(davPath(this.user, segments), this.baseUrl), { + method, + headers: { authorization: this.authorization, ...init.headers }, + body: init.body, + duplex: init.body instanceof ReadableStream ? "half" : undefined, + redirect: "error", + signal, + }); + if (method !== "PROPFIND") await res.body?.cancel(); + return res; + } +} diff --git a/apps/archive-bot/tsconfig.build.json b/apps/archive-bot/tsconfig.build.json new file mode 100644 index 0000000..0a11719 --- /dev/null +++ b/apps/archive-bot/tsconfig.build.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["src/**/*.test.ts"] +} diff --git a/apps/archive-bot/tsconfig.json b/apps/archive-bot/tsconfig.json new file mode 100644 index 0000000..2c0ddff --- /dev/null +++ b/apps/archive-bot/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext" + }, + "include": ["src"] +} diff --git a/apps/archive-bot/vitest.config.ts b/apps/archive-bot/vitest.config.ts new file mode 100644 index 0000000..ae847ff --- /dev/null +++ b/apps/archive-bot/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["src/**/*.test.ts"], + }, +}); diff --git a/apps/web/public/brand/nav-files.svg b/apps/web/public/brand/nav-files.svg new file mode 100644 index 0000000..7bdd812 --- /dev/null +++ b/apps/web/public/brand/nav-files.svg @@ -0,0 +1,4 @@ + + Nav Files + + diff --git a/apps/web/src/app/(main)/files/layout.tsx b/apps/web/src/app/(main)/files/layout.tsx new file mode 100644 index 0000000..1457feb --- /dev/null +++ b/apps/web/src/app/(main)/files/layout.tsx @@ -0,0 +1,13 @@ +"use client"; + +import type { ReactNode } from "react"; +import { FilesLangProvider } from "../../../components/files/i18n"; + +/** + * Files shell. Like the wiki, the language lives in localStorage per browser + * (lang-context explains why it is not an account setting) and is provided here + * so every Files screen and sub-component reads it through one context. + */ +export default function FilesLayout({ children }: { children: ReactNode }): React.ReactElement { + return {children}; +} diff --git a/apps/web/src/app/(main)/files/page.tsx b/apps/web/src/app/(main)/files/page.tsx new file mode 100644 index 0000000..5014f44 --- /dev/null +++ b/apps/web/src/app/(main)/files/page.tsx @@ -0,0 +1,383 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; +import { Button, Input } from "../../../components/ui/primitives"; +import { OfflineBadge } from "../../../components/ui/offline-badge"; +import { EmptyState } from "../../../components/brand/empty-state"; +import { Loading } from "../../../components/brand/loading"; +import { api, ApiUnreachableError, errorMessage } from "../../../lib/api-client"; +import { useAuth } from "../../../lib/auth-store"; +import { useApiResource } from "../../../lib/use-api"; +import { FileRow } from "../../../components/files/file-row"; +import { UploadDropzone } from "../../../components/files/upload-dropzone"; +import { ShareDialog } from "../../../components/files/share-dialog"; +import { RetentionDialog } from "../../../components/files/retention-dialog"; +import { downloadGroupFile } from "../../../components/files/files-client"; +import { useFilesLang } from "../../../components/files/i18n"; +import { MIME_CLASS_LABEL, ORIGIN_LABEL } from "../../../components/files/labels"; +import type { + FileOrigin, + GroupFileItem, + GroupFilePage, + GroupSummary, + MimeClass, + RetentionInput, +} from "../../../components/files/types"; + +/** + * Group file library — the per-group "group cloud". Live mode pages through + * GET /files/groups/:groupId/items by cursor (video library's load-more + * pattern) with a debounced server-side `q` and mimeClass/origin filters. + * Offline the screen renders sample data read-only per useApiResource rules. + */ + +const PAGE_LIMIT = 50; + +const SAMPLE_GROUPS: GroupSummary[] = [{ id: "sample-group", name: "Collective media" }]; +const SAMPLE_FILES: GroupFileItem[] = [ + { + id: "sample-1", groupId: "sample-group", origin: "matrix", senderMxid: "@ada:collective.example", + originalName: "press-kit.zip", safeName: "press-kit.zip", mime: "application/zip", mimeClass: "other", + byteSize: 4_812_441, sha256: "a".repeat(64), createdAt: "2026-10-08T09:12:00.000Z", available: true, + retentionMode: "keep", retentionUntil: null, + }, + { + id: "sample-2", groupId: "sample-group", origin: "upload", senderMxid: "@you:collective.example", + originalName: "launch-banner.png", safeName: "launch-banner.png", mime: "image/png", mimeClass: "image", + byteSize: 348_112, sha256: "b".repeat(64), createdAt: "2026-10-08T11:40:00.000Z", available: true, + retentionMode: "until", retentionUntil: "2026-11-30T00:00:00.000Z", + }, + { + id: "sample-3", groupId: "sample-group", origin: "bridge", senderMxid: "@bridge:collective.example", + originalName: "statement-final.pdf", safeName: "statement-final.pdf", mime: "application/pdf", mimeClass: "document", + byteSize: 1_204_998, sha256: "c".repeat(64), createdAt: "2026-10-07T18:05:00.000Z", available: true, + retentionMode: "legalHold", retentionUntil: null, + }, +]; + +const identityGroups = (groups: GroupSummary[]): GroupSummary[] => groups; + +export default function FilesPage(): React.ReactElement { + const { t, lang, setLang } = useFilesLang(); + const { status: authStatus, can, session } = useAuth(); + const groups = useApiResource("/groups/mine", identityGroups, SAMPLE_GROUPS); + + const [pickedGroup, setPickedGroup] = useState(null); + const [items, setItems] = useState([]); + const [source, setSource] = useState<"loading" | "live" | "sample">("loading"); + const [error, setError] = useState(null); + const [nextCursor, setNextCursor] = useState(null); + const [loading, setLoading] = useState(false); + const [loadingMore, setLoadingMore] = useState(false); + const [nonce, setNonce] = useState(0); + + const [qInput, setQInput] = useState(""); + const [q, setQ] = useState(""); + const [mimeClass, setMimeClass] = useState<"" | MimeClass>(""); + const [origin, setOrigin] = useState<"" | FileOrigin>(""); + + const [actionError, setActionError] = useState(null); + const [notice, setNotice] = useState(null); + const [busyId, setBusyId] = useState(null); + const [shareFor, setShareFor] = useState(null); + const [retentionFor, setRetentionFor] = useState(null); + const [shares, setShares] = useState>({}); + + const groupId = pickedGroup ?? groups.data[0]?.id ?? ""; + const live = source === "live"; + const isAdmin = can("workspace.members.manage") || session?.role === "Admin" || session?.role === "Owner"; + + // Debounce the search box; only the settled value drives the server query. + useEffect(() => { + const timer = window.setTimeout(() => setQ(qInput.trim()), 300); + return () => window.clearTimeout(timer); + }, [qInput]); + + useEffect(() => { + if (authStatus === "loading") return; + if (authStatus !== "authenticated") { + setItems(SAMPLE_FILES); + setNextCursor(null); + setSource("sample"); + setError(null); + return; + } + if (groupId === "") { + setItems([]); + setNextCursor(null); + setSource("live"); + return; + } + const controller = new AbortController(); + setLoading(true); + api + .get(`/files/groups/${encodeURIComponent(groupId)}/items`, { q, mimeClass, origin, limit: PAGE_LIMIT }, controller.signal) + .then((page) => { + setItems(page.items); + setNextCursor(page.nextCursor); + setSource("live"); + setError(null); + }) + .catch((err: unknown) => { + if (controller.signal.aborted) return; + if (err instanceof ApiUnreachableError) { + setItems(SAMPLE_FILES); + setNextCursor(null); + setSource("sample"); + return; + } + setSource("live"); + setError(errorMessage(err)); + }) + .finally(() => { + if (!controller.signal.aborted) setLoading(false); + }); + return () => controller.abort(); + }, [authStatus, groupId, q, mimeClass, origin, nonce]); + + async function loadMore(): Promise { + if (nextCursor === null || groupId === "") return; + setLoadingMore(true); + try { + const page = await api.get(`/files/groups/${encodeURIComponent(groupId)}/items`, { + q, mimeClass, origin, limit: PAGE_LIMIT, cursor: nextCursor, + }); + setItems((current) => { + const seen = new Set(current.map((item) => item.id)); + return [...current, ...page.items.filter((item) => !seen.has(item.id))]; + }); + setNextCursor(page.nextCursor); + } catch (err) { + setActionError(errorMessage(err)); + } finally { + setLoadingMore(false); + } + } + + const handleUploaded = useCallback((item: GroupFileItem): void => { + setItems((current) => [item, ...current.filter((existing) => existing.id !== item.id)]); + setNotice(t({ en: "File uploaded.", es: "Archivo subido." })); + }, [t]); + + async function handleDownload(item: GroupFileItem): Promise { + setActionError(null); + try { + await downloadGroupFile(item.id, item.safeName); + } catch (err) { + setActionError(errorMessage(err)); + } + } + + async function revokeShare(item: GroupFileItem): Promise { + const entry = shares[item.id]; + if (entry === undefined || entry.shareId === null || entry.shareId === "") { + setActionError(t({ en: "No share id to revoke — reopen the share dialog.", es: "No hay id de enlace que revocar — vuelve a abrir el cuadro de compartir." })); + return; + } + const previous = shares; + setBusyId(item.id); + setActionError(null); + // Optimistic: drop the link immediately, restore if the API rejects. + setShares((current) => { + const next = { ...current }; + delete next[item.id]; + return next; + }); + try { + await api.del(`/files/shares/${encodeURIComponent(entry.shareId)}`); + setNotice(t({ en: "Share link revoked.", es: "Enlace para compartir revocado." })); + } catch (err) { + setShares(previous); + setActionError(errorMessage(err)); + } finally { + setBusyId(null); + } + } + + function handleRetentionSaved(itemId: string, input: RetentionInput): void { + setItems((current) => + current.map((item) => + item.id === itemId ? { ...item, retentionMode: input.mode, retentionUntil: input.until ?? null } : item, + ), + ); + setNotice(t({ en: "Retention updated.", es: "Retención actualizada." })); + } + + const groupChosen = groupId !== ""; + const showList = source !== "loading"; + + return ( +
+
+

{t({ en: "Files", es: "Archivos" })}

+ + {groups.data.length > 0 && ( + + )} +
+ {(["en", "es"] as const).map((code) => ( + + ))} +
+
+ +

+ {t({ + en: "Files shared with this group, from Matrix rooms, bridges, direct uploads and pad exports. Download and share links stream the verified bytes back through the hub — nothing is stored on a third party.", + es: "Archivos compartidos con este grupo, desde salas Matrix, puentes, subidas directas y exportaciones de pad. La descarga y los enlaces de compartir sirven los bytes verificados a través del hub — nada queda en terceros.", + })} +

+ + {groups.error !== null &&

{groups.error}

} + {error !== null &&

{error}

} + {actionError !== null &&

{actionError}

} + {notice !== null &&

{notice}

} + + + +
+ setQInput(event.target.value)} + placeholder={t({ en: "Search files by name…", es: "Buscar archivos por nombre…" })} + aria-label={t({ en: "Search files", es: "Buscar archivos" })} + className="max-w-xs" + /> + + + {live && ( + + )} +
+ + {groups.data.length === 0 && source !== "sample" && ( + + )} + + {groups.data.length > 0 && !groupChosen && source !== "sample" && ( + + )} + + {source === "loading" && ( +
+ + {t({ en: "Loading files…", es: "Cargando archivos…" })} +
+ )} + + {showList && loading && items.length > 0 && ( +
+ + {t({ en: "Refreshing…", es: "Actualizando…" })} +
+ )} + + {showList && !loading && (groupChosen || source === "sample") && items.length === 0 && ( + + )} + + {showList && items.length > 0 && ( +
+ {items.map((item) => ( + void handleDownload(item)} + onShare={() => { setActionError(null); setShareFor(item); }} + onRevoke={() => void revokeShare(item)} + onRetention={() => { setActionError(null); setRetentionFor(item); }} + /> + ))} +
+ )} + + {live && nextCursor !== null && ( +
+ +
+ )} + +

+ {t({ + en: "Retention (keep / until / legal hold) is separate from link expiry: revoking or expiring a share never deletes the file, and legal hold blocks deletion until an admin releases it.", + es: "La retención (conservar / hasta / retención legal) es independiente de la caducidad del enlace: revocar o dejar caducar un enlace nunca borra el archivo, y la retención legal bloquea el borrado hasta que un admin la libere.", + })} +

+ + {shareFor !== null && ( + setShareFor(null)} + onCreated={(itemId, share) => setShares((current) => ({ ...current, [itemId]: share }))} + /> + )} + {retentionFor !== null && ( + setRetentionFor(null)} onSaved={handleRetentionSaved} /> + )} +
+ ); +} diff --git a/apps/web/src/app/(main)/layout.tsx b/apps/web/src/app/(main)/layout.tsx index c01602d..518bf76 100644 --- a/apps/web/src/app/(main)/layout.tsx +++ b/apps/web/src/app/(main)/layout.tsx @@ -15,7 +15,7 @@ import { useIsMobile } from "../../hooks/use-is-mobile"; const titles: Record = { streams: "Home", search: "Search", inbox: "Inbox", approvals: "Approvals", more: "More", compose: "New post", calendar: "Calendar", video: "Video", analytics: "Analytics", connections: "Connections", team: "Team", - audit: "Audit log", onboarding: "Welcome", settings: "Settings", archive: "Archive", + audit: "Audit log", onboarding: "Welcome", settings: "Settings", archive: "Archive", files: "Files", }; const rootTabs = new Set(["/streams", "/search", "/inbox", "/more"]); const scrollPositions = new Map(); diff --git a/apps/web/src/app/(main)/more/page.tsx b/apps/web/src/app/(main)/more/page.tsx index c7c25ea..d7b5d4a 100644 --- a/apps/web/src/app/(main)/more/page.tsx +++ b/apps/web/src/app/(main)/more/page.tsx @@ -63,6 +63,7 @@ export default function MorePage(): React.ReactElement { + diff --git a/apps/web/src/components/files/file-row.tsx b/apps/web/src/components/files/file-row.tsx new file mode 100644 index 0000000..2ca6f2d --- /dev/null +++ b/apps/web/src/components/files/file-row.tsx @@ -0,0 +1,65 @@ +"use client"; + +import { Badge, Button, Card } from "../ui/primitives"; +import { formatBytes } from "../video/format"; +import { MimeIcon } from "./mime-icon"; +import { mimeClassLabel, ORIGIN_LABEL, ORIGIN_TONE, RETENTION_LABEL, RETENTION_TONE } from "./labels"; +import { useFilesLang } from "./i18n"; +import type { GroupFileItem } from "./types"; + +/** One file row: identity + metadata on the left, actions on the right. */ +export function FileRow({ item, live, canRetention, busy, hasActiveShare, onDownload, onShare, onRevoke, onRetention }: { + item: GroupFileItem; + live: boolean; + canRetention: boolean; + busy: boolean; + hasActiveShare: boolean; + onDownload: () => void; + onShare: () => void; + onRevoke: () => void; + onRetention: () => void; +}): React.ReactElement { + const { t } = useFilesLang(); + const disabled = busy || !live || !item.available; + const retentionLabel = + item.retentionMode === "until" && item.retentionUntil !== null + ? `${t(RETENTION_LABEL.until)} ${new Date(item.retentionUntil).toLocaleDateString()}` + : t(RETENTION_LABEL[item.retentionMode]); + + return ( + + +
+
+ {item.safeName} + {t(ORIGIN_LABEL[item.origin])} + {retentionLabel} + {!item.available && {t({ en: "Unavailable", es: "No disponible" })}} +
+
+ {formatBytes(item.byteSize)} + + {t(mimeClassLabel(item.mimeClass))} + + {item.senderMxid} + + +
+
+
+ + + {hasActiveShare && ( + + )} + {canRetention && ( + + )} +
+ + ); +} diff --git a/apps/web/src/components/files/files-client.ts b/apps/web/src/components/files/files-client.ts new file mode 100644 index 0000000..bf66f57 --- /dev/null +++ b/apps/web/src/components/files/files-client.ts @@ -0,0 +1,40 @@ +import { API_URL, ApiError } from "../../lib/api-client"; +import { getSessionToken } from "../../lib/session-token"; + +/** + * Binary download for GET /files/items/:itemId/download. This cannot use + * `apiFetch` (it parses JSON) or a bare anchor (the endpoint needs the + * Authorization header, which anchors cannot send), so it replays the client's + * auth rules: same base URL, Bearer token, cookie credentials — then streams + * the body into a Blob and clicks a temporary anchor. + */ +export async function downloadGroupFile(itemId: string, fileName: string): Promise { + const token = getSessionToken(); + const response = await fetch(`${API_URL}/files/items/${encodeURIComponent(itemId)}/download`, { + credentials: "include", + headers: token === null ? {} : { Authorization: `Bearer ${token}` }, + }); + if (!response.ok) { + if (response.status === 401) throw new ApiError(401, "Your session expired — sign in again.", null); + throw new ApiError(response.status, `Download failed (HTTP ${response.status})`, null); + } + const blob = await response.blob(); + const url = URL.createObjectURL(blob); + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = fileName; + document.body.append(anchor); + anchor.click(); + anchor.remove(); + URL.revokeObjectURL(url); +} + +/** + * Derive the share id from a public tokenUrl of the form + * `/files/shares/:token/download`, used for DELETE /files/shares/:shareId when + * the POST response omits an explicit shareId. + */ +export function shareIdFromUrl(tokenUrl: string): string | null { + const match = /\/files\/shares\/([^/?#]+)/.exec(tokenUrl); + return match?.[1] !== undefined && match[1] !== "" ? match[1] : null; +} diff --git a/apps/web/src/components/files/i18n.tsx b/apps/web/src/components/files/i18n.tsx new file mode 100644 index 0000000..9fe7747 --- /dev/null +++ b/apps/web/src/components/files/i18n.tsx @@ -0,0 +1,66 @@ +"use client"; + +import { createContext, useCallback, useContext, useEffect, useMemo, useState, type ReactNode } from "react"; + +/** + * EN/ES language toggle for the Files section, mirroring + * app/(main)/wiki/lang-context.tsx exactly: a per-browser reading preference + * in localStorage (never sent to the server) and a `t({en, es})` helper every + * Files label goes through. Wired by app/(main)/files/layout.tsx. + */ + +export type FilesLang = "en" | "es"; + +export interface LocalizedText { + readonly en: string; + readonly es: string; +} + +interface FilesLangContextValue { + lang: FilesLang; + setLang: (lang: FilesLang) => void; + /** Pick the string for the current language — the one helper every Files screen uses. */ + t: (text: LocalizedText) => string; +} + +const STORAGE_KEY = "tmbh.files.lang"; +const FilesLangContext = createContext(null); + +function readStoredLang(): FilesLang { + if (typeof window === "undefined") return "en"; + try { + const stored = window.localStorage.getItem(STORAGE_KEY); + return stored === "es" ? "es" : "en"; + } catch { + return "en"; + } +} + +export function FilesLangProvider({ children }: { children: ReactNode }): React.ReactElement { + const [lang, setLangState] = useState("en"); + + useEffect(() => { + setLangState(readStoredLang()); + }, []); + + const setLang = useCallback((next: FilesLang) => { + setLangState(next); + try { + window.localStorage.setItem(STORAGE_KEY, next); + } catch { + // Private-browsing modes can refuse storage; the choice still holds for + // this screen, it just will not survive a reload. + } + }, []); + + const t = useCallback((text: LocalizedText) => text[lang], [lang]); + + const value = useMemo(() => ({ lang, setLang, t }), [lang, setLang, t]); + return {children}; +} + +export function useFilesLang(): FilesLangContextValue { + const context = useContext(FilesLangContext); + if (context === null) throw new Error("useFilesLang must be used inside FilesLangProvider"); + return context; +} diff --git a/apps/web/src/components/files/labels.ts b/apps/web/src/components/files/labels.ts new file mode 100644 index 0000000..ef98ba4 --- /dev/null +++ b/apps/web/src/components/files/labels.ts @@ -0,0 +1,43 @@ +import type { LocalizedText } from "./i18n"; +import type { FileOrigin, MimeClass, RetentionMode } from "./types"; + +/** Bilingual display labels for the group-files enums (values are fixed by the API). */ + +export const ORIGIN_LABEL: Record = { + matrix: { en: "Matrix", es: "Matrix" }, + bridge: { en: "Bridge", es: "Puente" }, + upload: { en: "Upload", es: "Subido" }, + "pad-export": { en: "Pad export", es: "Export. de pad" }, +}; + +export const ORIGIN_TONE: Record = { + matrix: "accent", + bridge: "live", + upload: "neutral", + "pad-export": "warn", +}; + +export const MIME_CLASS_LABEL: Record = { + image: { en: "Image", es: "Imagen" }, + video: { en: "Video", es: "Vídeo" }, + audio: { en: "Audio", es: "Audio" }, + document: { en: "Document", es: "Documento" }, + other: { en: "File", es: "Archivo" }, +}; + +export const RETENTION_LABEL: Record = { + keep: { en: "Keep", es: "Conservar" }, + until: { en: "Until", es: "Hasta" }, + legalHold: { en: "Legal hold", es: "Retención legal" }, +}; + +export const RETENTION_TONE: Record = { + keep: "neutral", + until: "accent", + legalHold: "danger", +}; + +/** Human label for a MIME class, tolerating an unknown value from the server. */ +export function mimeClassLabel(mimeClass: MimeClass): LocalizedText { + return MIME_CLASS_LABEL[mimeClass] ?? MIME_CLASS_LABEL.other; +} diff --git a/apps/web/src/components/files/mime-icon.tsx b/apps/web/src/components/files/mime-icon.tsx new file mode 100644 index 0000000..c03b322 --- /dev/null +++ b/apps/web/src/components/files/mime-icon.tsx @@ -0,0 +1,66 @@ +import type { MimeClass } from "./types"; + +/** + * MIME-class glyph for a file row. Inline SVG drawn with `currentColor` so it + * inherits the palette (text-accent in the row) instead of shipping four more + * brand assets — the sidebar nav icon is the only new static asset. + */ +export function MimeIcon({ mimeClass, size = 28, className = "" }: { + mimeClass: MimeClass; + size?: number; + className?: string; +}): React.ReactElement { + const common = { + width: size, + height: size, + viewBox: "0 0 24 24", + fill: "none", + stroke: "currentColor", + strokeWidth: 1.6, + strokeLinecap: "round" as const, + strokeLinejoin: "round" as const, + "aria-hidden": true, + className, + }; + if (mimeClass === "image") { + return ( + + + + + + ); + } + if (mimeClass === "video") { + return ( + + + + + ); + } + if (mimeClass === "audio") { + return ( + + + + + + ); + } + if (mimeClass === "document") { + return ( + + + + + + ); + } + return ( + + + + + ); +} diff --git a/apps/web/src/components/files/retention-dialog.tsx b/apps/web/src/components/files/retention-dialog.tsx new file mode 100644 index 0000000..22c97e2 --- /dev/null +++ b/apps/web/src/components/files/retention-dialog.tsx @@ -0,0 +1,105 @@ +"use client"; + +import { useState } from "react"; +import { Button, Input } from "../ui/primitives"; +import { api, errorMessage } from "../../lib/api-client"; +import { RETENTION_LABEL } from "./labels"; +import { useFilesLang } from "./i18n"; +import type { GroupFileItem, RetentionInput, RetentionMode } from "./types"; + +/** + * Admin-only retention change: POST /files/items/:itemId/retention + * {mode, until?}. Retention is separate from share expiry — legalHold blocks + * deletion, `until` requires a deadline, `keep` is the no-purge default. + */ + +export function RetentionDialog({ item, onClose, onSaved }: { + item: GroupFileItem; + onClose: () => void; + onSaved: (itemId: string, input: RetentionInput) => void; +}): React.ReactElement { + const { t } = useFilesLang(); + const [mode, setMode] = useState(item.retentionMode); + const [until, setUntil] = useState(item.retentionUntil?.slice(0, 16) ?? ""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + + const modes: RetentionMode[] = ["keep", "until", "legalHold"]; + + async function save(): Promise { + let iso: string | undefined; + if (mode === "until") { + if (until === "") { + setError(t({ en: "Pick a retention deadline.", es: "Elige una fecha límite de retención." })); + return; + } + const parsed = new Date(until); + if (!Number.isFinite(parsed.getTime())) { + setError(t({ en: "That deadline is not a valid date.", es: "Esa fecha límite no es válida." })); + return; + } + iso = parsed.toISOString(); + } + setBusy(true); + setError(null); + try { + const body: RetentionInput = mode === "until" ? { mode, until: iso } : { mode }; + await api.post(`/files/items/${item.id}/retention`, body); + onSaved(item.id, body); + onClose(); + } catch (err) { + setError(errorMessage(err)); + } finally { + setBusy(false); + } + } + + return ( +
+
e.stopPropagation()} + > +

{t({ en: "Set retention", es: "Definir retención" })}

+

{item.safeName}

+

+ {t({ + en: "Retention is independent of share expiry. Legal hold blocks deletion until an admin releases it.", + es: "La retención es independiente de la caducidad del enlace. La retención legal bloquea el borrado hasta que un admin la libere.", + })} +

+
+ {modes.map((value) => ( + + ))} +
+ {mode === "until" && ( + + )} + {error !== null &&

{error}

} +
+ + +
+
+
+ ); +} diff --git a/apps/web/src/components/files/share-dialog.tsx b/apps/web/src/components/files/share-dialog.tsx new file mode 100644 index 0000000..2788228 --- /dev/null +++ b/apps/web/src/components/files/share-dialog.tsx @@ -0,0 +1,158 @@ +"use client"; + +import { useMemo, useState } from "react"; +import { Button, Input } from "../ui/primitives"; +import { api, errorMessage } from "../../lib/api-client"; +import { shareIdFromUrl } from "./files-client"; +import { useFilesLang } from "./i18n"; +import type { GroupFileItem, ShareResponse } from "./types"; + +/** + * Mint a public share link for one file: POST /files/items/:itemId/share + * {expiresAt}. The token is returned once — after this dialog closes it can + * never be recovered, so the result view says so in plain language and offers + * a copy button before the user leaves. + */ + +type ExpiryKey = "1h" | "24h" | "7d" | "custom"; + +const EXPIRY_MS: Record, number> = { + "1h": 60 * 60 * 1000, + "24h": 24 * 60 * 60 * 1000, + "7d": 7 * 24 * 60 * 60 * 1000, +}; + +export function ShareDialog({ item, onClose, onCreated }: { + item: GroupFileItem; + onClose: () => void; + onCreated: (itemId: string, share: { shareId: string | null; tokenUrl: string }) => void; +}): React.ReactElement { + const { t } = useFilesLang(); + const [expiry, setExpiry] = useState("24h"); + const [customAt, setCustomAt] = useState(""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const [tokenUrl, setTokenUrl] = useState(null); + const [copied, setCopied] = useState(false); + + const options: Array<{ key: ExpiryKey; label: string }> = [ + { key: "1h", label: t({ en: "1 hour", es: "1 hora" }) }, + { key: "24h", label: t({ en: "24 hours", es: "24 horas" }) }, + { key: "7d", label: t({ en: "7 days", es: "7 días" }) }, + { key: "custom", label: t({ en: "Custom", es: "Personalizado" }) }, + ]; + + const minLocal = useMemo(() => { + const now = new Date(Date.now() - new Date().getTimezoneOffset() * 60_000); + return now.toISOString().slice(0, 16); + }, []); + + function expiresAtIso(): string | null { + if (expiry === "custom") { + if (customAt === "") return null; + const parsed = new Date(customAt); + return Number.isFinite(parsed.getTime()) ? parsed.toISOString() : null; + } + return new Date(Date.now() + EXPIRY_MS[expiry]).toISOString(); + } + + async function create(): Promise { + const expiresAt = expiresAtIso(); + if (expiresAt === null) { + setError(t({ en: "Pick a future expiry date.", es: "Elige una fecha de caducidad futura." })); + return; + } + setBusy(true); + setError(null); + try { + const response = await api.post(`/files/items/${item.id}/share`, { expiresAt }); + setTokenUrl(response.tokenUrl); + onCreated(item.id, { shareId: response.shareId ?? shareIdFromUrl(response.tokenUrl), tokenUrl: response.tokenUrl }); + } catch (err) { + setError(errorMessage(err)); + } finally { + setBusy(false); + } + } + + async function copy(): Promise { + if (tokenUrl === null) return; + try { + await navigator.clipboard.writeText(tokenUrl); + setCopied(true); + } catch { + setError(t({ en: "Could not copy — select the link and copy it manually.", es: "No se pudo copiar — selecciona el enlace y cópialo a mano." })); + } + } + + return ( +
+
e.stopPropagation()} + > +

{t({ en: "Share file", es: "Compartir archivo" })}

+

{item.safeName}

+ + {tokenUrl === null ? ( + <> +

+ {t({ + en: "Anyone with the link can download this file until it expires. Expiry only ends the link — it never deletes or hides the file.", + es: "Cualquiera con el enlace puede descargar este archivo hasta que caduque. La caducidad solo termina el enlace — nunca borra ni oculta el archivo.", + })} +

+
+ {options.map((option) => ( + + ))} +
+ {expiry === "custom" && ( + + )} + {error !== null &&

{error}

} +
+ + +
+ + ) : ( + <> +

+ {t({ + en: "Copy this link now — it is shown only once and cannot be recovered after you close this dialog.", + es: "Copia este enlace ahora — se muestra solo una vez y no se puede recuperar tras cerrar este cuadro.", + })} +

+ e.target.select()} className="mt-2 font-mono text-2xs" /> + {error !== null &&

{error}

} +
+ + +
+ + )} +
+
+ ); +} diff --git a/apps/web/src/components/files/types.ts b/apps/web/src/components/files/types.ts new file mode 100644 index 0000000..3a72de3 --- /dev/null +++ b/apps/web/src/components/files/types.ts @@ -0,0 +1,64 @@ +/** + * Wire shapes for the group-files HTTP surface (apps/api/src/group-files). + * Kept local to the Files feature so it does not collide with the shared + * api-types.ts other builders may be editing in parallel. Hand-synced with the + * controller: GET /files/groups/:groupId/items returns the flat item below. + */ + +export type FileOrigin = "matrix" | "bridge" | "upload" | "pad-export"; + +/** + * Coarse MIME class. NOTE: the API's DB CHECK (migration 029) and + * group-files.validation.ts MIME_CLASSES both use + * "image" | "video" | "audio" | "document" | "other" — the task brief said + * image/video/audio/file/doc; this follows the backend's real enum. + */ +export type MimeClass = "image" | "video" | "audio" | "document" | "other"; + +export type RetentionMode = "keep" | "until" | "legalHold"; + +/** One row of the group file library — the served item shape. */ +export interface GroupFileItem { + id: string; + groupId: string; + origin: FileOrigin; + senderMxid: string; + originalName: string; + safeName: string; + mime: string; + mimeClass: MimeClass; + byteSize: number; + sha256: string; + createdAt: string; + available: boolean; + retentionMode: RetentionMode; + retentionUntil: string | null; +} + +export interface GroupFilePage { + items: GroupFileItem[]; + nextCursor: string | null; +} + +/** GET /groups/mine (same shape the More page already consumes). */ +export interface GroupSummary { + id: string; + name: string; +} + +/** + * POST /files/items/:itemId/share response. The task surface documents only + * `tokenUrl`; the API may also return a `shareId` used by + * DELETE /files/shares/:shareId. When absent the client derives the id from + * the tokenUrl's path segment so revoke still works. + */ +export interface ShareResponse { + tokenUrl: string; + shareId?: string; +} + +/** POST /files/items/:itemId/retention body. */ +export interface RetentionInput { + mode: RetentionMode; + until?: string; +} diff --git a/apps/web/src/components/files/upload-dropzone.tsx b/apps/web/src/components/files/upload-dropzone.tsx new file mode 100644 index 0000000..2c5697e --- /dev/null +++ b/apps/web/src/components/files/upload-dropzone.tsx @@ -0,0 +1,160 @@ +"use client"; + +import { useCallback, useRef, useState } from "react"; +import { Button } from "../ui/primitives"; +import { Loading } from "../brand/loading"; +import { apiUploadFile, errorMessage } from "../../lib/api-client"; +import { formatBytes } from "../video/format"; +import { useFilesLang } from "./i18n"; +import type { GroupFileItem } from "./types"; + +/** + * Drag-drop + picker for POST /files/groups/:groupId/upload. The API accepts a + * single `file` field capped at 20 MiB; the cap is enforced here too so an + * oversized pick fails fast with a friendly message instead of a 413. + */ + +const MAX_UPLOAD_BYTES = 20 * 1024 * 1024; + +export function UploadDropzone({ groupId, live, onUploaded }: { + groupId: string; + live: boolean; + onUploaded: (item: GroupFileItem) => void; +}): React.ReactElement { + const { t } = useFilesLang(); + const inputRef = useRef(null); + const busyRef = useRef(false); + const [dragging, setDragging] = useState(false); + const [uploading, setUploading] = useState(null); + const [progress, setProgress] = useState(0); + const [error, setError] = useState(null); + + const disabled = !live || groupId === "" || uploading !== null; + + const register = useCallback( + async (file: File): Promise => { + if (disabled || busyRef.current) return; + if (file.size === 0) { + setError(t({ en: `${file.name}: the file is empty.`, es: `${file.name}: el archivo está vacío.` })); + return; + } + if (file.size > MAX_UPLOAD_BYTES) { + setError( + t({ + en: `${file.name} is ${formatBytes(file.size)} — the limit is 20 MiB.`, + es: `${file.name} pesa ${formatBytes(file.size)} — el límite es 20 MiB.`, + }), + ); + return; + } + setError(null); + busyRef.current = true; + setProgress(0); + setUploading(`${file.name} (${formatBytes(file.size)})`); + try { + const created = await apiUploadFile( + `/files/groups/${encodeURIComponent(groupId)}/upload`, + file, + setProgress, + ); + onUploaded(created); + } catch (err) { + setError(errorMessage(err)); + } finally { + setUploading(null); + busyRef.current = false; + } + }, + [disabled, groupId, onUploaded, t], + ); + + const hint = !live + ? t({ en: "Offline — connect to the API to upload.", es: "Sin conexión — conéctate a la API para subir." }) + : groupId === "" + ? t({ en: "Choose a group first.", es: "Elige un grupo primero." }) + : t({ + en: "Drop a file, paste one, or browse. Up to 20 MiB; stored with re-verified bytes.", + es: "Arrastra un archivo, pega uno o búscalo. Hasta 20 MiB; guardado con bytes reverificados.", + }); + + return ( +
+
{ + if (!disabled) inputRef.current?.click(); + }} + onKeyDown={(e) => { + if (disabled) return; + if (e.key === "Enter" || e.key === " ") { + e.preventDefault(); + inputRef.current?.click(); + } + }} + onDragOver={(e) => { + e.preventDefault(); + if (!disabled) setDragging(true); + }} + onDragLeave={() => setDragging(false)} + onDrop={(e) => { + e.preventDefault(); + setDragging(false); + if (disabled) return; + const file = e.dataTransfer.files[0]; + if (file !== undefined) void register(file); + }} + className={`flex flex-col items-center justify-center gap-2 rounded-lg border border-dashed px-4 py-6 text-center transition-colors duration-150 ${ + disabled + ? "cursor-not-allowed border-border-subtle opacity-50" + : dragging + ? "cursor-pointer border-accent bg-accent-dim" + : "cursor-pointer border-border hover:border-accent hover:bg-surface-2" + }`} + > + {uploading !== null ? ( +
+ + + {progress < 100 + ? t({ en: `Uploading ${progress}%`, es: `Subiendo ${progress}%` }) + : t({ en: "Upload sent — indexing…", es: "Subida enviada — indexando…" })}{" "} + {uploading} + + +
+ ) : ( + <> + {t({ en: "Drop a file here", es: "Suelta un archivo aquí" })} + {hint} + {!disabled && ( + + )} + + )} + { + const file = e.target.files?.[0]; + e.target.value = ""; + if (file !== undefined) void register(file); + }} + /> +
+ {error !== null &&

{error}

} +
+ ); +} diff --git a/apps/web/src/components/mobile/bottom-nav.tsx b/apps/web/src/components/mobile/bottom-nav.tsx index c53e60d..e28aaa5 100644 --- a/apps/web/src/components/mobile/bottom-nav.tsx +++ b/apps/web/src/components/mobile/bottom-nav.tsx @@ -38,6 +38,8 @@ function activeTab(pathname: string): string { ["/calendar", pathname.startsWith("/calendar") ? 1 : -1], ["/analytics", pathname.startsWith("/analytics") ? 1 : -1], ["/video", pathname.startsWith("/video") ? 1 : -1], + ["/files", pathname.startsWith("/files") ? 1 : -1], + ["/archive", pathname.startsWith("/archive") ? 1 : -1], ]; let best = ""; let bestScore = 0; diff --git a/apps/web/src/components/shell/command-palette.tsx b/apps/web/src/components/shell/command-palette.tsx index dd1463e..04b1349 100644 --- a/apps/web/src/components/shell/command-palette.tsx +++ b/apps/web/src/components/shell/command-palette.tsx @@ -12,6 +12,7 @@ const PAGES = [ { href: "/video", label: "Go to Video" }, { href: "/inbox", label: "Go to Inbox" }, { href: "/archive", label: "Go to Archive" }, + { href: "/files", label: "Go to Files" }, { href: "/connections", label: "Go to Channels" }, { href: "/team", label: "Go to Team" }, { href: "/audit", label: "Go to Audit" }, diff --git a/apps/web/src/components/shell/sidebar.tsx b/apps/web/src/components/shell/sidebar.tsx index 091aa85..1e822cb 100644 --- a/apps/web/src/components/shell/sidebar.tsx +++ b/apps/web/src/components/shell/sidebar.tsx @@ -17,6 +17,7 @@ const NAV = [ { href: "/inbox", label: "Inbox" }, { href: "/approvals", label: "Approvals" }, { href: "/archive", label: "Archive" }, + { href: "/files", label: "Files" }, { href: "/analytics", label: "Analytics" }, { href: "/connections", label: "Channels" }, { href: "/team", label: "Team" }, @@ -58,8 +59,8 @@ export function Sidebar(): React.ReactElement { }`} > - {item.href === "/video" || item.href === "/control-room" ? ( - + {item.href === "/video" || item.href === "/control-room" || item.href === "/files" ? ( + ) : ( )} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e5ff257..b2ab09a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -106,6 +106,34 @@ importers: specifier: ^2.0.5 version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + apps/archive-bot: + dependencies: + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../../packages/core + pg: + specifier: ^8.23.1 + version: 8.23.1 + pino: + specifier: ^9.3.2 + version: 9.14.0 + devDependencies: + '@types/node': + specifier: ^22.5.0 + version: 22.20.5 + '@types/pg': + specifier: ^8.23.1 + version: 8.23.1 + tsx: + specifier: ^4.17.0 + version: 4.23.15 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + apps/media-worker: dependencies: pino: