Add group cloud: files index api, web UI, archive sync bot
The Telegram-like group file contract (packages/core shared-files.ts) becomes a working subsystem, built in three parallel tracks: API (NestJS): migration 029 group_files (chat_media_archive metadata-only index with partial replay-unique + RLS, group_file_shares with sha256 token persistence, archive_sync_state/outbox durable-claim seam incl. a unique claim_key index), group-files module (list/search with cursor pagination, direct upload ≤20 MiB streamed+hashed to oCIS WebDAV, download with ACL + retention rechecks, share links with expiry, retention modes incl. legalHold) — RBAC group-membership authz + audit chain events. Web: Files section per group — group picker, drag-drop upload (20 MiB cap), search + MIME/origin filters, cursor pagination, download via authenticated blob fetch, share dialog with expiry + shown-once token, admin retention dialog; bilingual EN/ES (wiki i18n pattern); nav sidebar/palette/bottom-nav wiring; MIME classes aligned to the DB CHECK (document/other). Archive bot (apps/archive-bot): plain sync-loop Matrix bot mirroring attachments from explicitly-mapped unencrypted archive rooms — raw /sync with durable since-cursor, outbox-claimed ingestion (hash-verified stream, no temp files), magic-byte MIME sniffing, safe-name normalization, WebDAV upload, indexed + replied with link, redaction→unavailable; health endpoint; Dockerfile. Tests 74/74, typecheck clean across api/web/bot.
This commit is contained in:
parent
eb1b60f67d
commit
d58db63fe6
58 changed files with 5543 additions and 4 deletions
245
apps/api/migrations/029_group_files.sql
Normal file
245
apps/api/migrations/029_group_files.sql
Normal file
|
|
@ -0,0 +1,245 @@
|
|||
-- 029: group files index — the Telegram-like group file contract
|
||||
-- (docs/occupymedia-storage.md §"Telegram-like group file contract" and
|
||||
-- §"Metadata index and hub-archive seam").
|
||||
--
|
||||
-- Metadata ONLY. chat_media_archive is the index over remote oCIS payloads:
|
||||
-- it NEVER holds file bodies, encrypted-file keys, WOPI tokens or presigned
|
||||
-- URLs. Verified byte size + SHA-256 only — the payload always lives behind
|
||||
-- oCIS WebDAV, streamed through the API with the hash re-verified.
|
||||
--
|
||||
-- Attribution is by sender MXID (never a display name/alias) and origin is
|
||||
-- one of the four SharedFileOrigin kinds (packages/core/src/shared-files.ts).
|
||||
-- event_id is NULL for direct uploads/exports until a room announcement
|
||||
-- supplies one, so replay identity uniqueness is PARTIAL on event_id IS NOT
|
||||
-- NULL: "unique (workspace_id, room_id, event_id) for attachment events".
|
||||
--
|
||||
-- Retention is kept SEPARATE from link expiry: mode is 'keep' (phase-one
|
||||
-- default, no automated purge) or 'until' (requires retention_until), with an
|
||||
-- orthogonal legal_hold flag that blocks deletion. A share expiring never
|
||||
-- deletes or hides a file.
|
||||
--
|
||||
-- group_file_shares persists link IDs + sha256(token) ONLY: the plaintext
|
||||
-- token is returned once at creation and can never be recovered from the DB.
|
||||
-- No presigned object URLs are ever stored. public links require an explicit
|
||||
-- expiry (access='public' ⇒ expires_at is bounded below); the claim path uses
|
||||
-- the narrow SECURITY DEFINER lookup (017 invite-link pattern) because the
|
||||
-- claimer has no tenant context and the table is FORCE RLS.
|
||||
--
|
||||
-- archive_sync_state / archive_sync_outbox reserve the hub-archive seam
|
||||
-- (HubArchivePort): the sync bot (not implemented in this slice) commits the
|
||||
-- library reference and its outbox entry BEFORE advancing sync_cursor.
|
||||
--
|
||||
-- RLS / NULLIF / reject_workspace_change / sp_app grants follow the
|
||||
-- 001/008/021/026 conventions: no app.workspace_id => NULL => no rows; FORCE
|
||||
-- binds sp_app too. `origin`/`mime_class`/`retention_mode`/`legal_hold` CHECKs are coarse
|
||||
-- DB backstops — the authoritative typed validation is app-side
|
||||
-- (apps/api/src/group-files/group-files.validation.ts).
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- chat_media_archive — the metadata index
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE TABLE chat_media_archive (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT,
|
||||
group_id uuid NOT NULL REFERENCES groups(id) ON DELETE CASCADE ON UPDATE RESTRICT,
|
||||
-- Immutable Matrix room ID, never an alias. Tenant-pinned below.
|
||||
room_id text NOT NULL CHECK (btrim(room_id) <> '' AND room_id ~ '^!'),
|
||||
-- NULL for direct uploads/pad exports until a room announcement supplies one.
|
||||
event_id text CHECK (event_id IS NULL OR btrim(event_id) <> ''),
|
||||
origin text NOT NULL CHECK (origin IN ('matrix','bridge','upload','pad-export')),
|
||||
-- Sender attribution: MXID only, never a display name (mxc/handle spoofing).
|
||||
sender_mxid text NOT NULL CHECK (sender_mxid ~ '^@[^@\s]+:[^@\s]+$'),
|
||||
-- Redacted/superseded Matrix event context for the hub-archive seam; never
|
||||
-- file bodies or encrypted-file keys (jsonb_typeof backstop only).
|
||||
matrix_event_json jsonb CHECK (matrix_event_json IS NULL OR jsonb_typeof(matrix_event_json) = 'object'),
|
||||
original_name text NOT NULL CHECK (btrim(original_name) <> '' AND char_length(original_name) <= 1024),
|
||||
-- Normalized safe display filename (app-side sanitizeName).
|
||||
safe_name text NOT NULL CHECK (btrim(safe_name) <> '' AND char_length(safe_name) <= 255),
|
||||
mime text NOT NULL CHECK (mime <> ''),
|
||||
-- Coarse class derived from mime (image/video/audio/document/other).
|
||||
mime_class text NOT NULL CHECK (mime_class IN ('image','video','audio','document','other')),
|
||||
byte_size bigint NOT NULL CHECK (byte_size >= 0),
|
||||
-- SHA-256 of verified payload bytes, lowercase hex. Never an S3 ETag.
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'),
|
||||
-- Stable oCIS library resource identity (payload stays remote).
|
||||
ocis_resource_id text NOT NULL CHECK (btrim(ocis_resource_id) <> ''),
|
||||
ocis_resource_version text,
|
||||
created_at timestamptz NOT NULL,
|
||||
indexed_at timestamptz NOT NULL DEFAULT now(),
|
||||
available boolean NOT NULL DEFAULT true,
|
||||
-- FileRetention (packages/core): mode is keep|until; legalHold is a FLAG
|
||||
-- alongside the mode, not a third mode.
|
||||
retention_mode text NOT NULL DEFAULT 'keep' CHECK (retention_mode IN ('keep','until')),
|
||||
legal_hold boolean NOT NULL DEFAULT false,
|
||||
retention_until timestamptz,
|
||||
CONSTRAINT chat_media_archive_ws_group FOREIGN KEY (group_id, workspace_id)
|
||||
REFERENCES groups(id, workspace_id) ON DELETE CASCADE ON UPDATE RESTRICT,
|
||||
-- 'until' retention requires a deadline; 'keep' must not carry one.
|
||||
-- legalHold is orthogonal: it blocks deletion regardless of mode.
|
||||
CONSTRAINT chat_media_archive_retention_shape CHECK (
|
||||
(retention_mode = 'until' AND retention_until IS NOT NULL)
|
||||
OR (retention_mode = 'keep' AND retention_until IS NULL)
|
||||
)
|
||||
);
|
||||
|
||||
-- Tenant-pinned reference target for group_file_shares (015 style):
|
||||
-- the composite UNIQUE lets child FKs pin (id, workspace_id) so a share can
|
||||
-- never point across tenants even if ids collided.
|
||||
CREATE UNIQUE INDEX chat_media_archive_id_ws_key ON chat_media_archive (id, workspace_id);
|
||||
|
||||
CREATE TRIGGER trg_chat_media_archive_ws_immutable BEFORE UPDATE ON chat_media_archive
|
||||
FOR EACH ROW EXECUTE FUNCTION reject_workspace_change();
|
||||
|
||||
ALTER TABLE chat_media_archive ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE chat_media_archive FORCE ROW LEVEL SECURITY;
|
||||
|
||||
CREATE POLICY tenant_isolation ON chat_media_archive
|
||||
USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid)
|
||||
WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid);
|
||||
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON chat_media_archive TO sp_app;
|
||||
|
||||
-- Replay identity for ATTACHMENT events (029 note: uploads have NULL event_id,
|
||||
-- so the unique constraint must be partial). AttachmentArchiveKey's
|
||||
-- collision-free replay rule, enforced durably.
|
||||
CREATE UNIQUE INDEX chat_media_archive_replay_uq
|
||||
ON chat_media_archive (workspace_id, room_id, event_id)
|
||||
WHERE event_id IS NOT NULL;
|
||||
|
||||
-- Cursor pagination index: (createdAt,id) keyset scoped to workspace/group.
|
||||
-- Spec: "indexes on (workspace_id, group_id, created_at desc, id desc)".
|
||||
CREATE INDEX chat_media_archive_ws_group_time_idx
|
||||
ON chat_media_archive (workspace_id, group_id, created_at DESC, id DESC);
|
||||
|
||||
-- Sender/MIME-class/checksum facets (spec: "indexes on ... sender, MIME class,
|
||||
-- name and checksum"). MXID is indexed for per-sender listing/attribution.
|
||||
CREATE INDEX chat_media_archive_sender_idx ON chat_media_archive (workspace_id, sender_mxid, created_at DESC);
|
||||
CREATE INDEX chat_media_archive_mime_class_idx ON chat_media_archive (mime_class);
|
||||
CREATE INDEX chat_media_archive_sha256_idx ON chat_media_archive (sha256);
|
||||
CREATE INDEX chat_media_archive_name_idx ON chat_media_archive (safe_name text_pattern_ops);
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- group_file_shares — link IDs + sha256(token), never presigned URLs
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE TABLE group_file_shares (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT,
|
||||
item_id uuid NOT NULL,
|
||||
-- 'members' (default library link, ACL-checked) or 'public' (requires an
|
||||
-- explicit grant + bounded expiry; revoked on removal per the contract).
|
||||
access text NOT NULL DEFAULT 'members' CHECK (access IN ('members','public')),
|
||||
-- sha256(plaintext token). The plaintext is returned ONCE at creation; the
|
||||
-- DB can never re-derive it, so a leaked DB dump mints no downloads.
|
||||
token_hash text NOT NULL UNIQUE CHECK (token_hash ~ '^[0-9a-f]{64}$'),
|
||||
created_by uuid REFERENCES users(id) ON DELETE SET NULL,
|
||||
expires_at timestamptz NOT NULL,
|
||||
revoked_at timestamptz,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
CONSTRAINT group_file_shares_ws_item FOREIGN KEY (item_id, workspace_id)
|
||||
REFERENCES chat_media_archive(id, workspace_id) ON DELETE CASCADE ON UPDATE RESTRICT
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX group_file_shares_ws_item_key ON group_file_shares (item_id, workspace_id);
|
||||
|
||||
CREATE TRIGGER trg_group_file_shares_ws_immutable BEFORE UPDATE ON group_file_shares
|
||||
FOR EACH ROW EXECUTE FUNCTION reject_workspace_change();
|
||||
|
||||
ALTER TABLE group_file_shares ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE group_file_shares FORCE ROW LEVEL SECURITY;
|
||||
|
||||
CREATE POLICY tenant_isolation ON group_file_shares
|
||||
USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid)
|
||||
WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid);
|
||||
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON group_file_shares TO sp_app;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- share_for_public_claim — narrow SECURITY DEFINER lookup (017 pattern)
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Public downloads arrive with NO tenant context (the claimer holds only the
|
||||
-- token) and group_file_shares is FORCE RLS, so an ordinary SELECT sees zero
|
||||
-- rows. Exactly like invite_link_for_claim (017), this exposes ONE row keyed
|
||||
-- by the secret — workspace/file discovery only. It returns the row only when
|
||||
-- it is live: not revoked and not expired (the caller still rechecks the item
|
||||
-- row's availability/retention inside the tenant context it enters next).
|
||||
CREATE FUNCTION share_for_public_claim(p_token_hash text)
|
||||
RETURNS TABLE(id uuid, workspace_id uuid, item_id uuid, access text,
|
||||
expires_at timestamptz, revoked_at timestamptz)
|
||||
LANGUAGE sql STABLE SECURITY DEFINER SET search_path = public AS $$
|
||||
SELECT s.id, s.workspace_id, s.item_id, s.access, s.expires_at, s.revoked_at
|
||||
FROM group_file_shares s
|
||||
WHERE s.token_hash = p_token_hash
|
||||
AND s.revoked_at IS NULL
|
||||
AND s.expires_at > now()
|
||||
$$;
|
||||
|
||||
REVOKE ALL ON FUNCTION share_for_public_claim(text) FROM PUBLIC;
|
||||
GRANT EXECUTE ON FUNCTION share_for_public_claim(text) TO sp_app;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- archive_sync_state — per-room sync cursor (hub-archive seam)
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE TABLE archive_sync_state (
|
||||
room_id text PRIMARY KEY CHECK (btrim(room_id) <> ''),
|
||||
workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT,
|
||||
-- Last committed Matrix sync token; the bot never advances past a failed
|
||||
-- event (durable retry lives in archive_sync_outbox).
|
||||
sync_cursor text,
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TRIGGER trg_archive_sync_state_ws_immutable BEFORE UPDATE ON archive_sync_state
|
||||
FOR EACH ROW EXECUTE FUNCTION reject_workspace_change();
|
||||
|
||||
ALTER TABLE archive_sync_state ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE archive_sync_state FORCE ROW LEVEL SECURITY;
|
||||
|
||||
CREATE POLICY tenant_isolation ON archive_sync_state
|
||||
USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid)
|
||||
WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid);
|
||||
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON archive_sync_state TO sp_app;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- archive_sync_outbox — durable reply/retry records before cursor advance
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE TABLE archive_sync_outbox (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT,
|
||||
room_id text NOT NULL CHECK (btrim(room_id) <> ''),
|
||||
event_id text CHECK (event_id IS NULL OR btrim(event_id) <> ''),
|
||||
-- Idempotent claim key: attachmentArchiveKey(workspace,room,event) shape.
|
||||
claim_key text NOT NULL CHECK (btrim(claim_key) <> ''),
|
||||
status text NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','done','failed')),
|
||||
attempts int NOT NULL DEFAULT 0 CHECK (attempts >= 0),
|
||||
last_error text,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- DB-level idempotency for the durable claim: one outbox row per
|
||||
-- attachmentArchiveKey(workspace,room,event). The bot inserts with a
|
||||
-- NOT EXISTS guard + bare ON CONFLICT DO NOTHING; this index makes the
|
||||
-- race window impossible rather than unlikely.
|
||||
CREATE UNIQUE INDEX archive_sync_outbox_claim_key_uq ON archive_sync_outbox (claim_key);
|
||||
|
||||
CREATE TRIGGER trg_archive_sync_outbox_ws_immutable BEFORE UPDATE ON archive_sync_outbox
|
||||
FOR EACH ROW EXECUTE FUNCTION reject_workspace_change();
|
||||
|
||||
ALTER TABLE archive_sync_outbox ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE archive_sync_outbox FORCE ROW LEVEL SECURITY;
|
||||
|
||||
CREATE POLICY tenant_isolation ON archive_sync_outbox
|
||||
USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid)
|
||||
WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid);
|
||||
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON archive_sync_outbox TO sp_app;
|
||||
|
||||
-- Worker scan: pending-first, oldest-first.
|
||||
CREATE INDEX archive_sync_outbox_status_idx ON archive_sync_outbox (status, created_at);
|
||||
-- Replay/idempotency lookup by claim key.
|
||||
CREATE INDEX archive_sync_outbox_claim_idx ON archive_sync_outbox (claim_key);
|
||||
|
||||
COMMIT;
|
||||
Loading…
Add table
Add a link
Reference in a new issue