Add group cloud: files index api, web UI, archive sync bot
The Telegram-like group file contract (packages/core shared-files.ts) becomes a working subsystem, built in three parallel tracks: API (NestJS): migration 029 group_files (chat_media_archive metadata-only index with partial replay-unique + RLS, group_file_shares with sha256 token persistence, archive_sync_state/outbox durable-claim seam incl. a unique claim_key index), group-files module (list/search with cursor pagination, direct upload ≤20 MiB streamed+hashed to oCIS WebDAV, download with ACL + retention rechecks, share links with expiry, retention modes incl. legalHold) — RBAC group-membership authz + audit chain events. Web: Files section per group — group picker, drag-drop upload (20 MiB cap), search + MIME/origin filters, cursor pagination, download via authenticated blob fetch, share dialog with expiry + shown-once token, admin retention dialog; bilingual EN/ES (wiki i18n pattern); nav sidebar/palette/bottom-nav wiring; MIME classes aligned to the DB CHECK (document/other). Archive bot (apps/archive-bot): plain sync-loop Matrix bot mirroring attachments from explicitly-mapped unencrypted archive rooms — raw /sync with durable since-cursor, outbox-claimed ingestion (hash-verified stream, no temp files), magic-byte MIME sniffing, safe-name normalization, WebDAV upload, indexed + replied with link, redaction→unavailable; health endpoint; Dockerfile. Tests 74/74, typecheck clean across api/web/bot.
This commit is contained in:
parent
eb1b60f67d
commit
d58db63fe6
58 changed files with 5543 additions and 4 deletions
223
apps/archive-bot/src/store.ts
Normal file
223
apps/archive-bot/src/store.ts
Normal file
|
|
@ -0,0 +1,223 @@
|
|||
import { Pool, type PoolClient } from "pg";
|
||||
|
||||
export type OutboxStatus = "pending" | "done" | "failed";
|
||||
|
||||
export interface OutboxRow {
|
||||
id: string;
|
||||
status: OutboxStatus;
|
||||
attempts: number;
|
||||
lastError: string | null;
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
export interface ArchiveRecord {
|
||||
workspaceId: string;
|
||||
groupId: string;
|
||||
roomId: string;
|
||||
eventId: string;
|
||||
senderMxid: string;
|
||||
matrixEventJson: unknown;
|
||||
originalName: string;
|
||||
safeName: string;
|
||||
mime: string;
|
||||
mimeClass: string;
|
||||
byteSize: number;
|
||||
sha256: string;
|
||||
ocisResourceId: string;
|
||||
ocisResourceVersion: string;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
/** Durable state the processor depends on; faked in tests. */
|
||||
export interface ArchiveStore {
|
||||
loadCursor(workspaceIds: readonly string[]): Promise<string | null>;
|
||||
saveCursor(workspaceIds: readonly string[], cursor: string): Promise<void>;
|
||||
claim(workspaceId: string, roomId: string, eventId: string, claimKey: string): Promise<OutboxRow>;
|
||||
/** Index row + outbox done in one transaction. Returns the archive row id and indexed time. */
|
||||
commitArchived(record: ArchiveRecord, outboxId: string): Promise<{ id: string; indexedAt: Date }>;
|
||||
recordFailure(workspaceId: string, outboxId: string, error: string, terminal: boolean): Promise<OutboxRow>;
|
||||
/** Mark index rows unavailable and stop any not-yet-archived claim. Returns rows affected. */
|
||||
markRedacted(workspaceId: string, roomId: string, eventId: string, claimKey: string): Promise<number>;
|
||||
countPending(workspaceIds: readonly string[]): Promise<number>;
|
||||
}
|
||||
|
||||
interface OutboxDbRow {
|
||||
id: string;
|
||||
status: OutboxStatus;
|
||||
attempts: number;
|
||||
last_error: string | null;
|
||||
updated_at: Date;
|
||||
}
|
||||
|
||||
function toOutbox(row: OutboxDbRow): OutboxRow {
|
||||
return { id: row.id, status: row.status, attempts: row.attempts, lastError: row.last_error, updatedAt: row.updated_at };
|
||||
}
|
||||
|
||||
const OUTBOX_COLUMNS = "id, status, attempts, last_error, updated_at";
|
||||
const LAST_ERROR_MAX = 1000;
|
||||
|
||||
export class PgArchiveStore implements ArchiveStore {
|
||||
constructor(
|
||||
private readonly pool: Pool,
|
||||
private readonly cursorId: string,
|
||||
) {}
|
||||
|
||||
/** Single service connection; tenant context per transaction, like apps/api withTenant. */
|
||||
static create(databaseUrl: string, cursorId: string): PgArchiveStore {
|
||||
return new PgArchiveStore(new Pool({ connectionString: databaseUrl, max: 1 }), cursorId);
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this.pool.end();
|
||||
}
|
||||
|
||||
/**
|
||||
* The sync stream is global but rows are tenant-scoped, so each mapped
|
||||
* workspace holds a copy. A partial write leaves copies disagreeing: resume
|
||||
* from the least recently written (oldest) one — replay is idempotent.
|
||||
*/
|
||||
async loadCursor(workspaceIds: readonly string[]): Promise<string | null> {
|
||||
let oldest: { cursor: string; at: Date } | null = null;
|
||||
for (const workspaceId of workspaceIds) {
|
||||
const row = await this.withTenant(workspaceId, async (c) => {
|
||||
const res = await c.query<{ sync_cursor: string | null; updated_at: Date }>(
|
||||
"SELECT sync_cursor, updated_at FROM archive_sync_state WHERE id = $1 AND workspace_id = $2",
|
||||
[this.stateId(workspaceId), workspaceId],
|
||||
);
|
||||
return res.rows[0];
|
||||
});
|
||||
if (row?.sync_cursor == null) continue;
|
||||
if (oldest === null || row.updated_at < oldest.at) oldest = { cursor: row.sync_cursor, at: row.updated_at };
|
||||
}
|
||||
return oldest?.cursor ?? null;
|
||||
}
|
||||
|
||||
async saveCursor(workspaceIds: readonly string[], cursor: string): Promise<void> {
|
||||
for (const workspaceId of workspaceIds) {
|
||||
await this.withTenant(workspaceId, (c) => c.query(
|
||||
`INSERT INTO archive_sync_state (id, workspace_id, sync_cursor, updated_at)
|
||||
VALUES ($1, $2, $3, now())
|
||||
ON CONFLICT (id) DO UPDATE SET sync_cursor = EXCLUDED.sync_cursor, updated_at = now()`,
|
||||
[this.stateId(workspaceId), workspaceId, cursor],
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
async claim(workspaceId: string, roomId: string, eventId: string, claimKey: string): Promise<OutboxRow> {
|
||||
return this.withTenant(workspaceId, async (c) => {
|
||||
await c.query(
|
||||
`INSERT INTO archive_sync_outbox (workspace_id, room_id, event_id, claim_key)
|
||||
SELECT $1, $2, $3, $4
|
||||
WHERE NOT EXISTS (SELECT 1 FROM archive_sync_outbox WHERE workspace_id = $1 AND claim_key = $4)
|
||||
ON CONFLICT DO NOTHING`,
|
||||
[workspaceId, roomId, eventId, claimKey],
|
||||
);
|
||||
const res = await c.query<OutboxDbRow>(
|
||||
`SELECT ${OUTBOX_COLUMNS} FROM archive_sync_outbox
|
||||
WHERE workspace_id = $1 AND claim_key = $2 ORDER BY created_at LIMIT 1`,
|
||||
[workspaceId, claimKey],
|
||||
);
|
||||
const row = res.rows[0];
|
||||
if (row === undefined) throw new Error("outbox claim not visible after insert");
|
||||
return toOutbox(row);
|
||||
});
|
||||
}
|
||||
|
||||
async commitArchived(record: ArchiveRecord, outboxId: string): Promise<{ id: string; indexedAt: Date }> {
|
||||
return this.withTenant(record.workspaceId, async (c) => {
|
||||
await c.query(
|
||||
`INSERT INTO chat_media_archive (
|
||||
workspace_id, group_id, room_id, event_id, origin, sender_mxid, matrix_event_json,
|
||||
original_name, safe_name, mime, mime_class, byte_size, sha256,
|
||||
ocis_resource_id, ocis_resource_version, created_at
|
||||
) VALUES ($1, $2, $3, $4, 'matrix', $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
|
||||
ON CONFLICT DO NOTHING`,
|
||||
[
|
||||
record.workspaceId, record.groupId, record.roomId, record.eventId, record.senderMxid,
|
||||
JSON.stringify(record.matrixEventJson), record.originalName, record.safeName, record.mime,
|
||||
record.mimeClass, record.byteSize, record.sha256, record.ocisResourceId,
|
||||
record.ocisResourceVersion, record.createdAt,
|
||||
],
|
||||
);
|
||||
const indexed = await c.query<{ id: string; indexed_at: Date }>(
|
||||
`SELECT id, indexed_at FROM chat_media_archive
|
||||
WHERE workspace_id = $1 AND room_id = $2 AND event_id = $3 LIMIT 1`,
|
||||
[record.workspaceId, record.roomId, record.eventId],
|
||||
);
|
||||
const row = indexed.rows[0];
|
||||
if (row === undefined) throw new Error("archive row not visible after insert");
|
||||
await c.query(
|
||||
`UPDATE archive_sync_outbox SET status = 'done', last_error = NULL, updated_at = now()
|
||||
WHERE id = $1 AND workspace_id = $2`,
|
||||
[outboxId, record.workspaceId],
|
||||
);
|
||||
return { id: row.id, indexedAt: row.indexed_at };
|
||||
});
|
||||
}
|
||||
|
||||
async recordFailure(workspaceId: string, outboxId: string, error: string, terminal: boolean): Promise<OutboxRow> {
|
||||
return this.withTenant(workspaceId, async (c) => {
|
||||
const res = await c.query<OutboxDbRow>(
|
||||
`UPDATE archive_sync_outbox
|
||||
SET attempts = attempts + 1, last_error = $3, updated_at = now(),
|
||||
status = CASE WHEN $4 THEN 'failed' ELSE 'pending' END
|
||||
WHERE id = $1 AND workspace_id = $2 AND status = 'pending'
|
||||
RETURNING ${OUTBOX_COLUMNS}`,
|
||||
[outboxId, workspaceId, error.slice(0, LAST_ERROR_MAX), terminal],
|
||||
);
|
||||
const row = res.rows[0];
|
||||
if (row === undefined) throw new Error("outbox row not pending");
|
||||
return toOutbox(row);
|
||||
});
|
||||
}
|
||||
|
||||
async markRedacted(workspaceId: string, roomId: string, eventId: string, claimKey: string): Promise<number> {
|
||||
return this.withTenant(workspaceId, async (c) => {
|
||||
const res = await c.query(
|
||||
`UPDATE chat_media_archive SET available = false
|
||||
WHERE workspace_id = $1 AND room_id = $2 AND event_id = $3 AND available`,
|
||||
[workspaceId, roomId, eventId],
|
||||
);
|
||||
await c.query(
|
||||
`UPDATE archive_sync_outbox SET status = 'failed', last_error = 'redacted', updated_at = now()
|
||||
WHERE workspace_id = $1 AND claim_key = $2 AND status = 'pending'`,
|
||||
[workspaceId, claimKey],
|
||||
);
|
||||
return res.rowCount ?? 0;
|
||||
});
|
||||
}
|
||||
|
||||
async countPending(workspaceIds: readonly string[]): Promise<number> {
|
||||
let total = 0;
|
||||
for (const workspaceId of workspaceIds) {
|
||||
total += await this.withTenant(workspaceId, async (c) => {
|
||||
const res = await c.query<{ n: string }>(
|
||||
"SELECT count(*) AS n FROM archive_sync_outbox WHERE workspace_id = $1 AND status = 'pending'",
|
||||
[workspaceId],
|
||||
);
|
||||
return Number(res.rows[0]?.n ?? 0);
|
||||
});
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
private stateId(workspaceId: string): string {
|
||||
return `${this.cursorId}:${workspaceId}`;
|
||||
}
|
||||
|
||||
private async withTenant<T>(workspaceId: string, fn: (client: PoolClient) => Promise<T>): Promise<T> {
|
||||
const client = await this.pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]);
|
||||
const result = await fn(client);
|
||||
await client.query("COMMIT");
|
||||
return result;
|
||||
} catch (err) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw err;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue