diff --git a/apps/api/migrations/041_archive_storage_key.sql b/apps/api/migrations/041_archive_storage_key.sql new file mode 100644 index 0000000..6a3c25d --- /dev/null +++ b/apps/api/migrations/041_archive_storage_key.sql @@ -0,0 +1,4 @@ +-- Files uploads move to R2 object storage (SPENC1-sealed, content-addressed), +-- matching the user directive that object storage is the only media path. +-- Legacy rows keep their oCIS resource ids; storage_key is read first when set. +ALTER TABLE chat_media_archive ADD COLUMN storage_key text; diff --git a/apps/api/src/group-files/collective-share.service.ts b/apps/api/src/group-files/collective-share.service.ts index fc581a4..24a9618 100644 --- a/apps/api/src/group-files/collective-share.service.ts +++ b/apps/api/src/group-files/collective-share.service.ts @@ -31,10 +31,8 @@ import { mimeClassOf, noLibraryTargetMessage, openRoomNoticeContent, - suffixedName, wantsLibrary, wantsOpenRoom, - MAX_COLLISION_ATTEMPTS, type CollectiveRoomPoster, type CollectiveShareInput, type CollectiveShareItem, @@ -42,7 +40,8 @@ import { type CollectiveShareResult, type ShareErrorCode, } from "./collective-share.js"; -import { OcisNotFoundError, OcisPreconditionFailedError, OcisWebdavClient, ocisConfigFromEnv, type FetchLike } from "./ocis-webdav.js"; +import { OcisWebdavClient, ocisConfigFromEnv, type FetchLike } from "./ocis-webdav.js"; +import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js"; interface CollectiveRow { id: string; @@ -72,9 +71,24 @@ export class CollectiveShareService { constructor( @Optional() @Inject(COLLECTIVE_ROOM_POSTER) private readonly poster?: CollectiveRoomPoster, - @Optional() @Inject(COLLECTIVE_SHARE_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike }, + @Optional() @Inject(COLLECTIVE_SHARE_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike; storage?: StorageService }, ) {} + private store: StorageService | null = null; + + /** R2 object storage — the library payload store (041). */ + private storage(): StorageService { + if (this.store === null) { + this.store = this.deps?.storage ?? new S3Storage(s3ConfigFromEnv()); + } + return this.store; + } + + private libraryUrlFor(resourceId: string): string { + const base = (process.env.FILES_PUBLIC_BASE ?? "").replace(/\/+$/, ""); + return base === "" ? resourceId : `${base}/f/${encodeURIComponent(resourceId)}`; + } + /** Lazily bound: the oCIS env may be absent in dev/test contexts. */ private webdav(): OcisWebdavClient { if (this.client === null) { @@ -265,63 +279,50 @@ export class CollectiveShareService { }; } - const baseName = safeFileName(segments[segments.length - 1] ?? "attachment"); - let copied: Awaited> | null = null; - let finalName = baseName; - for (let attempt = 0; attempt <= MAX_COLLISION_ATTEMPTS; attempt += 1) { - const candidate = suffixedName(baseName, attempt); - try { - copied = await this.webdav().copyIntoGroup(collective.id, segments, folder, candidate); - finalName = candidate; - break; - } catch (err) { - if (err instanceof OcisPreconditionFailedError) continue; // try the next suffix - if (err instanceof OcisNotFoundError) { - return { - title: titleFromPath(item.path), - error: { - code: "source-not-found", - message: collective.language === "es" - ? `No se encontró el archivo de origen (${item.path}).` - : `The source file was not found (${item.path}).`, - }, - }; - } - return { - title: titleFromPath(item.path), - error: { - code: "copy-failed", - message: collective.language === "es" - ? "La copia en la biblioteca del colectivo falló." - : "Copying into the collective's library failed.", - }, - }; - } - } - if (copied === null) { + const name = safeFileName(segments[segments.length - 1] ?? "attachment"); + // oCIS stays the Drive source; the library destination is R2 (041). + const source = await this.webdav().download(segments.join("/")).catch(() => null); + if (source === null || source.stream === null) { return { title: titleFromPath(item.path), + error: { + code: "source-not-found", + message: collective.language === "es" + ? `No se encontró el archivo de origen (${item.path}).` + : `The source file was not found (${item.path}).`, + }, + }; + } + const chunks: Buffer[] = []; + for await (const chunk of source.stream as unknown as AsyncIterable) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array)); + } + const bytes = Buffer.concat(chunks); + const stored = await this.storage().putContent(bytes, source.contentType ?? "application/octet-stream").catch(() => null); + if (stored === null) { + return { + title: name, error: { code: "copy-failed", message: collective.language === "es" - ? `Ya existe un archivo llamado ${baseName} (y hasta 100 variantes); renómbralo primero.` - : `A file named ${baseName} already exists (plus 100 variants); rename it first.`, + ? "La copia en la biblioteca del colectivo falló." + : "Copying into the collective's library failed.", }, }; } const indexed = await this.indexCopy(workspaceId, collective, actor, { folder, - name: finalName, - resourceId: copied.resourceId, - version: copied.version, - sizeBytes: copied.sizeBytes, - contentType: copied.contentType, + name, + storageKey: stored.key, + sha256: stored.hash, + sizeBytes: stored.sizeBytes, + contentType: source.contentType ?? "application/octet-stream", sourcePath: item.path, senderMxid, }); - if ("error" in indexed) return { title: finalName, error: indexed.error }; - return { title: finalName, result: indexed.result }; + if ("error" in indexed) return { title: name, error: indexed.error }; + return { title: name, result: indexed.result }; } /** @@ -336,8 +337,8 @@ export class CollectiveShareService { copy: { folder: readonly string[]; name: string; - resourceId: string; - version: string | null; + storageKey: string; + sha256: string; sizeBytes: number; contentType: string; sourcePath: string; @@ -345,7 +346,7 @@ export class CollectiveShareService { }, ): Promise<{ result: { itemId: string; name: string; url: string } } | { error: { code: ShareErrorCode; message: string } }> { const mime = copy.contentType === "" ? "application/octet-stream" : copy.contentType; - const libraryUrl = this.webdav().libraryUrl(copy.resourceId); + const libraryUrl = this.libraryUrlFor(copy.storageKey); const folderPath = copy.folder.length === 0 ? "" : `${copy.folder.join("/")}/`; const origin = { kind: "upload", @@ -357,8 +358,8 @@ export class CollectiveShareService { `INSERT INTO chat_media_archive (workspace_id, group_id, room_id, event_id, origin, sender_mxid, matrix_event_json, original_name, safe_name, mime, mime_class, byte_size, sha256, ocis_resource_id, - ocis_resource_version, created_at, indexed_at) - VALUES ($1,$2,NULL,NULL,'upload',$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,now(),now()) + ocis_resource_version, created_at, indexed_at, storage_key) + VALUES ($1,$2,NULL,NULL,'upload',$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,now(),now(),$13) RETURNING id`, [ workspaceId, @@ -370,9 +371,10 @@ export class CollectiveShareService { mime, mimeClassOf(mime), copy.sizeBytes, - digestPlaceholder(copy.resourceId), - copy.resourceId, - copy.version, + copy.sha256, + copy.storageKey, + null, + copy.storageKey, ], ); return row.rows[0]?.id; diff --git a/apps/api/src/group-files/group-files.service.ts b/apps/api/src/group-files/group-files.service.ts index 186c617..02f38e1 100644 --- a/apps/api/src/group-files/group-files.service.ts +++ b/apps/api/src/group-files/group-files.service.ts @@ -29,6 +29,7 @@ import { withSystem, withTenant } from "../db/client.js"; import { isUuid, parseLimit } from "../http/validation.js"; import { decodeFileListCursor, encodeFileListCursor } from "./file-cursor.js"; import { ocisConfigFromEnv, OcisWebdavClient, type FetchLike, type OcisDownload } from "./ocis-webdav.js"; +import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js"; import { hashShareToken, mintShareToken } from "./share-token.js"; import { MIME_CLASSES, ORIGIN_KINDS, parseRetentionInput, type MimeClass, type OriginKind } from "./group-files.validation.js"; @@ -87,6 +88,7 @@ interface ArchiveRow { sha256: string; ocis_resource_id: string; ocis_resource_version: string | null; + storage_key: string | null; created_at: Date; indexed_at: Date; available: boolean; @@ -169,11 +171,12 @@ function rowToGroupFile(row: ArchiveRow, libraryUrl: (resourceId: string) => str indexedAt: iso(row.indexed_at), file: { spaceId: row.group_id, - resourceId: row.ocis_resource_id, + resourceId: row.storage_key ?? row.ocis_resource_id, version: row.ocis_resource_version ?? row.ocis_resource_id, + ...(row.storage_key === null ? {} : { storageKey: row.storage_key }), }, shareLink: { - url: libraryUrl(row.ocis_resource_id), + url: libraryUrl(row.storage_key ?? row.ocis_resource_id), access: row.share_access === "public" ? "public" : "members", expiresAt: row.share_expires_at === null ? null : iso(row.share_expires_at), }, @@ -185,10 +188,11 @@ function rowToGroupFile(row: ArchiveRow, libraryUrl: (resourceId: string) => str @Injectable() export class GroupFilesService { private client: OcisWebdavClient | null = null; + private store: StorageService | null = null; - constructor(@Optional() @Inject(GROUP_FILES_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike }) {} + constructor(@Optional() @Inject(GROUP_FILES_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike; storage?: StorageService }) {} - /** Lazily bound: the oCIS env may be absent in dev/test contexts. */ + /** Lazily bound: the oCIS env may be absent in dev/test contexts. Legacy rows only. */ private webdav(): OcisWebdavClient { if (this.client === null) { const fetcher: FetchLike = this.deps?.fetcher ?? ((url, init) => fetch(url, init)); @@ -197,6 +201,20 @@ export class GroupFilesService { return this.client; } + /** R2 object storage — the primary payload store (041). */ + private storage(): StorageService { + if (this.store === null) { + this.store = this.deps?.storage ?? new S3Storage(s3ConfigFromEnv()); + } + return this.store; + } + + /** Authenticated library link; never a persisted presigned R2 URL. */ + private libraryUrlFor(resourceId: string): string { + const base = (process.env.FILES_PUBLIC_BASE ?? "").replace(/\/+$/, ""); + return base === "" ? resourceId : `${base}/f/${encodeURIComponent(resourceId)}`; + } + // ------------------------------------------------------------------------- // Read paths // ------------------------------------------------------------------------- @@ -271,7 +289,7 @@ export class GroupFilesService { const result = await client.query(sql, values); const page = result.rows.slice(0, limit); const last = page.at(-1); - const lib = (resourceId: string): string => this.webdav().libraryUrl(resourceId); + const lib = (resourceId: string): string => this.libraryUrlFor(resourceId); return { items: page.map((row) => rowToGroupFile(row, lib)), nextCursor: result.rows.length > limit && last !== undefined @@ -292,7 +310,7 @@ export class GroupFilesService { const row = await this.requireRow(client, itemId); await this.requireAccess(client, userId, role, row.group_id); if (!row.available) throw new GoneException("file is revoked"); - return rowToGroupFile(row, (resourceId) => this.webdav().libraryUrl(resourceId)); + return rowToGroupFile(row, (resourceId) => this.libraryUrlFor(resourceId)); }); } @@ -300,12 +318,22 @@ export class GroupFilesService { async openDownload(workspaceId: string, userId: string, role: string, itemId: string): Promise { const item = await this.getFile(workspaceId, userId, role, itemId); if (item.status !== "available") throw new GoneException(`file is ${item.status}`); - const download = await this.webdav().download(item.file.resourceId).catch(() => { + const download = await this.openPayload(item.file.storageKey ?? null, item.file.resourceId, item.mimeType).catch(() => { throw new ServiceUnavailableException("file storage is unavailable"); }); return { item, download }; } + /** R2 for rows with a storage key (041+); oCIS WebDAV for legacy rows. */ + private async openPayload(storageKey: string | null | undefined, resourceId: string, mime: string): Promise { + const key = storageKey ?? null; + if (key !== null) { + const bytes = await this.storage().getObject(key); + return { stream: Readable.from(bytes), sizeBytes: bytes.byteLength, contentType: mime }; + } + return this.webdav().download(resourceId); + } + // ------------------------------------------------------------------------- // Retention / availability // ------------------------------------------------------------------------- @@ -441,8 +469,8 @@ export class GroupFilesService { if (row.retention_mode === "until" && row.retention_until !== null && row.retention_until.getTime() <= Date.now()) { throw new GoneException("file retention has expired"); } - const item = rowToGroupFile(row, (resourceId) => this.webdav().libraryUrl(resourceId)); - const download = await this.webdav().download(item.file.resourceId).catch(() => { + const item = rowToGroupFile(row, (resourceId) => this.libraryUrlFor(resourceId)); + const download = await this.openPayload(row.storage_key, item.file.resourceId, item.mimeType).catch(() => { throw new ServiceUnavailableException("file storage is unavailable"); }); return { item, download }; @@ -473,13 +501,14 @@ export class GroupFilesService { `INSERT INTO chat_media_archive (workspace_id, group_id, room_id, event_id, origin, sender_mxid, matrix_event_json, original_name, safe_name, mime, mime_class, byte_size, sha256, - ocis_resource_id, ocis_resource_version, created_at) - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) + ocis_resource_id, ocis_resource_version, created_at, storage_key) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17) RETURNING id`, [ workspaceId, entry.groupId, entry.roomId, entry.eventId, entry.originKind, entry.senderMxid, JSON.stringify(entry.origin), entry.originalName, entry.safeName, entry.mime, entry.mimeClass, entry.byteSize, entry.sha256, entry.ocisResourceId, entry.ocisResourceVersion, entry.createdAt, + entry.storageKey ?? null, ], ); } catch (err) { @@ -559,7 +588,12 @@ export class GroupFilesService { } const sniff = sniffMime(file.buffer.subarray(0, SNIFF_BYTES)); const safe = safeFileName(file.originalName, sniff.extension); - const uploaded = await this.webdav().upload(groupId, safe, Readable.from(file.buffer), sniff.mime); + // Object storage is the only payload path (user directive 2026-10-06): + // SPENC1-sealed, content-addressed, deduplicated. ocis_resource_id keeps + // carrying the resource handle (now the R2 key) for legacy readers. + const stored = await this.storage().putContent(file.buffer, sniff.mime).catch(() => { + throw new ServiceUnavailableException("file storage is unavailable"); + }); const username = await this.usernameFor(workspaceId, actor.userId); const { id } = await this.registerArchiveEntry(workspaceId, { groupId, @@ -572,10 +606,11 @@ export class GroupFilesService { safeName: safe, mime: sniff.mime, mimeClass: sniff.mimeClass, - byteSize: uploaded.sizeBytes, - sha256: uploaded.sha256, - ocisResourceId: uploaded.resourceId, - ocisResourceVersion: uploaded.version, + byteSize: stored.sizeBytes, + sha256: stored.hash, + ocisResourceId: stored.key, + ocisResourceVersion: null, + storageKey: stored.key, createdAt: new Date(), }); return this.getFile(workspaceId, actor.userId, actor.role, id); @@ -610,6 +645,7 @@ export interface ArchiveEntryInput { sha256: string; ocisResourceId: string; ocisResourceVersion: string | null; + storageKey?: string | null; createdAt: Date; } diff --git a/packages/core/src/shared-files.ts b/packages/core/src/shared-files.ts index a55f968..7c40c7a 100644 --- a/packages/core/src/shared-files.ts +++ b/packages/core/src/shared-files.ts @@ -43,7 +43,7 @@ export interface GroupFile { checksum: { algorithm: "sha256"; value: string }; createdAt: string; indexedAt: string; - file: { spaceId: string; resourceId: string; version: string }; + file: { spaceId: string; resourceId: string; version: string; storageKey?: string }; /** Authenticated library link by default; never a persisted presigned R2 URL. */ shareLink: { url: string; access: "members" | "public"; expiresAt: string | null }; retention: FileRetention;