feat: Files payloads move from oCIS WebDAV to R2 object storage
The Files tab was the only payload path not on object storage, violating the 2026-10-06 directive (object storage is the ONLY media I/O path) — and oCIS's WebDAV surface is currently 500ing server-side, which made every upload fail. Uploads now seal + dedupe into R2 via StorageService (SPENC1, content- addressed); downloads read R2 when storage_key is set, falling back to oCIS for legacy rows. Share-into-library copies oCIS Drive sources into R2 and records the real sha256 instead of a placeholder. Migration 041 adds chat_media_archive.storage_key.
This commit is contained in:
parent
5808e89dde
commit
18afff5e49
4 changed files with 114 additions and 72 deletions
|
|
@ -29,6 +29,7 @@ import { withSystem, withTenant } from "../db/client.js";
|
|||
import { isUuid, parseLimit } from "../http/validation.js";
|
||||
import { decodeFileListCursor, encodeFileListCursor } from "./file-cursor.js";
|
||||
import { ocisConfigFromEnv, OcisWebdavClient, type FetchLike, type OcisDownload } from "./ocis-webdav.js";
|
||||
import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js";
|
||||
import { hashShareToken, mintShareToken } from "./share-token.js";
|
||||
import { MIME_CLASSES, ORIGIN_KINDS, parseRetentionInput, type MimeClass, type OriginKind } from "./group-files.validation.js";
|
||||
|
||||
|
|
@ -87,6 +88,7 @@ interface ArchiveRow {
|
|||
sha256: string;
|
||||
ocis_resource_id: string;
|
||||
ocis_resource_version: string | null;
|
||||
storage_key: string | null;
|
||||
created_at: Date;
|
||||
indexed_at: Date;
|
||||
available: boolean;
|
||||
|
|
@ -169,11 +171,12 @@ function rowToGroupFile(row: ArchiveRow, libraryUrl: (resourceId: string) => str
|
|||
indexedAt: iso(row.indexed_at),
|
||||
file: {
|
||||
spaceId: row.group_id,
|
||||
resourceId: row.ocis_resource_id,
|
||||
resourceId: row.storage_key ?? row.ocis_resource_id,
|
||||
version: row.ocis_resource_version ?? row.ocis_resource_id,
|
||||
...(row.storage_key === null ? {} : { storageKey: row.storage_key }),
|
||||
},
|
||||
shareLink: {
|
||||
url: libraryUrl(row.ocis_resource_id),
|
||||
url: libraryUrl(row.storage_key ?? row.ocis_resource_id),
|
||||
access: row.share_access === "public" ? "public" : "members",
|
||||
expiresAt: row.share_expires_at === null ? null : iso(row.share_expires_at),
|
||||
},
|
||||
|
|
@ -185,10 +188,11 @@ function rowToGroupFile(row: ArchiveRow, libraryUrl: (resourceId: string) => str
|
|||
@Injectable()
|
||||
export class GroupFilesService {
|
||||
private client: OcisWebdavClient | null = null;
|
||||
private store: StorageService | null = null;
|
||||
|
||||
constructor(@Optional() @Inject(GROUP_FILES_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike }) {}
|
||||
constructor(@Optional() @Inject(GROUP_FILES_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike; storage?: StorageService }) {}
|
||||
|
||||
/** Lazily bound: the oCIS env may be absent in dev/test contexts. */
|
||||
/** Lazily bound: the oCIS env may be absent in dev/test contexts. Legacy rows only. */
|
||||
private webdav(): OcisWebdavClient {
|
||||
if (this.client === null) {
|
||||
const fetcher: FetchLike = this.deps?.fetcher ?? ((url, init) => fetch(url, init));
|
||||
|
|
@ -197,6 +201,20 @@ export class GroupFilesService {
|
|||
return this.client;
|
||||
}
|
||||
|
||||
/** R2 object storage — the primary payload store (041). */
|
||||
private storage(): StorageService {
|
||||
if (this.store === null) {
|
||||
this.store = this.deps?.storage ?? new S3Storage(s3ConfigFromEnv());
|
||||
}
|
||||
return this.store;
|
||||
}
|
||||
|
||||
/** Authenticated library link; never a persisted presigned R2 URL. */
|
||||
private libraryUrlFor(resourceId: string): string {
|
||||
const base = (process.env.FILES_PUBLIC_BASE ?? "").replace(/\/+$/, "");
|
||||
return base === "" ? resourceId : `${base}/f/${encodeURIComponent(resourceId)}`;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Read paths
|
||||
// -------------------------------------------------------------------------
|
||||
|
|
@ -271,7 +289,7 @@ export class GroupFilesService {
|
|||
const result = await client.query<ArchiveRow>(sql, values);
|
||||
const page = result.rows.slice(0, limit);
|
||||
const last = page.at(-1);
|
||||
const lib = (resourceId: string): string => this.webdav().libraryUrl(resourceId);
|
||||
const lib = (resourceId: string): string => this.libraryUrlFor(resourceId);
|
||||
return {
|
||||
items: page.map((row) => rowToGroupFile(row, lib)),
|
||||
nextCursor: result.rows.length > limit && last !== undefined
|
||||
|
|
@ -292,7 +310,7 @@ export class GroupFilesService {
|
|||
const row = await this.requireRow(client, itemId);
|
||||
await this.requireAccess(client, userId, role, row.group_id);
|
||||
if (!row.available) throw new GoneException("file is revoked");
|
||||
return rowToGroupFile(row, (resourceId) => this.webdav().libraryUrl(resourceId));
|
||||
return rowToGroupFile(row, (resourceId) => this.libraryUrlFor(resourceId));
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -300,12 +318,22 @@ export class GroupFilesService {
|
|||
async openDownload(workspaceId: string, userId: string, role: string, itemId: string): Promise<FileDownloadHandle> {
|
||||
const item = await this.getFile(workspaceId, userId, role, itemId);
|
||||
if (item.status !== "available") throw new GoneException(`file is ${item.status}`);
|
||||
const download = await this.webdav().download(item.file.resourceId).catch(() => {
|
||||
const download = await this.openPayload(item.file.storageKey ?? null, item.file.resourceId, item.mimeType).catch(() => {
|
||||
throw new ServiceUnavailableException("file storage is unavailable");
|
||||
});
|
||||
return { item, download };
|
||||
}
|
||||
|
||||
/** R2 for rows with a storage key (041+); oCIS WebDAV for legacy rows. */
|
||||
private async openPayload(storageKey: string | null | undefined, resourceId: string, mime: string): Promise<OcisDownload> {
|
||||
const key = storageKey ?? null;
|
||||
if (key !== null) {
|
||||
const bytes = await this.storage().getObject(key);
|
||||
return { stream: Readable.from(bytes), sizeBytes: bytes.byteLength, contentType: mime };
|
||||
}
|
||||
return this.webdav().download(resourceId);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Retention / availability
|
||||
// -------------------------------------------------------------------------
|
||||
|
|
@ -441,8 +469,8 @@ export class GroupFilesService {
|
|||
if (row.retention_mode === "until" && row.retention_until !== null && row.retention_until.getTime() <= Date.now()) {
|
||||
throw new GoneException("file retention has expired");
|
||||
}
|
||||
const item = rowToGroupFile(row, (resourceId) => this.webdav().libraryUrl(resourceId));
|
||||
const download = await this.webdav().download(item.file.resourceId).catch(() => {
|
||||
const item = rowToGroupFile(row, (resourceId) => this.libraryUrlFor(resourceId));
|
||||
const download = await this.openPayload(row.storage_key, item.file.resourceId, item.mimeType).catch(() => {
|
||||
throw new ServiceUnavailableException("file storage is unavailable");
|
||||
});
|
||||
return { item, download };
|
||||
|
|
@ -473,13 +501,14 @@ export class GroupFilesService {
|
|||
`INSERT INTO chat_media_archive
|
||||
(workspace_id, group_id, room_id, event_id, origin, sender_mxid, matrix_event_json,
|
||||
original_name, safe_name, mime, mime_class, byte_size, sha256,
|
||||
ocis_resource_id, ocis_resource_version, created_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)
|
||||
ocis_resource_id, ocis_resource_version, created_at, storage_key)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17)
|
||||
RETURNING id`,
|
||||
[
|
||||
workspaceId, entry.groupId, entry.roomId, entry.eventId, entry.originKind, entry.senderMxid,
|
||||
JSON.stringify(entry.origin), entry.originalName, entry.safeName, entry.mime, entry.mimeClass,
|
||||
entry.byteSize, entry.sha256, entry.ocisResourceId, entry.ocisResourceVersion, entry.createdAt,
|
||||
entry.storageKey ?? null,
|
||||
],
|
||||
);
|
||||
} catch (err) {
|
||||
|
|
@ -559,7 +588,12 @@ export class GroupFilesService {
|
|||
}
|
||||
const sniff = sniffMime(file.buffer.subarray(0, SNIFF_BYTES));
|
||||
const safe = safeFileName(file.originalName, sniff.extension);
|
||||
const uploaded = await this.webdav().upload(groupId, safe, Readable.from(file.buffer), sniff.mime);
|
||||
// Object storage is the only payload path (user directive 2026-10-06):
|
||||
// SPENC1-sealed, content-addressed, deduplicated. ocis_resource_id keeps
|
||||
// carrying the resource handle (now the R2 key) for legacy readers.
|
||||
const stored = await this.storage().putContent(file.buffer, sniff.mime).catch(() => {
|
||||
throw new ServiceUnavailableException("file storage is unavailable");
|
||||
});
|
||||
const username = await this.usernameFor(workspaceId, actor.userId);
|
||||
const { id } = await this.registerArchiveEntry(workspaceId, {
|
||||
groupId,
|
||||
|
|
@ -572,10 +606,11 @@ export class GroupFilesService {
|
|||
safeName: safe,
|
||||
mime: sniff.mime,
|
||||
mimeClass: sniff.mimeClass,
|
||||
byteSize: uploaded.sizeBytes,
|
||||
sha256: uploaded.sha256,
|
||||
ocisResourceId: uploaded.resourceId,
|
||||
ocisResourceVersion: uploaded.version,
|
||||
byteSize: stored.sizeBytes,
|
||||
sha256: stored.hash,
|
||||
ocisResourceId: stored.key,
|
||||
ocisResourceVersion: null,
|
||||
storageKey: stored.key,
|
||||
createdAt: new Date(),
|
||||
});
|
||||
return this.getFile(workspaceId, actor.userId, actor.role, id);
|
||||
|
|
@ -610,6 +645,7 @@ export interface ArchiveEntryInput {
|
|||
sha256: string;
|
||||
ocisResourceId: string;
|
||||
ocisResourceVersion: string | null;
|
||||
storageKey?: string | null;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue