hacklab/T-HUB
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat: Files payloads move from oCIS WebDAV to R2 object storage

Browse source 
The Files tab was the only payload path not on object storage, violating the
2026-10-06 directive (object storage is the ONLY media I/O path) — and oCIS's
WebDAV surface is currently 500ing server-side, which made every upload fail.

Uploads now seal + dedupe into R2 via StorageService (SPENC1, content-
addressed); downloads read R2 when storage_key is set, falling back to oCIS
for legacy rows. Share-into-library copies oCIS Drive sources into R2 and
records the real sha256 instead of a placeholder. Migration 041 adds
chat_media_archive.storage_key.
This commit is contained in:
glass beadian 2026-10-15 22:11:07 +00:00
parent 5808e89dde
commit 18afff5e49
4 changed files with 114 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

4
apps/api/migrations/041_archive_storage_key.sql Normal file
View file

@ -0,0 +1,4 @@
-- Files uploads move to R2 object storage (SPENC1-sealed, content-addressed),
-- matching the user directive that object storage is the only media path.
-- Legacy rows keep their oCIS resource ids; storage_key is read first when set.
ALTER TABLE chat_media_archive ADD COLUMN storage_key text;

112
apps/api/src/group-files/collective-share.service.ts
View file

@ -31,10 +31,8 @@ import {
mimeClassOf,
noLibraryTargetMessage,
openRoomNoticeContent,
suffixedName,
wantsLibrary,
wantsOpenRoom,
MAX_COLLISION_ATTEMPTS,
type CollectiveRoomPoster,
type CollectiveShareInput,
type CollectiveShareItem,
@ -42,7 +40,8 @@ import {
type CollectiveShareResult,
type ShareErrorCode,
} from "./collective-share.js";
import { OcisNotFoundError, OcisPreconditionFailedError, OcisWebdavClient, ocisConfigFromEnv, type FetchLike } from "./ocis-webdav.js";
import { OcisWebdavClient, ocisConfigFromEnv, type FetchLike } from "./ocis-webdav.js";
import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js";
interface CollectiveRow {
id: string;
@ -72,9 +71,24 @@ export class CollectiveShareService {
constructor(
@Optional() @Inject(COLLECTIVE_ROOM_POSTER) private readonly poster?: CollectiveRoomPoster,
@Optional() @Inject(COLLECTIVE_SHARE_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike },
@Optional() @Inject(COLLECTIVE_SHARE_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike; storage?: StorageService },
) {}
private store: StorageService | null = null;
/** R2 object storage — the library payload store (041). */
private storage(): StorageService {
if (this.store === null) {
this.store = this.deps?.storage ?? new S3Storage(s3ConfigFromEnv());
}
return this.store;
}
private libraryUrlFor(resourceId: string): string {
const base = (process.env.FILES_PUBLIC_BASE ?? "").replace(/\/+$/, "");
return base === "" ? resourceId : `${base}/f/${encodeURIComponent(resourceId)}`;
}
/** Lazily bound: the oCIS env may be absent in dev/test contexts. */
private webdav(): OcisWebdavClient {
if (this.client === null) {
@ -265,63 +279,50 @@ export class CollectiveShareService {
};
}
const baseName = safeFileName(segments[segments.length - 1] ?? "attachment");
let copied: Awaited<ReturnType<OcisWebdavClient["copyIntoGroup"]>> | null = null;
let finalName = baseName;
for (let attempt = 0; attempt <= MAX_COLLISION_ATTEMPTS; attempt += 1) {
const candidate = suffixedName(baseName, attempt);
try {
copied = await this.webdav().copyIntoGroup(collective.id, segments, folder, candidate);
finalName = candidate;
break;
} catch (err) {
if (err instanceof OcisPreconditionFailedError) continue; // try the next suffix
if (err instanceof OcisNotFoundError) {
return {
title: titleFromPath(item.path),
error: {
code: "source-not-found",
message: collective.language === "es"
? `No se encontró el archivo de origen (${item.path}).`
: `The source file was not found (${item.path}).`,
},
};
}
return {
title: titleFromPath(item.path),
error: {
code: "copy-failed",
message: collective.language === "es"
? "La copia en la biblioteca del colectivo falló."
: "Copying into the collective's library failed.",
},
};
}
}
if (copied === null) {
const name = safeFileName(segments[segments.length - 1] ?? "attachment");
// oCIS stays the Drive source; the library destination is R2 (041).
const source = await this.webdav().download(segments.join("/")).catch(() => null);
if (source === null || source.stream === null) {
return {
title: titleFromPath(item.path),
error: {
code: "source-not-found",
message: collective.language === "es"
? `No se encontró el archivo de origen (${item.path}).`
: `The source file was not found (${item.path}).`,
},
};
}
const chunks: Buffer[] = [];
for await (const chunk of source.stream as unknown as AsyncIterable<unknown>) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array));
}
const bytes = Buffer.concat(chunks);
const stored = await this.storage().putContent(bytes, source.contentType ?? "application/octet-stream").catch(() => null);
if (stored === null) {
return {
title: name,
error: {
code: "copy-failed",
message: collective.language === "es"
? `Ya existe un archivo llamado ${baseName} (y hasta 100 variantes); renómbralo primero.`
: `A file named ${baseName} already exists (plus 100 variants); rename it first.`,
? "La copia en la biblioteca del colectivo falló."
: "Copying into the collective's library failed.",
},
};
}
const indexed = await this.indexCopy(workspaceId, collective, actor, {
folder,
name: finalName,
resourceId: copied.resourceId,
version: copied.version,
sizeBytes: copied.sizeBytes,
contentType: copied.contentType,
name,
storageKey: stored.key,
sha256: stored.hash,
sizeBytes: stored.sizeBytes,
contentType: source.contentType ?? "application/octet-stream",
sourcePath: item.path,
senderMxid,
});
if ("error" in indexed) return { title: finalName, error: indexed.error };
return { title: finalName, result: indexed.result };
if ("error" in indexed) return { title: name, error: indexed.error };
return { title: name, result: indexed.result };
}
/**
@ -336,8 +337,8 @@ export class CollectiveShareService {
copy: {
folder: readonly string[];
name: string;
resourceId: string;
version: string | null;
storageKey: string;
sha256: string;
sizeBytes: number;
contentType: string;
sourcePath: string;
@ -345,7 +346,7 @@ export class CollectiveShareService {
},
): Promise<{ result: { itemId: string; name: string; url: string } } | { error: { code: ShareErrorCode; message: string } }> {
const mime = copy.contentType === "" ? "application/octet-stream" : copy.contentType;
const libraryUrl = this.webdav().libraryUrl(copy.resourceId);
const libraryUrl = this.libraryUrlFor(copy.storageKey);
const folderPath = copy.folder.length === 0 ? "" : `${copy.folder.join("/")}/`;
const origin = {
kind: "upload",
@ -357,8 +358,8 @@ export class CollectiveShareService {
`INSERT INTO chat_media_archive
(workspace_id, group_id, room_id, event_id, origin, sender_mxid, matrix_event_json,
original_name, safe_name, mime, mime_class, byte_size, sha256, ocis_resource_id,
ocis_resource_version, created_at, indexed_at)
VALUES ($1,$2,NULL,NULL,'upload',$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,now(),now())
ocis_resource_version, created_at, indexed_at, storage_key)
VALUES ($1,$2,NULL,NULL,'upload',$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,now(),now(),$13)
RETURNING id`,
[
workspaceId,
@ -370,9 +371,10 @@ export class CollectiveShareService {
mime,
mimeClassOf(mime),
copy.sizeBytes,
digestPlaceholder(copy.resourceId),
copy.resourceId,
copy.version,
copy.sha256,
copy.storageKey,
null,
copy.storageKey,
],
);
return row.rows[0]?.id;

68
apps/api/src/group-files/group-files.service.ts
View file

@ -29,6 +29,7 @@ import { withSystem, withTenant } from "../db/client.js";
import { isUuid, parseLimit } from "../http/validation.js";
import { decodeFileListCursor, encodeFileListCursor } from "./file-cursor.js";
import { ocisConfigFromEnv, OcisWebdavClient, type FetchLike, type OcisDownload } from "./ocis-webdav.js";
import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js";
import { hashShareToken, mintShareToken } from "./share-token.js";
import { MIME_CLASSES, ORIGIN_KINDS, parseRetentionInput, type MimeClass, type OriginKind } from "./group-files.validation.js";
@ -87,6 +88,7 @@ interface ArchiveRow {
sha256: string;
ocis_resource_id: string;
ocis_resource_version: string | null;
storage_key: string | null;
created_at: Date;
indexed_at: Date;
available: boolean;
@ -169,11 +171,12 @@ function rowToGroupFile(row: ArchiveRow, libraryUrl: (resourceId: string) => str
indexedAt: iso(row.indexed_at),
file: {
spaceId: row.group_id,
resourceId: row.ocis_resource_id,
resourceId: row.storage_key ?? row.ocis_resource_id,
version: row.ocis_resource_version ?? row.ocis_resource_id,
...(row.storage_key === null ? {} : { storageKey: row.storage_key }),
},
shareLink: {
url: libraryUrl(row.ocis_resource_id),
url: libraryUrl(row.storage_key ?? row.ocis_resource_id),
access: row.share_access === "public" ? "public" : "members",
expiresAt: row.share_expires_at === null ? null : iso(row.share_expires_at),
},
@ -185,10 +188,11 @@ function rowToGroupFile(row: ArchiveRow, libraryUrl: (resourceId: string) => str
@Injectable()
export class GroupFilesService {
private client: OcisWebdavClient | null = null;
private store: StorageService | null = null;
constructor(@Optional() @Inject(GROUP_FILES_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike }) {}
constructor(@Optional() @Inject(GROUP_FILES_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike; storage?: StorageService }) {}
/** Lazily bound: the oCIS env may be absent in dev/test contexts. */
/** Lazily bound: the oCIS env may be absent in dev/test contexts. Legacy rows only. */
private webdav(): OcisWebdavClient {
if (this.client === null) {
const fetcher: FetchLike = this.deps?.fetcher ?? ((url, init) => fetch(url, init));
@ -197,6 +201,20 @@ export class GroupFilesService {
return this.client;
}
/** R2 object storage — the primary payload store (041). */
private storage(): StorageService {
if (this.store === null) {
this.store = this.deps?.storage ?? new S3Storage(s3ConfigFromEnv());
}
return this.store;
}
/** Authenticated library link; never a persisted presigned R2 URL. */
private libraryUrlFor(resourceId: string): string {
const base = (process.env.FILES_PUBLIC_BASE ?? "").replace(/\/+$/, "");
return base === "" ? resourceId : `${base}/f/${encodeURIComponent(resourceId)}`;
}
// -------------------------------------------------------------------------
// Read paths
// -------------------------------------------------------------------------
@ -271,7 +289,7 @@ export class GroupFilesService {
const result = await client.query<ArchiveRow>(sql, values);
const page = result.rows.slice(0, limit);
const last = page.at(-1);
const lib = (resourceId: string): string => this.webdav().libraryUrl(resourceId);
const lib = (resourceId: string): string => this.libraryUrlFor(resourceId);
return {
items: page.map((row) => rowToGroupFile(row, lib)),
nextCursor: result.rows.length > limit && last !== undefined
@ -292,7 +310,7 @@ export class GroupFilesService {
const row = await this.requireRow(client, itemId);
await this.requireAccess(client, userId, role, row.group_id);
if (!row.available) throw new GoneException("file is revoked");
return rowToGroupFile(row, (resourceId) => this.webdav().libraryUrl(resourceId));
return rowToGroupFile(row, (resourceId) => this.libraryUrlFor(resourceId));
});
}
@ -300,12 +318,22 @@ export class GroupFilesService {
async openDownload(workspaceId: string, userId: string, role: string, itemId: string): Promise<FileDownloadHandle> {
const item = await this.getFile(workspaceId, userId, role, itemId);
if (item.status !== "available") throw new GoneException(`file is ${item.status}`);
const download = await this.webdav().download(item.file.resourceId).catch(() => {
const download = await this.openPayload(item.file.storageKey ?? null, item.file.resourceId, item.mimeType).catch(() => {
throw new ServiceUnavailableException("file storage is unavailable");
});
return { item, download };
}
/** R2 for rows with a storage key (041+); oCIS WebDAV for legacy rows. */
private async openPayload(storageKey: string | null | undefined, resourceId: string, mime: string): Promise<OcisDownload> {
const key = storageKey ?? null;
if (key !== null) {
const bytes = await this.storage().getObject(key);
return { stream: Readable.from(bytes), sizeBytes: bytes.byteLength, contentType: mime };
}
return this.webdav().download(resourceId);
}
// -------------------------------------------------------------------------
// Retention / availability
// -------------------------------------------------------------------------
@ -441,8 +469,8 @@ export class GroupFilesService {
if (row.retention_mode === "until" && row.retention_until !== null && row.retention_until.getTime() <= Date.now()) {
throw new GoneException("file retention has expired");
}
const item = rowToGroupFile(row, (resourceId) => this.webdav().libraryUrl(resourceId));
const download = await this.webdav().download(item.file.resourceId).catch(() => {
const item = rowToGroupFile(row, (resourceId) => this.libraryUrlFor(resourceId));
const download = await this.openPayload(row.storage_key, item.file.resourceId, item.mimeType).catch(() => {
throw new ServiceUnavailableException("file storage is unavailable");
});
return { item, download };
@ -473,13 +501,14 @@ export class GroupFilesService {
`INSERT INTO chat_media_archive
(workspace_id, group_id, room_id, event_id, origin, sender_mxid, matrix_event_json,
original_name, safe_name, mime, mime_class, byte_size, sha256,
ocis_resource_id, ocis_resource_version, created_at)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)
ocis_resource_id, ocis_resource_version, created_at, storage_key)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17)
RETURNING id`,
[
workspaceId, entry.groupId, entry.roomId, entry.eventId, entry.originKind, entry.senderMxid,
JSON.stringify(entry.origin), entry.originalName, entry.safeName, entry.mime, entry.mimeClass,
entry.byteSize, entry.sha256, entry.ocisResourceId, entry.ocisResourceVersion, entry.createdAt,
entry.storageKey ?? null,
],
);
} catch (err) {
@ -559,7 +588,12 @@ export class GroupFilesService {
}
const sniff = sniffMime(file.buffer.subarray(0, SNIFF_BYTES));
const safe = safeFileName(file.originalName, sniff.extension);
const uploaded = await this.webdav().upload(groupId, safe, Readable.from(file.buffer), sniff.mime);
// Object storage is the only payload path (user directive 2026-10-06):
// SPENC1-sealed, content-addressed, deduplicated. ocis_resource_id keeps
// carrying the resource handle (now the R2 key) for legacy readers.
const stored = await this.storage().putContent(file.buffer, sniff.mime).catch(() => {
throw new ServiceUnavailableException("file storage is unavailable");
});
const username = await this.usernameFor(workspaceId, actor.userId);
const { id } = await this.registerArchiveEntry(workspaceId, {
groupId,
@ -572,10 +606,11 @@ export class GroupFilesService {
safeName: safe,
mime: sniff.mime,
mimeClass: sniff.mimeClass,
byteSize: uploaded.sizeBytes,
sha256: uploaded.sha256,
ocisResourceId: uploaded.resourceId,
ocisResourceVersion: uploaded.version,
byteSize: stored.sizeBytes,
sha256: stored.hash,
ocisResourceId: stored.key,
ocisResourceVersion: null,
storageKey: stored.key,
createdAt: new Date(),
});
return this.getFile(workspaceId, actor.userId, actor.role, id);
@ -610,6 +645,7 @@ export interface ArchiveEntryInput {
sha256: string;
ocisResourceId: string;
ocisResourceVersion: string | null;
storageKey?: string | null;
createdAt: Date;
}

2
packages/core/src/shared-files.ts
View file

@ -43,7 +43,7 @@ export interface GroupFile {
checksum: { algorithm: "sha256"; value: string };
createdAt: string;
indexedAt: string;
file: { spaceId: string; resourceId: string; version: string };
file: { spaceId: string; resourceId: string; version: string; storageKey?: string };
/** Authenticated library link by default; never a persisted presigned R2 URL. */
shareLink: { url: string; access: "members" | "public"; expiresAt: string | null };
retention: FileRetention;