feat: retire oCIS — the hub runs entirely on R2 object storage
Files payloads moved to R2 in 041; this removes the remaining oCIS coupling: - delete the oCIS WebDAV client; verifyWhileStreaming moves to its own storage-agnostic module (verify-stream.ts) - group-files: storage_key is the only payload source; legacy rows answer 'retired storage' instead of touching oCIS - collective share: the ocis-resource item kind is gone (peertube links stay); share-into-library was R2-backed since the previous commit - import-drive endpoint removed (no web caller); import-chat (homeserver) stays - web: Drive tab + embedded oCIS page removed; Library is the Files view Legacy e2e rows (2) deleted; every remaining archive row is R2-backed.
This commit is contained in:
parent
4cbb63920a
commit
e7bf870de9
13 changed files with 88 additions and 750 deletions
|
|
@ -40,8 +40,6 @@ import {
|
||||||
type CollectiveShareResult,
|
type CollectiveShareResult,
|
||||||
type ShareErrorCode,
|
type ShareErrorCode,
|
||||||
} from "./collective-share.js";
|
} from "./collective-share.js";
|
||||||
import { OcisWebdavClient, ocisConfigFromEnv, type FetchLike } from "./ocis-webdav.js";
|
|
||||||
import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js";
|
|
||||||
|
|
||||||
interface CollectiveRow {
|
interface CollectiveRow {
|
||||||
id: string;
|
id: string;
|
||||||
|
|
@ -67,36 +65,10 @@ export const COLLECTIVE_SHARE_DEPENDENCIES: unique symbol = Symbol("COLLECTIVE_S
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class CollectiveShareService {
|
export class CollectiveShareService {
|
||||||
private client: OcisWebdavClient | null = null;
|
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
@Optional() @Inject(COLLECTIVE_ROOM_POSTER) private readonly poster?: CollectiveRoomPoster,
|
@Optional() @Inject(COLLECTIVE_ROOM_POSTER) private readonly poster?: CollectiveRoomPoster,
|
||||||
@Optional() @Inject(COLLECTIVE_SHARE_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike; storage?: StorageService },
|
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
private store: StorageService | null = null;
|
|
||||||
|
|
||||||
/** R2 object storage — the library payload store (041). */
|
|
||||||
private storage(): StorageService {
|
|
||||||
if (this.store === null) {
|
|
||||||
this.store = this.deps?.storage ?? new S3Storage(s3ConfigFromEnv());
|
|
||||||
}
|
|
||||||
return this.store;
|
|
||||||
}
|
|
||||||
|
|
||||||
private libraryUrlFor(resourceId: string): string {
|
|
||||||
const base = (process.env.FILES_PUBLIC_BASE ?? "").replace(/\/+$/, "");
|
|
||||||
return base === "" ? resourceId : `${base}/f/${encodeURIComponent(resourceId)}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Lazily bound: the oCIS env may be absent in dev/test contexts. */
|
|
||||||
private webdav(): OcisWebdavClient {
|
|
||||||
if (this.client === null) {
|
|
||||||
const fetcher: FetchLike = this.deps?.fetcher ?? ((url, init) => fetch(url, init));
|
|
||||||
this.client = new OcisWebdavClient(ocisConfigFromEnv(), fetcher);
|
|
||||||
}
|
|
||||||
return this.client;
|
|
||||||
}
|
|
||||||
|
|
||||||
private roomPoster(): CollectiveRoomPoster {
|
private roomPoster(): CollectiveRoomPoster {
|
||||||
if (this.poster === undefined) {
|
if (this.poster === undefined) {
|
||||||
|
|
@ -167,10 +139,9 @@ export class CollectiveShareService {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const senderMxid = `@${actor.username}:tmbh.local`;
|
|
||||||
const results: CollectiveShareItemResult[] = [];
|
const results: CollectiveShareItemResult[] = [];
|
||||||
for (const [index, item] of input.items.entries()) {
|
for (const [index, item] of input.items.entries()) {
|
||||||
results.push(await this.shareOne(workspaceId, collective, actor, input, item, index, senderMxid, doLibrary, doRoom));
|
results.push(await this.shareOne(workspaceId, collective, actor, input, item, index, doLibrary, doRoom));
|
||||||
}
|
}
|
||||||
|
|
||||||
await appendTenantAudit(workspaceId, {
|
await appendTenantAudit(workspaceId, {
|
||||||
|
|
@ -199,12 +170,11 @@ export class CollectiveShareService {
|
||||||
input: CollectiveShareInput,
|
input: CollectiveShareInput,
|
||||||
item: CollectiveShareItem,
|
item: CollectiveShareItem,
|
||||||
index: number,
|
index: number,
|
||||||
senderMxid: string,
|
|
||||||
doLibrary: boolean,
|
doLibrary: boolean,
|
||||||
doRoom: boolean,
|
doRoom: boolean,
|
||||||
): Promise<CollectiveShareItemResult> {
|
): Promise<CollectiveShareItemResult> {
|
||||||
if (item.kind === "peertube-video" && doLibrary && !doRoom) {
|
if (doLibrary && !doRoom) {
|
||||||
// A video has no library bytes to copy; the target combination is invalid for it.
|
// A video link has no library bytes to copy; the target combination is invalid for it.
|
||||||
return {
|
return {
|
||||||
index,
|
index,
|
||||||
kind: item.kind,
|
kind: item.kind,
|
||||||
|
|
@ -215,182 +185,21 @@ export class CollectiveShareService {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
let library: CollectiveShareItemResult["library"] = null;
|
|
||||||
if (doLibrary && item.kind === "ocis-resource") {
|
|
||||||
const copied = await this.copyIntoLibrary(workspaceId, collective, actor, item, input.folder, senderMxid);
|
|
||||||
if ("error" in copied) {
|
|
||||||
return { index, kind: item.kind, title: copied.title, library: null, openRoom: null, error: copied.error };
|
|
||||||
}
|
|
||||||
library = copied.result;
|
|
||||||
}
|
|
||||||
|
|
||||||
let openRoom: CollectiveShareItemResult["openRoom"] = null;
|
let openRoom: CollectiveShareItemResult["openRoom"] = null;
|
||||||
let failure: { code: ShareErrorCode; message: string } | null = null;
|
let failure: { code: ShareErrorCode; message: string } | null = null;
|
||||||
if (doRoom) {
|
if (doRoom) {
|
||||||
const title = item.kind === "peertube-video" ? item.title : library?.name ?? titleFromPath(item.path);
|
const posted = await this.postToOpenRoom(collective, { title: item.title, url: item.url, note: input.note, linkKind: "video" });
|
||||||
const url = item.kind === "peertube-video"
|
|
||||||
? item.url
|
|
||||||
: library?.url ?? libraryUrlFor(webdavSafePath(input.folder, item.path));
|
|
||||||
const posted = await this.postToOpenRoom(collective, { title, url, note: input.note, linkKind: item.kind === "peertube-video" ? "video" : "library" });
|
|
||||||
if ("error" in posted) failure = posted.error;
|
if ("error" in posted) failure = posted.error;
|
||||||
else openRoom = posted.result;
|
else openRoom = posted.result;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (failure !== null && library === null) {
|
if (failure !== null) {
|
||||||
return { index, kind: item.kind, title: item.kind === "peertube-video" ? item.title : titleFromPath(item.path), library: null, openRoom: null, error: failure };
|
return { index, kind: item.kind, title: item.title, library: null, openRoom: null, error: failure };
|
||||||
}
|
}
|
||||||
return {
|
return { index, kind: item.kind, title: item.title, library: null, openRoom, error: failure };
|
||||||
index,
|
|
||||||
kind: item.kind,
|
|
||||||
title: item.kind === "peertube-video" ? item.title : library?.name ?? titleFromPath(item.path),
|
|
||||||
library,
|
|
||||||
openRoom,
|
|
||||||
error: failure,
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* COPY + index for one oCIS resource. Collision naming (stem-1, stem-2 …)
|
|
||||||
* is applied here; the caller's chosen destination folder is honored
|
|
||||||
* relative to the collective's group root.
|
|
||||||
*/
|
|
||||||
private async copyIntoLibrary(
|
|
||||||
workspaceId: string,
|
|
||||||
collective: ResolvedCollective,
|
|
||||||
actor: { userId: string; role: string },
|
|
||||||
item: { kind: "ocis-resource"; path: string },
|
|
||||||
folder: readonly string[],
|
|
||||||
senderMxid: string,
|
|
||||||
): Promise<
|
|
||||||
| { title: string; result: { itemId: string; name: string; url: string } }
|
|
||||||
| { title: string; error: { code: ShareErrorCode; message: string } }
|
|
||||||
> {
|
|
||||||
const segments = davSegments(item.path, "path");
|
|
||||||
const stat = await this.webdav().statPath(segments).catch(() => null);
|
|
||||||
if (stat === null || !stat.exists) {
|
|
||||||
return {
|
|
||||||
title: titleFromPath(item.path),
|
|
||||||
error: {
|
|
||||||
code: "source-not-found",
|
|
||||||
message: collective.language === "es"
|
|
||||||
? `No se encontró el archivo de origen (${item.path}).`
|
|
||||||
: `The source file was not found (${item.path}).`,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const name = safeFileName(segments[segments.length - 1] ?? "attachment");
|
|
||||||
// oCIS stays the Drive source; the library destination is R2 (041).
|
|
||||||
const source = await this.webdav().download(segments.join("/")).catch(() => null);
|
|
||||||
if (source === null || source.stream === null) {
|
|
||||||
return {
|
|
||||||
title: titleFromPath(item.path),
|
|
||||||
error: {
|
|
||||||
code: "source-not-found",
|
|
||||||
message: collective.language === "es"
|
|
||||||
? `No se encontró el archivo de origen (${item.path}).`
|
|
||||||
: `The source file was not found (${item.path}).`,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
const chunks: Buffer[] = [];
|
|
||||||
for await (const chunk of source.stream as unknown as AsyncIterable<unknown>) {
|
|
||||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array));
|
|
||||||
}
|
|
||||||
const bytes = Buffer.concat(chunks);
|
|
||||||
const stored = await this.storage().putContent(bytes, source.contentType ?? "application/octet-stream").catch(() => null);
|
|
||||||
if (stored === null) {
|
|
||||||
return {
|
|
||||||
title: name,
|
|
||||||
error: {
|
|
||||||
code: "copy-failed",
|
|
||||||
message: collective.language === "es"
|
|
||||||
? "La copia en la biblioteca del colectivo falló."
|
|
||||||
: "Copying into the collective's library failed.",
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const indexed = await this.indexCopy(workspaceId, collective, actor, {
|
|
||||||
folder,
|
|
||||||
name,
|
|
||||||
storageKey: stored.key,
|
|
||||||
sha256: stored.hash,
|
|
||||||
sizeBytes: stored.sizeBytes,
|
|
||||||
contentType: source.contentType ?? "application/octet-stream",
|
|
||||||
sourcePath: item.path,
|
|
||||||
senderMxid,
|
|
||||||
});
|
|
||||||
if ("error" in indexed) return { title: name, error: indexed.error };
|
|
||||||
return { title: name, result: indexed.result };
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Index the copy in chat_media_archive (origin 'upload' — the caller put it
|
|
||||||
* there themselves) and mint the members library link row in
|
|
||||||
* group_file_shares so the collective's files index lists it.
|
|
||||||
*/
|
|
||||||
private async indexCopy(
|
|
||||||
workspaceId: string,
|
|
||||||
collective: ResolvedCollective,
|
|
||||||
actor: { userId: string; role: string },
|
|
||||||
copy: {
|
|
||||||
folder: readonly string[];
|
|
||||||
name: string;
|
|
||||||
storageKey: string;
|
|
||||||
sha256: string;
|
|
||||||
sizeBytes: number;
|
|
||||||
contentType: string;
|
|
||||||
sourcePath: string;
|
|
||||||
senderMxid: string;
|
|
||||||
},
|
|
||||||
): Promise<{ result: { itemId: string; name: string; url: string } } | { error: { code: ShareErrorCode; message: string } }> {
|
|
||||||
const mime = copy.contentType === "" ? "application/octet-stream" : copy.contentType;
|
|
||||||
const libraryUrl = this.libraryUrlFor(copy.storageKey);
|
|
||||||
const folderPath = copy.folder.length === 0 ? "" : `${copy.folder.join("/")}/`;
|
|
||||||
const origin = {
|
|
||||||
kind: "upload",
|
|
||||||
sharedFrom: copy.sourcePath,
|
|
||||||
sharedInto: `hub-groups/${collective.id}/${folderPath}${copy.name}`,
|
|
||||||
};
|
|
||||||
const inserted = await withTenant(workspaceId, async (client: PoolClient) => {
|
|
||||||
const row = await client.query<{ id: string }>(
|
|
||||||
`INSERT INTO chat_media_archive
|
|
||||||
(workspace_id, group_id, room_id, event_id, origin, sender_mxid, matrix_event_json,
|
|
||||||
original_name, safe_name, mime, mime_class, byte_size, sha256, ocis_resource_id,
|
|
||||||
ocis_resource_version, created_at, indexed_at, storage_key)
|
|
||||||
VALUES ($1,$2,NULL,NULL,'upload',$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,now(),now(),$13)
|
|
||||||
RETURNING id`,
|
|
||||||
[
|
|
||||||
workspaceId,
|
|
||||||
collective.id,
|
|
||||||
copy.senderMxid,
|
|
||||||
JSON.stringify(origin),
|
|
||||||
copy.name,
|
|
||||||
copy.name,
|
|
||||||
mime,
|
|
||||||
mimeClassOf(mime),
|
|
||||||
copy.sizeBytes,
|
|
||||||
copy.sha256,
|
|
||||||
copy.storageKey,
|
|
||||||
null,
|
|
||||||
copy.storageKey,
|
|
||||||
],
|
|
||||||
);
|
|
||||||
return row.rows[0]?.id;
|
|
||||||
}).catch(() => undefined);
|
|
||||||
if (inserted === undefined) {
|
|
||||||
return {
|
|
||||||
error: {
|
|
||||||
code: "index-failed",
|
|
||||||
message: collective.language === "es"
|
|
||||||
? "La copia se realizó pero no se pudo registrar en el índice."
|
|
||||||
: "The copy succeeded but indexing it failed.",
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
return { result: { itemId: inserted, name: copy.name, url: libraryUrl } };
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* JOIN-if-not-in-room, then one m.text notice. A closed room is refused
|
* JOIN-if-not-in-room, then one m.text notice. A closed room is refused
|
||||||
|
|
@ -470,12 +279,4 @@ function digestPlaceholder(resourceId: string): string {
|
||||||
return `copied:${resourceId}`;
|
return `copied:${resourceId}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function webdavSafePath(folder: readonly string[], path: string): string {
|
|
||||||
const segments = davSegments(path, "path");
|
|
||||||
return folder.length === 0 ? segments.join("/") : `${folder.join("/")}/${segments.join("/")}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function libraryUrlFor(path: string): string {
|
|
||||||
const base = process.env.FILES_PUBLIC_BASE ?? "";
|
|
||||||
return base === "" ? path : `${base.replace(/\/+$/, "")}/f/${encodeURIComponent(path)}`;
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -31,26 +31,13 @@ import type { MimeClass as ValidationMimeClass } from "./group-files.validation.
|
||||||
export const SHARE_TARGETS = ["library", "open-room", "both"] as const;
|
export const SHARE_TARGETS = ["library", "open-room", "both"] as const;
|
||||||
export type ShareTarget = (typeof SHARE_TARGETS)[number];
|
export type ShareTarget = (typeof SHARE_TARGETS)[number];
|
||||||
|
|
||||||
export const SHARE_ITEM_KINDS = ["ocis-resource", "peertube-video"] as const;
|
export const SHARE_ITEM_KINDS = ["peertube-video"] as const;
|
||||||
export type ShareItemKind = (typeof SHARE_ITEM_KINDS)[number];
|
export type ShareItemKind = (typeof SHARE_ITEM_KINDS)[number];
|
||||||
|
|
||||||
/** Per-request caps; a share is a person acting on a handful of items, not a bulk ingest. */
|
/** Per-request caps; a share is a person acting on a handful of items, not a bulk ingest. */
|
||||||
export const MAX_SHARE_ITEMS = 20;
|
export const MAX_SHARE_ITEMS = 20;
|
||||||
export const MAX_NOTE_LENGTH = 500;
|
export const MAX_NOTE_LENGTH = 500;
|
||||||
export const MAX_TITLE_LENGTH = 200;
|
export const MAX_TITLE_LENGTH = 200;
|
||||||
/** Collision suffix ceiling: stem-1 … stem-100, then a per-item error. */
|
|
||||||
export const MAX_COLLISION_ATTEMPTS = 100;
|
|
||||||
|
|
||||||
export interface OcisResourceItem {
|
|
||||||
readonly kind: "ocis-resource";
|
|
||||||
/**
|
|
||||||
* dav path RELATIVE to /remote.php/dav/files/<FILES_OCID_USER>/ — the hub's
|
|
||||||
* own conventions: `hub-resources/<resourceId>` (a row of the group files
|
|
||||||
* index) or a group folder path `hub-groups/<groupId>/<name>`.
|
|
||||||
*/
|
|
||||||
readonly path: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface PeertubeVideoItem {
|
export interface PeertubeVideoItem {
|
||||||
readonly kind: "peertube-video";
|
readonly kind: "peertube-video";
|
||||||
/** Canonical video URL (…/w/:id, /videos/watch/:id or /videos/embed/:id). */
|
/** Canonical video URL (…/w/:id, /videos/watch/:id or /videos/embed/:id). */
|
||||||
|
|
@ -58,7 +45,7 @@ export interface PeertubeVideoItem {
|
||||||
readonly title: string;
|
readonly title: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type CollectiveShareItem = OcisResourceItem | PeertubeVideoItem;
|
export type CollectiveShareItem = PeertubeVideoItem;
|
||||||
|
|
||||||
export interface CollectiveShareInput {
|
export interface CollectiveShareInput {
|
||||||
readonly target: ShareTarget;
|
readonly target: ShareTarget;
|
||||||
|
|
@ -148,14 +135,6 @@ function parseItem(value: unknown, index: number): CollectiveShareItem {
|
||||||
}
|
}
|
||||||
const record = value as Record<string, unknown>;
|
const record = value as Record<string, unknown>;
|
||||||
const kind = record.kind;
|
const kind = record.kind;
|
||||||
if (kind === "ocis-resource") {
|
|
||||||
if (typeof record.path !== "string" || record.path.trim() === "") {
|
|
||||||
throw new BadRequestException(`items[${index}].path must be a non-empty string`);
|
|
||||||
}
|
|
||||||
// Segment shape is validated here so a traversal attempt never reaches the service.
|
|
||||||
davSegments(record.path, `items[${index}].path`);
|
|
||||||
return { kind, path: record.path.trim() };
|
|
||||||
}
|
|
||||||
if (kind === "peertube-video") {
|
if (kind === "peertube-video") {
|
||||||
const title = boundedText(record.title, `items[${index}].title`, MAX_TITLE_LENGTH);
|
const title = boundedText(record.title, `items[${index}].title`, MAX_TITLE_LENGTH);
|
||||||
if (title === null) throw new BadRequestException(`items[${index}].title must be a non-empty string`);
|
if (title === null) throw new BadRequestException(`items[${index}].title must be a non-empty string`);
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac
|
||||||
import { requireObject, requirePositiveInt, requireSession, requireString } from "../http/validation.js";
|
import { requireObject, requirePositiveInt, requireSession, requireString } from "../http/validation.js";
|
||||||
import { GroupFilesService, type FilePage } from "./group-files.service.js";
|
import { GroupFilesService, type FilePage } from "./group-files.service.js";
|
||||||
import type { GroupFile } from "@tactical-media-broadcast-hub/core";
|
import type { GroupFile } from "@tactical-media-broadcast-hub/core";
|
||||||
import { verifyWhileStreaming } from "./ocis-webdav.js";
|
import { verifyWhileStreaming } from "./verify-stream.js";
|
||||||
|
|
||||||
const MAX_FILE_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024;
|
const MAX_FILE_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -322,12 +322,20 @@ describe("GroupFilesService invariants", () => {
|
||||||
const { createHash } = await import("node:crypto");
|
const { createHash } = await import("node:crypto");
|
||||||
const digest = createHash("sha256").update(payload).digest("hex");
|
const digest = createHash("sha256").update(payload).digest("hex");
|
||||||
const { Readable } = await import("node:stream");
|
const { Readable } = await import("node:stream");
|
||||||
const fetcher = (async () =>
|
// The R2-backed payload path: the storage stub serves the exact bytes.
|
||||||
new Response(Readable.toWeb(Readable.from([payload])) as unknown as ReadableStream, {
|
const storage = {
|
||||||
status: 200,
|
putContent: async () => { throw new Error("not used in this test"); },
|
||||||
headers: { "content-type": "application/octet-stream", "content-length": String(payload.byteLength) },
|
getObject: async () => payload,
|
||||||
})) as unknown as typeof fetch;
|
hasContent: async () => true,
|
||||||
const service = new GroupFilesService({ fetcher });
|
presign: async () => "",
|
||||||
|
};
|
||||||
|
behavior = (sql) => {
|
||||||
|
if (sql.includes("share_for_public_claim")) {
|
||||||
|
return { rows: [{ id: "s1", workspace_id: row.workspace_id, item_id: row.id, access: "public", expires_at: new Date(Date.now() + 3600_000), revoked_at: null }], rowCount: 1 };
|
||||||
|
}
|
||||||
|
return { rows: [{ ...row, sha256: digestOf("payload-bytes"), storage_key: "sha256/testing" }], rowCount: 1 };
|
||||||
|
};
|
||||||
|
const service = new GroupFilesService({ storage: storage as never });
|
||||||
const handle = await service.claimPublicDownload("some-token");
|
const handle = await service.claimPublicDownload("some-token");
|
||||||
expect(handle.item.id).toBe(row.id);
|
expect(handle.item.id).toBe(row.id);
|
||||||
expect(handle.item.checksum.value).toBe(digest);
|
expect(handle.item.checksum.value).toBe(digest);
|
||||||
|
|
|
||||||
|
|
@ -28,8 +28,8 @@ import { appendTenantAudit, type AuditEvent } from "../audit/audit.service.js";
|
||||||
import { withSystem, withTenant } from "../db/client.js";
|
import { withSystem, withTenant } from "../db/client.js";
|
||||||
import { isUuid, parseLimit } from "../http/validation.js";
|
import { isUuid, parseLimit } from "../http/validation.js";
|
||||||
import { decodeFileListCursor, encodeFileListCursor } from "./file-cursor.js";
|
import { decodeFileListCursor, encodeFileListCursor } from "./file-cursor.js";
|
||||||
import { ocisConfigFromEnv, OcisWebdavClient, type FetchLike, type OcisDownload } from "./ocis-webdav.js";
|
|
||||||
import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js";
|
import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js";
|
||||||
|
import type { PayloadDownload } from "./verify-stream.js";
|
||||||
import { hashShareToken, mintShareToken } from "./share-token.js";
|
import { hashShareToken, mintShareToken } from "./share-token.js";
|
||||||
import { MIME_CLASSES, ORIGIN_KINDS, parseRetentionInput, type MimeClass, type OriginKind } from "./group-files.validation.js";
|
import { MIME_CLASSES, ORIGIN_KINDS, parseRetentionInput, type MimeClass, type OriginKind } from "./group-files.validation.js";
|
||||||
|
|
||||||
|
|
@ -38,7 +38,7 @@ export const GROUP_FILES_DEPENDENCIES: unique symbol = Symbol("GROUP_FILES_DEPEN
|
||||||
/** Wraps the raw payload handle the controller streams out. */
|
/** Wraps the raw payload handle the controller streams out. */
|
||||||
export interface FileDownloadHandle {
|
export interface FileDownloadHandle {
|
||||||
item: GroupFile;
|
item: GroupFile;
|
||||||
download: OcisDownload;
|
download: PayloadDownload;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ShareCreated {
|
export interface ShareCreated {
|
||||||
|
|
@ -187,21 +187,11 @@ function rowToGroupFile(row: ArchiveRow, libraryUrl: (resourceId: string) => str
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class GroupFilesService {
|
export class GroupFilesService {
|
||||||
private client: OcisWebdavClient | null = null;
|
|
||||||
private store: StorageService | null = null;
|
private store: StorageService | null = null;
|
||||||
|
|
||||||
constructor(@Optional() @Inject(GROUP_FILES_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike; storage?: StorageService }) {}
|
constructor(@Optional() @Inject(GROUP_FILES_DEPENDENCIES) private readonly deps?: { storage?: StorageService }) {}
|
||||||
|
|
||||||
/** Lazily bound: the oCIS env may be absent in dev/test contexts. Legacy rows only. */
|
/** R2 object storage — the only payload store (041; oCIS retired). */
|
||||||
private webdav(): OcisWebdavClient {
|
|
||||||
if (this.client === null) {
|
|
||||||
const fetcher: FetchLike = this.deps?.fetcher ?? ((url, init) => fetch(url, init));
|
|
||||||
this.client = new OcisWebdavClient(ocisConfigFromEnv(), fetcher);
|
|
||||||
}
|
|
||||||
return this.client;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** R2 object storage — the primary payload store (041). */
|
|
||||||
private storage(): StorageService {
|
private storage(): StorageService {
|
||||||
if (this.store === null) {
|
if (this.store === null) {
|
||||||
this.store = this.deps?.storage ?? new S3Storage(s3ConfigFromEnv());
|
this.store = this.deps?.storage ?? new S3Storage(s3ConfigFromEnv());
|
||||||
|
|
@ -318,20 +308,18 @@ export class GroupFilesService {
|
||||||
async openDownload(workspaceId: string, userId: string, role: string, itemId: string): Promise<FileDownloadHandle> {
|
async openDownload(workspaceId: string, userId: string, role: string, itemId: string): Promise<FileDownloadHandle> {
|
||||||
const item = await this.getFile(workspaceId, userId, role, itemId);
|
const item = await this.getFile(workspaceId, userId, role, itemId);
|
||||||
if (item.status !== "available") throw new GoneException(`file is ${item.status}`);
|
if (item.status !== "available") throw new GoneException(`file is ${item.status}`);
|
||||||
const download = await this.openPayload(item.file.storageKey ?? null, item.file.resourceId, item.mimeType).catch(() => {
|
const download = await this.openPayload(item.file.storageKey, item.mimeType).catch(() => {
|
||||||
throw new ServiceUnavailableException("file storage is unavailable");
|
throw new ServiceUnavailableException("file storage is unavailable");
|
||||||
});
|
});
|
||||||
return { item, download };
|
return { item, download };
|
||||||
}
|
}
|
||||||
|
|
||||||
/** R2 for rows with a storage key (041+); oCIS WebDAV for legacy rows. */
|
private async openPayload(storageKey: string | null | undefined, mime: string): Promise<PayloadDownload> {
|
||||||
private async openPayload(storageKey: string | null | undefined, resourceId: string, mime: string): Promise<OcisDownload> {
|
if (storageKey === null || storageKey === undefined) {
|
||||||
const key = storageKey ?? null;
|
throw new ServiceUnavailableException("file was stored on retired storage and is no longer served");
|
||||||
if (key !== null) {
|
|
||||||
const bytes = await this.storage().getObject(key);
|
|
||||||
return { stream: Readable.from(bytes), sizeBytes: bytes.byteLength, contentType: mime };
|
|
||||||
}
|
}
|
||||||
return this.webdav().download(resourceId);
|
const bytes = await this.storage().getObject(storageKey);
|
||||||
|
return { stream: Readable.from(bytes), sizeBytes: bytes.byteLength, contentType: mime };
|
||||||
}
|
}
|
||||||
|
|
||||||
// -------------------------------------------------------------------------
|
// -------------------------------------------------------------------------
|
||||||
|
|
@ -358,7 +346,7 @@ export class GroupFilesService {
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const after = await this.requireRow(client, itemId);
|
const after = await this.requireRow(client, itemId);
|
||||||
return rowToGroupFile(after, (resourceId) => this.webdav().libraryUrl(resourceId));
|
return rowToGroupFile(after, (resourceId) => this.libraryUrlFor(resourceId));
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -375,7 +363,7 @@ export class GroupFilesService {
|
||||||
diff: { name: before.safe_name, roomId: before.room_id, eventId: before.event_id },
|
diff: { name: before.safe_name, roomId: before.room_id, eventId: before.event_id },
|
||||||
});
|
});
|
||||||
const after = await this.requireRow(client, itemId);
|
const after = await this.requireRow(client, itemId);
|
||||||
return rowToGroupFile(after, (resourceId) => this.webdav().libraryUrl(resourceId));
|
return rowToGroupFile(after, (resourceId) => this.libraryUrlFor(resourceId));
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -470,7 +458,7 @@ export class GroupFilesService {
|
||||||
throw new GoneException("file retention has expired");
|
throw new GoneException("file retention has expired");
|
||||||
}
|
}
|
||||||
const item = rowToGroupFile(row, (resourceId) => this.libraryUrlFor(resourceId));
|
const item = rowToGroupFile(row, (resourceId) => this.libraryUrlFor(resourceId));
|
||||||
const download = await this.openPayload(row.storage_key, item.file.resourceId, item.mimeType).catch(() => {
|
const download = await this.openPayload(row.storage_key, item.mimeType).catch(() => {
|
||||||
throw new ServiceUnavailableException("file storage is unavailable");
|
throw new ServiceUnavailableException("file storage is unavailable");
|
||||||
});
|
});
|
||||||
return { item, download };
|
return { item, download };
|
||||||
|
|
@ -590,7 +578,7 @@ export class GroupFilesService {
|
||||||
const safe = safeFileName(file.originalName, sniff.extension);
|
const safe = safeFileName(file.originalName, sniff.extension);
|
||||||
// Object storage is the only payload path (user directive 2026-10-06):
|
// Object storage is the only payload path (user directive 2026-10-06):
|
||||||
// SPENC1-sealed, content-addressed, deduplicated. ocis_resource_id keeps
|
// SPENC1-sealed, content-addressed, deduplicated. ocis_resource_id keeps
|
||||||
// carrying the resource handle (now the R2 key) for legacy readers.
|
// carrying the resource handle (the R2 key) for legacy readers.
|
||||||
const stored = await this.storage().putContent(file.buffer, sniff.mime).catch(() => {
|
const stored = await this.storage().putContent(file.buffer, sniff.mime).catch(() => {
|
||||||
throw new ServiceUnavailableException("file storage is unavailable");
|
throw new ServiceUnavailableException("file storage is unavailable");
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@ import {
|
||||||
requireRoomId,
|
requireRoomId,
|
||||||
requireSha256,
|
requireSha256,
|
||||||
} from "./group-files.validation.js";
|
} from "./group-files.validation.js";
|
||||||
import { OcisVerificationError, ocisConfigFromEnv, OcisWebdavClient, verifyWhileStreaming } from "./ocis-webdav.js";
|
import { OcisVerificationError, verifyWhileStreaming } from "./verify-stream.js";
|
||||||
import { hashShareToken, isPlausibleShareToken, mintShareToken, sameToken } from "./share-token.js";
|
import { hashShareToken, isPlausibleShareToken, mintShareToken, sameToken } from "./share-token.js";
|
||||||
|
|
||||||
/** Deterministic 64-hex helper: repeat a pattern to exactly 64 chars. */
|
/** Deterministic 64-hex helper: repeat a pattern to exactly 64 chars. */
|
||||||
|
|
@ -185,59 +185,7 @@ describe("share tokens", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("ocis-webdav config + streaming verification", () => {
|
describe("verify-stream (storage-agnostic streaming verification)", () => {
|
||||||
it("reads env lazily and throws 503 when unconfigured", () => {
|
|
||||||
expect(() => ocisConfigFromEnv({})).toThrow(ServiceUnavailableException);
|
|
||||||
expect(() => ocisConfigFromEnv({ FILES_OCID_WEBDAV_URL: "https://files.example.org" })).toThrow(ServiceUnavailableException);
|
|
||||||
const config = ocisConfigFromEnv({
|
|
||||||
FILES_OCID_WEBDAV_URL: "https://files.example.org/",
|
|
||||||
FILES_OCID_USER: "svc",
|
|
||||||
FILES_OCID_PASSWORD: "pw",
|
|
||||||
FILES_PUBLIC_BASE: "https://files.example.org/",
|
|
||||||
});
|
|
||||||
expect(config.webdavUrl).toBe("https://files.example.org");
|
|
||||||
expect(config.publicBase).toBe("https://files.example.org");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("uploads hash the actual streamed bytes and surface oc-fileid/oc-etag", async () => {
|
|
||||||
const bytes = Buffer.from("group file payload");
|
|
||||||
const calls: string[] = [];
|
|
||||||
const fetcher = (async (url: string, init: RequestInit) => {
|
|
||||||
calls.push(`${init.method} ${url}`);
|
|
||||||
return new Response(null, {
|
|
||||||
status: 201,
|
|
||||||
headers: init.method === "MKCOL" ? new Headers() : { "oc-fileid": "res-1", "oc-etag": "v-9" },
|
|
||||||
});
|
|
||||||
}) as unknown as typeof fetch;
|
|
||||||
const client = new OcisWebdavClient(
|
|
||||||
{ webdavUrl: "https://files.example.org", user: "svc", password: "pw", publicBase: "https://files.example.org" },
|
|
||||||
fetcher,
|
|
||||||
);
|
|
||||||
const result = await client.upload("group-1", "clip.mp4", Readable.from([bytes]), "video/mp4");
|
|
||||||
const { createHash } = await import("node:crypto");
|
|
||||||
expect(result.sha256).toBe(createHash("sha256").update(bytes).digest("hex"));
|
|
||||||
expect(result.sizeBytes).toBe(bytes.byteLength);
|
|
||||||
expect(result.resourceId).toBe("res-1");
|
|
||||||
expect(result.version).toBe("v-9");
|
|
||||||
expect(calls[0]).toBe("MKCOL https://files.example.org/remote.php/dav/files/svc/hub-groups");
|
|
||||||
expect(calls[1]).toBe("MKCOL https://files.example.org/remote.php/dav/files/svc/hub-groups/group-1");
|
|
||||||
expect(calls[2]).toBe("PUT https://files.example.org/remote.php/dav/files/svc/hub-groups/group-1/clip.mp4");
|
|
||||||
expect(client.uploadUrl("g", "n.mp4")).toContain("/remote.php/dav/files/svc/hub-groups/g/n.mp4");
|
|
||||||
expect(client.libraryUrl("res-1")).toBe("https://files.example.org/f/res-1");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("fails the upload when the group folder cannot be created", async () => {
|
|
||||||
const fetcher = (async (url: string, init: RequestInit) =>
|
|
||||||
new Response(null, { status: init.method === "MKCOL" ? 500 : 201 })) as unknown as typeof fetch;
|
|
||||||
const client = new OcisWebdavClient(
|
|
||||||
{ webdavUrl: "https://files.example.org", user: "svc", password: "pw", publicBase: "https://files.example.org" },
|
|
||||||
fetcher,
|
|
||||||
);
|
|
||||||
await expect(
|
|
||||||
client.upload("group-1", "clip.mp4", Readable.from(["x"]), "text/plain"),
|
|
||||||
).rejects.toThrow("oCIS folder creation failed with 500");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("verifyWhileStreaming passes on match and aborts on tamper", async () => {
|
it("verifyWhileStreaming passes on match and aborts on tamper", async () => {
|
||||||
const { createHash } = await import("node:crypto");
|
const { createHash } = await import("node:crypto");
|
||||||
const payload = Buffer.from("0123456789");
|
const payload = Buffer.from("0123456789");
|
||||||
|
|
|
||||||
|
|
@ -1,323 +0,0 @@
|
||||||
/**
|
|
||||||
* oCIS WebDAV port — the ONLY payload I/O path for group files.
|
|
||||||
*
|
|
||||||
* The API streams bytes through itself on both upload and download: it NEVER
|
|
||||||
* mints or persists a presigned object URL (docs/occupymedia-storage.md
|
|
||||||
* "Persist library links/share IDs and expiry, not presigned object URLs").
|
|
||||||
* Downloads re-verify the SHA-256 while streaming so storage-level tampering
|
|
||||||
* is detected instead of silently relayed.
|
|
||||||
*
|
|
||||||
* Server-side COPY ("share into a collective's library") moves no bytes
|
|
||||||
* through the API at all: oCIS performs the copy in the storage backend and
|
|
||||||
* the API only learns the resulting file id / etag / length, which is exactly
|
|
||||||
* the metadata the group files index stores. The copy is verified by oCIS
|
|
||||||
* itself; the index records what the server committed, never a claimed length.
|
|
||||||
*
|
|
||||||
* ENV CONTRACT (lazy getters — read per call so tests/parallel deployments
|
|
||||||
* can reconfigure without a rebuild; nothing is read at import time):
|
|
||||||
*
|
|
||||||
* FILES_OCID_WEBDAV_URL oCIS WebDAV root, e.g. https://files.example.org
|
|
||||||
* FILES_OCID_USER service account username (the scoped oCIS identity)
|
|
||||||
* FILES_OCID_PASSWORD service account password
|
|
||||||
* FILES_PUBLIC_BASE public base of the file library, used to build the
|
|
||||||
* persistent (authenticated) library link returned in
|
|
||||||
* GroupFile.shareLink.url — a stable resource URL,
|
|
||||||
* NEVER a presigned object URL.
|
|
||||||
*
|
|
||||||
* The oCIS deployment lands in parallel; when the env is unset the port
|
|
||||||
* reports unconfigured and routes fail 503 rather than silently no-op.
|
|
||||||
*
|
|
||||||
* WebDAV shape: PUT returns the stable resource id (oc-fileid) and ETag
|
|
||||||
* (oc-etag / etag) this index needs; plain PUT/GET carry the bytes. The
|
|
||||||
* transport is injectable (OCIS_FETCH token) so tests stub it without HTTP.
|
|
||||||
*/
|
|
||||||
import { ServiceUnavailableException } from "@nestjs/common";
|
|
||||||
import { createHash } from "node:crypto";
|
|
||||||
import { Readable } from "node:stream";
|
|
||||||
|
|
||||||
export interface OcisEnvConfig {
|
|
||||||
webdavUrl: string;
|
|
||||||
user: string;
|
|
||||||
password: string;
|
|
||||||
publicBase: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function ocisConfigFromEnv(env: Readonly<Record<string, string | undefined>> = process.env): OcisEnvConfig {
|
|
||||||
const webdavUrl = env.FILES_OCID_WEBDAV_URL;
|
|
||||||
const user = env.FILES_OCID_USER;
|
|
||||||
const password = env.FILES_OCID_PASSWORD;
|
|
||||||
if (webdavUrl === undefined || webdavUrl === "") throw new ServiceUnavailableException("FILES_OCID_WEBDAV_URL is not configured");
|
|
||||||
if (user === undefined || user === "") throw new ServiceUnavailableException("FILES_OCID_USER is not configured");
|
|
||||||
if (password === undefined || password === "") throw new ServiceUnavailableException("FILES_OCID_PASSWORD is not configured");
|
|
||||||
return {
|
|
||||||
webdavUrl: webdavUrl.replace(/\/+$/, ""),
|
|
||||||
user,
|
|
||||||
password,
|
|
||||||
publicBase: (env.FILES_PUBLIC_BASE ?? "").replace(/\/+$/, ""),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Result of a committed upload: stable identity + digest over bytes sent. */
|
|
||||||
export interface OcisUploadResult {
|
|
||||||
resourceId: string;
|
|
||||||
version: string | null;
|
|
||||||
sha256: string;
|
|
||||||
sizeBytes: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** An opened download: verifyWhileStreaming checks the digest before relay. */
|
|
||||||
export interface OcisDownload {
|
|
||||||
stream: Readable;
|
|
||||||
sizeBytes: number | null;
|
|
||||||
contentType: string | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** A server-side COPY result: identity + size, with the digest oCIS reports. */
|
|
||||||
export interface OcisCopyResult {
|
|
||||||
resourceId: string;
|
|
||||||
version: string | null;
|
|
||||||
sizeBytes: number;
|
|
||||||
sha256: string | null;
|
|
||||||
contentType: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class OcisVerificationError extends Error {
|
|
||||||
constructor(expected: string, actual: string) {
|
|
||||||
super(`oCIS payload verification failed: expected sha256 ${expected}, got ${actual}`);
|
|
||||||
this.name = "OcisVerificationError";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Minimal fetch surface so tests can stub transport without HTTP. */
|
|
||||||
export type FetchLike = (url: string, init: RequestInit) => Promise<Response>;
|
|
||||||
|
|
||||||
export const OCIS_FETCH: unique symbol = Symbol("OCIS_FETCH");
|
|
||||||
|
|
||||||
/** Bytes allowed in flight to the transport before the pump pauses. */
|
|
||||||
export const UPLOAD_BUFFER_CAP = 8 * 1024 * 1024;
|
|
||||||
|
|
||||||
/** Node's undici fetch needs this hint when a stream body is supplied. */
|
|
||||||
type FetchInitWithDuplex = RequestInit & { duplex?: "half" };
|
|
||||||
|
|
||||||
/** Distinguishing marker for "the source path does not exist (404)". */
|
|
||||||
export class OcisNotFoundError extends Error {
|
|
||||||
constructor(message: string) {
|
|
||||||
super(message);
|
|
||||||
this.name = "OcisNotFoundError";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Distinguishing marker for "destination exists and Overwrite:F was sent (412)". */
|
|
||||||
export class OcisPreconditionFailedError extends Error {
|
|
||||||
constructor(message: string) {
|
|
||||||
super(message);
|
|
||||||
this.name = "OcisPreconditionFailedError";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function encodeSegments(segments: readonly string[]): string {
|
|
||||||
return segments.map((segment) => encodeURIComponent(segment)).join("/");
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* WebDAV client. `fetcher` defaults to the global fetch.
|
|
||||||
*/
|
|
||||||
export class OcisWebdavClient {
|
|
||||||
constructor(
|
|
||||||
private readonly config: OcisEnvConfig,
|
|
||||||
private readonly fetcher: FetchLike = (url, init) => fetch(url, init),
|
|
||||||
) {}
|
|
||||||
|
|
||||||
/** Group folder path under the service identity's WebDAV root. */
|
|
||||||
uploadUrl(groupId: string, safeName: string): string {
|
|
||||||
return `${this.config.webdavUrl}/remote.php/dav/files/${encodeURIComponent(this.config.user)}/hub-groups/${encodeURIComponent(groupId)}/${encodeURIComponent(safeName)}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
downloadUrl(resourceId: string): string {
|
|
||||||
return `${this.config.webdavUrl}/remote.php/dav/files/${encodeURIComponent(this.config.user)}/hub-resources/${encodeURIComponent(resourceId)}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Any path, as sanitized segments, under the service identity's WebDAV root. */
|
|
||||||
private rootUrl(segments: readonly string[]): string {
|
|
||||||
return `${this.config.webdavUrl}/remote.php/dav/files/${encodeURIComponent(this.config.user)}/${encodeSegments(segments)}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** MKCOL the group folder; 405 (already exists) is fine. */
|
|
||||||
private async ensureGroupFolder(groupId: string): Promise<void> {
|
|
||||||
const base = `${this.config.webdavUrl}/remote.php/dav/files/${encodeURIComponent(this.config.user)}`;
|
|
||||||
// MKCOL returns 409 when the parent collection is missing — create the
|
|
||||||
// hub-groups root (405 = already exists), then the per-group folder.
|
|
||||||
const root = await this.fetcher(`${base}/hub-groups`, { method: "MKCOL", headers: this.authHeaders() });
|
|
||||||
if (!root.ok && root.status !== 405) {
|
|
||||||
throw new Error(`oCIS folder creation failed with ${root.status}`);
|
|
||||||
}
|
|
||||||
const url = `${base}/hub-groups/${encodeURIComponent(groupId)}`;
|
|
||||||
const response = await this.fetcher(url, { method: "MKCOL", headers: this.authHeaders() });
|
|
||||||
if (response.ok || response.status === 405) return;
|
|
||||||
throw new Error(`oCIS folder creation failed with ${response.status}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** MKCOL a folder path (segments already sanitized); 405 = already there. */
|
|
||||||
async ensureFolderPath(segments: readonly string[]): Promise<void> {
|
|
||||||
if (segments.length === 0) return;
|
|
||||||
const response = await this.fetcher(this.rootUrl(segments), { method: "MKCOL", headers: this.authHeaders() });
|
|
||||||
if (response.ok || response.status === 405) return;
|
|
||||||
throw new Error(`oCIS folder creation failed with ${response.status}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
private authHeaders(): Record<string, string> {
|
|
||||||
const basic = Buffer.from(`${this.config.user}:${this.config.password}`, "utf8").toString("base64");
|
|
||||||
return { authorization: `Basic ${basic}` };
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Stream-to-storage PUT, hashing while streaming (contract: "Hash actual
|
|
||||||
* bytes while streaming"). The digest covers the exact bytes handed to the
|
|
||||||
* store — never a claimed Content-Length.
|
|
||||||
*
|
|
||||||
* Backpressure: the pump pauses when more than UPLOAD_BUFFER_CAP bytes are
|
|
||||||
* queued for the transport, so RAM stays bounded for large payloads. The
|
|
||||||
* pump is driven by our own loop, so the digest completes even if the
|
|
||||||
* transport returns without draining a small body (a 2xx store that never
|
|
||||||
* reads its request body is broken and will surface as an integrity error).
|
|
||||||
*/
|
|
||||||
async upload(groupId: string, safeName: string, body: Readable, contentType: string): Promise<OcisUploadResult> {
|
|
||||||
await this.ensureGroupFolder(groupId);
|
|
||||||
// reva rejects chunked PUTs, so the source drains into a bounded buffer
|
|
||||||
// (the upload cap is 2 GiB) and goes out with an explicit content-length.
|
|
||||||
const hash = createHash("sha256");
|
|
||||||
const chunks: Buffer[] = [];
|
|
||||||
let sizeBytes = 0;
|
|
||||||
for await (const chunk of body) {
|
|
||||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array | string);
|
|
||||||
hash.update(buffer);
|
|
||||||
sizeBytes += buffer.byteLength;
|
|
||||||
chunks.push(buffer);
|
|
||||||
}
|
|
||||||
const payload = Buffer.concat(chunks);
|
|
||||||
const response = await this.fetcher(this.uploadUrl(groupId, safeName), {
|
|
||||||
method: "PUT",
|
|
||||||
headers: { ...this.authHeaders(), "content-type": contentType, "content-length": String(payload.byteLength) },
|
|
||||||
body: payload,
|
|
||||||
});
|
|
||||||
if (!response.ok) throw new Error(`oCIS upload failed with ${response.status}`);
|
|
||||||
const etag = response.headers.get("oc-etag") ?? response.headers.get("etag");
|
|
||||||
const resourceId = response.headers.get("oc-fileid");
|
|
||||||
return {
|
|
||||||
resourceId: resourceId ?? safeName,
|
|
||||||
version: etag,
|
|
||||||
sha256: hash.digest("hex"),
|
|
||||||
sizeBytes,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Open a GET. The digest is re-checked by verifyWhileStreaming downstream. */
|
|
||||||
async download(resourceId: string): Promise<OcisDownload> {
|
|
||||||
const response = await this.fetcher(this.downloadUrl(resourceId), { method: "GET", headers: this.authHeaders() });
|
|
||||||
if (!response.ok || response.body === null) throw new Error(`oCIS download failed with ${response.status}`);
|
|
||||||
const length = response.headers.get("content-length");
|
|
||||||
return {
|
|
||||||
stream: response.body as unknown as Readable,
|
|
||||||
sizeBytes: length === null ? null : Number(length),
|
|
||||||
contentType: response.headers.get("content-type"),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* PROPFIND (Depth: 0) on a path: existence + Content-Length + Content-Type.
|
|
||||||
* Used by the share-into-library path to size and type the copy without
|
|
||||||
* streaming a single payload byte through the API.
|
|
||||||
*/
|
|
||||||
async statPath(segments: readonly string[]): Promise<{ exists: boolean; sizeBytes: number; contentType: string }> {
|
|
||||||
const body = `<?xml version="1.0" encoding="utf-8"?>\n<d:propfind xmlns:d="DAV:"><d:prop><d:getcontentlength/><d:getcontenttype/></d:prop></d:propfind>`;
|
|
||||||
const response = await this.fetcher(this.rootUrl(segments), {
|
|
||||||
method: "PROPFIND",
|
|
||||||
headers: { ...this.authHeaders(), depth: "0", "content-type": "application/xml" },
|
|
||||||
body,
|
|
||||||
});
|
|
||||||
if (response.status === 404) return { exists: false, sizeBytes: 0, contentType: "" };
|
|
||||||
if (!response.ok) throw new Error(`oCIS PROPFIND failed with ${response.status}`);
|
|
||||||
const xml = await response.text();
|
|
||||||
const sizeBytes = Number(firstTag(xml, "getcontentlength") ?? "0");
|
|
||||||
const contentType = firstTag(xml, "getcontenttype") ?? "application/octet-stream";
|
|
||||||
if (!Number.isSafeInteger(sizeBytes) || sizeBytes < 0) throw new Error("oCIS PROPFIND returned an invalid content length");
|
|
||||||
return { exists: true, sizeBytes, contentType };
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Server-side COPY of a source path into the collective's group folder
|
|
||||||
* (optionally inside a sub-folder). `Overwrite: F` makes a collision a 412
|
|
||||||
* the caller can retry with a suffixed name — a share never clobbers an
|
|
||||||
* existing library file.
|
|
||||||
*/
|
|
||||||
async copyIntoGroup(
|
|
||||||
groupId: string,
|
|
||||||
sourceSegments: readonly string[],
|
|
||||||
destinationFolder: readonly string[],
|
|
||||||
name: string,
|
|
||||||
): Promise<OcisCopyResult> {
|
|
||||||
await this.ensureFolderPath(["hub-groups", groupId, ...destinationFolder]);
|
|
||||||
const destination = this.rootUrl(["hub-groups", groupId, ...destinationFolder, name]);
|
|
||||||
const response = await this.fetcher(destination, {
|
|
||||||
method: "COPY",
|
|
||||||
headers: {
|
|
||||||
...this.authHeaders(),
|
|
||||||
destination,
|
|
||||||
"overwrite": "F",
|
|
||||||
},
|
|
||||||
});
|
|
||||||
if (response.status === 404) throw new OcisNotFoundError(`oCIS source not found: /${encodeSegments(sourceSegments)}`);
|
|
||||||
if (response.status === 412) throw new OcisPreconditionFailedError(`oCIS destination already exists: ${name}`);
|
|
||||||
if (!response.ok) throw new Error(`oCIS copy failed with ${response.status}`);
|
|
||||||
const stat = await this.statPath(["hub-groups", groupId, ...destinationFolder, name]);
|
|
||||||
if (!stat.exists) throw new Error("oCIS copy reported success but the destination is not present");
|
|
||||||
const etag = response.headers.get("oc-etag") ?? response.headers.get("etag");
|
|
||||||
const resourceId = response.headers.get("oc-fileid") ?? ["hub-groups", groupId, ...destinationFolder, name].join("/");
|
|
||||||
return {
|
|
||||||
resourceId,
|
|
||||||
version: etag,
|
|
||||||
sizeBytes: stat.sizeBytes,
|
|
||||||
sha256: null,
|
|
||||||
contentType: stat.contentType,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Stable resource link for GroupFile.shareLink.url — never presigned. */
|
|
||||||
libraryUrl(resourceId: string): string {
|
|
||||||
return this.config.publicBase === "" ? resourceId : `${this.config.publicBase}/f/${encodeURIComponent(resourceId)}`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** First <tag>…</tag> value in a small XML body, namespace-agnostic. */
|
|
||||||
function firstTag(xml: string, tag: string): string | null {
|
|
||||||
const match = xml.match(new RegExp(`<(?:[\\w.-]+:)?${tag}(?:\\s[^>]*)?>([^<]*)</(?:[\\w.-]+:)?${tag}>`, "i"));
|
|
||||||
return match === null ? null : match[1]!.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Pipe a download forward, verifying sha256 chunk-by-chunk. Contract:
|
|
||||||
* "SHA-256 verified while streaming" — a mismatch aborts the relay rather
|
|
||||||
* than handing over tampered bytes (the sink sees them, but the caller
|
|
||||||
* treats the error as fatal and closes the response).
|
|
||||||
*/
|
|
||||||
export async function verifyWhileStreaming(
|
|
||||||
download: OcisDownload,
|
|
||||||
expectedSha256: string,
|
|
||||||
sink: (chunk: Buffer) => void,
|
|
||||||
): Promise<{ sizeBytes: number }> {
|
|
||||||
const hash = createHash("sha256");
|
|
||||||
let sizeBytes = 0;
|
|
||||||
for await (const chunk of download.stream) {
|
|
||||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array);
|
|
||||||
hash.update(buffer);
|
|
||||||
sizeBytes += buffer.byteLength;
|
|
||||||
sink(buffer);
|
|
||||||
}
|
|
||||||
const actual = hash.digest("hex");
|
|
||||||
if (actual !== expectedSha256) throw new OcisVerificationError(expectedSha256, actual);
|
|
||||||
return { sizeBytes };
|
|
||||||
}
|
|
||||||
|
|
||||||
export function isOcisVerificationError(err: unknown): err is OcisVerificationError {
|
|
||||||
return err instanceof OcisVerificationError;
|
|
||||||
}
|
|
||||||
44
apps/api/src/group-files/verify-stream.ts
Normal file
44
apps/api/src/group-files/verify-stream.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
||||||
|
/**
|
||||||
|
* Streaming payload verification: sha256 is computed over exactly the bytes
|
||||||
|
* served, and a mismatch aborts the relay rather than handing over tampered
|
||||||
|
* bytes. Storage-agnostic since 041 (was the oCIS client's contract; kept for
|
||||||
|
* R2-backed rows).
|
||||||
|
*/
|
||||||
|
import { createHash } from "node:crypto";
|
||||||
|
import type { Readable } from "node:stream";
|
||||||
|
|
||||||
|
/** An opened payload: streamed to the caller with metadata for the relay. */
|
||||||
|
export interface PayloadDownload {
|
||||||
|
stream: Readable;
|
||||||
|
sizeBytes: number | null;
|
||||||
|
contentType: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class OcisVerificationError extends Error {
|
||||||
|
constructor(expected: string, actual: string) {
|
||||||
|
super(`payload verification failed: expected sha256 ${expected}, got ${actual}`);
|
||||||
|
this.name = "OcisVerificationError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isOcisVerificationError(err: unknown): err is OcisVerificationError {
|
||||||
|
return err instanceof OcisVerificationError;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function verifyWhileStreaming(
|
||||||
|
download: PayloadDownload,
|
||||||
|
expectedSha256: string,
|
||||||
|
sink: (chunk: Buffer) => void,
|
||||||
|
): Promise<{ sizeBytes: number }> {
|
||||||
|
const hash = createHash("sha256");
|
||||||
|
let sizeBytes = 0;
|
||||||
|
for await (const chunk of download.stream) {
|
||||||
|
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array);
|
||||||
|
hash.update(buffer);
|
||||||
|
sizeBytes += buffer.byteLength;
|
||||||
|
sink(buffer);
|
||||||
|
}
|
||||||
|
const actual = hash.digest("hex");
|
||||||
|
if (actual !== expectedSha256) throw new OcisVerificationError(expectedSha256, actual);
|
||||||
|
return { sizeBytes };
|
||||||
|
}
|
||||||
|
|
@ -1,6 +1,5 @@
|
||||||
import { Readable } from "node:stream";
|
import { Readable } from "node:stream";
|
||||||
import { BadRequestException, ServiceUnavailableException } from "@nestjs/common";
|
import { BadRequestException, ServiceUnavailableException } from "@nestjs/common";
|
||||||
import { OcisNotFoundError, OcisWebdavClient, ocisConfigFromEnv } from "../group-files/ocis-webdav.js";
|
|
||||||
/** Imports buffer the whole file in memory; 256 MiB is the v1 cap (streaming is a follow-up). */
|
/** Imports buffer the whole file in memory; 256 MiB is the v1 cap (streaming is a follow-up). */
|
||||||
export const IMPORT_MAX_BYTES = 256 * 1024 * 1024;
|
export const IMPORT_MAX_BYTES = 256 * 1024 * 1024;
|
||||||
|
|
||||||
|
|
@ -34,19 +33,6 @@ export async function bufferWithCap(stream: Readable, capBytes: number): Promise
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let ocisClient: OcisWebdavClient | null = null;
|
|
||||||
|
|
||||||
/** Drive import: pull one oCIS resource through the service-account WebDAV client. */
|
|
||||||
export async function loadDriveBytes(resourceId: string): Promise<{ bytes: Buffer; mime: string }> {
|
|
||||||
ocisClient ??= new OcisWebdavClient(ocisConfigFromEnv());
|
|
||||||
const download = await ocisClient.download(resourceId).catch((error: unknown) => {
|
|
||||||
if (error instanceof OcisNotFoundError) throw new BadRequestException("no such Drive resource");
|
|
||||||
throw new ServiceUnavailableException("Drive is unreachable — try again later");
|
|
||||||
});
|
|
||||||
const bytes = await bufferWithCap(download.stream, IMPORT_MAX_BYTES);
|
|
||||||
return { bytes, mime: download.contentType ?? "application/octet-stream" };
|
|
||||||
}
|
|
||||||
|
|
||||||
export function matrixMediaConfig(): { homeserver: string; token: string } | null {
|
export function matrixMediaConfig(): { homeserver: string; token: string } | null {
|
||||||
const token = process.env.MATRIX_MEDIA_TOKEN?.trim() ?? "";
|
const token = process.env.MATRIX_MEDIA_TOKEN?.trim() ?? "";
|
||||||
if (token === "") return null;
|
if (token === "") return null;
|
||||||
|
|
|
||||||
|
|
@ -38,7 +38,7 @@ import {
|
||||||
type ChunkReceipt,
|
type ChunkReceipt,
|
||||||
type InitChunkedUploadRequest,
|
type InitChunkedUploadRequest,
|
||||||
} from "./chunked-upload.js";
|
} from "./chunked-upload.js";
|
||||||
import { loadDriveBytes, loadMatrixBytes, parseMxcUri } from "./import-sources.js";
|
import { loadMatrixBytes, parseMxcUri } from "./import-sources.js";
|
||||||
|
|
||||||
interface VideoFile {
|
interface VideoFile {
|
||||||
buffer: Buffer;
|
buffer: Buffer;
|
||||||
|
|
@ -148,16 +148,6 @@ export class VideoController {
|
||||||
return this.video.uploadVideo(session.workspaceId, dto.storageKey, dto.mimeType, dto.sizeBytes);
|
return this.video.uploadVideo(session.workspaceId, dto.storageKey, dto.mimeType, dto.sizeBytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Post("import-drive")
|
|
||||||
@RequirePermission("media.manage")
|
|
||||||
async importDrive(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise<VideoUploadResult> {
|
|
||||||
const session = requireSession(request);
|
|
||||||
const input = requireObject(body, "body");
|
|
||||||
const resourceId = requireString(input.resourceId, "resourceId");
|
|
||||||
const { bytes, mime } = await loadDriveBytes(resourceId);
|
|
||||||
return this.video.importVideoBytes(session.workspaceId, bytes, mime, "drive", resourceId);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Post("import-chat")
|
@Post("import-chat")
|
||||||
@RequirePermission("media.manage")
|
@RequirePermission("media.manage")
|
||||||
async importChat(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise<VideoUploadResult> {
|
async importChat(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise<VideoUploadResult> {
|
||||||
|
|
|
||||||
|
|
@ -1,79 +0,0 @@
|
||||||
"use client";
|
|
||||||
|
|
||||||
import { EmbeddedApp } from "../../../../components/embed/embedded-app";
|
|
||||||
import { useFilesLang } from "../../../../components/files/i18n";
|
|
||||||
import { useApiResource } from "../../../../lib/use-api";
|
|
||||||
import { useIsMobile } from "../../../../hooks/use-is-mobile";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Drive — the oCIS web UI (personal space, shares, office documents) embedded
|
|
||||||
* full height under the Files tabs. The group Library tab keeps the Hub-owned
|
|
||||||
* features (upload into a collective, share with a collective, retention).
|
|
||||||
*
|
|
||||||
* Sign-in follows GET /drive/session's authMode: "interactive" means oCIS's
|
|
||||||
* own identity provider — the user signs in once inside the frame and oCIS
|
|
||||||
* web keeps its tokens (tokenStorageLocal) on the files origin, which this
|
|
||||||
* frame shares because app.* and files.* are the same site. "oidc" means oCIS
|
|
||||||
* and the Hub share an issuer and the frame signs in silently.
|
|
||||||
*/
|
|
||||||
|
|
||||||
/** Same host FILES_PUBLIC_BASE defaults to; used only while the API is offline. */
|
|
||||||
const FALLBACK_DRIVE_URL = "https://files.occupymedia.cc";
|
|
||||||
|
|
||||||
interface DriveSession {
|
|
||||||
driveUrl: string | null;
|
|
||||||
authMode: "interactive" | "oidc";
|
|
||||||
}
|
|
||||||
|
|
||||||
const driveView = (session: DriveSession): DriveSession | null => session;
|
|
||||||
|
|
||||||
export default function DrivePage(): React.ReactElement {
|
|
||||||
const { t } = useFilesLang();
|
|
||||||
const drive = useApiResource<DriveSession, DriveSession | null>("/drive/session", driveView, null);
|
|
||||||
const isMobile = useIsMobile();
|
|
||||||
|
|
||||||
const driveUrl = drive.source === "sample" ? FALLBACK_DRIVE_URL : drive.data?.driveUrl ?? null;
|
|
||||||
const notConfigured = drive.source === "live" && drive.error === null && driveUrl === null;
|
|
||||||
const error =
|
|
||||||
drive.error ??
|
|
||||||
(notConfigured ? t({ en: "The drive is not configured on this server (FILES_DRIVE_URL).", es: "La unidad no está configurada en este servidor (FILES_DRIVE_URL)." }) : null);
|
|
||||||
const notice =
|
|
||||||
drive.data?.authMode === "oidc"
|
|
||||||
? t({ en: "Collective files live in the Library tab.", es: "Los archivos de los colectivos están en la pestaña Biblioteca." })
|
|
||||||
: t({
|
|
||||||
en: "Sign in once inside the drive — it stays signed in. Collective files live in the Library tab.",
|
|
||||||
es: "Inicia sesión una vez dentro de la unidad y quedará abierta. Los archivos de los colectivos están en la pestaña Biblioteca.",
|
|
||||||
});
|
|
||||||
|
|
||||||
if (isMobile) {
|
|
||||||
// Mobile: the oCIS file browser under the bottom tab bar breaks (nested
|
|
||||||
// scrolling, double chrome) — hand off to the standalone drive instead
|
|
||||||
// (panel consensus: gateway pattern). The Library tab is fully native.
|
|
||||||
return (
|
|
||||||
<div className="flex h-full flex-col justify-center px-6 py-10 text-center">
|
|
||||||
<h1 className="text-lg font-bold">{t({ en: "Drive", es: "Unidad" })}</h1>
|
|
||||||
<p className="mx-auto mt-2 max-w-xs text-xs leading-relaxed text-text-dim">{error ?? notice}</p>
|
|
||||||
<div className="mx-auto mt-8 w-full max-w-xs">
|
|
||||||
<a
|
|
||||||
href={`${driveUrl ?? FALLBACK_DRIVE_URL}/`}
|
|
||||||
className="flex min-h-12 items-center justify-center rounded-xl bg-accent px-4 text-sm font-semibold text-white transition-opacity hover:opacity-90"
|
|
||||||
>
|
|
||||||
{t({ en: "Open drive ↗", es: "Abrir unidad ↗" })}
|
|
||||||
</a>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
|
||||||
<EmbeddedApp
|
|
||||||
title={t({ en: "Drive", es: "Unidad" })}
|
|
||||||
src={drive.source === "loading" || error !== null ? null : `${driveUrl ?? FALLBACK_DRIVE_URL}/`}
|
|
||||||
externalHref={`${driveUrl ?? FALLBACK_DRIVE_URL}/`}
|
|
||||||
error={error}
|
|
||||||
notice={notice}
|
|
||||||
onRetry={drive.error !== null ? drive.reload : undefined}
|
|
||||||
allow="fullscreen; clipboard-read; clipboard-write"
|
|
||||||
/>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
@ -8,7 +8,7 @@ import { FilesTabs } from "../../../components/files/files-tabs";
|
||||||
* Files shell. Like the wiki, the language lives in localStorage per browser
|
* Files shell. Like the wiki, the language lives in localStorage per browser
|
||||||
* (lang-context explains why it is not an account setting) and is provided here
|
* (lang-context explains why it is not an account setting) and is provided here
|
||||||
* so every Files screen and sub-component reads it through one context. The tab
|
* so every Files screen and sub-component reads it through one context. The tab
|
||||||
* strip switches between the group Library and the embedded oCIS Drive.
|
* strip switches between the group Library views.
|
||||||
*/
|
*/
|
||||||
export default function FilesLayout({ children }: { children: ReactNode }): React.ReactElement {
|
export default function FilesLayout({ children }: { children: ReactNode }): React.ReactElement {
|
||||||
return (
|
return (
|
||||||
|
|
|
||||||
|
|
@ -6,20 +6,16 @@ import { useFilesLang, type LocalizedText } from "./i18n";
|
||||||
|
|
||||||
const TABS: { href: string; label: LocalizedText }[] = [
|
const TABS: { href: string; label: LocalizedText }[] = [
|
||||||
{ href: "/files", label: { en: "Library", es: "Biblioteca" } },
|
{ href: "/files", label: { en: "Library", es: "Biblioteca" } },
|
||||||
{ href: "/files/drive", label: { en: "Drive", es: "Unidad" } },
|
|
||||||
];
|
];
|
||||||
|
|
||||||
/**
|
/** Files section switcher: the Hub's group library (upload, share, retention). */
|
||||||
* Files section switcher: the Hub's group library (upload, share with a
|
|
||||||
* collective, retention) and the embedded oCIS drive side by side.
|
|
||||||
*/
|
|
||||||
export function FilesTabs(): React.ReactElement {
|
export function FilesTabs(): React.ReactElement {
|
||||||
const pathname = usePathname();
|
const pathname = usePathname();
|
||||||
const { t } = useFilesLang();
|
const { t } = useFilesLang();
|
||||||
return (
|
return (
|
||||||
<nav aria-label={t({ en: "Files views", es: "Vistas de archivos" })} className="flex shrink-0 gap-1 border-b border-border-subtle bg-surface px-3 pt-2">
|
<nav aria-label={t({ en: "Files views", es: "Vistas de archivos" })} className="flex shrink-0 gap-1 border-b border-border-subtle bg-surface px-3 pt-2">
|
||||||
{TABS.map((tab) => {
|
{TABS.map((tab) => {
|
||||||
const active = tab.href === "/files" ? pathname === "/files" : pathname.startsWith(tab.href);
|
const active = pathname === "/files";
|
||||||
return (
|
return (
|
||||||
<Link
|
<Link
|
||||||
key={tab.href}
|
key={tab.href}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue