Compare commits
43 commits
main
...
moxexley-m
| Author | SHA1 | Date | |
|---|---|---|---|
| f4ed6281c5 | |||
| 74f0802f1a | |||
| e486e62220 | |||
| 434e490f28 | |||
| 8f01765df3 | |||
| 57762f543c | |||
| 4f0b69f5aa | |||
| 0ff1484a6e | |||
| b24ebc5862 | |||
| e475402b4d | |||
| 04f5f7e3aa | |||
| 31c97a0b0a | |||
| 215952fe5e | |||
| adaa621129 | |||
| 7776569cf1 | |||
| fe483a0822 | |||
| 0bdf13816d | |||
| 98cf36fc51 | |||
| ad912828ea | |||
| f3631307aa | |||
| 8401aa3d37 | |||
| 6373715ffe | |||
| 2a813eebe6 | |||
| 78d0d0c15e | |||
| d8de9f5d82 | |||
| 10dfda3b7c | |||
| df62aa88d4 | |||
| 11b7619653 | |||
| 1429e4e9af | |||
| a7c3f26a4f | |||
| 0a1c3b392b | |||
| b3c2085436 | |||
| 2d881f5e41 | |||
| e7bf870de9 | |||
| 4cbb63920a | |||
| 93edeab5ca | |||
| 18afff5e49 | |||
| 5808e89dde | |||
| 613aba236b | |||
| ef8d28bc26 | |||
| 3e4fb970c3 | |||
| b8115eec09 | |||
| 9e8653aca5 |
147 changed files with 13668 additions and 1150 deletions
223
README.md
223
README.md
|
|
@ -20,6 +20,229 @@ Every feature is multi-tenant with row-level security.
|
|||
- **Archive** — content-addressed encrypted S3, dedup, full-text search, CSV/JSON export. Deleted platform content stays citable.
|
||||
- **Analytics** — month-partitioned snapshots, current + previous month with deltas, per-provider breakdown.
|
||||
|
||||
## Architecture
|
||||
|
||||
Three flows explain the whole system.
|
||||
|
||||
### 1. One account
|
||||
|
||||
One person connects a collective's account once. From then on: draft, someone
|
||||
approves, the Hub publishes. The social network's password never passes through
|
||||
the Hub — the network returns a token and the Hub stores it encrypted.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
P([Person]) -->|1. Connect| H[Hub]
|
||||
H -->|2. Permission screen| R[(Social network)]
|
||||
R -->|3. Token| H
|
||||
P -->|4. Drafts| H
|
||||
H -->|5. Approval| A([Another crew member])
|
||||
A -->|6. Sign-off| H
|
||||
H -->|7. Publishes on time| R
|
||||
```
|
||||
|
||||
### 2. Several shared accounts
|
||||
|
||||
Each collective has its own accounts and approval rules. A draft can go to
|
||||
several accounts at once: the Hub creates one copy per destination and each
|
||||
passes its own account rule (automatic, one person, majority or consensus).
|
||||
Anything meant for the whole network rises to the parent collective's queue.
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
subgraph TTS[NETWORK QUEUE]
|
||||
Q[Shared queue] --> IG1[Instagram international]
|
||||
Q --> M1[Mastodon @network]
|
||||
end
|
||||
subgraph S26[COLLECTIVE A]
|
||||
D[Draft] --> IG2[Instagram @collectiveA]
|
||||
D --> M2[Mastodon @collectiveA]
|
||||
D --> T2[Telegram collectiveA]
|
||||
end
|
||||
subgraph SUB[Local collectives]
|
||||
B[Barcelona] --> IG3[Own Instagram]
|
||||
S[Sevilla] --> IG4[Own Instagram]
|
||||
end
|
||||
S26 -->|Uploads for the network| Q
|
||||
SUB -->|Uploads to A| D
|
||||
IG2 -.->|Rule: 2 approvals| G{{Approval}}
|
||||
M2 -.->|Rule: automatic| G
|
||||
```
|
||||
|
||||
### 3. The technical half
|
||||
|
||||
The web only talks to the API. The API keeps state in Postgres with
|
||||
per-collective row-level security, credentials live in an encrypted vault, and
|
||||
scheduled publishing runs through the scheduler with Temporal. Each network is
|
||||
a connector; outside the connectors there is no code that distinguishes
|
||||
networks. Media bytes are end-to-end encrypted (SPENC1 server-sealed,
|
||||
SPENC2 browser-sealed) in content-addressed R2 storage; heavy processing runs
|
||||
in workers that pull sealed jobs — the api box never touches plaintext.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
subgraph Client
|
||||
W[apps/web — Next.js]
|
||||
end
|
||||
subgraph Control
|
||||
API[apps/api — NestJS]
|
||||
SCH[apps/scheduler — Temporal]
|
||||
REG[ConnectorRegistry — packages/connector-*]
|
||||
PW[produce-worker — probe / compose / assemble]
|
||||
end
|
||||
subgraph Data
|
||||
PG[(PostgreSQL — RLS per collective)]
|
||||
RD[(Redis — OAuth state 10 min)]
|
||||
V[Vault — AES-256-GCM + KEK]
|
||||
S3[(R2 — SPENC1/SPENC2 sealed media)]
|
||||
end
|
||||
subgraph Media
|
||||
MW[apps/media-worker — FFmpeg]
|
||||
PT[PeerTube — tube.occupymedia.cc]
|
||||
end
|
||||
subgraph Networks
|
||||
MA[Mastodon API]
|
||||
IG[Instagram Graph API]
|
||||
TG[Telegram Bot API]
|
||||
end
|
||||
|
||||
W -->|HTTPS + session| API
|
||||
API --> PG
|
||||
API --> RD
|
||||
API --> V
|
||||
V --> PG
|
||||
API -->|connect / identity| REG
|
||||
SCH -->|publish| REG
|
||||
SCH --> PG
|
||||
W -->|SPENC2 sealed| S3
|
||||
PW -->|decrypt + render| S3
|
||||
PW --> PG
|
||||
REG -->|providerFetch egress control| MA
|
||||
REG --> IG
|
||||
REG --> TG
|
||||
MW --> S3
|
||||
MW -->|RTMP| PT
|
||||
```
|
||||
|
||||
Connecting an account (OAuth with one-time `state` and PKCE where the network
|
||||
supports it):
|
||||
|
||||
1. `POST /connections/authorize`: the API stores the request state encrypted in
|
||||
Redis and returns the network's URL.
|
||||
2. The person accepts on the network, which returns to
|
||||
`GET /connections/callback`.
|
||||
3. The API checks `state`, cookie and permissions, redeems the code and stores
|
||||
the token in the vault.
|
||||
|
||||
## Network status
|
||||
|
||||
| Network | Connect | Publish | Notes |
|
||||
|---|---|---|---|
|
||||
| Mastodon | Yes | Text | No server keys; each account registers the app on its instance. |
|
||||
| Telegram | Yes | Text | Bot token from BotFather. |
|
||||
| Instagram | Yes, with `INSTAGRAM_CLIENT_ID/SECRET` | Not yet | Meta fetches the image from a public URL and media are encrypted: a temporary endpoint is pending. See [docs/oauth-connect.md](docs/oauth-connect.md). |
|
||||
| PeerTube | No | Live only | The OAuth bridge on tube.occupymedia.cc is pending. |
|
||||
| X | No | No | No connector. Paid per-use API. |
|
||||
| TikTok | No | No | Out of scope: the API requires an audit; without it only private publishing. |
|
||||
|
||||
## Outreach (hand-written DMs)
|
||||
|
||||
A list of public accounts (artists, collectives, media) to write to privately
|
||||
asking for amplification. Each contact has its drafted message, fillable from a
|
||||
template (`{name}`, `{handle}`), marked sent / replied / declined. CSV import:
|
||||
|
||||
```
|
||||
name,platform,handle,profileUrl,category,language
|
||||
Climáximo,bluesky,climaximopt.bsky.social,https://bsky.app/profile/climaximopt.bsky.social,climate,pt
|
||||
```
|
||||
|
||||
The Hub **does not send** the messages. "Copy & open profile" copies the text
|
||||
and opens the profile; the person pastes and sends it from the network's own
|
||||
DM box. Reasons:
|
||||
|
||||
| Network | Open a DM via API | Problem |
|
||||
|---|---|---|
|
||||
| Instagram | No | The API only allows replying to whoever wrote you in the last 24 h. |
|
||||
| X | Yes, paid | Mass DMs to strangers break automation rules: suspension. |
|
||||
| Mastodon | Yes (private mention) | Instances mute or block accounts doing it in bulk; the whole instance can suffer. |
|
||||
| Bluesky | Yes | Same spam criteria; stranger chats are usually closed. |
|
||||
|
||||
The screen counter warns from 20 first messages per day per collective.
|
||||
Spread them across people and days, and personalise each one.
|
||||
|
||||
## Requirements
|
||||
|
||||
Measured on the current deployment. Media goes to R2, so the disk does not
|
||||
grow with videos.
|
||||
|
||||
| Tier | CPU | RAM | Disk | For |
|
||||
|---|---|---|---|---|
|
||||
| Minimum | 4 vCPU | 8 GB | 160 GB SSD | The whole stack, transcoding off-box |
|
||||
| Recommended | 8 vCPU | 16 GB | 240 GB SSD | Livestreams, on-box transcoding, Collabora |
|
||||
| Render node (optional) | 4+ cores or a GPU with NVENC | 8 GB | 50 GB | Joins over Tailscale; never holds keys |
|
||||
|
||||
Real usage today: ~2.6 GB of RAM across all services.
|
||||
|
||||
Bandwidth:
|
||||
|
||||
| Use | Needed |
|
||||
|---|---|
|
||||
| Panel, publishing, network APIs | < 1 Mbps sustained |
|
||||
| One outgoing live (1080p30 to PeerTube) | 6 Mbps upload per live; 3 Mbps at 720p |
|
||||
| Viewers | Served by Cloudflare from the HLS cache; the origin does not scale with the audience |
|
||||
| Video uploads | Bursts as fast as the line allows; 100 GiB per file via chunked ingest |
|
||||
|
||||
## Network and ports
|
||||
|
||||
Inbound from the internet: **none direct**. Everything enters through the
|
||||
Cloudflare tunnel (with an ingress failover to a standby proxy); the edge
|
||||
nginx only accepts Cloudflare ranges on 80/443.
|
||||
|
||||
| Port | Service | Expose to |
|
||||
|---|---|---|
|
||||
| 443/tcp | Edge nginx | Cloudflare ranges only |
|
||||
| 80/tcp | Edge nginx (redirect) | Cloudflare ranges only |
|
||||
| 3000/tcp | API | 127.0.0.1 only |
|
||||
| 3001/tcp | Web | 127.0.0.1 only |
|
||||
| 9000/tcp | PeerTube | 127.0.0.1 or Tailscale |
|
||||
| 1935/tcp | PeerTube RTMP | Tailscale only (media worker) |
|
||||
| 6167/tcp | Matrix (conduwuit) | 127.0.0.1 |
|
||||
| 9200, 9980/tcp | oCIS, Collabora | 127.0.0.1 |
|
||||
| 5432, 6379, 7233 | Postgres, Redis, Temporal | Never outside the Docker network |
|
||||
|
||||
Required egress: 443/tcp to the networks' APIs and to R2, 7844/tcp+udp for
|
||||
`cloudflared`, 41641/udp for Tailscale.
|
||||
|
||||
Docker publishes ports bypassing ufw. To close them: publish on
|
||||
`127.0.0.1:PORT`, on the Tailscale IP, or add rules to the `DOCKER-USER` chain.
|
||||
|
||||
## Security and monitoring
|
||||
|
||||
Practices for any deployment. Report security failures privately to the
|
||||
maintainers — never in public issues.
|
||||
|
||||
- **Invite-only registration.** Open registration hands anyone a collective of their own with Owner rights on it.
|
||||
- **One secret per service.** The web and workers never need `SUPERPOSTER_KEK`, `S3_KEK` or `SESSION_SECRET`. Different keys for the vault and for media.
|
||||
- **FFmpeg always sandboxed** (`NODE_ENV=production`, `MEDIA_SANDBOX=on`), no network, no inherited environment variables.
|
||||
- **Headers**: strict CSP, HSTS, `X-Content-Type-Options`, `Referrer-Policy` on web and API.
|
||||
- **Request limits** on `/auth/*`, invitations, OIDC and public uploads, keyed by `CF-Connecting-IP`.
|
||||
- **Metadata out**: strip EXIF/GPS from photos and videos before storing or publishing.
|
||||
- **Encrypted backups** (`age` or GPG) kept off-machine; test the restore.
|
||||
- **Tamper-proof audit**: revoke `UPDATE`/`DELETE` on `audit_log` from the `sp_app` role and anchor the chain head off-box.
|
||||
|
||||
What to watch (Prometheus + Grafana ship in `infra/`; Grafana never anonymous):
|
||||
|
||||
| Signal | Alert if |
|
||||
|---|---|
|
||||
| `401`/`403` on `/auth/login` per IP | > 20 in 5 min |
|
||||
| New registrations / redeemed invitations | Anything unplanned |
|
||||
| `audit_log` events: `member.*`, `connection.removed`, `group.deleted`, `outreach.removed` | Spikes or odd actors |
|
||||
| `connections` in `needs_credentials` state | Any: token revoked or expired |
|
||||
| API 5xx rate | > 1 % over 5 min |
|
||||
| API and worker memory | > 80 % |
|
||||
| Publishing queue `failed`/`unknown` | Any |
|
||||
| Cloudflare panel logins and DNS changes | Always |
|
||||
|
||||
## Quickstart
|
||||
|
||||
```bash
|
||||
|
|
|
|||
4
apps/api/migrations/041_archive_storage_key.sql
Normal file
4
apps/api/migrations/041_archive_storage_key.sql
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
-- Files uploads move to R2 object storage (SPENC1-sealed, content-addressed),
|
||||
-- matching the user directive that object storage is the only media path.
|
||||
-- Legacy rows keep their oCIS resource ids; storage_key is read first when set.
|
||||
ALTER TABLE chat_media_archive ADD COLUMN storage_key text;
|
||||
11
apps/api/migrations/042_media_assets_rls.sql
Normal file
11
apps/api/migrations/042_media_assets_rls.sql
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
-- 042: media_assets row-level security.
|
||||
--
|
||||
-- Enable and force the tenant_isolation policy declared in 038, with an
|
||||
-- explicit WITH CHECK. All media_assets queries run inside withTenant.
|
||||
|
||||
ALTER TABLE media_assets ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE media_assets FORCE ROW LEVEL SECURITY;
|
||||
DROP POLICY IF EXISTS tenant_isolation ON media_assets;
|
||||
CREATE POLICY tenant_isolation ON media_assets
|
||||
USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid)
|
||||
WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid);
|
||||
48
apps/api/migrations/042_probe_jobs_claimable.sql
Normal file
48
apps/api/migrations/042_probe_jobs_claimable.sql
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
-- 042: the produce worker claims PROBE jobs too.
|
||||
--
|
||||
-- Direct-to-R2 video uploads register an unprobed asset plus a queued probe
|
||||
-- job and return immediately — the api never touches the bytes, so it cannot
|
||||
-- probe inline the way 009's ingest path does. The worker (which can decrypt
|
||||
-- SPENC2 frames via DIRECT_EC_PRIVATE_KEY) owns the probe from here on.
|
||||
--
|
||||
-- Same body as 040 with the kind filter widened; 'transcoding' remains the
|
||||
-- lease state (the UI's in-flight set already covers queued/probing/transcoding).
|
||||
|
||||
CREATE OR REPLACE FUNCTION produce_job_claim(p_lane text, p_stale_after interval)
|
||||
RETURNS TABLE (
|
||||
id uuid,
|
||||
workspace_id uuid,
|
||||
kind text,
|
||||
input_json jsonb,
|
||||
video_asset_id uuid
|
||||
)
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = public AS $$
|
||||
BEGIN
|
||||
UPDATE video_processing_jobs stale
|
||||
SET state = 'queued', progress_percent = 0, error = NULL, updated_at = now()
|
||||
WHERE stale.state = 'transcoding'
|
||||
AND stale.updated_at < now() - p_stale_after;
|
||||
|
||||
RETURN QUERY
|
||||
UPDATE video_processing_jobs j
|
||||
SET state = 'transcoding', progress_percent = 5, error = NULL, updated_at = now()
|
||||
WHERE j.id IN (
|
||||
SELECT c.id
|
||||
FROM video_processing_jobs c
|
||||
WHERE c.kind IN ('probe', 'compose', 'mix')
|
||||
AND c.state = 'queued'
|
||||
AND (
|
||||
(p_lane = 'remote' AND c.input_json->>'lane' = 'remote')
|
||||
OR (p_lane <> 'remote' AND c.input_json->>'lane' IS DISTINCT FROM 'remote')
|
||||
)
|
||||
ORDER BY c.created_at, c.id
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING j.id, j.workspace_id, j.kind, j.input_json, j.video_asset_id;
|
||||
END
|
||||
$$;
|
||||
|
||||
-- Ownership unchanged from 040.
|
||||
REVOKE ALL ON FUNCTION produce_job_claim(text, interval) FROM PUBLIC;
|
||||
GRANT EXECUTE ON FUNCTION produce_job_claim(text, interval) TO sp_app;
|
||||
56
apps/api/migrations/043_edl_projects.sql
Normal file
56
apps/api/migrations/043_edl_projects.sql
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
-- 043: EDL projects — the intake half of the editor-v2 assemble feature.
|
||||
--
|
||||
-- edl_projects stores a validated editor timeline (the EdlProject shape from
|
||||
-- apps/api/src/media-control/edl.ts) as jsonb. The api validates strictly
|
||||
-- (closed enums, integer ms, unknown keys rejected, filtergraph metacharacters
|
||||
-- blocked in card text) before anything is written; the media-worker
|
||||
-- re-validates the job's input_json on the assemble lane (044).
|
||||
--
|
||||
-- video_processing_jobs gains kind 'assemble' (044's claim already admits it)
|
||||
-- and video_asset_id becomes nullable: an assemble job is queued from an EDL
|
||||
-- project, not a single source asset, so there is no asset to point at yet —
|
||||
-- the produced output is a derived video_assets row like compose/mix.
|
||||
--
|
||||
-- RLS / NULLIF / reject_workspace_change / sp_app grants follow the
|
||||
-- 001/008/021/026/029 conventions: no app.workspace_id => NULL => no rows;
|
||||
-- FORCE binds sp_app too. The jsonb column is content, not a schema facet:
|
||||
-- the authoritative typed validation is app-side (edl.ts).
|
||||
|
||||
BEGIN;
|
||||
|
||||
CREATE TABLE edl_projects (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT,
|
||||
group_id uuid NULL,
|
||||
project jsonb NOT NULL,
|
||||
created_by uuid NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (workspace_id, id)
|
||||
);
|
||||
|
||||
-- Tenant-pinned reference target (029 style): the composite UNIQUE lets child
|
||||
-- rows pin (id, workspace_id) so an EDL project can never point across tenants.
|
||||
CREATE INDEX edl_projects_ws_updated_idx ON edl_projects (workspace_id, updated_at DESC);
|
||||
|
||||
CREATE TRIGGER trg_edl_projects_ws_immutable BEFORE UPDATE ON edl_projects
|
||||
FOR EACH ROW EXECUTE FUNCTION reject_workspace_change();
|
||||
|
||||
ALTER TABLE edl_projects ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE edl_projects FORCE ROW LEVEL SECURITY;
|
||||
|
||||
CREATE POLICY tenant_isolation ON edl_projects
|
||||
USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid)
|
||||
WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid);
|
||||
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON edl_projects TO sp_app;
|
||||
|
||||
-- The assemble lane: widen the 039 kind CHECK so an undecodable row cannot
|
||||
-- exist, and let an assemble job carry no source asset yet.
|
||||
ALTER TABLE video_processing_jobs DROP CONSTRAINT video_processing_jobs_kind_check;
|
||||
ALTER TABLE video_processing_jobs
|
||||
ADD CONSTRAINT video_processing_jobs_kind_check
|
||||
CHECK (kind IN ('probe', 'compose', 'mix', 'assemble'));
|
||||
ALTER TABLE video_processing_jobs ALTER COLUMN video_asset_id DROP NOT NULL;
|
||||
|
||||
COMMIT;
|
||||
54
apps/api/migrations/044_assemble_claim.sql
Normal file
54
apps/api/migrations/044_assemble_claim.sql
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
-- 044: the produce worker claims ASSEMBLE jobs (editor-v2 L1, the worker assemble lane).
|
||||
--
|
||||
-- An assemble row is an editor-v2 timeline render: input_json carries
|
||||
-- { edlProjectId, project: EdlProject } (the EDL shapes live in
|
||||
-- apps/api/src/media-control/edl.ts), video_asset_id points at the project's
|
||||
-- first source clip, and the produced output is a derived video_assets row
|
||||
-- exactly like a compose/mix produce. 043 owns the intake half of the feature
|
||||
-- (edl_projects + widening video_processing_jobs.kind's CHECK to admit
|
||||
-- 'assemble'); THIS migration is only the worker's half — the claim's kind
|
||||
-- filter widened so the lane can pick those rows up.
|
||||
--
|
||||
-- Same body as 042 with 'assemble' added to the kind list; 'transcoding'
|
||||
-- remains the lease state and the stale-lease sweep is unchanged, so a worker
|
||||
-- that dies mid-assembly still hands its row back through the same path a
|
||||
-- dead compose/mix encode does today.
|
||||
|
||||
CREATE OR REPLACE FUNCTION produce_job_claim(p_lane text, p_stale_after interval)
|
||||
RETURNS TABLE (
|
||||
id uuid,
|
||||
workspace_id uuid,
|
||||
kind text,
|
||||
input_json jsonb,
|
||||
video_asset_id uuid
|
||||
)
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = public AS $$
|
||||
BEGIN
|
||||
UPDATE video_processing_jobs stale
|
||||
SET state = 'queued', progress_percent = 0, error = NULL, updated_at = now()
|
||||
WHERE stale.state = 'transcoding'
|
||||
AND stale.updated_at < now() - p_stale_after;
|
||||
|
||||
RETURN QUERY
|
||||
UPDATE video_processing_jobs j
|
||||
SET state = 'transcoding', progress_percent = 5, error = NULL, updated_at = now()
|
||||
WHERE j.id IN (
|
||||
SELECT c.id
|
||||
FROM video_processing_jobs c
|
||||
WHERE c.kind IN ('probe', 'compose', 'mix', 'assemble')
|
||||
AND c.state = 'queued'
|
||||
AND (
|
||||
(p_lane = 'remote' AND c.input_json->>'lane' = 'remote')
|
||||
OR (p_lane <> 'remote' AND c.input_json->>'lane' IS DISTINCT FROM 'remote')
|
||||
)
|
||||
ORDER BY c.created_at, c.id
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING j.id, j.workspace_id, j.kind, j.input_json, j.video_asset_id;
|
||||
END
|
||||
$$;
|
||||
|
||||
-- Ownership unchanged from 040/042.
|
||||
REVOKE ALL ON FUNCTION produce_job_claim(text, interval) FROM PUBLIC;
|
||||
GRANT EXECUTE ON FUNCTION produce_job_claim(text, interval) TO sp_app;
|
||||
1
apps/api/migrations/045_file_delete_setting.sql
Normal file
1
apps/api/migrations/045_file_delete_setting.sql
Normal file
|
|
@ -0,0 +1 @@
|
|||
ALTER TABLE groups ADD COLUMN member_delete_own boolean NOT NULL DEFAULT true;
|
||||
65
apps/api/migrations/046_file_folders.sql
Normal file
65
apps/api/migrations/046_file_folders.sql
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
-- Folder trees and personal files are metadata over the existing payload keys.
|
||||
CREATE TABLE file_folders (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE,
|
||||
group_id uuid NULL REFERENCES groups(id) ON DELETE CASCADE,
|
||||
owner_id uuid NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
parent_id uuid NULL REFERENCES file_folders(id) ON DELETE CASCADE,
|
||||
name text NOT NULL CHECK (btrim(name) <> '' AND char_length(name) <= 255),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
CONSTRAINT file_folders_scope CHECK ((group_id IS NULL) <> (owner_id IS NULL)),
|
||||
UNIQUE (id, workspace_id),
|
||||
FOREIGN KEY (group_id, workspace_id) REFERENCES groups(id, workspace_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TRIGGER trg_file_folders_ws_immutable BEFORE UPDATE ON file_folders
|
||||
FOR EACH ROW EXECUTE FUNCTION reject_workspace_change();
|
||||
ALTER TABLE file_folders ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE file_folders FORCE ROW LEVEL SECURITY;
|
||||
CREATE POLICY tenant_isolation ON file_folders
|
||||
USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid)
|
||||
WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid);
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON file_folders TO sp_app;
|
||||
|
||||
ALTER TABLE chat_media_archive
|
||||
ALTER COLUMN group_id DROP NOT NULL,
|
||||
ADD COLUMN folder_id uuid NULL REFERENCES file_folders(id) ON DELETE SET NULL,
|
||||
ADD COLUMN created_by uuid NULL,
|
||||
-- Required even at the personal root, which has no folder from which to infer ownership.
|
||||
ADD COLUMN owner_id uuid NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
ADD CONSTRAINT chat_media_archive_scope CHECK ((group_id IS NULL) <> (owner_id IS NULL));
|
||||
|
||||
-- 045 may already have introduced this setting.
|
||||
ALTER TABLE groups ADD COLUMN IF NOT EXISTS member_delete_own boolean NOT NULL DEFAULT true;
|
||||
|
||||
-- Only exact, known identities are mapped. New authenticated writes supply
|
||||
-- created_by; Matrix ingest also resolves the configured homeserver in code.
|
||||
UPDATE chat_media_archive a SET created_by = (
|
||||
SELECT u.id FROM users u
|
||||
WHERE (a.sender_mxid = '@' || u.username || ':tmbh.local' OR a.sender_mxid = u.external_id)
|
||||
AND EXISTS (SELECT 1 FROM memberships m WHERE m.workspace_id = a.workspace_id AND m.user_id = u.id)
|
||||
ORDER BY (a.sender_mxid = u.external_id) DESC, u.id LIMIT 1
|
||||
) WHERE a.created_by IS NULL;
|
||||
|
||||
-- Other ingest clients (including archive-bot) write this table directly.
|
||||
-- Keep exact known sender mappings attributed on those inserts too. Unknown
|
||||
-- remote identities stay NULL, so they never acquire member delete rights.
|
||||
CREATE FUNCTION archive_file_creator() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = public AS $$
|
||||
BEGIN
|
||||
IF NEW.created_by IS NULL THEN
|
||||
SELECT u.id INTO NEW.created_by FROM users u
|
||||
WHERE (NEW.sender_mxid = '@' || u.username || ':tmbh.local' OR NEW.sender_mxid = u.external_id)
|
||||
AND EXISTS (SELECT 1 FROM memberships m WHERE m.workspace_id = NEW.workspace_id AND m.user_id = u.id)
|
||||
ORDER BY (NEW.sender_mxid = u.external_id) DESC, u.id LIMIT 1;
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
CREATE TRIGGER trg_chat_media_archive_creator BEFORE INSERT ON chat_media_archive
|
||||
FOR EACH ROW EXECUTE FUNCTION archive_file_creator();
|
||||
|
||||
CREATE INDEX file_folders_scope_parent_idx ON file_folders (workspace_id, group_id, owner_id, parent_id);
|
||||
CREATE INDEX chat_media_archive_folder_idx ON chat_media_archive (workspace_id, folder_id);
|
||||
CREATE INDEX chat_media_archive_personal_idx ON chat_media_archive (workspace_id, owner_id, created_at DESC, id DESC)
|
||||
WHERE group_id IS NULL;
|
||||
51
apps/api/migrations/047_outreach.sql
Normal file
51
apps/api/migrations/047_outreach.sql
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
-- 041: outreach — per-workspace list of public accounts to contact by hand.
|
||||
--
|
||||
-- The Hub never sends these messages. A person opens the contact's profile,
|
||||
-- pastes the drafted text into the platform's own DM box and marks the row
|
||||
-- sent; replies are marked the same way. Only public accounts belong here:
|
||||
-- names and contact details of private persons stay out of the Hub.
|
||||
|
||||
CREATE TABLE outreach_contacts (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT,
|
||||
name text NOT NULL CHECK (char_length(name) BETWEEN 1 AND 200),
|
||||
platform text NOT NULL CHECK (platform ~ '^[a-z0-9][a-z0-9-]{1,31}$'),
|
||||
handle text NOT NULL CHECK (char_length(handle) BETWEEN 1 AND 200),
|
||||
profile_url text NOT NULL CHECK (profile_url ~ '^https://[^\s]+$' AND char_length(profile_url) <= 500),
|
||||
category text CHECK (category IS NULL OR char_length(category) <= 100),
|
||||
language text CHECK (language IS NULL OR language ~ '^[a-z]{2,3}(-[A-Za-z0-9]{2,8})?$'),
|
||||
message text NOT NULL DEFAULT '' CHECK (char_length(message) <= 4000),
|
||||
notes text NOT NULL DEFAULT '' CHECK (char_length(notes) <= 2000),
|
||||
status text NOT NULL DEFAULT 'todo' CHECK (status IN ('todo', 'sent', 'replied', 'declined')),
|
||||
sent_at timestamptz,
|
||||
sent_by uuid REFERENCES users(id) ON DELETE SET NULL,
|
||||
replied_at timestamptz,
|
||||
created_by uuid REFERENCES users(id) ON DELETE SET NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (id, workspace_id)
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX outreach_contacts_account ON outreach_contacts (workspace_id, platform, lower(handle));
|
||||
CREATE INDEX outreach_contacts_status ON outreach_contacts (workspace_id, status);
|
||||
|
||||
CREATE TRIGGER trg_outreach_contacts_ws_immutable BEFORE UPDATE ON outreach_contacts
|
||||
FOR EACH ROW EXECUTE FUNCTION reject_workspace_change();
|
||||
|
||||
ALTER TABLE outreach_contacts ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE outreach_contacts FORCE ROW LEVEL SECURITY;
|
||||
CREATE POLICY tenant_isolation ON outreach_contacts
|
||||
USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid)
|
||||
WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid);
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON outreach_contacts TO sp_app;
|
||||
|
||||
-- Reads for anyone who already sees the inbox; edits for the roles that write drafts.
|
||||
INSERT INTO role_permissions (role, permission)
|
||||
SELECT 'Owner', 'outreach.view'
|
||||
UNION ALL SELECT 'Admin', 'outreach.view'
|
||||
UNION ALL SELECT 'Editor', 'outreach.view'
|
||||
UNION ALL SELECT 'Moderator', 'outreach.view'
|
||||
UNION ALL SELECT 'Owner', 'outreach.manage'
|
||||
UNION ALL SELECT 'Admin', 'outreach.manage'
|
||||
UNION ALL SELECT 'Editor', 'outreach.manage'
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
|
@ -13,6 +13,7 @@
|
|||
"dependencies": {
|
||||
"@tactical-media-broadcast-hub/connector-sdk": "workspace:*",
|
||||
"@tactical-media-broadcast-hub/connector-telegram": "workspace:*",
|
||||
"@tactical-media-broadcast-hub/connector-instagram": "workspace:*",
|
||||
"@tactical-media-broadcast-hub/core": "workspace:*",
|
||||
"@aws-sdk/client-s3": "^3.1146.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.1146.0",
|
||||
|
|
|
|||
|
|
@ -20,8 +20,11 @@ import { GroupsController } from "./groups/groups.controller";
|
|||
import { GroupsService } from "./groups/groups.service";
|
||||
import { VideoController } from "./video/video.controller";
|
||||
import { VideoService } from "./video/video.service";
|
||||
import { ChunkedUploadService } from "./video/chunked-upload.js";
|
||||
import { LiveMonitorModule } from "./live-monitor/live-monitor.module";
|
||||
import { MediaControlModule } from "./media-control/media-control.module";
|
||||
import { EdlController } from "./media-control/edl.controller";
|
||||
import { EdlService } from "./media-control/edl.service";
|
||||
import { ProductionModule } from "./production/production.module";
|
||||
import { RealtimeModule } from "./realtime/realtime.module";
|
||||
import { LiveChatModule } from "./live-chat/live-chat.module";
|
||||
|
|
@ -43,9 +46,10 @@ import { MatrixSsoModule } from "./matrix-sso/matrix-sso.module.js";
|
|||
import { OidcModule } from "./oidc/oidc.module.js";
|
||||
import { SubmissionsModule } from "./submissions/submissions.module.js";
|
||||
import { MediaModule } from "./media/media.module.js";
|
||||
import { OutreachModule } from "./outreach/outreach.module.js";
|
||||
|
||||
@Module({
|
||||
imports: [HealthModule, LiveMonitorModule, MediaControlModule, ProductionModule, RealtimeModule, LiveChatModule, LiveModule, VirtualChannelsModule, ChatgovModule, IntakeModule, GroupFilesModule, SettingsModule, CollectivesModule, MatrixSsoModule, OidcModule, SubmissionsModule, MediaModule],
|
||||
imports: [HealthModule, LiveMonitorModule, MediaControlModule, ProductionModule, RealtimeModule, LiveChatModule, LiveModule, VirtualChannelsModule, ChatgovModule, IntakeModule, GroupFilesModule, SettingsModule, CollectivesModule, MatrixSsoModule, OidcModule, SubmissionsModule, MediaModule, OutreachModule],
|
||||
controllers: [
|
||||
DemoController,
|
||||
PublishingController,
|
||||
|
|
@ -58,6 +62,7 @@ import { MediaModule } from "./media/media.module.js";
|
|||
VideoController,
|
||||
GroupsController,
|
||||
AnalyticsController,
|
||||
EdlController,
|
||||
SettingsController,
|
||||
BrandingController,
|
||||
PublicBrandingController,
|
||||
|
|
@ -66,6 +71,6 @@ import { MediaModule } from "./media/media.module.js";
|
|||
{ provide: PublishingExecutionRepository, useFactory: () => new PublishingExecutionRepository(pool) },
|
||||
TemporalWorkflowStarter,
|
||||
{ provide: PUBLISHING_DEPENDENCIES, inject: [PublishingExecutionRepository, TemporalWorkflowStarter, ApprovalGateService], useFactory: (repository: PublishingExecutionRepository, workflowStarter: TemporalWorkflowStarter, approvalGate: ApprovalGateService) => ({ repository, workflowStarter, approvalGate, tenantTransaction: withTenant, assertReady: () => assertPublishingSchema(pool) }) },
|
||||
PublishingService, ApprovalGateService, AuthService, ConnectionsService, TeamService, InviteLinkService, AuditQueryService, VideoService, GroupsService, SnapshotService, SettingsService, BrandingService],
|
||||
PublishingService, ApprovalGateService, AuthService, ConnectionsService, TeamService, InviteLinkService, AuditQueryService, VideoService, ChunkedUploadService, GroupsService, SnapshotService, SettingsService, BrandingService, EdlService],
|
||||
})
|
||||
export class AppModule {}
|
||||
|
|
|
|||
|
|
@ -51,12 +51,13 @@ export class AuthController {
|
|||
/** Signed session present ⇒ its server-side row is revoked (phase 2). Legacy tokens: client-side only. */
|
||||
@Post("logout")
|
||||
@HttpCode(200)
|
||||
logout(@Req() request: Request): { ok: true } {
|
||||
async logout(@Req() request: Request): Promise<{ ok: true }> {
|
||||
const authorization = request.headers.authorization;
|
||||
const session = authorization !== undefined && authorization.startsWith("Bearer ")
|
||||
? verifySession(authorization.slice("Bearer ".length))
|
||||
: null;
|
||||
void this.auth.logout(session);
|
||||
if (session === null) return { ok: true };
|
||||
await this.auth.logout(session);
|
||||
return { ok: true };
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ import { ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk";
|
|||
import { createMastodonConnector } from "../../../../packages/connector-mastodon/src/index.js";
|
||||
import { createTelegramConnector } from "@tactical-media-broadcast-hub/connector-telegram";
|
||||
import { createPeerTubeConnector } from "../../../../packages/connector-peertube/src/index.js";
|
||||
import { createInstagramConnector } from "@tactical-media-broadcast-hub/connector-instagram";
|
||||
import { connectionRegistry } from "./connector-registry.js";
|
||||
import { Vault, type SealedCredential } from "../vault/vault.js";
|
||||
import type { SessionPayload } from "../auth/session.js";
|
||||
import { RbacGuard } from "../rbac/rbac.guard.js";
|
||||
|
|
@ -105,12 +107,13 @@ describe("HTTP OAuth connect with real connectors and mock authorization server"
|
|||
let issued = 0;
|
||||
let failExchange = false;
|
||||
let bridgeConfigured = false;
|
||||
let instagramConfigured = false;
|
||||
const botToken = "12345:bot_token_secret";
|
||||
|
||||
beforeEach(() => {
|
||||
store.rows.clear(); store.audit.length = 0; store.authorized = true;
|
||||
redis.entries.clear(); activeTokens.clear(); grants.clear(); revokedTokens.length = 0;
|
||||
nextAccount = "account-1"; issued = 0; failExchange = false; bridgeConfigured = false;
|
||||
nextAccount = "account-1"; issued = 0; failExchange = false; bridgeConfigured = false; instagramConfigured = false;
|
||||
warn.mockClear(); info.mockClear();
|
||||
});
|
||||
|
||||
|
|
@ -180,6 +183,9 @@ describe("HTTP OAuth connect with real connectors and mock authorization server"
|
|||
}));
|
||||
const botFetch: typeof fetch = (input, init) => fetch(String(input).replace("https://api.telegram.org", origin), init);
|
||||
registry.register(createTelegramConnector({ credentials, fetchImpl: botFetch, requestsPerSecond: 100_000 }));
|
||||
registry.register(createInstagramConnector({ credentials, fetchImpl: mockFetch, requestsPerSecond: 100_000,
|
||||
env: instagramConfigured ? { INSTAGRAM_CLIENT_ID: "hub-app-id", INSTAGRAM_CLIENT_SECRET: "hub-app-secret" } : {},
|
||||
}));
|
||||
return registry;
|
||||
},
|
||||
};
|
||||
|
|
@ -342,6 +348,25 @@ describe("HTTP OAuth connect with real connectors and mock authorization server"
|
|||
bridgeConfigured = false;
|
||||
});
|
||||
|
||||
it("offers Instagram only with the Hub's Facebook app configured, and keeps the app secret out of the browser", async () => {
|
||||
expect(connectionRegistry({} as never).has("instagram")).toBe(true);
|
||||
const providersOf = async () => await (await api("/providers")).json() as Array<{ id: string; enabled: boolean; note?: string }>;
|
||||
expect((await providersOf()).find((provider) => provider.id === "instagram")).toMatchObject({ enabled: false, note: expect.stringContaining("INSTAGRAM_CLIENT_ID") });
|
||||
expect((await api("/authorize", "POST", { provider: "instagram" })).status).toBe(400);
|
||||
instagramConfigured = true;
|
||||
expect((await providersOf()).find((provider) => provider.id === "instagram")?.enabled).toBe(true);
|
||||
const response = await api("/authorize", "POST", { provider: "instagram" });
|
||||
expect(response.status).toBe(201);
|
||||
const { redirectUrl } = await response.json() as { redirectUrl: string };
|
||||
const dialog = new URL(redirectUrl);
|
||||
expect(dialog.origin + dialog.pathname).toBe("https://www.facebook.com/v21.0/dialog/oauth");
|
||||
expect(dialog.searchParams.get("client_id")).toBe("hub-app-id");
|
||||
expect(dialog.searchParams.get("redirect_uri")).toBe(`${apiOrigin}/connections/callback`);
|
||||
expect(dialog.searchParams.get("state")).toMatch(/.{32,}/);
|
||||
expect(redirectUrl).not.toContain("hub-app-secret");
|
||||
instagramConfigured = false;
|
||||
});
|
||||
|
||||
it("sets, validates and clears per-connection publish governance", async () => {
|
||||
await finish(await begin());
|
||||
const id = (await listed())[0]?.id ?? "";
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import { ConnectorRegistry, type CredentialAccessor } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
// eslint-disable-next-line no-restricted-syntax -- Connector composition boundary.
|
||||
import { createInstagramConnector } from "@tactical-media-broadcast-hub/connector-instagram";
|
||||
// eslint-disable-next-line no-restricted-syntax -- Connector composition boundary.
|
||||
import { createMastodonConnector } from "@tactical-media-broadcast-hub/connector-mastodon";
|
||||
// eslint-disable-next-line no-restricted-syntax -- Connector composition boundary.
|
||||
import { createPeerTubeConnector } from "../../../../packages/connector-peertube/src/index.js";
|
||||
|
|
@ -13,5 +15,6 @@ export function connectionRegistry(credentials: CredentialAccessor, fetchImpl: t
|
|||
registry.register(createMastodonConnector(options));
|
||||
registry.register(createPeerTubeConnector(options));
|
||||
registry.register(createTelegramConnector(options));
|
||||
registry.register(createInstagramConnector(options));
|
||||
return registry;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,10 +31,8 @@ import {
|
|||
mimeClassOf,
|
||||
noLibraryTargetMessage,
|
||||
openRoomNoticeContent,
|
||||
suffixedName,
|
||||
wantsLibrary,
|
||||
wantsOpenRoom,
|
||||
MAX_COLLISION_ATTEMPTS,
|
||||
type CollectiveRoomPoster,
|
||||
type CollectiveShareInput,
|
||||
type CollectiveShareItem,
|
||||
|
|
@ -42,7 +40,6 @@ import {
|
|||
type CollectiveShareResult,
|
||||
type ShareErrorCode,
|
||||
} from "./collective-share.js";
|
||||
import { OcisNotFoundError, OcisPreconditionFailedError, OcisWebdavClient, ocisConfigFromEnv, type FetchLike } from "./ocis-webdav.js";
|
||||
|
||||
interface CollectiveRow {
|
||||
id: string;
|
||||
|
|
@ -68,21 +65,10 @@ export const COLLECTIVE_SHARE_DEPENDENCIES: unique symbol = Symbol("COLLECTIVE_S
|
|||
|
||||
@Injectable()
|
||||
export class CollectiveShareService {
|
||||
private client: OcisWebdavClient | null = null;
|
||||
|
||||
constructor(
|
||||
@Optional() @Inject(COLLECTIVE_ROOM_POSTER) private readonly poster?: CollectiveRoomPoster,
|
||||
@Optional() @Inject(COLLECTIVE_SHARE_DEPENDENCIES) private readonly deps?: { fetcher?: FetchLike },
|
||||
) {}
|
||||
|
||||
/** Lazily bound: the oCIS env may be absent in dev/test contexts. */
|
||||
private webdav(): OcisWebdavClient {
|
||||
if (this.client === null) {
|
||||
const fetcher: FetchLike = this.deps?.fetcher ?? ((url, init) => fetch(url, init));
|
||||
this.client = new OcisWebdavClient(ocisConfigFromEnv(), fetcher);
|
||||
}
|
||||
return this.client;
|
||||
}
|
||||
|
||||
private roomPoster(): CollectiveRoomPoster {
|
||||
if (this.poster === undefined) {
|
||||
|
|
@ -153,10 +139,9 @@ export class CollectiveShareService {
|
|||
);
|
||||
}
|
||||
|
||||
const senderMxid = `@${actor.username}:tmbh.local`;
|
||||
const results: CollectiveShareItemResult[] = [];
|
||||
for (const [index, item] of input.items.entries()) {
|
||||
results.push(await this.shareOne(workspaceId, collective, actor, input, item, index, senderMxid, doLibrary, doRoom));
|
||||
results.push(await this.shareOne(workspaceId, collective, actor, input, item, index, doLibrary, doRoom));
|
||||
}
|
||||
|
||||
await appendTenantAudit(workspaceId, {
|
||||
|
|
@ -185,12 +170,11 @@ export class CollectiveShareService {
|
|||
input: CollectiveShareInput,
|
||||
item: CollectiveShareItem,
|
||||
index: number,
|
||||
senderMxid: string,
|
||||
doLibrary: boolean,
|
||||
doRoom: boolean,
|
||||
): Promise<CollectiveShareItemResult> {
|
||||
if (item.kind === "peertube-video" && doLibrary && !doRoom) {
|
||||
// A video has no library bytes to copy; the target combination is invalid for it.
|
||||
if (doLibrary && !doRoom) {
|
||||
// A video link has no library bytes to copy; the target combination is invalid for it.
|
||||
return {
|
||||
index,
|
||||
kind: item.kind,
|
||||
|
|
@ -201,194 +185,21 @@ export class CollectiveShareService {
|
|||
};
|
||||
}
|
||||
|
||||
let library: CollectiveShareItemResult["library"] = null;
|
||||
if (doLibrary && item.kind === "ocis-resource") {
|
||||
const copied = await this.copyIntoLibrary(workspaceId, collective, actor, item, input.folder, senderMxid);
|
||||
if ("error" in copied) {
|
||||
return { index, kind: item.kind, title: copied.title, library: null, openRoom: null, error: copied.error };
|
||||
}
|
||||
library = copied.result;
|
||||
}
|
||||
|
||||
let openRoom: CollectiveShareItemResult["openRoom"] = null;
|
||||
let failure: { code: ShareErrorCode; message: string } | null = null;
|
||||
if (doRoom) {
|
||||
const title = item.kind === "peertube-video" ? item.title : library?.name ?? titleFromPath(item.path);
|
||||
const url = item.kind === "peertube-video"
|
||||
? item.url
|
||||
: library?.url ?? libraryUrlFor(webdavSafePath(input.folder, item.path));
|
||||
const posted = await this.postToOpenRoom(collective, { title, url, note: input.note, linkKind: item.kind === "peertube-video" ? "video" : "library" });
|
||||
const posted = await this.postToOpenRoom(collective, { title: item.title, url: item.url, note: input.note, linkKind: "video" });
|
||||
if ("error" in posted) failure = posted.error;
|
||||
else openRoom = posted.result;
|
||||
}
|
||||
|
||||
if (failure !== null && library === null) {
|
||||
return { index, kind: item.kind, title: item.kind === "peertube-video" ? item.title : titleFromPath(item.path), library: null, openRoom: null, error: failure };
|
||||
if (failure !== null) {
|
||||
return { index, kind: item.kind, title: item.title, library: null, openRoom: null, error: failure };
|
||||
}
|
||||
return {
|
||||
index,
|
||||
kind: item.kind,
|
||||
title: item.kind === "peertube-video" ? item.title : library?.name ?? titleFromPath(item.path),
|
||||
library,
|
||||
openRoom,
|
||||
error: failure,
|
||||
};
|
||||
return { index, kind: item.kind, title: item.title, library: null, openRoom, error: failure };
|
||||
}
|
||||
|
||||
/**
|
||||
* COPY + index for one oCIS resource. Collision naming (stem-1, stem-2 …)
|
||||
* is applied here; the caller's chosen destination folder is honored
|
||||
* relative to the collective's group root.
|
||||
*/
|
||||
private async copyIntoLibrary(
|
||||
workspaceId: string,
|
||||
collective: ResolvedCollective,
|
||||
actor: { userId: string; role: string },
|
||||
item: { kind: "ocis-resource"; path: string },
|
||||
folder: readonly string[],
|
||||
senderMxid: string,
|
||||
): Promise<
|
||||
| { title: string; result: { itemId: string; name: string; url: string } }
|
||||
| { title: string; error: { code: ShareErrorCode; message: string } }
|
||||
> {
|
||||
const segments = davSegments(item.path, "path");
|
||||
const stat = await this.webdav().statPath(segments).catch(() => null);
|
||||
if (stat === null || !stat.exists) {
|
||||
return {
|
||||
title: titleFromPath(item.path),
|
||||
error: {
|
||||
code: "source-not-found",
|
||||
message: collective.language === "es"
|
||||
? `No se encontró el archivo de origen (${item.path}).`
|
||||
: `The source file was not found (${item.path}).`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const baseName = safeFileName(segments[segments.length - 1] ?? "attachment");
|
||||
let copied: Awaited<ReturnType<OcisWebdavClient["copyIntoGroup"]>> | null = null;
|
||||
let finalName = baseName;
|
||||
for (let attempt = 0; attempt <= MAX_COLLISION_ATTEMPTS; attempt += 1) {
|
||||
const candidate = suffixedName(baseName, attempt);
|
||||
try {
|
||||
copied = await this.webdav().copyIntoGroup(collective.id, segments, folder, candidate);
|
||||
finalName = candidate;
|
||||
break;
|
||||
} catch (err) {
|
||||
if (err instanceof OcisPreconditionFailedError) continue; // try the next suffix
|
||||
if (err instanceof OcisNotFoundError) {
|
||||
return {
|
||||
title: titleFromPath(item.path),
|
||||
error: {
|
||||
code: "source-not-found",
|
||||
message: collective.language === "es"
|
||||
? `No se encontró el archivo de origen (${item.path}).`
|
||||
: `The source file was not found (${item.path}).`,
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
title: titleFromPath(item.path),
|
||||
error: {
|
||||
code: "copy-failed",
|
||||
message: collective.language === "es"
|
||||
? "La copia en la biblioteca del colectivo falló."
|
||||
: "Copying into the collective's library failed.",
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
if (copied === null) {
|
||||
return {
|
||||
title: titleFromPath(item.path),
|
||||
error: {
|
||||
code: "copy-failed",
|
||||
message: collective.language === "es"
|
||||
? `Ya existe un archivo llamado ${baseName} (y hasta 100 variantes); renómbralo primero.`
|
||||
: `A file named ${baseName} already exists (plus 100 variants); rename it first.`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const indexed = await this.indexCopy(workspaceId, collective, actor, {
|
||||
folder,
|
||||
name: finalName,
|
||||
resourceId: copied.resourceId,
|
||||
version: copied.version,
|
||||
sizeBytes: copied.sizeBytes,
|
||||
contentType: copied.contentType,
|
||||
sourcePath: item.path,
|
||||
senderMxid,
|
||||
});
|
||||
if ("error" in indexed) return { title: finalName, error: indexed.error };
|
||||
return { title: finalName, result: indexed.result };
|
||||
}
|
||||
|
||||
/**
|
||||
* Index the copy in chat_media_archive (origin 'upload' — the caller put it
|
||||
* there themselves) and mint the members library link row in
|
||||
* group_file_shares so the collective's files index lists it.
|
||||
*/
|
||||
private async indexCopy(
|
||||
workspaceId: string,
|
||||
collective: ResolvedCollective,
|
||||
actor: { userId: string; role: string },
|
||||
copy: {
|
||||
folder: readonly string[];
|
||||
name: string;
|
||||
resourceId: string;
|
||||
version: string | null;
|
||||
sizeBytes: number;
|
||||
contentType: string;
|
||||
sourcePath: string;
|
||||
senderMxid: string;
|
||||
},
|
||||
): Promise<{ result: { itemId: string; name: string; url: string } } | { error: { code: ShareErrorCode; message: string } }> {
|
||||
const mime = copy.contentType === "" ? "application/octet-stream" : copy.contentType;
|
||||
const libraryUrl = this.webdav().libraryUrl(copy.resourceId);
|
||||
const folderPath = copy.folder.length === 0 ? "" : `${copy.folder.join("/")}/`;
|
||||
const origin = {
|
||||
kind: "upload",
|
||||
sharedFrom: copy.sourcePath,
|
||||
sharedInto: `hub-groups/${collective.id}/${folderPath}${copy.name}`,
|
||||
};
|
||||
const inserted = await withTenant(workspaceId, async (client: PoolClient) => {
|
||||
const row = await client.query<{ id: string }>(
|
||||
`INSERT INTO chat_media_archive
|
||||
(workspace_id, group_id, room_id, event_id, origin, sender_mxid, matrix_event_json,
|
||||
original_name, safe_name, mime, mime_class, byte_size, sha256, ocis_resource_id,
|
||||
ocis_resource_version, created_at, indexed_at)
|
||||
VALUES ($1,$2,NULL,NULL,'upload',$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,now(),now())
|
||||
RETURNING id`,
|
||||
[
|
||||
workspaceId,
|
||||
collective.id,
|
||||
copy.senderMxid,
|
||||
JSON.stringify(origin),
|
||||
copy.name,
|
||||
copy.name,
|
||||
mime,
|
||||
mimeClassOf(mime),
|
||||
copy.sizeBytes,
|
||||
digestPlaceholder(copy.resourceId),
|
||||
copy.resourceId,
|
||||
copy.version,
|
||||
],
|
||||
);
|
||||
return row.rows[0]?.id;
|
||||
}).catch(() => undefined);
|
||||
if (inserted === undefined) {
|
||||
return {
|
||||
error: {
|
||||
code: "index-failed",
|
||||
message: collective.language === "es"
|
||||
? "La copia se realizó pero no se pudo registrar en el índice."
|
||||
: "The copy succeeded but indexing it failed.",
|
||||
},
|
||||
};
|
||||
}
|
||||
return { result: { itemId: inserted, name: copy.name, url: libraryUrl } };
|
||||
}
|
||||
|
||||
/**
|
||||
* JOIN-if-not-in-room, then one m.text notice. A closed room is refused
|
||||
|
|
@ -468,12 +279,4 @@ function digestPlaceholder(resourceId: string): string {
|
|||
return `copied:${resourceId}`;
|
||||
}
|
||||
|
||||
function webdavSafePath(folder: readonly string[], path: string): string {
|
||||
const segments = davSegments(path, "path");
|
||||
return folder.length === 0 ? segments.join("/") : `${folder.join("/")}/${segments.join("/")}`;
|
||||
}
|
||||
|
||||
function libraryUrlFor(path: string): string {
|
||||
const base = process.env.FILES_PUBLIC_BASE ?? "";
|
||||
return base === "" ? path : `${base.replace(/\/+$/, "")}/f/${encodeURIComponent(path)}`;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,26 +31,13 @@ import type { MimeClass as ValidationMimeClass } from "./group-files.validation.
|
|||
export const SHARE_TARGETS = ["library", "open-room", "both"] as const;
|
||||
export type ShareTarget = (typeof SHARE_TARGETS)[number];
|
||||
|
||||
export const SHARE_ITEM_KINDS = ["ocis-resource", "peertube-video"] as const;
|
||||
export const SHARE_ITEM_KINDS = ["peertube-video"] as const;
|
||||
export type ShareItemKind = (typeof SHARE_ITEM_KINDS)[number];
|
||||
|
||||
/** Per-request caps; a share is a person acting on a handful of items, not a bulk ingest. */
|
||||
export const MAX_SHARE_ITEMS = 20;
|
||||
export const MAX_NOTE_LENGTH = 500;
|
||||
export const MAX_TITLE_LENGTH = 200;
|
||||
/** Collision suffix ceiling: stem-1 … stem-100, then a per-item error. */
|
||||
export const MAX_COLLISION_ATTEMPTS = 100;
|
||||
|
||||
export interface OcisResourceItem {
|
||||
readonly kind: "ocis-resource";
|
||||
/**
|
||||
* dav path RELATIVE to /remote.php/dav/files/<FILES_OCID_USER>/ — the hub's
|
||||
* own conventions: `hub-resources/<resourceId>` (a row of the group files
|
||||
* index) or a group folder path `hub-groups/<groupId>/<name>`.
|
||||
*/
|
||||
readonly path: string;
|
||||
}
|
||||
|
||||
export interface PeertubeVideoItem {
|
||||
readonly kind: "peertube-video";
|
||||
/** Canonical video URL (…/w/:id, /videos/watch/:id or /videos/embed/:id). */
|
||||
|
|
@ -58,7 +45,7 @@ export interface PeertubeVideoItem {
|
|||
readonly title: string;
|
||||
}
|
||||
|
||||
export type CollectiveShareItem = OcisResourceItem | PeertubeVideoItem;
|
||||
export type CollectiveShareItem = PeertubeVideoItem;
|
||||
|
||||
export interface CollectiveShareInput {
|
||||
readonly target: ShareTarget;
|
||||
|
|
@ -148,14 +135,6 @@ function parseItem(value: unknown, index: number): CollectiveShareItem {
|
|||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
const kind = record.kind;
|
||||
if (kind === "ocis-resource") {
|
||||
if (typeof record.path !== "string" || record.path.trim() === "") {
|
||||
throw new BadRequestException(`items[${index}].path must be a non-empty string`);
|
||||
}
|
||||
// Segment shape is validated here so a traversal attempt never reaches the service.
|
||||
davSegments(record.path, `items[${index}].path`);
|
||||
return { kind, path: record.path.trim() };
|
||||
}
|
||||
if (kind === "peertube-video") {
|
||||
const title = boundedText(record.title, `items[${index}].title`, MAX_TITLE_LENGTH);
|
||||
if (title === null) throw new BadRequestException(`items[${index}].title must be a non-empty string`);
|
||||
|
|
|
|||
|
|
@ -1,13 +1,14 @@
|
|||
import { BadRequestException, Body, Controller, Delete, Get, HttpCode, Inject, Param, ParseUUIDPipe, Post, Query, Req, Res, UploadedFile, UseGuards, UseInterceptors } from "@nestjs/common";
|
||||
import { BadRequestException, Body, Controller, Delete, Get, Headers, HttpCode, Inject, Param, ParseUUIDPipe, Patch, Post, Query, Req, Res, UploadedFile, UseGuards, UseInterceptors } from "@nestjs/common";
|
||||
import { FileInterceptor } from "@nestjs/platform-express";
|
||||
import type { Request, Response } from "express";
|
||||
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import { requireObject, requirePositiveInt, requireSession, requireString } from "../http/validation.js";
|
||||
import { GroupFilesService, type FilePage } from "./group-files.service.js";
|
||||
import type { GroupFile } from "@tactical-media-broadcast-hub/core";
|
||||
import { verifyWhileStreaming } from "./ocis-webdav.js";
|
||||
import type { FileFolder, GroupFile } from "./group-files.types.js";
|
||||
import { FileRangeNotSatisfiableException } from "./playback-range.js";
|
||||
import { verifyWhileStreaming } from "./verify-stream.js";
|
||||
|
||||
const MAX_FILE_UPLOAD_BYTES = 20 * 1024 * 1024;
|
||||
const MAX_FILE_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024;
|
||||
|
||||
interface FileUpload {
|
||||
buffer: Buffer;
|
||||
|
|
@ -20,7 +21,14 @@ interface FileUpload {
|
|||
*
|
||||
* GET /files list/search, keyset (groupId, senderMxid,
|
||||
* mimeClass, origin, name, from, to, cursor, limit)
|
||||
* POST /files/groups/:groupId/upload direct web upload (≤20 MiB, multipart)
|
||||
* GET /files/direct/config direct-upload capability + server public JWK
|
||||
* POST /files/direct/init presigned PUT for a browser-sealed upload
|
||||
* POST /files/direct/complete HEAD-checked commit + index row
|
||||
* POST /files/:id/playback mint a 10-minute playback capability
|
||||
* GET /files/play/:ticket public capability-authenticated byte range
|
||||
* DELETE /files/:id delete own upload (group setting) or Owner/Admin
|
||||
* POST /files/:id/direct-download presigned GET + unwrapped CEK
|
||||
* POST /files/groups/:groupId/upload direct web upload (≤2 GiB, multipart)
|
||||
* GET /files/:id one GroupFile
|
||||
* GET /files/:id/download stream payload (sha256 re-verified)
|
||||
* POST /files/:id/retention set retention (keep/until/legalHold)
|
||||
|
|
@ -49,6 +57,177 @@ export class GroupFilesController {
|
|||
return this.files.listFiles(session.workspaceId, session.userId, session.role, query);
|
||||
}
|
||||
|
||||
@Post("folders")
|
||||
@RequirePermission("media.view")
|
||||
async createFolder(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise<FileFolder> {
|
||||
const s = requireSession(request);
|
||||
return this.files.createFolder(s.workspaceId, s, requireObject(body, "body"));
|
||||
}
|
||||
|
||||
@Get("folders")
|
||||
@RequirePermission("media.view")
|
||||
async listFolders(@Req() request: Request & AuthenticatedRequest, @Query() query: Record<string, unknown>): Promise<FileFolder[]> {
|
||||
const s = requireSession(request);
|
||||
return this.files.listFolders(s.workspaceId, s, query);
|
||||
}
|
||||
|
||||
@Patch("folders/:id")
|
||||
@RequirePermission("media.view")
|
||||
async updateFolder(@Req() request: Request & AuthenticatedRequest, @Param("id", ParseUUIDPipe) id: string, @Body() body: unknown): Promise<FileFolder> {
|
||||
const s = requireSession(request);
|
||||
return this.files.updateFolder(s.workspaceId, id, s, requireObject(body, "body"));
|
||||
}
|
||||
|
||||
@Delete("folders/:id")
|
||||
@RequirePermission("media.view")
|
||||
async deleteFolder(@Req() request: Request & AuthenticatedRequest, @Param("id", ParseUUIDPipe) id: string): Promise<{ ok: true }> {
|
||||
const s = requireSession(request);
|
||||
return this.files.deleteFolder(s.workspaceId, id, s);
|
||||
}
|
||||
|
||||
@Get("groups/:groupId/settings")
|
||||
@RequirePermission("media.view")
|
||||
async groupSettings(@Req() request: Request & AuthenticatedRequest, @Param("groupId", ParseUUIDPipe) groupId: string): Promise<{ memberDeleteOwn: boolean }> {
|
||||
const s = requireSession(request);
|
||||
return this.files.getGroupFileSettings(s.workspaceId, groupId, s);
|
||||
}
|
||||
|
||||
@Patch("groups/:groupId/settings")
|
||||
@RequirePermission("media.view")
|
||||
async updateGroupSettings(@Req() request: Request & AuthenticatedRequest, @Param("groupId", ParseUUIDPipe) groupId: string, @Body() body: unknown): Promise<{ memberDeleteOwn: boolean }> {
|
||||
const s = requireSession(request);
|
||||
return this.files.setGroupFileSettings(s.workspaceId, groupId, s, requireObject(body, "body"));
|
||||
}
|
||||
|
||||
@Post("personal/upload")
|
||||
@RequirePermission("media.view")
|
||||
@UseInterceptors(FileInterceptor("file", {
|
||||
limits: { fileSize: MAX_FILE_UPLOAD_BYTES, files: 1, fields: 3, fieldSize: 1024, parts: 4 },
|
||||
}))
|
||||
async uploadPersonal(@Req() request: Request & AuthenticatedRequest, @UploadedFile() file?: FileUpload, @Body() body?: Record<string, unknown>): Promise<GroupFile> {
|
||||
const s = requireSession(request);
|
||||
if (file === undefined) throw new BadRequestException("a file is required (field: file)");
|
||||
return this.files.uploadForGroup(s.workspaceId, null, s,
|
||||
{ buffer: file.buffer, mimetype: file.mimetype, originalName: file.originalname },
|
||||
{ personal: true, folderId: body?.folderId });
|
||||
}
|
||||
|
||||
/**
|
||||
* Direct-upload capability. Declared before the :id routes so the literal
|
||||
* "direct/config" path wins route matching over ":id".
|
||||
*/
|
||||
@Get("direct/config")
|
||||
@RequirePermission("media.view")
|
||||
async directConfig(): Promise<ReturnType<GroupFilesService["directUploadConfig"]>> {
|
||||
return this.files.directUploadConfig();
|
||||
}
|
||||
|
||||
@Post("direct/init")
|
||||
@RequirePermission("media.view")
|
||||
async directInit(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Body() body: unknown,
|
||||
): Promise<ReturnType<GroupFilesService["initDirectUpload"]>> {
|
||||
const session = requireSession(request);
|
||||
const dto = requireObject(body, "body");
|
||||
return this.files.initDirectUpload(session.workspaceId, dto.groupId == null ? null : requireUuidValue(dto.groupId, "groupId"), {
|
||||
userId: session.userId,
|
||||
role: session.role,
|
||||
}, {
|
||||
filename: requireString(dto.filename, "filename"),
|
||||
mimeType: requireString(dto.mimeType, "mimeType"),
|
||||
sizeBytes: requireNumber(dto.sizeBytes, "sizeBytes"),
|
||||
personal: dto.personal,
|
||||
folderId: dto.folderId,
|
||||
});
|
||||
}
|
||||
|
||||
@Post("direct/complete")
|
||||
@RequirePermission("media.view")
|
||||
async directComplete(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Body() body: unknown,
|
||||
): Promise<GroupFile> {
|
||||
const session = requireSession(request);
|
||||
const dto = requireObject(body, "body");
|
||||
return this.files.completeDirectUpload(session.workspaceId, { userId: session.userId, role: session.role }, {
|
||||
uploadId: requireUuidValue(dto.uploadId, "uploadId"),
|
||||
objectKey: requireString(dto.objectKey, "objectKey"),
|
||||
groupId: dto.groupId == null ? null : requireUuidValue(dto.groupId, "groupId"),
|
||||
personal: dto.personal,
|
||||
folderId: dto.folderId,
|
||||
filename: requireString(dto.filename, "filename"),
|
||||
mimeType: requireString(dto.mimeType, "mimeType"),
|
||||
sizeBytes: requireNumber(dto.sizeBytes, "sizeBytes"),
|
||||
sha256: requireString(dto.sha256, "sha256"),
|
||||
});
|
||||
}
|
||||
|
||||
/** Segment-aligned multipart upload; upload IDs come from S3. */
|
||||
@Post("direct/multipart/init")
|
||||
@RequirePermission("media.view")
|
||||
async directMultipartInit(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Body() body: unknown,
|
||||
): Promise<ReturnType<GroupFilesService["initDirectMultipartUpload"]>> {
|
||||
const session = requireSession(request);
|
||||
const dto = requireObject(body, "body");
|
||||
return this.files.initDirectMultipartUpload(session.workspaceId,
|
||||
dto.groupId == null ? null : requireUuidValue(dto.groupId, "groupId"),
|
||||
{ userId: session.userId, role: session.role }, {
|
||||
filename: requireString(dto.filename, "filename"),
|
||||
mimeType: requireString(dto.mimeType, "mimeType"),
|
||||
sizeBytes: requireNumber(dto.sizeBytes, "sizeBytes"),
|
||||
personal: dto.personal, folderId: dto.folderId,
|
||||
});
|
||||
}
|
||||
|
||||
@Post("direct/multipart/complete")
|
||||
@RequirePermission("media.view")
|
||||
async directMultipartComplete(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Body() body: unknown,
|
||||
): Promise<GroupFile> {
|
||||
const session = requireSession(request);
|
||||
const dto = requireObject(body, "body");
|
||||
return this.files.completeDirectMultipartUpload(session.workspaceId,
|
||||
{ userId: session.userId, role: session.role }, {
|
||||
uploadId: requireString(dto.uploadId, "uploadId"),
|
||||
objectKey: requireString(dto.objectKey, "objectKey"),
|
||||
groupId: dto.groupId == null ? null : requireUuidValue(dto.groupId, "groupId"),
|
||||
personal: dto.personal, folderId: dto.folderId,
|
||||
filename: requireString(dto.filename, "filename"),
|
||||
mimeType: requireString(dto.mimeType, "mimeType"),
|
||||
plainSizeBytes: requireNumber(dto.plainSizeBytes, "plainSizeBytes"),
|
||||
sha256: requireString(dto.sha256, "sha256"), parts: dto.parts,
|
||||
});
|
||||
}
|
||||
|
||||
/** Same ACL/availability checks as the streaming route; direct objects only. */
|
||||
@Post(":id/direct-download")
|
||||
@RequirePermission("media.view")
|
||||
async directDownload(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Param("id", ParseUUIDPipe) id: string,
|
||||
): Promise<ReturnType<GroupFilesService["openDirectDownload"]>> {
|
||||
const session = requireSession(request);
|
||||
return this.files.openDirectDownload(session.workspaceId, session.userId, session.role, id);
|
||||
}
|
||||
|
||||
@Post(":id/playback")
|
||||
@HttpCode(200)
|
||||
@RequirePermission("media.view")
|
||||
async openPlayback(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Param("id", ParseUUIDPipe) id: string,
|
||||
@Body() body: unknown,
|
||||
): Promise<ReturnType<GroupFilesService["openRangePlayback"]>> {
|
||||
const session = requireSession(request);
|
||||
const dto = requireObject(body, "body");
|
||||
return this.files.openRangePlayback(session.workspaceId, session.userId, session.role, id,
|
||||
requireNumber(dto.start, "start"), requireNumber(dto.length, "length"));
|
||||
}
|
||||
|
||||
@Post("groups/:groupId/upload")
|
||||
@RequirePermission("media.view")
|
||||
@UseInterceptors(FileInterceptor("file", {
|
||||
|
|
@ -58,6 +237,7 @@ export class GroupFilesController {
|
|||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Param("groupId", ParseUUIDPipe) groupId: string,
|
||||
@UploadedFile() file?: FileUpload,
|
||||
@Body() body?: Record<string, unknown>,
|
||||
): Promise<GroupFile> {
|
||||
const session = requireSession(request);
|
||||
if (file === undefined) throw new BadRequestException("a file is required (field: file)");
|
||||
|
|
@ -66,9 +246,32 @@ export class GroupFilesController {
|
|||
groupId,
|
||||
{ userId: session.userId, role: session.role },
|
||||
{ buffer: file.buffer, mimetype: file.mimetype, originalName: file.originalname },
|
||||
{ folderId: body?.folderId },
|
||||
);
|
||||
}
|
||||
|
||||
@Post(":id/move")
|
||||
@HttpCode(200)
|
||||
@RequirePermission("media.view")
|
||||
async move(@Req() request: Request & AuthenticatedRequest, @Param("id", ParseUUIDPipe) id: string, @Body() body: unknown): Promise<GroupFile> {
|
||||
const s = requireSession(request);
|
||||
return this.files.moveFile(s.workspaceId, id, s, requireObject(body, "body"));
|
||||
}
|
||||
|
||||
@Post(":id/copy")
|
||||
@RequirePermission("media.view")
|
||||
async copy(@Req() request: Request & AuthenticatedRequest, @Param("id", ParseUUIDPipe) id: string, @Body() body: unknown): Promise<GroupFile> {
|
||||
const s = requireSession(request);
|
||||
return this.files.copyFile(s.workspaceId, id, s, requireObject(body, "body"));
|
||||
}
|
||||
|
||||
@Post(":id/share-to-group")
|
||||
@RequirePermission("media.view")
|
||||
async shareToGroup(@Req() request: Request & AuthenticatedRequest, @Param("id", ParseUUIDPipe) id: string, @Body() body: unknown): Promise<GroupFile> {
|
||||
const s = requireSession(request);
|
||||
return this.files.shareToGroup(s.workspaceId, id, s, requireObject(body, "body"));
|
||||
}
|
||||
|
||||
@Get(":id")
|
||||
@RequirePermission("media.view")
|
||||
async get(@Req() request: Request & AuthenticatedRequest, @Param("id", ParseUUIDPipe) id: string): Promise<GroupFile> {
|
||||
|
|
@ -97,6 +300,13 @@ export class GroupFilesController {
|
|||
response.end();
|
||||
}
|
||||
|
||||
@Delete(":id")
|
||||
@RequirePermission("media.view")
|
||||
async deleteFile(@Req() request: Request & AuthenticatedRequest, @Param("id", ParseUUIDPipe) id: string): Promise<GroupFile> {
|
||||
const session = requireSession(request);
|
||||
return this.files.deleteFile(session.workspaceId, id, { userId: session.userId, role: session.role });
|
||||
}
|
||||
|
||||
@Post(":id/retention")
|
||||
@RequirePermission("media.manage")
|
||||
async setRetention(
|
||||
|
|
@ -153,6 +363,40 @@ export class GroupFilesController {
|
|||
}
|
||||
}
|
||||
|
||||
/** No session guard: the short-lived, single-file ticket is the credential. */
|
||||
@Controller("files/play")
|
||||
export class FilePlaybackController {
|
||||
constructor(@Inject(GroupFilesService) private readonly files: GroupFilesService) {}
|
||||
|
||||
@Get(":ticket")
|
||||
async play(
|
||||
@Param("ticket") ticket: string,
|
||||
@Query() query: Record<string, unknown>,
|
||||
@Headers("range") range: string | undefined,
|
||||
@Res() response: Response,
|
||||
): Promise<void> {
|
||||
response.setHeader("Cache-Control", "no-store");
|
||||
response.setHeader("Accept-Ranges", "bytes");
|
||||
try {
|
||||
const slice = await this.files.readPlaybackTicket(ticket, query.start, query.end, range);
|
||||
response.status(206);
|
||||
response.setHeader("Content-Range", `bytes ${slice.start}-${slice.end}/${slice.total}`);
|
||||
response.setHeader("Content-Type", slice.item.mimeType);
|
||||
response.setHeader("Content-Length", String(slice.bytes.length));
|
||||
response.flushHeaders();
|
||||
// The storage reader verifies each segment's GCM tag before returning bytes.
|
||||
response.write(slice.bytes);
|
||||
response.end();
|
||||
} catch (error) {
|
||||
if (!(error instanceof FileRangeNotSatisfiableException)) throw error;
|
||||
response.status(416);
|
||||
response.setHeader("Content-Range", `bytes */${error.total}`);
|
||||
response.setHeader("Content-Length", "0");
|
||||
response.end();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Public share-claim surface: NO RbacGuard (the claimer holds only the
|
||||
* token). Token lives in the path — same shape as the invite claim routes.
|
||||
|
|
@ -176,3 +420,18 @@ export class PublicFileShareController {
|
|||
response.end();
|
||||
}
|
||||
}
|
||||
|
||||
function requireUuidValue(value: unknown, field: string): string {
|
||||
const text = requireString(value, field);
|
||||
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(text)) {
|
||||
throw new BadRequestException(`${field} must be a UUID`);
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
function requireNumber(value: unknown, field: string): number {
|
||||
if (typeof value !== "number" || !Number.isSafeInteger(value)) {
|
||||
throw new BadRequestException(`${field} must be a safe integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
|
|
|||
372
apps/api/src/group-files/group-files.folders.test.ts
Normal file
372
apps/api/src/group-files/group-files.folders.test.ts
Normal file
|
|
@ -0,0 +1,372 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { PoolClient } from "pg";
|
||||
import { GroupFilesService, type ActorRef, type ArchiveEntryInput } from "./group-files.service.js";
|
||||
import { GroupFilesController } from "./group-files.controller.js";
|
||||
import { PATH_METADATA, METHOD_METADATA } from "@nestjs/common/constants.js";
|
||||
import { RequestMethod } from "@nestjs/common";
|
||||
import type { Request } from "express";
|
||||
import { directStorageDisabled, type StorageService } from "../storage/storage.js";
|
||||
|
||||
const ws = "22222222-2222-4222-8222-222222222222";
|
||||
const group = "33333333-3333-4333-8333-333333333333";
|
||||
const otherGroup = "44444444-4444-4444-8444-444444444444";
|
||||
const alice = "55555555-5555-4555-8555-555555555555";
|
||||
const bob = "66666666-6666-4666-8666-666666666666";
|
||||
const now = new Date("2026-01-02T03:04:05Z");
|
||||
const member: ActorRef = { userId: alice, role: "Publisher" };
|
||||
type Row = Record<string, unknown>;
|
||||
let folders: Row[];
|
||||
let files: Row[];
|
||||
let statements: { sql: string; values: unknown[] }[];
|
||||
let memberOf: string[];
|
||||
let memberDeleteOwn: boolean;
|
||||
let storage: StorageService;
|
||||
let service: GroupFilesService;
|
||||
let mappedCreator: string | null;
|
||||
|
||||
function folder(patch: Row = {}): Row {
|
||||
return { id: randomUUID(), workspace_id: ws, group_id: group, owner_id: null, parent_id: null, name: "Footage", created_at: now, ...patch };
|
||||
}
|
||||
function file(patch: Row = {}): Row {
|
||||
return { id: randomUUID(), workspace_id: ws, group_id: group, owner_id: null, folder_id: null, created_by: alice,
|
||||
room_id: null, event_id: null, origin: "upload", sender_mxid: "@alice:tmbh.local", matrix_event_json: { kind: "upload" },
|
||||
original_name: "clip.mp4", safe_name: "clip.mp4", mime: "video/mp4", mime_class: "video", byte_size: "10",
|
||||
sha256: "a".repeat(64), ocis_resource_id: "sha256/same.mp4", ocis_resource_version: null, storage_key: "sha256/same.mp4",
|
||||
created_at: now, indexed_at: now, available: true, retention_mode: "keep", legal_hold: false, retention_until: null,
|
||||
share_access: null, share_expires_at: null, share_revoked_at: null, ...patch };
|
||||
}
|
||||
|
||||
// Stateful metadata fake: exercise the service through public operations and
|
||||
// independently inspect row identity, placement, provenance and storage calls.
|
||||
async function query(sql: string, values: unknown[] = []): Promise<{ rows: Row[]; rowCount: number }> {
|
||||
statements.push({ sql, values });
|
||||
let rows: Row[] = [];
|
||||
if (sql.includes("pg_advisory_xact_lock") || sql.startsWith("SELECT audit_append")) rows = [{}];
|
||||
else if (sql.startsWith("SELECT 1 FROM groups")) rows = values[0] === group || values[0] === otherGroup ? [{}] : [];
|
||||
else if (sql.includes("FROM group_members")) rows = memberOf.includes(String(values[0])) ? [{}] : [];
|
||||
else if (sql.includes("member_delete_own FROM groups")) rows = [{ member_delete_own: memberDeleteOwn }];
|
||||
else if (sql.startsWith("UPDATE groups SET member_delete_own")) { memberDeleteOwn = Boolean(values[1]); rows = [{}]; }
|
||||
else if (sql.startsWith("SELECT u.id FROM users")) rows = mappedCreator === null ? [] : [{ id: mappedCreator }];
|
||||
else if (sql.startsWith("SELECT username FROM users")) rows = [{ username: "alice" }];
|
||||
else if (sql.startsWith("SELECT * FROM file_folders WHERE id")) rows = folders.filter((f) => f.id === values[0]);
|
||||
else if (sql.startsWith("SELECT * FROM file_folders WHERE workspace_id")) rows = folders.filter((f) => f.group_id === values[1] && f.owner_id === values[2]);
|
||||
else if (sql.startsWith("INSERT INTO file_folders")) {
|
||||
const f = folder({ workspace_id: values[0], group_id: values[1], owner_id: values[2], parent_id: values[3], name: values[4] });
|
||||
folders.push(f); rows = [f];
|
||||
} else if (sql.startsWith("UPDATE file_folders")) {
|
||||
rows = folders.filter((f) => f.id === values[0]);
|
||||
rows.forEach((f) => Object.assign(f, { name: values[1], parent_id: values[2] }));
|
||||
} else if (sql.includes("WITH RECURSIVE descendants")) {
|
||||
const ids = new Set([values[0]]);
|
||||
let size = 0;
|
||||
while (ids.size !== size) { size = ids.size; folders.forEach((f) => { if (ids.has(f.parent_id)) ids.add(f.id); }); }
|
||||
rows = ids.has(values[1]) ? [{ id: values[1] }] : [];
|
||||
} else if (sql.startsWith("SELECT id FROM file_folders WHERE parent_id")) {
|
||||
rows = [...folders.filter((f) => f.parent_id === values[0]), ...files.filter((f) => f.folder_id === values[0])];
|
||||
} else if (sql.startsWith("DELETE FROM file_folders")) {
|
||||
folders = folders.filter((f) => f.id !== values[0]);
|
||||
files.filter((f) => f.folder_id === values[0]).forEach((f) => { f.folder_id = null; });
|
||||
} else if (sql.includes("FROM chat_media_archive a")) {
|
||||
if (sql.includes("WHERE a.id = $1")) rows = files.filter((f) => f.id === values[0]);
|
||||
else rows = files; // Listing predicates are asserted separately below.
|
||||
} else if (sql.startsWith("UPDATE chat_media_archive SET folder_id")) {
|
||||
files.filter((f) => f.id === values[0]).forEach((f) => { f.folder_id = values[1]; });
|
||||
} else if (sql.startsWith("UPDATE chat_media_archive SET available")) {
|
||||
files.filter((f) => f.id === values[0]).forEach((f) => { f.available = false; });
|
||||
} else if (sql.startsWith("INSERT INTO chat_media_archive")) {
|
||||
let f: Row;
|
||||
if (sql.includes("SELECT workspace_id")) {
|
||||
const source = files.find((item) => item.id === values[0])!;
|
||||
f = { ...source, id: randomUUID(), group_id: values[1], owner_id: values[2], folder_id: values[3], created_by: values[4],
|
||||
room_id: null, event_id: null, origin: values[5], matrix_event_json: JSON.parse(String(values[6])), original_name: values[7], safe_name: values[8] };
|
||||
} else {
|
||||
f = file({ workspace_id: values[0], group_id: values[1], room_id: values[2], event_id: values[3], origin: values[4], sender_mxid: values[5],
|
||||
matrix_event_json: JSON.parse(String(values[6])), original_name: values[7], safe_name: values[8], mime: values[9], mime_class: values[10], byte_size: values[11],
|
||||
sha256: values[12], ocis_resource_id: values[13], ocis_resource_version: values[14], created_at: values[15], storage_key: values[16], folder_id: values[17], created_by: values[18], owner_id: values[19] });
|
||||
}
|
||||
files.push(f); rows = [{ id: f.id }];
|
||||
} else if (sql.startsWith("UPDATE group_file_shares")) rows = [];
|
||||
else throw new Error(`unhandled SQL: ${sql}`);
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
vi.mock("../db/client.js", () => ({
|
||||
withTenant: async (_ws: string, fn: (client: PoolClient) => Promise<unknown>) => fn({ query } as unknown as PoolClient),
|
||||
withSystem: vi.fn(),
|
||||
}));
|
||||
|
||||
beforeEach(() => {
|
||||
folders = []; files = []; statements = []; memberOf = [group]; memberDeleteOwn = true; mappedCreator = null;
|
||||
storage = { ...directStorageDisabled(), putContent: vi.fn(async () => ({ key: "sha256/same.mp4", hash: "a".repeat(64), sizeBytes: 10, contentType: "video/mp4", deduplicated: true })),
|
||||
getObject: vi.fn(), hasContent: vi.fn(), presign: vi.fn(), getSealedObject: vi.fn() };
|
||||
service = new GroupFilesService({ storage });
|
||||
});
|
||||
afterEach(() => { vi.unstubAllEnvs(); });
|
||||
|
||||
describe("folder trees", () => {
|
||||
it("creates, lists, renames, reparents and deletes empty folders", async () => {
|
||||
const parent = await service.createFolder(ws, member, { groupId: group, name: " Parent " });
|
||||
const child = await service.createFolder(ws, member, { groupId: group, name: "Child", parentId: parent.id });
|
||||
expect(parent.name).toBe("Parent");
|
||||
expect(child).toMatchObject({ parentId: parent.id, groupId: group, ownerId: null });
|
||||
expect(await service.listFolders(ws, member, { scope: "group", groupId: group })).toHaveLength(2);
|
||||
await expect(service.deleteFolder(ws, parent.id, member)).rejects.toMatchObject({ status: 409 });
|
||||
const renamed = await service.updateFolder(ws, child.id, member, { name: "New" });
|
||||
expect(renamed).toMatchObject({ name: "New", parentId: parent.id });
|
||||
expect(await service.updateFolder(ws, child.id, member, { parentId: null })).toMatchObject({ parentId: null, name: "New" });
|
||||
await expect(service.deleteFolder(ws, parent.id, member)).resolves.toEqual({ ok: true });
|
||||
expect(folders.map((f) => f.id)).toEqual([child.id]);
|
||||
});
|
||||
|
||||
it.each(["Publisher", "Owner", "Admin"])("allows %s group folder CRUD with the group ACL", async (role) => {
|
||||
const actor = { userId: alice, role };
|
||||
if (role !== "Publisher") memberOf = [];
|
||||
const f = await service.createFolder(ws, actor, { groupId: group, name: "Test" });
|
||||
expect(await service.listFolders(ws, actor, { scope: "group", groupId: group })).toHaveLength(1);
|
||||
await service.updateFolder(ws, f.id, actor, { name: "Changed" });
|
||||
await expect(service.deleteFolder(ws, f.id, actor)).resolves.toEqual({ ok: true });
|
||||
});
|
||||
|
||||
it.each(["create", "list", "update", "delete"])("denies non-member folder %s", async (operation) => {
|
||||
const f = folder(); folders.push(f); memberOf = [];
|
||||
const actions = {
|
||||
create: () => service.createFolder(ws, member, { groupId: group, name: "Test" }),
|
||||
list: () => service.listFolders(ws, member, { scope: "group", groupId: group }),
|
||||
update: () => service.updateFolder(ws, String(f.id), member, { name: "Changed" }),
|
||||
delete: () => service.deleteFolder(ws, String(f.id), member),
|
||||
};
|
||||
await expect(actions[operation as keyof typeof actions]()).rejects.toMatchObject({ status: 403 });
|
||||
expect(statements.some((s) => /^(INSERT|UPDATE|DELETE)/.test(s.sql))).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps personal trees private, including from other workspace Owner/Admin", async () => {
|
||||
const f = await service.createFolder(ws, member, { personal: true, name: "Private" });
|
||||
expect(f).toMatchObject({ groupId: null, ownerId: alice });
|
||||
expect(await service.listFolders(ws, member, { scope: "personal" })).toHaveLength(1);
|
||||
for (const role of ["Publisher", "Owner", "Admin"]) {
|
||||
const actor = { userId: bob, role };
|
||||
expect(await service.listFolders(ws, actor, { scope: "personal" })).toEqual([]);
|
||||
await expect(service.updateFolder(ws, f.id, actor, { name: "Stolen" })).rejects.toMatchObject({ status: 403 });
|
||||
await expect(service.deleteFolder(ws, f.id, actor)).rejects.toMatchObject({ status: 403 });
|
||||
}
|
||||
await service.updateFolder(ws, f.id, member, { name: "Renamed" });
|
||||
await expect(service.deleteFolder(ws, f.id, member)).resolves.toEqual({ ok: true });
|
||||
});
|
||||
|
||||
it("rejects cycles, cross-scope parents, invalid names, missing folders and nonempty deletion", async () => {
|
||||
const parent = await service.createFolder(ws, member, { groupId: group, name: "Parent" });
|
||||
const child = await service.createFolder(ws, member, { groupId: group, name: "Child", parentId: parent.id });
|
||||
await expect(service.updateFolder(ws, parent.id, member, { parentId: parent.id })).rejects.toMatchObject({ status: 409 });
|
||||
await expect(service.updateFolder(ws, parent.id, member, { parentId: child.id })).rejects.toMatchObject({ status: 409 });
|
||||
const personal = await service.createFolder(ws, member, { personal: true, name: "Private" });
|
||||
await expect(service.updateFolder(ws, parent.id, member, { parentId: personal.id })).rejects.toMatchObject({ status: 400 });
|
||||
await expect(service.createFolder(ws, member, { groupId: group, parentId: personal.id, name: "Wrong" })).rejects.toMatchObject({ status: 400 });
|
||||
for (const name of [" ", "a/b", "a\\b", "a".repeat(256)]) {
|
||||
await expect(service.createFolder(ws, member, { groupId: group, name })).rejects.toMatchObject({ status: 400 });
|
||||
}
|
||||
await expect(service.updateFolder(ws, randomUUID(), member, { name: "X" })).rejects.toMatchObject({ status: 404 });
|
||||
files.push(file({ folder_id: child.id }));
|
||||
await expect(service.deleteFolder(ws, child.id, member)).rejects.toMatchObject({ status: 409 });
|
||||
files[0]!.available = false;
|
||||
await expect(service.deleteFolder(ws, child.id, member)).rejects.toMatchObject({ status: 409 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("file scopes, moves, copies and sharing", () => {
|
||||
it("binds personal listings to caller and folder, and default listings to group roots", async () => {
|
||||
const f = folder({ group_id: null, owner_id: alice }); folders.push(f);
|
||||
await service.listFiles(ws, alice, "Owner", { scope: "personal", folderId: f.id });
|
||||
const listing = statements.find((s) => s.sql.includes("FROM chat_media_archive a"))!;
|
||||
expect(listing.sql).toContain("a.group_id IS NULL AND a.owner_id = $2::uuid");
|
||||
expect(listing.sql).toContain("a.folder_id = $3::uuid");
|
||||
expect(listing.values).toEqual([ws, alice, f.id, 51]);
|
||||
statements = [];
|
||||
await service.listFiles(ws, alice, "Owner", {});
|
||||
const roots = statements.find((s) => s.sql.includes("FROM chat_media_archive a"))!;
|
||||
expect(roots.sql).toContain("a.group_id IS NOT NULL");
|
||||
expect(roots.sql).toContain("a.folder_id IS NULL");
|
||||
await expect(service.listFiles(ws, alice, "Owner", { scope: "personal", groupId: group })).rejects.toMatchObject({ status: 400 });
|
||||
await expect(service.listFiles(ws, bob, "Owner", { scope: "personal", folderId: f.id })).rejects.toMatchObject({ status: 403 });
|
||||
});
|
||||
|
||||
it.each(["Publisher", "Owner", "Admin"])("allows %s metadata moves in groups and returns to root", async (role) => {
|
||||
const f = folder(); folders.push(f); const source = file(); files.push(source);
|
||||
const moved = await service.moveFile(ws, String(source.id), { userId: alice, role }, { folderId: f.id });
|
||||
expect(moved.folderId).toBe(f.id);
|
||||
expect(await service.moveFile(ws, String(source.id), { userId: alice, role }, { folderId: null })).toMatchObject({ folderId: null });
|
||||
expect(files).toHaveLength(1);
|
||||
expect(storage.putContent).not.toHaveBeenCalled(); expect(storage.getObject).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("copies a new row with identical payload key and checksum while move preserves identity", async () => {
|
||||
const f = folder(); folders.push(f); const source = file(); files.push(source);
|
||||
const snapshot = structuredClone(source);
|
||||
const copy = await service.copyFile(ws, String(source.id), member, { targetFolderId: f.id });
|
||||
expect(copy).toMatchObject({ name: "clip.mp4 (copy)", folderId: f.id, createdBy: alice, checksum: { value: source.sha256 }, file: { storageKey: source.storage_key } });
|
||||
expect(copy.id).not.toBe(source.id); expect(files).toHaveLength(2); expect(source).toEqual(snapshot);
|
||||
await service.moveFile(ws, String(source.id), member, { folderId: f.id });
|
||||
expect(files).toHaveLength(2); expect(source.id).toBe(snapshot.id); expect(source.folder_id).toBe(f.id);
|
||||
expect(storage.putContent).not.toHaveBeenCalled(); expect(storage.getObject).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("copies into another accessible group root and personal folder", async () => {
|
||||
memberOf.push(otherGroup);
|
||||
const source = file(); files.push(source);
|
||||
expect(await service.copyFile(ws, String(source.id), member, { targetGroupId: otherGroup })).toMatchObject({ groupId: otherGroup, folderId: null });
|
||||
const personal = folder({ group_id: null, owner_id: alice }); folders.push(personal);
|
||||
expect(await service.copyFile(ws, String(source.id), member, { targetFolderId: personal.id })).toMatchObject({ groupId: null, ownerId: alice, folderId: personal.id });
|
||||
await expect(service.copyFile(ws, String(source.id), member, {})).rejects.toMatchObject({ status: 400 });
|
||||
await expect(service.copyFile(ws, String(source.id), member, { targetGroupId: group, targetFolderId: personal.id })).rejects.toMatchObject({ status: 400 });
|
||||
});
|
||||
|
||||
it("shares a personal file into group root, preserving original and payload", async () => {
|
||||
const source = file({ group_id: null, owner_id: alice }); files.push(source);
|
||||
const snapshot = structuredClone(source);
|
||||
const shared = await service.shareToGroup(ws, String(source.id), member, { groupId: group });
|
||||
expect(shared).toMatchObject({ groupId: group, ownerId: null, folderId: null, shared: true, origin: { kind: "upload", sharedFrom: "personal" }, name: source.original_name,
|
||||
checksum: { value: source.sha256 }, file: { storageKey: source.storage_key }, createdBy: alice });
|
||||
expect(shared.id).not.toBe(source.id); expect(files).toHaveLength(2); expect(source).toEqual(snapshot);
|
||||
expect(storage.putContent).not.toHaveBeenCalled(); expect(storage.getObject).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("enforces source and destination ACLs, same-scope moves and personal ownership", async () => {
|
||||
const source = file(); files.push(source);
|
||||
const foreign = folder({ group_id: otherGroup }); folders.push(foreign);
|
||||
await expect(service.copyFile(ws, String(source.id), member, { targetGroupId: otherGroup })).rejects.toMatchObject({ status: 403 });
|
||||
await expect(service.moveFile(ws, String(source.id), member, { folderId: foreign.id })).rejects.toMatchObject({ status: 403 });
|
||||
memberOf.push(otherGroup);
|
||||
await expect(service.moveFile(ws, String(source.id), member, { folderId: foreign.id })).rejects.toMatchObject({ status: 400 });
|
||||
await expect(service.moveFile(ws, String(source.id), member, {})).rejects.toMatchObject({ status: 400 });
|
||||
await expect(service.shareToGroup(ws, String(source.id), member, { groupId: group })).rejects.toMatchObject({ status: 400 });
|
||||
memberOf = [];
|
||||
await expect(service.copyFile(ws, String(source.id), member, { targetGroupId: otherGroup })).rejects.toMatchObject({ status: 403 });
|
||||
const personal = file({ group_id: null, owner_id: alice }); files.push(personal);
|
||||
for (const role of ["Publisher", "Owner", "Admin"]) {
|
||||
const stranger = { userId: bob, role };
|
||||
await expect(service.getFile(ws, bob, role, String(personal.id))).rejects.toMatchObject({ status: 403 });
|
||||
await expect(service.moveFile(ws, String(personal.id), stranger, { folderId: null })).rejects.toMatchObject({ status: 403 });
|
||||
await expect(service.copyFile(ws, String(personal.id), stranger, { targetGroupId: group })).rejects.toMatchObject({ status: 403 });
|
||||
await expect(service.shareToGroup(ws, String(personal.id), stranger, { groupId: group })).rejects.toMatchObject({ status: 403 });
|
||||
await expect(service.deleteFile(ws, String(personal.id), stranger)).rejects.toMatchObject({ status: 403 });
|
||||
}
|
||||
await expect(service.shareToGroup(ws, String(personal.id), member, { groupId: group })).rejects.toMatchObject({ status: 403 });
|
||||
const privateFolder = folder({ group_id: null, owner_id: alice }); folders.push(privateFolder);
|
||||
expect(await service.moveFile(ws, String(personal.id), member, { folderId: privateFolder.id })).toMatchObject({ folderId: privateFolder.id });
|
||||
memberDeleteOwn = false;
|
||||
await expect(service.deleteFile(ws, String(personal.id), member)).resolves.toMatchObject({ status: "revoked" });
|
||||
});
|
||||
|
||||
it.each([{ available: false }, { retention_mode: "until", retention_until: new Date("2020-01-01") }])("refuses copies/moves/shares of unavailable files: %j", async (patch) => {
|
||||
const source = file({ group_id: null, owner_id: alice, ...patch }); files.push(source);
|
||||
await expect(service.moveFile(ws, String(source.id), member, { folderId: null })).rejects.toMatchObject({ status: 410 });
|
||||
await expect(service.copyFile(ws, String(source.id), member, { targetGroupId: group })).rejects.toMatchObject({ status: 410 });
|
||||
await expect(service.shareToGroup(ws, String(source.id), member, { groupId: group })).rejects.toMatchObject({ status: 410 });
|
||||
expect(files).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("creator attribution, upload destinations and delete settings", () => {
|
||||
function entry(patch: Partial<ArchiveEntryInput> = {}): ArchiveEntryInput {
|
||||
return { groupId: group, roomId: null, eventId: null, origin: { kind: "upload" }, originKind: "upload", senderMxid: "@alice:tmbh.local",
|
||||
originalName: "clip.mp4", safeName: "clip.mp4", mime: "video/mp4", mimeClass: "video", byteSize: 10, sha256: "a".repeat(64),
|
||||
ocisResourceId: "sha256/same.mp4", storageKey: "sha256/same.mp4", ocisResourceVersion: null, createdAt: now, ...patch };
|
||||
}
|
||||
|
||||
it("maps sender identities for ingest and preserves explicit authenticated creators", async () => {
|
||||
mappedCreator = alice;
|
||||
const { id } = await service.registerArchiveEntry(ws, entry({ originKind: "matrix" }));
|
||||
expect(files.find((f) => f.id === id)?.created_by).toBe(alice);
|
||||
const mapping = statements.find((s) => s.sql.startsWith("SELECT u.id"))!;
|
||||
expect(mapping.values).toEqual([ws, "@alice:tmbh.local", "alice"]);
|
||||
mappedCreator = null;
|
||||
const explicit = await service.registerArchiveEntry(ws, entry({ createdBy: bob }));
|
||||
expect(files.find((f) => f.id === explicit.id)?.created_by).toBe(bob);
|
||||
const unknown = await service.registerArchiveEntry(ws, entry({ senderMxid: "@alice:foreign.example" }));
|
||||
expect(files.find((f) => f.id === unknown.id)?.created_by).toBeNull();
|
||||
});
|
||||
|
||||
it("uploads into group/personal folders or roots and attributes the authenticated creator", async () => {
|
||||
const f = folder(); const personal = folder({ group_id: null, owner_id: alice }); folders.push(f, personal);
|
||||
const input = { buffer: Buffer.from("hello"), mimetype: "text/plain", originalName: "hello.txt" };
|
||||
expect(await service.uploadForGroup(ws, group, member, input, { folderId: f.id })).toMatchObject({ groupId: group, folderId: f.id, createdBy: alice });
|
||||
expect(await service.uploadForGroup(ws, null, member, input, { personal: true, folderId: personal.id })).toMatchObject({ groupId: null, ownerId: alice, folderId: personal.id, createdBy: alice });
|
||||
expect(await service.uploadForGroup(ws, null, member, input, { personal: true })).toMatchObject({ groupId: null, ownerId: alice, folderId: null });
|
||||
await expect(service.uploadForGroup(ws, group, member, input, { folderId: personal.id })).rejects.toMatchObject({ status: 400 });
|
||||
expect(storage.putContent).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it("resolves configured Hub Matrix identities without trusting foreign localparts", async () => {
|
||||
vi.stubEnv("MATRIX_SSO_HOMESERVER_URL", "https://chat.example.org");
|
||||
vi.stubEnv("MATRIX_SSO_LOCALPART_PREFIX", "hub_");
|
||||
mappedCreator = alice;
|
||||
await service.registerArchiveEntry(ws, entry({ senderMxid: "@hub_alice:chat.example.org", originKind: "matrix" }));
|
||||
const own = statements.find((s) => s.sql.startsWith("SELECT u.id"))!;
|
||||
expect(own.values).toEqual([ws, "@hub_alice:chat.example.org", "alice"]);
|
||||
statements = [];
|
||||
await service.registerArchiveEntry(ws, entry({ senderMxid: "@hub_alice:foreign.example", originKind: "matrix" }));
|
||||
const foreign = statements.find((s) => s.sql.startsWith("SELECT u.id"))!;
|
||||
expect(foreign.values).toEqual([ws, "@hub_alice:foreign.example", null]);
|
||||
});
|
||||
|
||||
it("accepts personal direct upload init/complete and rejects cross-scope destinations", async () => {
|
||||
const f = folder({ group_id: null, owner_id: alice }); folders.push(f);
|
||||
storage.directPublicJwk = vi.fn(async () => ({ kty: "EC" as const, crv: "P-256" as const, x: "x", y: "y" }));
|
||||
storage.presignPut = vi.fn(async () => "https://storage.example/put");
|
||||
storage.headObject = vi.fn(async () => ({ exists: true, contentLength: 177 }));
|
||||
const upload = await service.initDirectUpload(ws, null, member, { filename: "clip.mp4", mimeType: "video/mp4", sizeBytes: 177, personal: true, folderId: f.id });
|
||||
const result = await service.completeDirectUpload(ws, member, { ...upload, personal: true, folderId: f.id, filename: "clip.mp4", mimeType: "video/mp4", sizeBytes: 177, sha256: "a".repeat(64) });
|
||||
expect(result).toMatchObject({ groupId: null, ownerId: alice, folderId: f.id, createdBy: alice, file: { storageKey: upload.objectKey } });
|
||||
await expect(service.initDirectUpload(ws, group, member, { filename: "clip.mp4", mimeType: "video/mp4", sizeBytes: 177, folderId: f.id })).rejects.toMatchObject({ status: 400 });
|
||||
});
|
||||
|
||||
it.each(["matrix", "bridge", "pad-export", "upload"])("allows deleting creator-owned %s rows regardless of sender text", async (origin) => {
|
||||
const source = file({ origin, created_by: alice, sender_mxid: "@someone:else.example" }); files.push(source);
|
||||
await expect(service.deleteFile(ws, String(source.id), member)).resolves.toMatchObject({ status: "revoked" });
|
||||
expect(files).toHaveLength(1); expect(source.storage_key).toBe("sha256/same.mp4");
|
||||
});
|
||||
|
||||
it("gives clear disabled/other-creator errors and exposes settings only to authorized callers", async () => {
|
||||
const source = file(); files.push(source);
|
||||
memberDeleteOwn = false;
|
||||
await expect(service.deleteFile(ws, String(source.id), member)).rejects.toThrow("member file deletion is disabled");
|
||||
expect(await service.getGroupFileSettings(ws, group, member)).toEqual({ memberDeleteOwn: false });
|
||||
await expect(service.setGroupFileSettings(ws, group, member, { memberDeleteOwn: true })).rejects.toMatchObject({ status: 403 });
|
||||
expect(await service.setGroupFileSettings(ws, group, { userId: alice, role: "Admin" }, { memberDeleteOwn: true })).toEqual({ memberDeleteOwn: true });
|
||||
source.created_by = bob;
|
||||
await expect(service.deleteFile(ws, String(source.id), member)).rejects.toThrow("members may only delete files they created");
|
||||
source.created_by = null;
|
||||
await expect(service.deleteFile(ws, String(source.id), member)).rejects.toMatchObject({ status: 403 });
|
||||
await expect(service.setGroupFileSettings(ws, group, { userId: alice, role: "Owner" }, { memberDeleteOwn: "false" })).rejects.toMatchObject({ status: 400 });
|
||||
memberOf = [];
|
||||
await expect(service.getGroupFileSettings(ws, group, member)).rejects.toMatchObject({ status: 403 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("controller contract", () => {
|
||||
it("declares literal folder routes ahead of file-id routes and binds the expected verbs", () => {
|
||||
const prototype = GroupFilesController.prototype;
|
||||
expect(Object.getOwnPropertyNames(prototype).indexOf("listFolders")).toBeLessThan(Object.getOwnPropertyNames(prototype).indexOf("get"));
|
||||
for (const [method, path, verb] of [
|
||||
["createFolder", "folders", RequestMethod.POST], ["listFolders", "folders", RequestMethod.GET],
|
||||
["updateFolder", "folders/:id", RequestMethod.PATCH], ["deleteFolder", "folders/:id", RequestMethod.DELETE],
|
||||
["move", ":id/move", RequestMethod.POST], ["copy", ":id/copy", RequestMethod.POST], ["shareToGroup", ":id/share-to-group", RequestMethod.POST],
|
||||
] as const) {
|
||||
expect(Reflect.getMetadata(PATH_METADATA, prototype[method])).toBe(path);
|
||||
expect(Reflect.getMetadata(METHOD_METADATA, prototype[method])).toBe(verb);
|
||||
}
|
||||
});
|
||||
|
||||
it("forwards authenticated scope and optional upload folder from both multipart routes", async () => {
|
||||
const stub = { uploadForGroup: vi.fn(async (..._args: unknown[]) => ({})), initDirectUpload: vi.fn(async () => ({})), completeDirectUpload: vi.fn(async () => ({})) };
|
||||
const controller = new GroupFilesController(stub as unknown as GroupFilesService);
|
||||
const session = { workspaceId: ws, userId: alice, role: "Publisher" };
|
||||
const request = { session } as unknown as Request;
|
||||
const upload = { buffer: Buffer.from("x"), originalname: "x.txt", mimetype: "text/plain" };
|
||||
const folderId = randomUUID();
|
||||
await controller.upload(request, group, upload, { folderId });
|
||||
await controller.uploadPersonal(request, upload, { folderId });
|
||||
expect(stub.uploadForGroup.mock.calls[0]?.slice(0, 3)).toEqual([ws, group, member]);
|
||||
expect(stub.uploadForGroup.mock.calls[0]?.[4]).toEqual({ folderId });
|
||||
expect(stub.uploadForGroup.mock.calls[1]?.[4]).toEqual({ personal: true, folderId });
|
||||
});
|
||||
});
|
||||
|
|
@ -255,4 +255,88 @@ describe.skipIf(appUrl === undefined || ownerUrl === undefined)("group-files met
|
|||
});
|
||||
await expect(svc.getFile(wsA, owner, "Owner", otherTenant)).rejects.toMatchObject({ status: 404 });
|
||||
});
|
||||
|
||||
it("persists folder trees, blocks cycles/nonempty deletion and protects personal owners", async () => {
|
||||
const actor = { userId: publisher, role: "Publisher" };
|
||||
const parent = await svc.createFolder(wsA, actor, { groupId: groupA1, name: "Parent" });
|
||||
const child = await svc.createFolder(wsA, actor, { groupId: groupA1, parentId: parent.id, name: "Child" });
|
||||
expect(await svc.listFolders(wsA, actor, { scope: "group", groupId: groupA1 })).toEqual(expect.arrayContaining([parent, child]));
|
||||
await expect(svc.updateFolder(wsA, parent.id, actor, { parentId: child.id })).rejects.toMatchObject({ status: 409 });
|
||||
await expect(svc.deleteFolder(wsA, parent.id, actor)).rejects.toMatchObject({ status: 409 });
|
||||
await svc.updateFolder(wsA, child.id, actor, { parentId: null, name: "Renamed" });
|
||||
await svc.deleteFolder(wsA, parent.id, actor);
|
||||
const personal = await svc.createFolder(wsA, actor, { personal: true, name: "Private" });
|
||||
await expect(svc.updateFolder(wsA, personal.id, { userId: owner, role: "Owner" }, { name: "No" })).rejects.toMatchObject({ status: 403 });
|
||||
await expect(svc.createFolder(wsA, actor, { groupId: groupA2, name: "No" })).rejects.toMatchObject({ status: 403 });
|
||||
await svc.deleteFolder(wsA, personal.id, actor);
|
||||
await svc.deleteFolder(wsA, child.id, actor);
|
||||
});
|
||||
|
||||
async function personalEntry(): Promise<string> {
|
||||
const { id } = await svc.registerArchiveEntry(wsA, {
|
||||
groupId: null, ownerId: publisher, createdBy: publisher, roomId: null, eventId: null,
|
||||
origin: { kind: "upload" }, originKind: "upload", senderMxid: "@publisher:tmbh.local",
|
||||
originalName: "private.txt", safeName: "private.txt", mime: "text/plain", mimeClass: "document", byteSize: 3,
|
||||
sha256: "a".repeat(64), ocisResourceId: "sha256/metadata-only.txt", storageKey: "sha256/metadata-only.txt",
|
||||
ocisResourceVersion: null, createdAt: new Date(),
|
||||
});
|
||||
return id;
|
||||
}
|
||||
|
||||
it("moves one row, copies/shares new rows with identical keys, and leaves the personal original intact", async () => {
|
||||
const actor = { userId: publisher, role: "Publisher" };
|
||||
const id = await personalEntry();
|
||||
const folder = await svc.createFolder(wsA, actor, { personal: true, name: "Personal files" });
|
||||
const original = await svc.getFile(wsA, publisher, "Publisher", id);
|
||||
const copy = await svc.copyFile(wsA, id, actor, { targetFolderId: folder.id });
|
||||
expect(copy.id).not.toBe(id);
|
||||
expect(copy).toMatchObject({ name: "private.txt (copy)", folderId: folder.id, file: original.file, checksum: original.checksum });
|
||||
const shared = await svc.shareToGroup(wsA, id, actor, { groupId: groupA1 });
|
||||
expect(shared).toMatchObject({ groupId: groupA1, ownerId: null, shared: true, origin: { kind: "upload", sharedFrom: "personal" }, file: { storageKey: original.file.storageKey }, checksum: original.checksum });
|
||||
expect(await svc.getFile(wsA, publisher, "Publisher", id)).toEqual(original);
|
||||
await expect(svc.getFile(wsA, owner, "Owner", id)).rejects.toMatchObject({ status: 403 });
|
||||
expect((await svc.moveFile(wsA, id, actor, { folderId: folder.id })).id).toBe(id);
|
||||
const root = await svc.listFiles(wsA, publisher, "Publisher", { scope: "personal" });
|
||||
expect(root.items.some((f) => f.id === id)).toBe(false);
|
||||
const inFolder = await svc.listFiles(wsA, publisher, "Publisher", { scope: "personal", folderId: folder.id });
|
||||
expect(inFolder.items.map((f) => f.id).sort()).toEqual([id, copy.id].sort());
|
||||
await expect(svc.deleteFolder(wsA, folder.id, actor)).rejects.toMatchObject({ status: 409 });
|
||||
const count = await inTenant(wsA, (c) => c.query("SELECT count(*)::int AS n FROM chat_media_archive WHERE storage_key = $1", [original.file.storageKey]));
|
||||
expect(count.rows[0]?.n).toBe(3);
|
||||
});
|
||||
|
||||
it("enforces member creator/settings deletion while Owner/Admin keep soft-revoke access", async () => {
|
||||
const actor = { userId: publisher, role: "Publisher" };
|
||||
const personal = await personalEntry();
|
||||
const shared = await svc.shareToGroup(wsA, personal, actor, { groupId: groupA1 });
|
||||
await svc.setGroupFileSettings(wsA, groupA1, { userId: owner, role: "Owner" }, { memberDeleteOwn: false });
|
||||
await expect(svc.deleteFile(wsA, shared.id, actor)).rejects.toMatchObject({ status: 403 });
|
||||
await svc.setGroupFileSettings(wsA, groupA1, { userId: owner, role: "Admin" }, { memberDeleteOwn: true });
|
||||
await expect(svc.deleteFile(wsA, shared.id, actor)).resolves.toMatchObject({ status: "revoked" });
|
||||
const other = await svc.copyFile(wsA, personal, { userId: publisher, role: "Publisher" }, { targetGroupId: groupA1 });
|
||||
await inTenant(wsA, (c) => c.query("UPDATE chat_media_archive SET created_by = $2 WHERE id = $1", [other.id, owner]));
|
||||
await expect(svc.deleteFile(wsA, other.id, actor)).rejects.toMatchObject({ status: 403 });
|
||||
await svc.setGroupFileSettings(wsA, groupA1, { userId: owner, role: "Owner" }, { memberDeleteOwn: false });
|
||||
await expect(svc.deleteFile(wsA, other.id, { userId: owner, role: "Owner" })).resolves.toMatchObject({ status: "revoked" });
|
||||
const raw = await inTenant(wsA, (c) => c.query("SELECT available, storage_key FROM chat_media_archive WHERE id = $1", [shared.id]));
|
||||
expect(raw.rows[0]).toEqual({ available: false, storage_key: "sha256/metadata-only.txt" });
|
||||
});
|
||||
|
||||
it("applies FORCE tenant RLS to folders and attributes direct database inserts", async () => {
|
||||
const folder = await svc.createFolder(wsB, { userId: owner, role: "Owner" }, { groupId: groupB, name: "Other tenant" });
|
||||
await expect(svc.updateFolder(wsA, folder.id, { userId: owner, role: "Owner" }, { name: "No" })).rejects.toMatchObject({ status: 404 });
|
||||
const withoutTenant = await m.db.pool.query("SELECT id FROM file_folders WHERE id = $1", [folder.id]);
|
||||
expect(withoutTenant.rows).toEqual([]);
|
||||
const id = await personalEntry();
|
||||
const username = await inTenant(wsA, (c) => c.query("SELECT username FROM users WHERE id = $1", [publisher]));
|
||||
const inserted = await inTenant(wsA, (c) => c.query(
|
||||
`INSERT INTO chat_media_archive (workspace_id, group_id, origin, sender_mxid, original_name, safe_name,
|
||||
mime, mime_class, byte_size, sha256, ocis_resource_id, created_at)
|
||||
SELECT workspace_id, $2, origin, $3, original_name, safe_name, mime, mime_class, byte_size, sha256, ocis_resource_id, now()
|
||||
FROM chat_media_archive WHERE id = $1 RETURNING created_by`,
|
||||
[id, groupA1, `@${String(username.rows[0]?.username)}:tmbh.local`],
|
||||
));
|
||||
expect(inserted.rows[0]?.created_by).toBe(publisher);
|
||||
});
|
||||
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { Module } from "@nestjs/common";
|
||||
import { GroupFilesController, PublicFileShareController } from "./group-files.controller.js";
|
||||
import { GroupFilesController, FilePlaybackController, PublicFileShareController } from "./group-files.controller.js";
|
||||
import { GroupFilesService } from "./group-files.service.js";
|
||||
import { DriveController } from "./drive.controller.js";
|
||||
|
||||
|
|
@ -21,7 +21,7 @@ import { DriveController } from "./drive.controller.js";
|
|||
* deployment is absent instead of breaking startup.
|
||||
*/
|
||||
@Module({
|
||||
controllers: [GroupFilesController, PublicFileShareController, DriveController],
|
||||
controllers: [FilePlaybackController, GroupFilesController, PublicFileShareController, DriveController],
|
||||
providers: [GroupFilesService],
|
||||
exports: [GroupFilesService],
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,7 +1,13 @@
|
|||
import type { PoolClient } from "pg";
|
||||
import { GroupFilesService } from "./group-files.service.js";
|
||||
import type { Response } from "express";
|
||||
import { GUARDS_METADATA } from "@nestjs/common/constants.js";
|
||||
import { FilePlaybackController, GroupFilesController } from "./group-files.controller.js";
|
||||
import { RbacGuard } from "../rbac/rbac.guard.js";
|
||||
import { FileRangeNotSatisfiableException } from "./playback-range.js";
|
||||
import { DIRECT_SEGMENT_BYTES, directSealedSize, directStorageDisabled, type StorageService } from "../storage/storage.js";
|
||||
import { hashShareToken } from "./share-token.js";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { BadRequestException, ConflictException, ForbiddenException, GoneException, NotFoundException } from "@nestjs/common";
|
||||
|
||||
/** Deterministic 64-hex helper: repeat a pattern to exactly 64 chars. */
|
||||
|
|
@ -322,12 +328,20 @@ describe("GroupFilesService invariants", () => {
|
|||
const { createHash } = await import("node:crypto");
|
||||
const digest = createHash("sha256").update(payload).digest("hex");
|
||||
const { Readable } = await import("node:stream");
|
||||
const fetcher = (async () =>
|
||||
new Response(Readable.toWeb(Readable.from([payload])) as unknown as ReadableStream, {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/octet-stream", "content-length": String(payload.byteLength) },
|
||||
})) as unknown as typeof fetch;
|
||||
const service = new GroupFilesService({ fetcher });
|
||||
// The R2-backed payload path: the storage stub serves the exact bytes.
|
||||
const storage = {
|
||||
putContent: async () => { throw new Error("not used in this test"); },
|
||||
getObject: async () => payload,
|
||||
hasContent: async () => true,
|
||||
presign: async () => "",
|
||||
};
|
||||
behavior = (sql) => {
|
||||
if (sql.includes("share_for_public_claim")) {
|
||||
return { rows: [{ id: "s1", workspace_id: row.workspace_id, item_id: row.id, access: "public", expires_at: new Date(Date.now() + 3600_000), revoked_at: null }], rowCount: 1 };
|
||||
}
|
||||
return { rows: [{ ...row, sha256: digestOf("payload-bytes"), storage_key: "sha256/testing" }], rowCount: 1 };
|
||||
};
|
||||
const service = new GroupFilesService({ storage: storage as never });
|
||||
const handle = await service.claimPublicDownload("some-token");
|
||||
expect(handle.item.id).toBe(row.id);
|
||||
expect(handle.item.checksum.value).toBe(digest);
|
||||
|
|
@ -347,3 +361,310 @@ describe("GroupFilesService invariants", () => {
|
|||
await expect(service.claimPublicDownload("nope")).rejects.toBeInstanceOf(NotFoundException);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe("direct multipart uploads", () => {
|
||||
const actor = { userId: "user-1", role: "Publisher" };
|
||||
const objectKey = "direct/multipart/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa.enc";
|
||||
const plainSizeBytes = DIRECT_SEGMENT_BYTES + 1;
|
||||
const input = {
|
||||
uploadId: "opaque-s3-upload-id+/=", objectKey, groupId: row.group_id,
|
||||
filename: "clip.mp4", mimeType: "video/mp4", plainSizeBytes, sha256: "a".repeat(64),
|
||||
parts: [{ ETag: '"etag-1"', PartNumber: 1 }, { ETag: '"etag-2"', PartNumber: 2 }],
|
||||
};
|
||||
let storage: StorageService;
|
||||
let service: GroupFilesService;
|
||||
|
||||
beforeEach(() => {
|
||||
behavior = () => ({ rows: [row], rowCount: 1 });
|
||||
storage = {
|
||||
...directStorageDisabled(), putContent: vi.fn(), getObject: vi.fn(), hasContent: vi.fn(), presign: vi.fn(),
|
||||
directPublicJwk: vi.fn(async () => ({ kty: "EC", crv: "P-256", x: "x", y: "y" } as const)),
|
||||
createMultipart: vi.fn(async () => ({ uploadId: input.uploadId })),
|
||||
presignPart: vi.fn(async (_key, _upload, part) => `https://storage.example/part/${part}`),
|
||||
completeMultipart: vi.fn(async () => undefined), abortMultipart: vi.fn(async () => undefined),
|
||||
headObject: vi.fn(async () => ({ exists: true, contentLength: directSealedSize(plainSizeBytes) })),
|
||||
};
|
||||
service = new GroupFilesService({ storage });
|
||||
vi.spyOn(service, "registerArchiveEntry").mockResolvedValue({ id: row.id, claimKey: "claim" });
|
||||
});
|
||||
|
||||
it("initializes the full 2 GiB as 205 ordered part URLs", async () => {
|
||||
const result = await service.initDirectMultipartUpload(row.workspace_id, row.group_id, actor, {
|
||||
filename: "clip.mp4", mimeType: "video/mp4", sizeBytes: 2 * 1024 ** 3,
|
||||
});
|
||||
expect(result.partCount).toBe(205);
|
||||
expect(result.partUrls).toHaveLength(205);
|
||||
expect(result.uploadId).toBe(input.uploadId);
|
||||
expect(result.objectKey).toMatch(/^direct\/multipart\/[a-f0-9-]+\.enc$/);
|
||||
expect(storage.createMultipart).toHaveBeenCalledWith(result.objectKey, "application/octet-stream");
|
||||
expect(storage.presignPart).toHaveBeenNthCalledWith(205, result.objectKey, input.uploadId, 205, 900);
|
||||
});
|
||||
|
||||
it("completes before HEAD and registers plaintext bytes after exact size verification", async () => {
|
||||
await service.completeDirectMultipartUpload(row.workspace_id, actor, input);
|
||||
expect(storage.completeMultipart).toHaveBeenCalledWith(objectKey, input.uploadId, input.parts);
|
||||
expect(vi.mocked(storage.completeMultipart).mock.invocationCallOrder[0])
|
||||
.toBeLessThan(vi.mocked(storage.headObject).mock.invocationCallOrder[0]!);
|
||||
expect(service.registerArchiveEntry).toHaveBeenCalledWith(row.workspace_id, expect.objectContaining({
|
||||
byteSize: plainSizeBytes, storageKey: objectKey, groupId: row.group_id, sha256: input.sha256,
|
||||
}));
|
||||
expect(storage.abortMultipart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
[], [input.parts[0]], [...input.parts, { ETag: "extra", PartNumber: 3 }],
|
||||
[{ ETag: "", PartNumber: 1 }, input.parts[1]],
|
||||
[{ ETag: "one", PartNumber: 1 }, { ETag: "two", PartNumber: 1 }],
|
||||
[input.parts[1], input.parts[0]], null,
|
||||
].map((parts) => ({ parts })))("rejects an invalid ETag list and aborts before completion: $parts", async ({ parts }) => {
|
||||
await expect(service.completeDirectMultipartUpload(row.workspace_id, actor, { ...input, parts }))
|
||||
.rejects.toBeInstanceOf(BadRequestException);
|
||||
expect(storage.abortMultipart).toHaveBeenCalledWith(objectKey, input.uploadId);
|
||||
expect(storage.completeMultipart).not.toHaveBeenCalled();
|
||||
expect(service.registerArchiveEntry).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([null, directSealedSize(plainSizeBytes) - 1, directSealedSize(plainSizeBytes) + 1, plainSizeBytes])(
|
||||
"rejects missing or inexact sealed size %s without registering", async (contentLength) => {
|
||||
vi.mocked(storage.headObject).mockResolvedValue({ exists: contentLength !== null, contentLength });
|
||||
await expect(service.completeDirectMultipartUpload(row.workspace_id, actor, input)).rejects.toMatchObject({ status: 400 });
|
||||
expect(storage.abortMultipart).toHaveBeenCalledWith(objectKey, input.uploadId);
|
||||
expect(service.registerArchiveEntry).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("aborts when presigning a part fails", async () => {
|
||||
vi.mocked(storage.presignPart).mockRejectedValue(new Error("unavailable"));
|
||||
await expect(service.initDirectMultipartUpload(row.workspace_id, row.group_id, actor, {
|
||||
filename: "clip.mp4", mimeType: "video/mp4", sizeBytes: plainSizeBytes,
|
||||
})).rejects.toMatchObject({ status: 503 });
|
||||
expect(storage.abortMultipart).toHaveBeenCalledWith(expect.stringMatching(/^direct\/multipart\//), input.uploadId);
|
||||
});
|
||||
|
||||
it("rejects an oversize file before allocating an upload", async () => {
|
||||
await expect(service.initDirectMultipartUpload(row.workspace_id, row.group_id, actor, {
|
||||
filename: "clip.mp4", mimeType: "video/mp4", sizeBytes: 2 * 1024 ** 3 + 1,
|
||||
})).rejects.toMatchObject({ status: 400 });
|
||||
expect(storage.createMultipart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps multipart plaintext metadata intact for range playback", async () => {
|
||||
behavior = () => ({ rows: [{ ...row, byte_size: String(plainSizeBytes), storage_key: objectKey }], rowCount: 1 });
|
||||
const result = await service.openRangePlayback(row.workspace_id, actor.userId, actor.role, row.id, plainSizeBytes - 1, 1);
|
||||
const { token } = result.ticket;
|
||||
storage.readDirectRange = vi.fn(async () => Buffer.from("x"));
|
||||
const slice = await service.readPlaybackTicket(token, String(plainSizeBytes - 1), String(plainSizeBytes - 1));
|
||||
expect(slice.total).toBe(plainSizeBytes);
|
||||
expect(storage.readDirectRange).toHaveBeenCalledWith(objectKey, plainSizeBytes - 1, 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("range playback capabilities", () => {
|
||||
const payload = Buffer.from("0123456789");
|
||||
let service: GroupFilesService;
|
||||
let storage: StorageService;
|
||||
|
||||
beforeEach(() => {
|
||||
behavior = () => ({ rows: [{ ...row, byte_size: "177", storage_key: "direct/video.enc" }], rowCount: 1 });
|
||||
storage = {
|
||||
...directStorageDisabled(),
|
||||
putContent: vi.fn(), hasContent: vi.fn(), presign: vi.fn(), getObject: vi.fn(), getSealedObject: vi.fn(),
|
||||
readDirectRange: vi.fn(async (_key, start, length) => payload.subarray(start, start + length)),
|
||||
};
|
||||
service = new GroupFilesService({ storage });
|
||||
});
|
||||
|
||||
afterEach(() => { vi.useRealTimers(); });
|
||||
|
||||
async function issue(): Promise<string> {
|
||||
const result = await service.openRangePlayback(row.workspace_id, "user-1", "Publisher", row.id, 2, 4);
|
||||
return result.ticket.token;
|
||||
}
|
||||
|
||||
function responseFixture(): { response: Response; headers: Record<string, string>; chunks: Buffer[]; status: () => number } {
|
||||
const headers: Record<string, string> = {};
|
||||
const chunks: Buffer[] = [];
|
||||
let code = 0;
|
||||
const response = {
|
||||
status(value: number) { code = value; return this; },
|
||||
setHeader(key: string, value: string) { headers[key] = value; return this; },
|
||||
flushHeaders: vi.fn(), write: (bytes: Buffer) => chunks.push(bytes), end: vi.fn(),
|
||||
} as unknown as Response;
|
||||
return { response, headers, chunks, status: () => code };
|
||||
}
|
||||
|
||||
it("issues a 256-bit single-file ticket with exactly a 10-minute TTL without reading payloads", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-10-18T12:00:00Z"));
|
||||
const { ticket } = await service.openRangePlayback(row.workspace_id, "user-1", "Publisher", row.id, 2, 4);
|
||||
expect(ticket).toEqual({ token: expect.stringMatching(/^[a-f0-9]{64}$/), itemId: row.id, expiresAt: "2026-10-18T12:10:00.000Z" });
|
||||
expect(storage.readDirectRange).not.toHaveBeenCalled();
|
||||
// The ticket covers its one file, allowing later seeks beyond the initial range.
|
||||
const slice = await service.readPlaybackTicket(ticket.token, "6", "8");
|
||||
expect(slice.bytes.toString()).toBe("678");
|
||||
expect(slice.item.id).toBe(row.id);
|
||||
expect(storage.readDirectRange).toHaveBeenCalledWith("direct/video.enc", 6, 3);
|
||||
expect(storage.getObject).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("expires tickets at the 10-minute boundary and rejects unknown tokens before any DB/storage access", async () => {
|
||||
vi.useFakeTimers();
|
||||
const token = await issue();
|
||||
vi.advanceTimersByTime(10 * 60_000 - 1);
|
||||
await expect(service.readPlaybackTicket(token, "0", "1")).resolves.toMatchObject({ start: 0, end: 1 });
|
||||
const count = statements.length;
|
||||
vi.advanceTimersByTime(1);
|
||||
await expect(service.readPlaybackTicket(token, "0", "1")).rejects.toBeInstanceOf(NotFoundException);
|
||||
await expect(service.readPlaybackTicket("unknown", "0", "1")).rejects.toBeInstanceOf(NotFoundException);
|
||||
expect(statements).toHaveLength(count);
|
||||
expect(storage.readDirectRange).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("checks group membership before issuance and again on use, and rejects legacy objects", async () => {
|
||||
behavior = (sql) => sql.includes("FROM group_members")
|
||||
? { rows: [], rowCount: 0 } : { rows: [{ ...row, byte_size: "177", storage_key: "direct/video.enc" }], rowCount: 1 };
|
||||
await expect(issue()).rejects.toBeInstanceOf(ForbiddenException);
|
||||
behavior = () => ({ rows: [{ ...row, storage_key: "sha256/legacy.mp4" }], rowCount: 1 });
|
||||
await expect(issue()).rejects.toBeInstanceOf(BadRequestException);
|
||||
behavior = () => ({ rows: [{ ...row, byte_size: "177", storage_key: "direct/video.enc" }], rowCount: 1 });
|
||||
const token = await issue();
|
||||
behavior = (sql) => sql.includes("FROM group_members")
|
||||
? { rows: [], rowCount: 0 } : { rows: [{ ...row, byte_size: "177", storage_key: "direct/video.enc" }], rowCount: 1 };
|
||||
await expect(service.readPlaybackTicket(token, "0", "1")).rejects.toBeInstanceOf(ForbiddenException);
|
||||
expect(storage.readDirectRange).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ available: false },
|
||||
{ retention_mode: "until", retention_until: new Date("2020-01-01T00:00:00Z") },
|
||||
])("stops playback of unavailable files: %j", async (patch) => {
|
||||
const token = await issue();
|
||||
behavior = () => ({ rows: [{ ...row, byte_size: "177", storage_key: "direct/video.enc", ...patch }], rowCount: 1 });
|
||||
await expect(service.readPlaybackTicket(token, "0", "1")).rejects.toBeInstanceOf(GoneException);
|
||||
await expect(issue()).rejects.toBeInstanceOf(GoneException);
|
||||
expect(storage.readDirectRange).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("serves 206 with plaintext size (sealed metadata overhead removed) and no auth guard", async () => {
|
||||
expect(Reflect.getMetadata(GUARDS_METADATA, FilePlaybackController)).toBeUndefined();
|
||||
expect(Reflect.getMetadata(GUARDS_METADATA, FilePlaybackController.prototype.play)).toBeUndefined();
|
||||
expect(Reflect.getMetadata(GUARDS_METADATA, GroupFilesController)).toEqual([RbacGuard]);
|
||||
const token = await issue();
|
||||
const out = responseFixture();
|
||||
await new FilePlaybackController(service).play(token, { start: "2", end: "5" }, undefined, out.response);
|
||||
expect(out.status()).toBe(206);
|
||||
expect(out.headers).toEqual({
|
||||
"Cache-Control": "no-store", "Accept-Ranges": "bytes", "Content-Type": "video/mp4",
|
||||
"Content-Range": "bytes 2-5/10", "Content-Length": "4",
|
||||
});
|
||||
expect(Buffer.concat(out.chunks).toString()).toBe("2345");
|
||||
expect(out.response.flushHeaders).toHaveBeenCalled();
|
||||
expect(out.response.end).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["bytes=7-99", "789", 7, 9],
|
||||
["bytes=8-", "89", 8, 9],
|
||||
["bytes=-3", "789", 7, 9],
|
||||
["bytes=-100", "0123456789", 0, 9],
|
||||
[undefined, "0123456789", 0, 9],
|
||||
])("supports browser Range header %s and clips to EOF", async (range, expected, start, end) => {
|
||||
const slice = await service.readPlaybackTicket(await issue(), undefined, undefined, range);
|
||||
expect(slice.bytes.toString()).toBe(expected);
|
||||
expect(slice).toMatchObject({ start, end, total: 10 });
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ start: "10", end: "20" }, { start: "4", end: "3" }, { start: "-1", end: "4" },
|
||||
{ start: "0.5", end: "4" }, { start: "9007199254740992" }, { start: ["0", "1"] },
|
||||
])("returns 416 and bytes */total for invalid/unsatisfiable query %j", async (query) => {
|
||||
const out = responseFixture();
|
||||
await new FilePlaybackController(service).play(await issue(), query, undefined, out.response);
|
||||
expect(out.status()).toBe(416);
|
||||
expect(out.headers["Content-Range"]).toBe("bytes */10");
|
||||
expect(out.headers["Accept-Ranges"]).toBe("bytes");
|
||||
expect(out.headers["Content-Length"]).toBe("0");
|
||||
expect(out.chunks).toEqual([]);
|
||||
expect(storage.readDirectRange).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["bytes=10-", "bytes=3-2", "bytes=-0", "bytes=", "bytes=0-1,3-4", "bytes=a-b"])("rejects invalid/unsatisfiable Range %s", async (range) => {
|
||||
await expect(service.readPlaybackTicket(await issue(), undefined, undefined, range)).rejects.toBeInstanceOf(FileRangeNotSatisfiableException);
|
||||
expect(storage.readDirectRange).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns 416 before storage for an unsatisfiable initial range", async () => {
|
||||
await expect(service.openRangePlayback(row.workspace_id, "user-1", "Owner", row.id, 10, 1)).rejects.toMatchObject({ status: 416 });
|
||||
expect(storage.readDirectRange).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("unwraps browser-download CEKs from a header-only fetch", async () => {
|
||||
storage.presign = vi.fn(async () => "https://storage.example/video");
|
||||
storage.openDirectHeaderBlob = vi.fn(async () => Buffer.alloc(135));
|
||||
storage.openDirectHeader = vi.fn(async () => Buffer.alloc(32, 1));
|
||||
const ticket = await service.openDirectDownload(row.workspace_id, "u", "Owner", row.id);
|
||||
expect(ticket.frame).toBe("SPENC2");
|
||||
expect(storage.openDirectHeaderBlob).toHaveBeenCalledWith("direct/video.enc");
|
||||
expect(storage.getSealedObject).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("member-delete-own permissions", () => {
|
||||
// Default-true is enforced by migrations 045/046; both explicit setting values
|
||||
// are exercised here for members and the Owner/Admin override.
|
||||
const cases = ["Publisher", "Editor", "Owner", "Admin"].flatMap((role) =>
|
||||
[true, false].flatMap((enabled) => [true, false].map((own) => ({ role, enabled, own,
|
||||
allowed: role === "Owner" || role === "Admin" || (enabled && own) }))));
|
||||
|
||||
it.each(cases)("$role setting=$enabled own=$own => allowed=$allowed", async ({ role, enabled, own, allowed }) => {
|
||||
let revoked = false;
|
||||
behavior = (sql) => {
|
||||
if (sql.includes("SELECT g.member_delete_own")) return { rows: [{ member_delete_own: enabled, username: "alice" }], rowCount: 1 };
|
||||
if (sql.includes("FROM group_members")) return { rows: [{}], rowCount: 1 };
|
||||
if (sql.startsWith("UPDATE chat_media_archive")) revoked = true;
|
||||
return { rows: [{ ...row, origin: "upload", created_by: own ? "alice-id" : "bob-id", sender_mxid: own ? "@alice:tmbh.local" : "@bob:tmbh.local", available: !revoked }], rowCount: 1 };
|
||||
};
|
||||
const service = new GroupFilesService();
|
||||
const result = service.deleteFile(row.workspace_id, row.id, { userId: "alice-id", role });
|
||||
if (allowed) {
|
||||
await expect(result).resolves.toMatchObject({ status: "revoked" });
|
||||
expect(statements.some((q) => q.sql.startsWith("UPDATE group_file_shares SET revoked_at"))).toBe(true);
|
||||
expect(statements.find((q) => q.sql.startsWith("SELECT audit_append"))?.values[3]).toBe("file.deleted");
|
||||
} else {
|
||||
await expect(result).rejects.toBeInstanceOf(ForbiddenException);
|
||||
expect(statements.some((q) => q.sql.startsWith("UPDATE"))).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["matrix", "bridge", "pad-export"])("does not infer ownership of %s attachments from sender_mxid", async (origin) => {
|
||||
behavior = (sql) => {
|
||||
if (sql.includes("SELECT g.member_delete_own")) return { rows: [{ member_delete_own: true, username: "alice" }], rowCount: 1 };
|
||||
return { rows: [{ ...row, origin, sender_mxid: "@alice:tmbh.local" }], rowCount: 1 };
|
||||
};
|
||||
await expect(new GroupFilesService().deleteFile(row.workspace_id, row.id, { userId: "alice-id", role: "Publisher" }))
|
||||
.rejects.toBeInstanceOf(ForbiddenException);
|
||||
});
|
||||
|
||||
it("denies former/non-members even for their own uploads", async () => {
|
||||
behavior = (sql) => sql.includes("FROM group_members") ? { rows: [], rowCount: 0 }
|
||||
: { rows: [{ ...row, origin: "upload", sender_mxid: "@alice:tmbh.local" }], rowCount: 1 };
|
||||
await expect(new GroupFilesService().deleteFile(row.workspace_id, row.id, { userId: "alice-id", role: "Publisher" }))
|
||||
.rejects.toBeInstanceOf(ForbiddenException);
|
||||
expect(statements.some((q) => q.sql.startsWith("UPDATE"))).toBe(false);
|
||||
});
|
||||
|
||||
it("applies the setting to the revoke route too", async () => {
|
||||
behavior = (sql) => sql.includes("SELECT g.member_delete_own")
|
||||
? { rows: [{ member_delete_own: false, username: "alice" }], rowCount: 1 }
|
||||
: { rows: [{ ...row, origin: "upload", sender_mxid: "@alice:tmbh.local" }], rowCount: 1 };
|
||||
await expect(new GroupFilesService().markRevoked(row.workspace_id, row.id, { userId: "alice-id", role: "Publisher" }))
|
||||
.rejects.toBeInstanceOf(ForbiddenException);
|
||||
});
|
||||
|
||||
it.each(["Owner", "Admin"])("preserves legal hold even for %s deletion", async (role) => {
|
||||
behavior = () => ({ rows: [{ ...row, legal_hold: true }], rowCount: 1 });
|
||||
await expect(new GroupFilesService().deleteFile(row.workspace_id, row.id, { userId: "u", role }))
|
||||
.rejects.toThrow("legal hold");
|
||||
expect(statements.some((q) => q.sql.startsWith("UPDATE"))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -16,7 +16,7 @@ import {
|
|||
requireRoomId,
|
||||
requireSha256,
|
||||
} from "./group-files.validation.js";
|
||||
import { OcisVerificationError, ocisConfigFromEnv, OcisWebdavClient, verifyWhileStreaming } from "./ocis-webdav.js";
|
||||
import { OcisVerificationError, verifyWhileStreaming } from "./verify-stream.js";
|
||||
import { hashShareToken, isPlausibleShareToken, mintShareToken, sameToken } from "./share-token.js";
|
||||
|
||||
/** Deterministic 64-hex helper: repeat a pattern to exactly 64 chars. */
|
||||
|
|
@ -129,6 +129,7 @@ describe("group-files runtime validation (shared contracts)", () => {
|
|||
id: "b3a1c2d4-1111-4222-8333-444455556666",
|
||||
workspaceId: "c3a1c2d4-1111-4222-8333-444455556666",
|
||||
groupId: "d3a1c2d4-1111-4222-8333-444455556666",
|
||||
ownerId: null, folderId: null, createdBy: null, shared: false,
|
||||
roomId: "!r:example.org",
|
||||
senderId: "@s:example.org",
|
||||
eventId: "$evt",
|
||||
|
|
@ -146,6 +147,11 @@ describe("group-files runtime validation (shared contracts)", () => {
|
|||
status: "available",
|
||||
};
|
||||
expect(parseGroupFile(value)).toEqual(value);
|
||||
const personal = { ...value, groupId: null, ownerId: value.id, folderId: value.id, createdBy: value.id,
|
||||
file: { ...value.file, spaceId: null, storageKey: "sha256/key.mp4" } };
|
||||
expect(parseGroupFile(personal)).toEqual(personal);
|
||||
const shared = { ...value, shared: true, origin: { kind: "upload", sharedFrom: "personal" } };
|
||||
expect(parseGroupFile(shared)).toEqual(shared);
|
||||
expect(() => parseGroupFile({ ...value, sizeBytes: -1 })).toThrow(BadRequestException);
|
||||
expect(() => parseGroupFile({ ...value, senderId: "Bob" })).toThrow(BadRequestException);
|
||||
expect(() => parseGroupFile({ ...value, checksum: { algorithm: "sha256", value: "zz" } })).toThrow(BadRequestException);
|
||||
|
|
@ -185,59 +191,7 @@ describe("share tokens", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("ocis-webdav config + streaming verification", () => {
|
||||
it("reads env lazily and throws 503 when unconfigured", () => {
|
||||
expect(() => ocisConfigFromEnv({})).toThrow(ServiceUnavailableException);
|
||||
expect(() => ocisConfigFromEnv({ FILES_OCID_WEBDAV_URL: "https://files.example.org" })).toThrow(ServiceUnavailableException);
|
||||
const config = ocisConfigFromEnv({
|
||||
FILES_OCID_WEBDAV_URL: "https://files.example.org/",
|
||||
FILES_OCID_USER: "svc",
|
||||
FILES_OCID_PASSWORD: "pw",
|
||||
FILES_PUBLIC_BASE: "https://files.example.org/",
|
||||
});
|
||||
expect(config.webdavUrl).toBe("https://files.example.org");
|
||||
expect(config.publicBase).toBe("https://files.example.org");
|
||||
});
|
||||
|
||||
it("uploads hash the actual streamed bytes and surface oc-fileid/oc-etag", async () => {
|
||||
const bytes = Buffer.from("group file payload");
|
||||
const calls: string[] = [];
|
||||
const fetcher = (async (url: string, init: RequestInit) => {
|
||||
calls.push(`${init.method} ${url}`);
|
||||
return new Response(null, {
|
||||
status: 201,
|
||||
headers: init.method === "MKCOL" ? new Headers() : { "oc-fileid": "res-1", "oc-etag": "v-9" },
|
||||
});
|
||||
}) as unknown as typeof fetch;
|
||||
const client = new OcisWebdavClient(
|
||||
{ webdavUrl: "https://files.example.org", user: "svc", password: "pw", publicBase: "https://files.example.org" },
|
||||
fetcher,
|
||||
);
|
||||
const result = await client.upload("group-1", "clip.mp4", Readable.from([bytes]), "video/mp4");
|
||||
const { createHash } = await import("node:crypto");
|
||||
expect(result.sha256).toBe(createHash("sha256").update(bytes).digest("hex"));
|
||||
expect(result.sizeBytes).toBe(bytes.byteLength);
|
||||
expect(result.resourceId).toBe("res-1");
|
||||
expect(result.version).toBe("v-9");
|
||||
expect(calls[0]).toBe("MKCOL https://files.example.org/remote.php/dav/files/svc/hub-groups");
|
||||
expect(calls[1]).toBe("MKCOL https://files.example.org/remote.php/dav/files/svc/hub-groups/group-1");
|
||||
expect(calls[2]).toBe("PUT https://files.example.org/remote.php/dav/files/svc/hub-groups/group-1/clip.mp4");
|
||||
expect(client.uploadUrl("g", "n.mp4")).toContain("/remote.php/dav/files/svc/hub-groups/g/n.mp4");
|
||||
expect(client.libraryUrl("res-1")).toBe("https://files.example.org/f/res-1");
|
||||
});
|
||||
|
||||
it("fails the upload when the group folder cannot be created", async () => {
|
||||
const fetcher = (async (url: string, init: RequestInit) =>
|
||||
new Response(null, { status: init.method === "MKCOL" ? 500 : 201 })) as unknown as typeof fetch;
|
||||
const client = new OcisWebdavClient(
|
||||
{ webdavUrl: "https://files.example.org", user: "svc", password: "pw", publicBase: "https://files.example.org" },
|
||||
fetcher,
|
||||
);
|
||||
await expect(
|
||||
client.upload("group-1", "clip.mp4", Readable.from(["x"]), "text/plain"),
|
||||
).rejects.toThrow("oCIS folder creation failed with 500");
|
||||
});
|
||||
|
||||
describe("verify-stream (storage-agnostic streaming verification)", () => {
|
||||
it("verifyWhileStreaming passes on match and aborts on tamper", async () => {
|
||||
const { createHash } = await import("node:crypto");
|
||||
const payload = Buffer.from("0123456789");
|
||||
|
|
|
|||
28
apps/api/src/group-files/group-files.types.ts
Normal file
28
apps/api/src/group-files/group-files.types.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
import type { GroupFile as CoreGroupFile } from "@tactical-media-broadcast-hub/core";
|
||||
|
||||
/** Files wire shape, including personal roots and folder placement. */
|
||||
export interface GroupFile extends Omit<CoreGroupFile, "groupId" | "file" | "origin"> {
|
||||
groupId: string | null;
|
||||
ownerId: string | null;
|
||||
folderId: string | null;
|
||||
createdBy: string | null;
|
||||
shared: boolean;
|
||||
origin: Exclude<CoreGroupFile["origin"], { kind: "upload" }> | { kind: "upload"; sharedFrom?: "personal" };
|
||||
file: Omit<CoreGroupFile["file"], "spaceId"> & { spaceId: string | null };
|
||||
}
|
||||
|
||||
export interface FileFolder {
|
||||
id: string;
|
||||
workspaceId: string;
|
||||
groupId: string | null;
|
||||
ownerId: string | null;
|
||||
parentId: string | null;
|
||||
name: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface FileScopeInput {
|
||||
groupId?: unknown;
|
||||
personal?: unknown;
|
||||
folderId?: unknown;
|
||||
}
|
||||
|
|
@ -16,12 +16,11 @@ import { BadRequestException } from "@nestjs/common";
|
|||
import { MIME_CLASSES } from "@tactical-media-broadcast-hub/core";
|
||||
import type {
|
||||
FileRetention,
|
||||
GroupFile,
|
||||
GroupFileMapping,
|
||||
MatrixAttachmentCandidate,
|
||||
MimeClass,
|
||||
SharedFileOrigin,
|
||||
} from "@tactical-media-broadcast-hub/core";
|
||||
import type { GroupFile } from "./group-files.types.js";
|
||||
|
||||
// FileRetention (packages/core/src/shared-files.ts): mode is keep|until and
|
||||
// legalHold is an orthogonal boolean flag — NOT a third mode.
|
||||
|
|
@ -142,7 +141,7 @@ export function requireIsoTimestamp(value: unknown, field: string): string {
|
|||
}
|
||||
|
||||
/** The four SharedFileOrigin variants, discriminated by `kind`. */
|
||||
export function parseSharedFileOrigin(value: unknown, field = "origin"): SharedFileOrigin {
|
||||
export function parseSharedFileOrigin(value: unknown, field = "origin"): GroupFile["origin"] {
|
||||
if (!isRecord(value)) throw new BadRequestException(`${field} must be an object`);
|
||||
const kind = enumField(value, "kind", ORIGIN_KINDS, `${field}.kind`);
|
||||
if (kind === "matrix") return { kind, mxcUri: requireMxcUri(value.mxcUri, `${field}.mxcUri`) };
|
||||
|
|
@ -154,7 +153,10 @@ export function parseSharedFileOrigin(value: unknown, field = "origin"): SharedF
|
|||
...(value.remoteEventId === undefined || value.remoteEventId === null ? {} : { remoteEventId: stringField(value, "remoteEventId", `${field}.remoteEventId`) }),
|
||||
};
|
||||
}
|
||||
if (kind === "upload") return { kind };
|
||||
if (kind === "upload") {
|
||||
if (value.sharedFrom !== undefined && value.sharedFrom !== "personal") throw new BadRequestException(`${field}.sharedFrom must be personal`);
|
||||
return { kind, ...(value.sharedFrom === "personal" ? { sharedFrom: "personal" as const } : {}) };
|
||||
}
|
||||
return { kind: "pad-export", padId: stringField(value, "padId", `${field}.padId`), revision: stringField(value, "revision", `${field}.revision`) };
|
||||
}
|
||||
|
||||
|
|
@ -215,7 +217,11 @@ export function parseGroupFile(value: unknown, field = "item"): GroupFile {
|
|||
return {
|
||||
id: stringField(value, "id", `${field}.id`),
|
||||
workspaceId: stringField(value, "workspaceId", `${field}.workspaceId`),
|
||||
groupId: stringField(value, "groupId", `${field}.groupId`),
|
||||
groupId: value.groupId === null ? null : stringField(value, "groupId", `${field}.groupId`),
|
||||
ownerId: nullableUuid(value.ownerId, `${field}.ownerId`),
|
||||
folderId: nullableUuid(value.folderId, `${field}.folderId`),
|
||||
createdBy: nullableUuid(value.createdBy, `${field}.createdBy`),
|
||||
shared: value.shared === undefined ? false : booleanField(value.shared, `${field}.shared`),
|
||||
roomId: value.roomId === undefined || value.roomId === null ? null : requireRoomId(value.roomId, `${field}.roomId`),
|
||||
senderId: requireMxid(value.senderId, `${field}.senderId`),
|
||||
eventId: eventId === undefined || eventId === null ? null : eventId,
|
||||
|
|
@ -228,9 +234,10 @@ export function parseGroupFile(value: unknown, field = "item"): GroupFile {
|
|||
createdAt: isoField(value, "createdAt", `${field}.createdAt`),
|
||||
indexedAt: isoField(value, "indexedAt", `${field}.indexedAt`),
|
||||
file: {
|
||||
spaceId: stringField(file, "spaceId", `${field}.file.spaceId`),
|
||||
spaceId: file.spaceId === null ? null : stringField(file, "spaceId", `${field}.file.spaceId`),
|
||||
resourceId: stringField(file, "resourceId", `${field}.file.resourceId`),
|
||||
version: stringField(file, "version", `${field}.file.version`),
|
||||
...(file.storageKey === undefined ? {} : { storageKey: stringField(file, "storageKey", `${field}.file.storageKey`) }),
|
||||
},
|
||||
shareLink: {
|
||||
url: stringField(shareLink, "url", `${field}.shareLink.url`),
|
||||
|
|
@ -241,3 +248,16 @@ export function parseGroupFile(value: unknown, field = "item"): GroupFile {
|
|||
status: enumField(value, "status", FILE_STATUSES, `${field}.status`),
|
||||
};
|
||||
}
|
||||
|
||||
function nullableUuid(value: unknown, field: string): string | null {
|
||||
if (value === undefined || value === null) return null;
|
||||
if (typeof value !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value)) {
|
||||
throw new BadRequestException(`${field} must be a UUID or null`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function booleanField(value: unknown, field: string): boolean {
|
||||
if (typeof value !== "boolean") throw new BadRequestException(`${field} must be a boolean`);
|
||||
return value;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,323 +0,0 @@
|
|||
/**
|
||||
* oCIS WebDAV port — the ONLY payload I/O path for group files.
|
||||
*
|
||||
* The API streams bytes through itself on both upload and download: it NEVER
|
||||
* mints or persists a presigned object URL (docs/occupymedia-storage.md
|
||||
* "Persist library links/share IDs and expiry, not presigned object URLs").
|
||||
* Downloads re-verify the SHA-256 while streaming so storage-level tampering
|
||||
* is detected instead of silently relayed.
|
||||
*
|
||||
* Server-side COPY ("share into a collective's library") moves no bytes
|
||||
* through the API at all: oCIS performs the copy in the storage backend and
|
||||
* the API only learns the resulting file id / etag / length, which is exactly
|
||||
* the metadata the group files index stores. The copy is verified by oCIS
|
||||
* itself; the index records what the server committed, never a claimed length.
|
||||
*
|
||||
* ENV CONTRACT (lazy getters — read per call so tests/parallel deployments
|
||||
* can reconfigure without a rebuild; nothing is read at import time):
|
||||
*
|
||||
* FILES_OCID_WEBDAV_URL oCIS WebDAV root, e.g. https://files.example.org
|
||||
* FILES_OCID_USER service account username (the scoped oCIS identity)
|
||||
* FILES_OCID_PASSWORD service account password
|
||||
* FILES_PUBLIC_BASE public base of the file library, used to build the
|
||||
* persistent (authenticated) library link returned in
|
||||
* GroupFile.shareLink.url — a stable resource URL,
|
||||
* NEVER a presigned object URL.
|
||||
*
|
||||
* The oCIS deployment lands in parallel; when the env is unset the port
|
||||
* reports unconfigured and routes fail 503 rather than silently no-op.
|
||||
*
|
||||
* WebDAV shape: PUT returns the stable resource id (oc-fileid) and ETag
|
||||
* (oc-etag / etag) this index needs; plain PUT/GET carry the bytes. The
|
||||
* transport is injectable (OCIS_FETCH token) so tests stub it without HTTP.
|
||||
*/
|
||||
import { ServiceUnavailableException } from "@nestjs/common";
|
||||
import { createHash } from "node:crypto";
|
||||
import { Readable } from "node:stream";
|
||||
|
||||
export interface OcisEnvConfig {
|
||||
webdavUrl: string;
|
||||
user: string;
|
||||
password: string;
|
||||
publicBase: string;
|
||||
}
|
||||
|
||||
export function ocisConfigFromEnv(env: Readonly<Record<string, string | undefined>> = process.env): OcisEnvConfig {
|
||||
const webdavUrl = env.FILES_OCID_WEBDAV_URL;
|
||||
const user = env.FILES_OCID_USER;
|
||||
const password = env.FILES_OCID_PASSWORD;
|
||||
if (webdavUrl === undefined || webdavUrl === "") throw new ServiceUnavailableException("FILES_OCID_WEBDAV_URL is not configured");
|
||||
if (user === undefined || user === "") throw new ServiceUnavailableException("FILES_OCID_USER is not configured");
|
||||
if (password === undefined || password === "") throw new ServiceUnavailableException("FILES_OCID_PASSWORD is not configured");
|
||||
return {
|
||||
webdavUrl: webdavUrl.replace(/\/+$/, ""),
|
||||
user,
|
||||
password,
|
||||
publicBase: (env.FILES_PUBLIC_BASE ?? "").replace(/\/+$/, ""),
|
||||
};
|
||||
}
|
||||
|
||||
/** Result of a committed upload: stable identity + digest over bytes sent. */
|
||||
export interface OcisUploadResult {
|
||||
resourceId: string;
|
||||
version: string | null;
|
||||
sha256: string;
|
||||
sizeBytes: number;
|
||||
}
|
||||
|
||||
/** An opened download: verifyWhileStreaming checks the digest before relay. */
|
||||
export interface OcisDownload {
|
||||
stream: Readable;
|
||||
sizeBytes: number | null;
|
||||
contentType: string | null;
|
||||
}
|
||||
|
||||
/** A server-side COPY result: identity + size, with the digest oCIS reports. */
|
||||
export interface OcisCopyResult {
|
||||
resourceId: string;
|
||||
version: string | null;
|
||||
sizeBytes: number;
|
||||
sha256: string | null;
|
||||
contentType: string;
|
||||
}
|
||||
|
||||
export class OcisVerificationError extends Error {
|
||||
constructor(expected: string, actual: string) {
|
||||
super(`oCIS payload verification failed: expected sha256 ${expected}, got ${actual}`);
|
||||
this.name = "OcisVerificationError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Minimal fetch surface so tests can stub transport without HTTP. */
|
||||
export type FetchLike = (url: string, init: RequestInit) => Promise<Response>;
|
||||
|
||||
export const OCIS_FETCH: unique symbol = Symbol("OCIS_FETCH");
|
||||
|
||||
/** Bytes allowed in flight to the transport before the pump pauses. */
|
||||
export const UPLOAD_BUFFER_CAP = 8 * 1024 * 1024;
|
||||
|
||||
/** Node's undici fetch needs this hint when a stream body is supplied. */
|
||||
type FetchInitWithDuplex = RequestInit & { duplex?: "half" };
|
||||
|
||||
/** Distinguishing marker for "the source path does not exist (404)". */
|
||||
export class OcisNotFoundError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "OcisNotFoundError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Distinguishing marker for "destination exists and Overwrite:F was sent (412)". */
|
||||
export class OcisPreconditionFailedError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "OcisPreconditionFailedError";
|
||||
}
|
||||
}
|
||||
|
||||
function encodeSegments(segments: readonly string[]): string {
|
||||
return segments.map((segment) => encodeURIComponent(segment)).join("/");
|
||||
}
|
||||
|
||||
/**
|
||||
* WebDAV client. `fetcher` defaults to the global fetch.
|
||||
*/
|
||||
export class OcisWebdavClient {
|
||||
constructor(
|
||||
private readonly config: OcisEnvConfig,
|
||||
private readonly fetcher: FetchLike = (url, init) => fetch(url, init),
|
||||
) {}
|
||||
|
||||
/** Group folder path under the service identity's WebDAV root. */
|
||||
uploadUrl(groupId: string, safeName: string): string {
|
||||
return `${this.config.webdavUrl}/remote.php/dav/files/${encodeURIComponent(this.config.user)}/hub-groups/${encodeURIComponent(groupId)}/${encodeURIComponent(safeName)}`;
|
||||
}
|
||||
|
||||
downloadUrl(resourceId: string): string {
|
||||
return `${this.config.webdavUrl}/remote.php/dav/files/${encodeURIComponent(this.config.user)}/hub-resources/${encodeURIComponent(resourceId)}`;
|
||||
}
|
||||
|
||||
/** Any path, as sanitized segments, under the service identity's WebDAV root. */
|
||||
private rootUrl(segments: readonly string[]): string {
|
||||
return `${this.config.webdavUrl}/remote.php/dav/files/${encodeURIComponent(this.config.user)}/${encodeSegments(segments)}`;
|
||||
}
|
||||
|
||||
/** MKCOL the group folder; 405 (already exists) is fine. */
|
||||
private async ensureGroupFolder(groupId: string): Promise<void> {
|
||||
const base = `${this.config.webdavUrl}/remote.php/dav/files/${encodeURIComponent(this.config.user)}`;
|
||||
// MKCOL returns 409 when the parent collection is missing — create the
|
||||
// hub-groups root (405 = already exists), then the per-group folder.
|
||||
const root = await this.fetcher(`${base}/hub-groups`, { method: "MKCOL", headers: this.authHeaders() });
|
||||
if (!root.ok && root.status !== 405) {
|
||||
throw new Error(`oCIS folder creation failed with ${root.status}`);
|
||||
}
|
||||
const url = `${base}/hub-groups/${encodeURIComponent(groupId)}`;
|
||||
const response = await this.fetcher(url, { method: "MKCOL", headers: this.authHeaders() });
|
||||
if (response.ok || response.status === 405) return;
|
||||
throw new Error(`oCIS folder creation failed with ${response.status}`);
|
||||
}
|
||||
|
||||
/** MKCOL a folder path (segments already sanitized); 405 = already there. */
|
||||
async ensureFolderPath(segments: readonly string[]): Promise<void> {
|
||||
if (segments.length === 0) return;
|
||||
const response = await this.fetcher(this.rootUrl(segments), { method: "MKCOL", headers: this.authHeaders() });
|
||||
if (response.ok || response.status === 405) return;
|
||||
throw new Error(`oCIS folder creation failed with ${response.status}`);
|
||||
}
|
||||
|
||||
private authHeaders(): Record<string, string> {
|
||||
const basic = Buffer.from(`${this.config.user}:${this.config.password}`, "utf8").toString("base64");
|
||||
return { authorization: `Basic ${basic}` };
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream-to-storage PUT, hashing while streaming (contract: "Hash actual
|
||||
* bytes while streaming"). The digest covers the exact bytes handed to the
|
||||
* store — never a claimed Content-Length.
|
||||
*
|
||||
* Backpressure: the pump pauses when more than UPLOAD_BUFFER_CAP bytes are
|
||||
* queued for the transport, so RAM stays bounded for large payloads. The
|
||||
* pump is driven by our own loop, so the digest completes even if the
|
||||
* transport returns without draining a small body (a 2xx store that never
|
||||
* reads its request body is broken and will surface as an integrity error).
|
||||
*/
|
||||
async upload(groupId: string, safeName: string, body: Readable, contentType: string): Promise<OcisUploadResult> {
|
||||
await this.ensureGroupFolder(groupId);
|
||||
// reva rejects chunked PUTs, so the source drains into a bounded buffer
|
||||
// (the upload cap is 20 MiB) and goes out with an explicit content-length.
|
||||
const hash = createHash("sha256");
|
||||
const chunks: Buffer[] = [];
|
||||
let sizeBytes = 0;
|
||||
for await (const chunk of body) {
|
||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array | string);
|
||||
hash.update(buffer);
|
||||
sizeBytes += buffer.byteLength;
|
||||
chunks.push(buffer);
|
||||
}
|
||||
const payload = Buffer.concat(chunks);
|
||||
const response = await this.fetcher(this.uploadUrl(groupId, safeName), {
|
||||
method: "PUT",
|
||||
headers: { ...this.authHeaders(), "content-type": contentType, "content-length": String(payload.byteLength) },
|
||||
body: payload,
|
||||
});
|
||||
if (!response.ok) throw new Error(`oCIS upload failed with ${response.status}`);
|
||||
const etag = response.headers.get("oc-etag") ?? response.headers.get("etag");
|
||||
const resourceId = response.headers.get("oc-fileid");
|
||||
return {
|
||||
resourceId: resourceId ?? safeName,
|
||||
version: etag,
|
||||
sha256: hash.digest("hex"),
|
||||
sizeBytes,
|
||||
};
|
||||
}
|
||||
|
||||
/** Open a GET. The digest is re-checked by verifyWhileStreaming downstream. */
|
||||
async download(resourceId: string): Promise<OcisDownload> {
|
||||
const response = await this.fetcher(this.downloadUrl(resourceId), { method: "GET", headers: this.authHeaders() });
|
||||
if (!response.ok || response.body === null) throw new Error(`oCIS download failed with ${response.status}`);
|
||||
const length = response.headers.get("content-length");
|
||||
return {
|
||||
stream: response.body as unknown as Readable,
|
||||
sizeBytes: length === null ? null : Number(length),
|
||||
contentType: response.headers.get("content-type"),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* PROPFIND (Depth: 0) on a path: existence + Content-Length + Content-Type.
|
||||
* Used by the share-into-library path to size and type the copy without
|
||||
* streaming a single payload byte through the API.
|
||||
*/
|
||||
async statPath(segments: readonly string[]): Promise<{ exists: boolean; sizeBytes: number; contentType: string }> {
|
||||
const body = `<?xml version="1.0" encoding="utf-8"?>\n<d:propfind xmlns:d="DAV:"><d:prop><d:getcontentlength/><d:getcontenttype/></d:prop></d:propfind>`;
|
||||
const response = await this.fetcher(this.rootUrl(segments), {
|
||||
method: "PROPFIND",
|
||||
headers: { ...this.authHeaders(), depth: "0", "content-type": "application/xml" },
|
||||
body,
|
||||
});
|
||||
if (response.status === 404) return { exists: false, sizeBytes: 0, contentType: "" };
|
||||
if (!response.ok) throw new Error(`oCIS PROPFIND failed with ${response.status}`);
|
||||
const xml = await response.text();
|
||||
const sizeBytes = Number(firstTag(xml, "getcontentlength") ?? "0");
|
||||
const contentType = firstTag(xml, "getcontenttype") ?? "application/octet-stream";
|
||||
if (!Number.isSafeInteger(sizeBytes) || sizeBytes < 0) throw new Error("oCIS PROPFIND returned an invalid content length");
|
||||
return { exists: true, sizeBytes, contentType };
|
||||
}
|
||||
|
||||
/**
|
||||
* Server-side COPY of a source path into the collective's group folder
|
||||
* (optionally inside a sub-folder). `Overwrite: F` makes a collision a 412
|
||||
* the caller can retry with a suffixed name — a share never clobbers an
|
||||
* existing library file.
|
||||
*/
|
||||
async copyIntoGroup(
|
||||
groupId: string,
|
||||
sourceSegments: readonly string[],
|
||||
destinationFolder: readonly string[],
|
||||
name: string,
|
||||
): Promise<OcisCopyResult> {
|
||||
await this.ensureFolderPath(["hub-groups", groupId, ...destinationFolder]);
|
||||
const destination = this.rootUrl(["hub-groups", groupId, ...destinationFolder, name]);
|
||||
const response = await this.fetcher(destination, {
|
||||
method: "COPY",
|
||||
headers: {
|
||||
...this.authHeaders(),
|
||||
destination,
|
||||
"overwrite": "F",
|
||||
},
|
||||
});
|
||||
if (response.status === 404) throw new OcisNotFoundError(`oCIS source not found: /${encodeSegments(sourceSegments)}`);
|
||||
if (response.status === 412) throw new OcisPreconditionFailedError(`oCIS destination already exists: ${name}`);
|
||||
if (!response.ok) throw new Error(`oCIS copy failed with ${response.status}`);
|
||||
const stat = await this.statPath(["hub-groups", groupId, ...destinationFolder, name]);
|
||||
if (!stat.exists) throw new Error("oCIS copy reported success but the destination is not present");
|
||||
const etag = response.headers.get("oc-etag") ?? response.headers.get("etag");
|
||||
const resourceId = response.headers.get("oc-fileid") ?? ["hub-groups", groupId, ...destinationFolder, name].join("/");
|
||||
return {
|
||||
resourceId,
|
||||
version: etag,
|
||||
sizeBytes: stat.sizeBytes,
|
||||
sha256: null,
|
||||
contentType: stat.contentType,
|
||||
};
|
||||
}
|
||||
|
||||
/** Stable resource link for GroupFile.shareLink.url — never presigned. */
|
||||
libraryUrl(resourceId: string): string {
|
||||
return this.config.publicBase === "" ? resourceId : `${this.config.publicBase}/f/${encodeURIComponent(resourceId)}`;
|
||||
}
|
||||
}
|
||||
|
||||
/** First <tag>…</tag> value in a small XML body, namespace-agnostic. */
|
||||
function firstTag(xml: string, tag: string): string | null {
|
||||
const match = xml.match(new RegExp(`<(?:[\\w.-]+:)?${tag}(?:\\s[^>]*)?>([^<]*)</(?:[\\w.-]+:)?${tag}>`, "i"));
|
||||
return match === null ? null : match[1]!.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Pipe a download forward, verifying sha256 chunk-by-chunk. Contract:
|
||||
* "SHA-256 verified while streaming" — a mismatch aborts the relay rather
|
||||
* than handing over tampered bytes (the sink sees them, but the caller
|
||||
* treats the error as fatal and closes the response).
|
||||
*/
|
||||
export async function verifyWhileStreaming(
|
||||
download: OcisDownload,
|
||||
expectedSha256: string,
|
||||
sink: (chunk: Buffer) => void,
|
||||
): Promise<{ sizeBytes: number }> {
|
||||
const hash = createHash("sha256");
|
||||
let sizeBytes = 0;
|
||||
for await (const chunk of download.stream) {
|
||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array);
|
||||
hash.update(buffer);
|
||||
sizeBytes += buffer.byteLength;
|
||||
sink(buffer);
|
||||
}
|
||||
const actual = hash.digest("hex");
|
||||
if (actual !== expectedSha256) throw new OcisVerificationError(expectedSha256, actual);
|
||||
return { sizeBytes };
|
||||
}
|
||||
|
||||
export function isOcisVerificationError(err: unknown): err is OcisVerificationError {
|
||||
return err instanceof OcisVerificationError;
|
||||
}
|
||||
37
apps/api/src/group-files/playback-range.ts
Normal file
37
apps/api/src/group-files/playback-range.ts
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
import { HttpException, HttpStatus } from "@nestjs/common";
|
||||
|
||||
/** Carries the plaintext size for the controller’s unsatisfied Content-Range header. */
|
||||
export class FileRangeNotSatisfiableException extends HttpException {
|
||||
constructor(readonly total: number) {
|
||||
super("requested range is not satisfiable", HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE);
|
||||
}
|
||||
}
|
||||
|
||||
/** Inclusive HTTP byte bounds, clipped to the plaintext EOF. */
|
||||
export function playbackRange(total: number, start: number, end: number): { start: number; end: number } {
|
||||
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || start >= total || end < start) {
|
||||
throw new FileRangeNotSatisfiableException(total);
|
||||
}
|
||||
return { start, end: Math.min(end, total - 1) };
|
||||
}
|
||||
|
||||
/** Query bounds take precedence; also accept the single Range browsers send. */
|
||||
export function requestedPlaybackRange(total: number, start: unknown, end: unknown, range?: string): { start: number; end: number } {
|
||||
const fail = (): never => { throw new FileRangeNotSatisfiableException(total); };
|
||||
const integer = (value: unknown): number => {
|
||||
if (typeof value !== "string" || !/^\d+$/.test(value) || !Number.isSafeInteger(Number(value))) return fail();
|
||||
return Number(value);
|
||||
};
|
||||
if (start !== undefined || end !== undefined) {
|
||||
return playbackRange(total, start === undefined ? 0 : integer(start), end === undefined ? total - 1 : integer(end));
|
||||
}
|
||||
if (range === undefined) return playbackRange(total, 0, total - 1);
|
||||
const match = /^bytes=(\d*)-(\d*)$/.exec(range);
|
||||
if (match === null || (match[1] === "" && match[2] === "")) return fail();
|
||||
if (match[1] === "") {
|
||||
const suffix = integer(match[2]);
|
||||
if (suffix === 0) return fail();
|
||||
return playbackRange(total, Math.max(0, total - suffix), total - 1);
|
||||
}
|
||||
return playbackRange(total, integer(match[1]), match[2] === "" ? total - 1 : integer(match[2]));
|
||||
}
|
||||
44
apps/api/src/group-files/verify-stream.ts
Normal file
44
apps/api/src/group-files/verify-stream.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
/**
|
||||
* Streaming payload verification: sha256 is computed over exactly the bytes
|
||||
* served, and a mismatch aborts the relay rather than handing over tampered
|
||||
* bytes. Storage-agnostic since 041 (was the oCIS client's contract; kept for
|
||||
* R2-backed rows).
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import type { Readable } from "node:stream";
|
||||
|
||||
/** An opened payload: streamed to the caller with metadata for the relay. */
|
||||
export interface PayloadDownload {
|
||||
stream: Readable;
|
||||
sizeBytes: number | null;
|
||||
contentType: string | null;
|
||||
}
|
||||
|
||||
export class OcisVerificationError extends Error {
|
||||
constructor(expected: string, actual: string) {
|
||||
super(`payload verification failed: expected sha256 ${expected}, got ${actual}`);
|
||||
this.name = "OcisVerificationError";
|
||||
}
|
||||
}
|
||||
|
||||
export function isOcisVerificationError(err: unknown): err is OcisVerificationError {
|
||||
return err instanceof OcisVerificationError;
|
||||
}
|
||||
|
||||
export async function verifyWhileStreaming(
|
||||
download: PayloadDownload,
|
||||
expectedSha256: string,
|
||||
sink: (chunk: Buffer) => void,
|
||||
): Promise<{ sizeBytes: number }> {
|
||||
const hash = createHash("sha256");
|
||||
let sizeBytes = 0;
|
||||
for await (const chunk of download.stream) {
|
||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array);
|
||||
hash.update(buffer);
|
||||
sizeBytes += buffer.byteLength;
|
||||
sink(buffer);
|
||||
}
|
||||
const actual = hash.digest("hex");
|
||||
if (actual !== expectedSha256) throw new OcisVerificationError(expectedSha256, actual);
|
||||
return { sizeBytes };
|
||||
}
|
||||
|
|
@ -25,7 +25,7 @@ async function bootstrap(): Promise<void> {
|
|||
app.enableCors({
|
||||
origin: corsOrigins(),
|
||||
credentials: true,
|
||||
methods: ["GET", "POST", "PATCH", "DELETE"],
|
||||
methods: ["GET", "POST", "PUT", "PATCH", "DELETE"],
|
||||
allowedHeaders: ["Authorization", "Content-Type"],
|
||||
});
|
||||
const port = Number(process.env.PORT ?? 3000);
|
||||
|
|
|
|||
70
apps/api/src/media-control/edl.controller.ts
Normal file
70
apps/api/src/media-control/edl.controller.ts
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
import { BadRequestException, Body, Controller, Get, Inject, Param, Patch, Post, Req, UseGuards } from "@nestjs/common";
|
||||
import type { Request } from "express";
|
||||
import { parseId, requireObject, requireSession } from "../http/validation.js";
|
||||
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import { validateEdlProject } from "./edl.js";
|
||||
import { EdlService, type EdlLane, type EdlProjectRecord, type EdlProjectView } from "./edl.service.js";
|
||||
|
||||
/**
|
||||
* EDL projects (editor-v2 intake): the timeline is the resource, the render is
|
||||
* a queued assemble job. Bodies are parsed with validateEdlProject — closed
|
||||
* enums, integer ms, unknown keys rejected — before anything touches the DB.
|
||||
*/
|
||||
@Controller("video/edl")
|
||||
@UseGuards(RbacGuard)
|
||||
export class EdlController {
|
||||
constructor(@Inject(EdlService) private readonly edl: EdlService) {}
|
||||
|
||||
@Post()
|
||||
@RequirePermission("media.manage")
|
||||
create(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Body() body: unknown,
|
||||
): Promise<EdlProjectRecord> {
|
||||
const session = requireSession(request);
|
||||
const project = validateEdlProject(requireObject(body, "body").project);
|
||||
return this.edl.create(session.workspaceId, project, session.userId);
|
||||
}
|
||||
|
||||
@Patch(":id")
|
||||
@RequirePermission("media.manage")
|
||||
update(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Param("id") id: string,
|
||||
@Body() body: unknown,
|
||||
): Promise<EdlProjectRecord> {
|
||||
const session = requireSession(request);
|
||||
const project = validateEdlProject(requireObject(body, "body").project);
|
||||
return this.edl.update(session.workspaceId, parseId(id, "id"), project, session.userId);
|
||||
}
|
||||
|
||||
@Get(":id")
|
||||
@RequirePermission("media.view")
|
||||
get(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Param("id") id: string,
|
||||
): Promise<EdlProjectView> {
|
||||
return this.edl.get(requireSession(request).workspaceId, parseId(id, "id"));
|
||||
}
|
||||
|
||||
@Post(":id/render")
|
||||
@RequirePermission("media.manage")
|
||||
render(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Param("id") id: string,
|
||||
@Body() body: unknown,
|
||||
): Promise<{ jobId: string }> {
|
||||
const session = requireSession(request);
|
||||
const input = requireObject(body, "body");
|
||||
const lane: EdlLane | undefined =
|
||||
input.lane === undefined || input.lane === null ? undefined : parseLane(input.lane);
|
||||
return this.edl.render(session.workspaceId, parseId(id, "id"), lane, session.userId);
|
||||
}
|
||||
}
|
||||
|
||||
function parseLane(value: unknown): EdlLane {
|
||||
if (value !== "local" && value !== "remote") {
|
||||
throw new BadRequestException("lane must be one of local, remote");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
180
apps/api/src/media-control/edl.service.ts
Normal file
180
apps/api/src/media-control/edl.service.ts
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
import { BadRequestException, Inject, Injectable, NotFoundException, Optional } from "@nestjs/common";
|
||||
import type { PoolClient } from "pg";
|
||||
import { appendTenantAudit, type AuditEvent } from "../audit/audit.service.js";
|
||||
import { withTenant } from "../db/client.js";
|
||||
import { isUuid } from "../http/validation.js";
|
||||
import type { EdlProject } from "./edl.js";
|
||||
|
||||
type TenantTransaction = <T>(workspaceId: string, fn: (client: PoolClient) => Promise<T>) => Promise<T>;
|
||||
type AuditAppender = (workspaceId: string, event: AuditEvent) => Promise<string>;
|
||||
|
||||
export interface EdlServiceDependencies {
|
||||
tenantTransaction: TenantTransaction;
|
||||
appendAudit: AuditAppender;
|
||||
}
|
||||
|
||||
export const EDL_SERVICE_DEPENDENCIES = Symbol("EDL_SERVICE_DEPENDENCIES");
|
||||
|
||||
const DEFAULT_DEPENDENCIES: EdlServiceDependencies = {
|
||||
tenantTransaction: withTenant,
|
||||
appendAudit: appendTenantAudit,
|
||||
};
|
||||
|
||||
export interface EdlProjectRecord {
|
||||
id: string;
|
||||
project: EdlProject;
|
||||
}
|
||||
|
||||
export interface EdlJobSummary {
|
||||
id: string;
|
||||
state: string;
|
||||
progressPercent: number;
|
||||
}
|
||||
|
||||
export interface EdlProjectView extends EdlProjectRecord {
|
||||
job: EdlJobSummary | null;
|
||||
}
|
||||
|
||||
export type EdlLane = "local" | "remote";
|
||||
|
||||
/**
|
||||
* EDL project persistence (editor-v2 intake). Every read/write runs inside the
|
||||
* caller's tenant transaction (withTenant), so RLS pins the rows to the
|
||||
* session's workspace and the audit trail appends to the tenant chain.
|
||||
* The project is only ever stored after validateEdlProject accepted it.
|
||||
*/
|
||||
@Injectable()
|
||||
export class EdlService {
|
||||
private readonly dependencies: EdlServiceDependencies;
|
||||
|
||||
constructor(@Optional() @Inject(EDL_SERVICE_DEPENDENCIES) dependencies?: Partial<EdlServiceDependencies>) {
|
||||
this.dependencies = { ...DEFAULT_DEPENDENCIES, ...dependencies };
|
||||
}
|
||||
|
||||
async create(workspaceId: string, project: EdlProject, actorUserId: string): Promise<EdlProjectRecord> {
|
||||
if (!isUuid(workspaceId)) throw new NotFoundException("workspace not found");
|
||||
const created = await this.dependencies.tenantTransaction(workspaceId, (client) =>
|
||||
client
|
||||
.query(
|
||||
`INSERT INTO edl_projects (workspace_id, project, created_by)
|
||||
VALUES ($1, $2::jsonb, $3) RETURNING id, project`,
|
||||
[workspaceId, JSON.stringify(project), actorUserId],
|
||||
)
|
||||
.then((result) => {
|
||||
const row = result.rows[0];
|
||||
if (row === undefined) throw new Error("database did not return the created EDL project");
|
||||
return { id: String(row.id), project: row.project as EdlProject };
|
||||
}),
|
||||
);
|
||||
await this.audit(workspaceId, "edl.saved", created.id, { actorUserId, operation: "create" });
|
||||
return created;
|
||||
}
|
||||
|
||||
async update(workspaceId: string, id: string, project: EdlProject, actorUserId: string): Promise<EdlProjectRecord> {
|
||||
if (!isUuid(workspaceId) || !isUuid(id)) throw new NotFoundException("EDL project not found");
|
||||
const updated = await this.dependencies.tenantTransaction(workspaceId, (client) =>
|
||||
client
|
||||
.query(
|
||||
`UPDATE edl_projects
|
||||
SET project = $2::jsonb, updated_at = now()
|
||||
WHERE id = $3
|
||||
RETURNING id, project`,
|
||||
[workspaceId, JSON.stringify(project), id],
|
||||
)
|
||||
.then((result) => {
|
||||
const row = result.rows[0];
|
||||
if (row === undefined) throw new NotFoundException(`EDL project ${id} not found`);
|
||||
return { id: String(row.id), project: row.project as EdlProject };
|
||||
}),
|
||||
);
|
||||
await this.audit(workspaceId, "edl.saved", updated.id, { actorUserId, operation: "update" });
|
||||
return updated;
|
||||
}
|
||||
|
||||
async get(workspaceId: string, id: string): Promise<EdlProjectView> {
|
||||
if (!isUuid(workspaceId) || !isUuid(id)) throw new NotFoundException("EDL project not found");
|
||||
return this.dependencies.tenantTransaction(workspaceId, (client) =>
|
||||
client
|
||||
.query(
|
||||
`SELECT e.id, e.project,
|
||||
j.id AS job_id, j.state AS job_state, j.progress_percent AS job_progress_percent
|
||||
FROM edl_projects e
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT id, state, progress_percent
|
||||
FROM video_processing_jobs
|
||||
WHERE input_json ->> 'edlProjectId' = e.id::text
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT 1
|
||||
) j ON true
|
||||
WHERE e.id = $1`,
|
||||
[id],
|
||||
)
|
||||
.then((result) => {
|
||||
const row = result.rows[0];
|
||||
if (row === undefined) throw new NotFoundException(`EDL project ${id} not found`);
|
||||
const job: EdlJobSummary | null =
|
||||
row.job_id === null || row.job_id === undefined
|
||||
? null
|
||||
: {
|
||||
id: String(row.job_id),
|
||||
state: String(row.job_state),
|
||||
progressPercent: typeof row.job_progress_percent === "number" ? row.job_progress_percent : Number(row.job_progress_percent),
|
||||
};
|
||||
return { id: String(row.id), project: row.project as EdlProject, job };
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue the assemble lane: the job row carries the validated project so the
|
||||
* worker re-validates exactly what the api persisted (044's claim picks the
|
||||
* row up by kind 'assemble'). video_asset_id pins the project's first
|
||||
* output is a derived video_assets row written when the job completes.
|
||||
*/
|
||||
async render(workspaceId: string, id: string, lane: EdlLane | undefined, actorUserId: string): Promise<{ jobId: string }> {
|
||||
if (!isUuid(workspaceId) || !isUuid(id)) throw new NotFoundException("EDL project not found");
|
||||
const created = await this.dependencies.tenantTransaction(workspaceId, (client) => {
|
||||
const found = client.query("SELECT project FROM edl_projects WHERE id = $1", [id]);
|
||||
return found.then((result) => {
|
||||
const row = result.rows[0];
|
||||
if (row === undefined) throw new NotFoundException(`EDL project ${id} not found`);
|
||||
const project = row.project as EdlProject;
|
||||
const input = { edlProjectId: id, project, ...(lane === undefined ? {} : { lane }) };
|
||||
// The claim path resolves the row's source asset (the project's FIRST
|
||||
// clip); the runner resolves the remaining clips through asset().
|
||||
const firstClip = project.items.find((item) => item.kind === "clip");
|
||||
if (firstClip === undefined || firstClip.kind !== "clip") {
|
||||
throw new BadRequestException("project has no clip to render");
|
||||
}
|
||||
return client
|
||||
.query(
|
||||
`INSERT INTO video_processing_jobs (workspace_id, video_asset_id, kind, state, input_json)
|
||||
VALUES ($1, $3, 'assemble', 'queued', $2::jsonb) RETURNING id`,
|
||||
[workspaceId, JSON.stringify(input), firstClip.assetId],
|
||||
)
|
||||
.then((jobResult) => {
|
||||
const jobRow = jobResult.rows[0];
|
||||
if (jobRow === undefined) throw new Error("database did not return the queued assemble job");
|
||||
return { jobId: String(jobRow.id) };
|
||||
});
|
||||
});
|
||||
});
|
||||
await this.audit(workspaceId, "edl.render", id, {
|
||||
actorUserId,
|
||||
jobId: created.jobId,
|
||||
lane: lane ?? null,
|
||||
});
|
||||
return created;
|
||||
}
|
||||
|
||||
private audit(workspaceId: string, action: "edl.saved" | "edl.render", targetId: string, diff: Record<string, unknown>): Promise<string> {
|
||||
return this.dependencies.appendAudit(workspaceId, {
|
||||
actorUserId: diff.actorUserId === null ? null : String(diff.actorUserId),
|
||||
actorRole: "member",
|
||||
action,
|
||||
targetType: "edl_project",
|
||||
targetId,
|
||||
diff,
|
||||
});
|
||||
}
|
||||
}
|
||||
172
apps/api/src/media-control/edl.test.ts
Normal file
172
apps/api/src/media-control/edl.test.ts
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
import { BadRequestException } from "@nestjs/common";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
EDL_CARD_MAX_MS,
|
||||
EDL_CARD_MIN_MS,
|
||||
EDL_CARD_TEXT_MAX,
|
||||
EDL_LABEL_MAX,
|
||||
EDL_MAX_ITEMS,
|
||||
EDL_MAX_TOTAL_MS,
|
||||
EDL_RESOLUTIONS,
|
||||
validateEdlProject,
|
||||
} from "./edl.js";
|
||||
|
||||
const ASSET_A = "10000000-0000-4000-8000-000000000001";
|
||||
const ASSET_B = "10000000-0000-4000-8000-000000000002";
|
||||
const AUDIO = "20000000-0000-4000-8000-000000000001";
|
||||
|
||||
function clip(overrides: Record<string, unknown> = {}): Record<string, unknown> {
|
||||
return { kind: "clip", assetId: ASSET_A, trimStartMs: 0, trimEndMs: 10_000, rotate: 0, volumeDb: 0, ...overrides };
|
||||
}
|
||||
function card(overrides: Record<string, unknown> = {}): Record<string, unknown> {
|
||||
return { kind: "card", text: "Chapter one", durationMs: 5000, style: "title", ...overrides };
|
||||
}
|
||||
function overlayInput(overrides: Record<string, unknown> = {}): Record<string, unknown> {
|
||||
return { overlayType: "lower_third", fields: { name: "A. Reporter" }, startMs: 0, endMs: 5000, ...overrides };
|
||||
}
|
||||
function voiceoverInput(overrides: Record<string, unknown> = {}): Record<string, unknown> {
|
||||
return { audioAssetId: AUDIO, gainDb: -12, duckOriginal: true, offsetMs: 500, ...overrides };
|
||||
}
|
||||
function project(overrides: Record<string, unknown> = {}): Record<string, unknown> {
|
||||
return {
|
||||
version: 1,
|
||||
aspect: "16:9",
|
||||
fit: "cover",
|
||||
fps: 30,
|
||||
items: [clip()],
|
||||
overlays: [],
|
||||
voiceover: null,
|
||||
outputLabel: "EDL test",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function rejectError(value: unknown): BadRequestException {
|
||||
try {
|
||||
validateEdlProject(value);
|
||||
} catch (error) {
|
||||
expect(error).toBeInstanceOf(BadRequestException);
|
||||
return error as BadRequestException;
|
||||
}
|
||||
throw new Error(`validateEdlProject accepted ${JSON.stringify(value)}`);
|
||||
}
|
||||
|
||||
function messageOf(error: BadRequestException): string {
|
||||
const response = error.getResponse();
|
||||
if (typeof response === "string") return response;
|
||||
const record = response as Record<string, unknown>;
|
||||
if (typeof record.message === "string") return record.message;
|
||||
if (Array.isArray(record.message)) return record.message.join(" ");
|
||||
return String(response);
|
||||
}
|
||||
|
||||
/** Each rejection names the offending field path in the 400 message. */
|
||||
const rejections: Array<[string, unknown, RegExp]> = [
|
||||
// -- closed enums -------------------------------------------------------
|
||||
["version 2", project({ version: 2 }), /project\.version must be 1/],
|
||||
["aspect 4:3", project({ aspect: "4:3" }), /project\.aspect must be one of 16:9, 9:16/],
|
||||
["fit fill", project({ fit: "fill" }), /project\.fit must be one of cover, contain/],
|
||||
["fps 25", project({ fps: 25 }), /project\.fps must be 30/],
|
||||
["clip rotate 45", project({ items: [clip({ rotate: 45 })] }), /items\[0\]\.rotate must be one of 0, 90, 180, 270/],
|
||||
["card style lower", project({ items: [card({ style: "lower" })] }), /items\[0\]\.style must be one of title, chapter/],
|
||||
["overlay type watermark", project({ overlays: [overlayInput({ overlayType: "watermark" })] }), /overlays\[0\]\.overlayType must be one of lower_third, chyron, logo/],
|
||||
// -- unknown keys are rejected, never ignored ---------------------------
|
||||
["unknown project key", project({ filter: "drawtext" }), /project\.filter is not allowed/],
|
||||
["unknown clip key", project({ items: [clip({ speed: 2 })] }), /items\[0\]\.speed is not allowed/],
|
||||
["unknown card key", project({ items: [card({ font: "sans" })] }), /items\[0\]\.font is not allowed/],
|
||||
["unknown overlay key", project({ overlays: [overlayInput({ font: "sans" })] }), /overlays\[0\]\.font is not allowed/],
|
||||
["unknown voiceover key", project({ voiceover: voiceoverInput({ streamKey: "x" }) }), /voiceover\.streamKey is not allowed/],
|
||||
// -- items bounds ---------------------------------------------------------
|
||||
["empty items", project({ items: [] }), /project\.items must contain at least 1 item/],
|
||||
["51 items", project({ items: Array.from({ length: 51 }, () => clip()) }), /project\.items allows at most 50 items/],
|
||||
// -- total duration bound --------------------------------------------------
|
||||
["total 15 min + 1 ms", project({ items: [clip({ trimStartMs: 0, trimEndMs: 900_001 })] }), /total duration must be at most 900000 ms/],
|
||||
// -- clip bounds -------------------------------------------------------------
|
||||
["clip trimStartMs -1", project({ items: [clip({ trimStartMs: -1 })] }), /items\[0\]\.trimStartMs must be at least 0 ms/],
|
||||
["clip trimEndMs 0 (trimStartMs 0)", project({ items: [clip({ trimEndMs: 0 })] }), /project\.items\[0\]\.trimEndMs must be greater than project\.items\[0\]\.trimStartMs/],
|
||||
["clip trimEndMs == trimStartMs", project({ items: [clip({ trimStartMs: 1000, trimEndMs: 1000 })] }), /project\.items\[0\]\.trimEndMs must be greater than project\.items\[0\]\.trimStartMs/],
|
||||
["clip trimEndMs < trimStartMs", project({ items: [clip({ trimStartMs: 2000, trimEndMs: 1000 })] }), /project\.items\[0\]\.trimEndMs must be greater than project\.items\[0\]\.trimStartMs/],
|
||||
["clip trimStartMs not an integer", project({ items: [clip({ trimStartMs: 10.5 })] }), /items\[0\]\.trimStartMs must be an integer number of milliseconds/],
|
||||
["clip volumeDb -39", project({ items: [clip({ volumeDb: -39 })] }), /items\[0\]\.volumeDb must be between -36 and 6 dB/],
|
||||
["clip volumeDb 9", project({ items: [clip({ volumeDb: 9 })] }), /items\[0\]\.volumeDb must be between -36 and 6 dB/],
|
||||
["clip volumeDb 1 (not a 3 dB step)", project({ items: [clip({ volumeDb: 1 })] }), /items\[0\]\.volumeDb must be a multiple of 3 dB/],
|
||||
// -- card bounds -------------------------------------------------------------
|
||||
["card durationMs 999", project({ items: [card({ durationMs: 999 })] }), /items\[0\]\.durationMs must be at least 1000 ms/],
|
||||
["card durationMs 10001", project({ items: [card({ durationMs: 10_001 })] }), /items\[0\]\.durationMs must be at most 10000 ms/],
|
||||
["card text 121 chars", project({ items: [card({ text: "x".repeat(121) })] }), /items\[0\]\.text must be at most 120 characters/],
|
||||
// -- overlay bounds -----------------------------------------------------------
|
||||
["overlay startMs -1", project({ overlays: [overlayInput({ startMs: -1 })] }), /overlays\[0\]\.startMs must be at least 0 ms/],
|
||||
["overlay endMs 0", project({ overlays: [overlayInput({ startMs: 0, endMs: 0 })] }), /project\.overlays\[0\]\.endMs must be greater than project\.overlays\[0\]\.startMs/],
|
||||
["overlay endMs < startMs", project({ overlays: [overlayInput({ startMs: 5000, endMs: 4000 })] }), /project\.overlays\[0\]\.endMs must be greater than project\.overlays\[0\]\.startMs/],
|
||||
["overlay fields value not a string", project({ overlays: [overlayInput({ fields: { name: 42 } })] }), /overlays\[0\]\.fields\.name must be a string/],
|
||||
// -- voiceover bounds -----------------------------------------------------------
|
||||
["voiceover gainDb -39", project({ voiceover: voiceoverInput({ gainDb: -39 }) }), /voiceover\.gainDb must be between -36 and 6 dB/],
|
||||
["voiceover gainDb 1 (not a 3 dB step)", project({ voiceover: voiceoverInput({ gainDb: 1 }) }), /voiceover\.gainDb must be a multiple of 3 dB/],
|
||||
["voiceover offsetMs -1", project({ voiceover: voiceoverInput({ offsetMs: -1 }) }), /voiceover\.offsetMs must be at least 0 ms/],
|
||||
["voiceover duckOriginal not a boolean", project({ voiceover: voiceoverInput({ duckOriginal: "yes" }) }), /voiceover\.duckOriginal must be a boolean/],
|
||||
// -- output label bound -----------------------------------------------------------
|
||||
["outputLabel 81 chars", project({ outputLabel: "x".repeat(81) }), /outputLabel must be at most 80 characters/],
|
||||
// -- uuids ---------------------------------------------------------------------------
|
||||
["clip assetId not a uuid", project({ items: [clip({ assetId: "not-a-uuid" })] }), /items\[0\]\.assetId must be a UUID/],
|
||||
["voiceover audioAssetId not a uuid", project({ voiceover: voiceoverInput({ audioAssetId: "nope" }) }), /voiceover\.audioAssetId must be a UUID/],
|
||||
];
|
||||
|
||||
describe("validateEdlProject", () => {
|
||||
it("exports the spec'd resolution table", () => {
|
||||
expect(EDL_RESOLUTIONS).toEqual({ "16:9": { w: 1280, h: 720 }, "9:16": { w: 1080, h: 1920 } });
|
||||
});
|
||||
|
||||
it("accepts the minimal project and canonicalizes it", () => {
|
||||
expect(validateEdlProject(project())).toEqual(project());
|
||||
});
|
||||
|
||||
it("accepts a full project (both aspects, all item/overlay kinds, voiceover)", () => {
|
||||
const full = project({
|
||||
aspect: "9:16",
|
||||
fit: "contain",
|
||||
items: [
|
||||
clip({ assetId: ASSET_B, trimStartMs: 100, trimEndMs: 11_000, rotate: 90, volumeDb: -6 }),
|
||||
card({ text: "Chapter one", durationMs: 10_000, style: "chapter" }),
|
||||
],
|
||||
overlays: [
|
||||
overlayInput(),
|
||||
overlayInput({ overlayType: "logo", fields: {}, startMs: 5_000, endMs: 11_100 }),
|
||||
overlayInput({ overlayType: "chyron", fields: { headline: "Market close" }, startMs: 2_000, endMs: 8_000 }),
|
||||
],
|
||||
voiceover: voiceoverInput(),
|
||||
outputLabel: "Full EDL",
|
||||
});
|
||||
expect(validateEdlProject(full)).toEqual(full);
|
||||
});
|
||||
|
||||
it("accepts every boundary value", () => {
|
||||
expect(validateEdlProject(project({ items: Array.from({ length: 50 }, () => clip()) }))).toBeTruthy(); // 50 items
|
||||
expect(validateEdlProject(project({ items: [clip({ trimStartMs: 0, trimEndMs: 900_000 })] }))).toBeTruthy(); // exactly 15 min
|
||||
expect(validateEdlProject(project({ items: [card({ durationMs: 1000 })] }))).toBeTruthy(); // card floor
|
||||
expect(validateEdlProject(project({ items: [card({ durationMs: 10_000 })] }))).toBeTruthy(); // card ceiling
|
||||
expect(validateEdlProject(project({ items: [card({ text: "x".repeat(120) })] }))).toBeTruthy(); // card text ceiling
|
||||
expect(validateEdlProject(project({ outputLabel: "x".repeat(80) }))).toBeTruthy(); // label ceiling
|
||||
expect(validateEdlProject(project({ items: [clip({ volumeDb: -36 })] }))).toBeTruthy(); // dB floor
|
||||
expect(validateEdlProject(project({ items: [clip({ volumeDb: 6 })] }))).toBeTruthy(); // dB ceiling
|
||||
expect(validateEdlProject(project({ items: [clip({ trimStartMs: 0 })] }))).toBeTruthy(); // trimStart floor
|
||||
expect(validateEdlProject(project({ overlays: [overlayInput({ startMs: 0, endMs: 1 })] }))).toBeTruthy(); // overlay floor
|
||||
expect(validateEdlProject(project({ voiceover: voiceoverInput({ gainDb: -36, offsetMs: 0 }) }))).toBeTruthy(); // voiceover floors
|
||||
});
|
||||
|
||||
it("throws BadRequestException (HTTP 400) with the field path in the message", () => {
|
||||
const error = rejectError(project({ fit: "fill" }));
|
||||
expect(error.getStatus()).toBe(400);
|
||||
expect(messageOf(error)).toContain("project.fit");
|
||||
});
|
||||
|
||||
it.each(rejections)("%s", (_label, value, pattern) => {
|
||||
expect(messageOf(rejectError(value))).toMatch(pattern);
|
||||
});
|
||||
|
||||
it("rejects every filtergraph metacharacter in card text (produce-request.ts blocklist)", () => {
|
||||
for (const character of [";", "[", "]", "'", '"', "`", "$", "\\", "{", "}", "=", ","]) {
|
||||
const error = rejectError(project({ items: [card({ text: `ab${character}cd` })] }));
|
||||
expect(messageOf(error)).toMatch(/items\[0\]\.text must not contain the filtergraph metacharacter/);
|
||||
}
|
||||
});
|
||||
});
|
||||
167
apps/api/src/media-control/edl.ts
Normal file
167
apps/api/src/media-control/edl.ts
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
import { BadRequestException } from "@nestjs/common";
|
||||
import { isUuid } from "../http/validation.js";
|
||||
|
||||
/**
|
||||
* EDL (editor timeline) projects: the editor-v2 intake shape.
|
||||
*
|
||||
* Everything accepted here is persisted verbatim into edl_projects.project
|
||||
* and later reaches the media-worker's ffmpeg assemble lane, so the rules
|
||||
* follow produce-request.ts: closed enums, exhaustive key allowlists (an
|
||||
* unknown key is rejected, never ignored — that is what keeps a stray
|
||||
* `movie=`/`enable=` field from riding into a filtergraph), integer
|
||||
* milliseconds, bounded text, and no filtergraph or shell metacharacter
|
||||
* inside card text.
|
||||
*
|
||||
* validateEdlProject is total: it either returns a canonicalized project
|
||||
* (fixed key order, so the stored jsonb bytes are deterministic) or throws
|
||||
* BadRequestException with a field path.
|
||||
*/
|
||||
|
||||
export type EdlAspect = "16:9" | "9:16";
|
||||
export type EdlFit = "cover" | "contain";
|
||||
export const EDL_RESOLUTIONS = { "16:9": { w: 1280, h: 720 }, "9:16": { w: 1080, h: 1920 } } as const;
|
||||
|
||||
export interface EdlClip { kind: "clip"; assetId: string; trimStartMs: number; trimEndMs: number; rotate: 0 | 90 | 180 | 270; volumeDb: number }
|
||||
export interface EdlCard { kind: "card"; text: string; durationMs: number; style: "title" | "chapter" }
|
||||
export type EdlItem = EdlClip | EdlCard;
|
||||
export interface EdlOverlay { overlayType: "lower_third" | "chyron" | "logo"; fields: Record<string, string>; startMs: number; endMs: number }
|
||||
export interface EdlVoiceover { audioAssetId: string; gainDb: number; duckOriginal: boolean; offsetMs: number }
|
||||
export interface EdlProject { version: 1; aspect: EdlAspect; fit: EdlFit; fps: 30; items: EdlItem[]; overlays: EdlOverlay[]; voiceover: EdlVoiceover | null; outputLabel: string }
|
||||
|
||||
export const EDL_MAX_ITEMS = 50;
|
||||
export const EDL_MAX_TOTAL_MS = 900_000;
|
||||
export const EDL_CARD_MIN_MS = 1000;
|
||||
export const EDL_CARD_MAX_MS = 10_000;
|
||||
export const EDL_CARD_TEXT_MAX = 120;
|
||||
export const EDL_LABEL_MAX = 80;
|
||||
export const EDL_DB_MIN = -36;
|
||||
export const EDL_DB_MAX = 6;
|
||||
export const EDL_FPS = 30;
|
||||
|
||||
/**
|
||||
* Same blocklist as produce-request.ts: characters that change an ffmpeg
|
||||
* filtergraph (`;` splits, `[`/`]` label pads, `,` separates args, `'` quotes,
|
||||
* `=` sets an option) or a shell string. Card text is operator prose, so the
|
||||
* check is a blocklist of these rather than a tight allowlist — prose keeps
|
||||
* its punctuation and no metacharacter survives.
|
||||
*/
|
||||
const TEXT_FORBIDDEN = new Set([";", "[", "]", "'", '"', "`", "$", "\\", "{", "}", "=", ","]);
|
||||
|
||||
function fail(message: string): never {
|
||||
throw new BadRequestException(message);
|
||||
}
|
||||
|
||||
/** Allowlist gate: unknown keys are rejected, never ignored. */
|
||||
function record(value: unknown, field: string, allowed: readonly string[]): Record<string, unknown> {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${field} must be an object`);
|
||||
for (const key of Object.keys(value)) {
|
||||
if (!allowed.includes(key)) fail(`${field}.${key} is not allowed`);
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** Integer milliseconds within an inclusive range. */
|
||||
function integerMs(value: unknown, field: string, min: number, max?: number): number {
|
||||
if (typeof value !== "number" || !Number.isSafeInteger(value)) fail(`${field} must be an integer number of milliseconds`);
|
||||
if (value < min) fail(`${field} must be at least ${min} ms`);
|
||||
if (max !== undefined && value > max) fail(`${field} must be at most ${max} ms`);
|
||||
return value;
|
||||
}
|
||||
|
||||
/** dB values are stepped in 3 dB increments inside the -36..+6 window. */
|
||||
function dbValue(value: unknown, field: string): number {
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) fail(`${field} must be a number`);
|
||||
if (value < EDL_DB_MIN || value > EDL_DB_MAX) fail(`${field} must be between ${EDL_DB_MIN} and ${EDL_DB_MAX} dB`);
|
||||
if (value % 3 !== 0) fail(`${field} must be a multiple of 3 dB`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function boundedText(value: unknown, field: string, max: number): string {
|
||||
if (typeof value !== "string") fail(`${field} must be a string`);
|
||||
if (value.length > max) fail(`${field} must be at most ${max} characters`);
|
||||
for (const character of value) {
|
||||
if (TEXT_FORBIDDEN.has(character)) fail(`${field} must not contain the filtergraph metacharacter ${JSON.stringify(character)}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function item(value: unknown, field: string): EdlItem {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${field} must be an object`);
|
||||
const kind = (value as Record<string, unknown>).kind;
|
||||
if (kind === "clip") {
|
||||
const clip = record(value, field, ["kind", "assetId", "trimStartMs", "trimEndMs", "rotate", "volumeDb"]);
|
||||
const assetId = clip.assetId;
|
||||
if (typeof assetId !== "string" || !isUuid(assetId)) fail(`${field}.assetId must be a UUID`);
|
||||
const trimStartMs = integerMs(clip.trimStartMs, `${field}.trimStartMs`, 0);
|
||||
const trimEndMs = integerMs(clip.trimEndMs, `${field}.trimEndMs`, 0);
|
||||
if (trimEndMs <= trimStartMs) fail(`${field}.trimEndMs must be greater than ${field}.trimStartMs`);
|
||||
const rotate = clip.rotate;
|
||||
if (rotate !== 0 && rotate !== 90 && rotate !== 180 && rotate !== 270) fail(`${field}.rotate must be one of 0, 90, 180, 270`);
|
||||
const volumeDb = dbValue(clip.volumeDb, `${field}.volumeDb`);
|
||||
return { kind: "clip", assetId, trimStartMs, trimEndMs, rotate: rotate as 0 | 90 | 180 | 270, volumeDb };
|
||||
}
|
||||
if (kind === "card") {
|
||||
const card = record(value, field, ["kind", "text", "durationMs", "style"]);
|
||||
const style = card.style;
|
||||
if (style !== "title" && style !== "chapter") fail(`${field}.style must be one of title, chapter`);
|
||||
const text = boundedText(card.text, `${field}.text`, EDL_CARD_TEXT_MAX);
|
||||
const durationMs = integerMs(card.durationMs, `${field}.durationMs`, EDL_CARD_MIN_MS, EDL_CARD_MAX_MS);
|
||||
return { kind: "card", text, durationMs, style: style as "title" | "chapter" };
|
||||
}
|
||||
fail(`${field}.kind must be one of clip, card`);
|
||||
}
|
||||
|
||||
function overlay(value: unknown, field: string): EdlOverlay {
|
||||
const o = record(value, field, ["overlayType", "fields", "startMs", "endMs"]);
|
||||
const overlayType = o.overlayType;
|
||||
if (overlayType !== "lower_third" && overlayType !== "chyron" && overlayType !== "logo") {
|
||||
fail(`${field}.overlayType must be one of lower_third, chyron, logo`);
|
||||
}
|
||||
if (typeof o.fields !== "object" || o.fields === null || Array.isArray(o.fields)) fail(`${field}.fields must be an object`);
|
||||
const fields: Record<string, string> = {};
|
||||
for (const [key, entry] of Object.entries(o.fields)) {
|
||||
if (typeof entry !== "string") fail(`${field}.fields.${key} must be a string`);
|
||||
fields[key] = entry;
|
||||
}
|
||||
const startMs = integerMs(o.startMs, `${field}.startMs`, 0);
|
||||
const endMs = integerMs(o.endMs, `${field}.endMs`, 0);
|
||||
if (endMs <= startMs) fail(`${field}.endMs must be greater than ${field}.startMs`);
|
||||
return { overlayType: overlayType as "lower_third" | "chyron" | "logo", fields, startMs, endMs };
|
||||
}
|
||||
|
||||
function voiceover(value: unknown, field: string): EdlVoiceover {
|
||||
const v = record(value, field, ["audioAssetId", "gainDb", "duckOriginal", "offsetMs"]);
|
||||
const audioAssetId = v.audioAssetId;
|
||||
if (typeof audioAssetId !== "string" || !isUuid(audioAssetId)) fail(`${field}.audioAssetId must be a UUID`);
|
||||
const gainDb = dbValue(v.gainDb, `${field}.gainDb`);
|
||||
if (typeof v.duckOriginal !== "boolean") fail(`${field}.duckOriginal must be a boolean`);
|
||||
const offsetMs = integerMs(v.offsetMs, `${field}.offsetMs`, 0);
|
||||
return { audioAssetId, gainDb, duckOriginal: v.duckOriginal, offsetMs };
|
||||
}
|
||||
|
||||
/** Parse and canonicalize an EDL project; throws BadRequestException on the first problem. */
|
||||
export function validateEdlProject(value: unknown): EdlProject {
|
||||
const project = record(value, "project", ["version", "aspect", "fit", "fps", "items", "overlays", "voiceover", "outputLabel"]);
|
||||
if (project.version !== 1) fail("project.version must be 1");
|
||||
const aspect = project.aspect;
|
||||
if (aspect !== "16:9" && aspect !== "9:16") fail(`project.aspect must be one of ${Object.keys(EDL_RESOLUTIONS).join(", ")}`);
|
||||
const fit = project.fit;
|
||||
if (fit !== "cover" && fit !== "contain") fail("project.fit must be one of cover, contain");
|
||||
if (project.fps !== EDL_FPS) fail(`project.fps must be ${EDL_FPS}`);
|
||||
if (!Array.isArray(project.items)) fail("project.items must be an array");
|
||||
if (project.items.length < 1) fail("project.items must contain at least 1 item");
|
||||
if (project.items.length > EDL_MAX_ITEMS) fail(`project.items allows at most ${EDL_MAX_ITEMS} items`);
|
||||
const items = project.items.map((entry, index) => item(entry, `project.items[${index}]`));
|
||||
let totalMs = 0;
|
||||
for (const entry of items) {
|
||||
totalMs += entry.kind === "clip" ? entry.trimEndMs - entry.trimStartMs : entry.durationMs;
|
||||
}
|
||||
if (totalMs > EDL_MAX_TOTAL_MS) fail(`project.items total duration must be at most ${EDL_MAX_TOTAL_MS} ms`);
|
||||
if (project.overlays !== undefined && project.overlays !== null && !Array.isArray(project.overlays)) {
|
||||
fail("project.overlays must be an array");
|
||||
}
|
||||
const overlays = (Array.isArray(project.overlays) ? project.overlays : []).map((entry, index) => overlay(entry, `project.overlays[${index}]`));
|
||||
const voiceoverValue = project.voiceover === undefined || project.voiceover === null ? null : voiceover(project.voiceover, "project.voiceover");
|
||||
const outputLabel = boundedText(project.outputLabel, "project.outputLabel", EDL_LABEL_MAX);
|
||||
return { version: 1, aspect: aspect as EdlAspect, fit: fit as EdlFit, fps: 30, items, overlays, voiceover: voiceoverValue, outputLabel };
|
||||
}
|
||||
|
|
@ -50,8 +50,16 @@ export function sniffMediaAsset(bytes: Buffer): SniffedMedia | null {
|
|||
if (bytes.subarray(0, 4).toString("latin1") === "RIFF" && bytes.subarray(8, 12).toString("latin1") === "WEBP") {
|
||||
return { kind: "image", mime: "image/webp" };
|
||||
}
|
||||
// mp4: "ftyp" at offset 4
|
||||
if (bytes.subarray(4, 8).toString("latin1") === "ftyp") return { kind: "video", mime: "video/mp4" };
|
||||
// mp4: "ftyp" at offset 4, but only ISOBMFF video brands — HEIC/AVIF photos
|
||||
// and QuickTime .mov share the box and would be mislabelled as video/mp4.
|
||||
if (bytes.subarray(4, 8).toString("latin1") === "ftyp") {
|
||||
const brand = bytes.subarray(8, 12).toString("latin1").toLowerCase();
|
||||
if (brand.startsWith("hei") || brand.startsWith("avif") || brand.startsWith("avis") || brand.startsWith("mif1") || brand.startsWith("msf1")) {
|
||||
return { kind: "image", mime: "image/avif" };
|
||||
}
|
||||
if (brand.startsWith("qt")) return { kind: "video", mime: "video/quicktime" };
|
||||
return { kind: "video", mime: "video/mp4" };
|
||||
}
|
||||
// webm/mkv: EBML magic 1A 45 DF A3
|
||||
if (bytes.subarray(0, 4).toString("hex") === "1a45dfa3") return { kind: "video", mime: "video/webm" };
|
||||
return null;
|
||||
|
|
@ -170,7 +178,7 @@ export class MediaService {
|
|||
/** Auth-scoped read: RLS already fences this to the caller's workspace. */
|
||||
async readAsset(workspaceId: string, id: string): Promise<MediaAssetRecord & { bytes: Buffer }> {
|
||||
const asset = await withTenant(workspaceId, async (client) => {
|
||||
const result = await client.query<MediaRow>("SELECT * FROM media_assets WHERE id = $1", [id]);
|
||||
const result = await client.query<MediaRow>("SELECT * FROM media_assets WHERE id = $1 AND workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid", [id]);
|
||||
const row = result.rows[0];
|
||||
if (row === undefined) throw new NotFoundException(`media asset ${id} not found`);
|
||||
return fromRow(row);
|
||||
|
|
@ -192,7 +200,7 @@ export class MediaService {
|
|||
): Promise<void> {
|
||||
if (mediaAssetIds.length === 0) return;
|
||||
const result = await client.query(
|
||||
"SELECT count(*)::integer AS count FROM media_assets WHERE id = ANY($1::uuid[])",
|
||||
"SELECT count(*)::integer AS count FROM media_assets WHERE id = ANY($1::uuid[]) AND workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid",
|
||||
[mediaAssetIds],
|
||||
);
|
||||
const count = Number(result.rows[0]?.count ?? 0);
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import {
|
|||
sniffMediaAsset,
|
||||
} from "./media.service.js";
|
||||
import { appendTenantAudit } from "../audit/audit.service.js";
|
||||
import { sha256, type StorageService } from "../storage/storage.js";
|
||||
import { sha256, type StorageService, directStorageDisabled } from "../storage/storage.js";
|
||||
import { MediaController } from "./media.controller.js";
|
||||
import type { AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import type { Request, Response } from "express";
|
||||
|
|
@ -68,6 +68,7 @@ function fakeStorage(): StorageService & {
|
|||
}),
|
||||
hasContent: vi.fn(async () => false),
|
||||
presign: vi.fn(async () => ""),
|
||||
...directStorageDisabled(),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -161,11 +161,14 @@ export function validateAuthorize(params: AuthorizeParams): AuthorizeValidation
|
|||
}
|
||||
|
||||
/** `redirectUri?error=…&error_description=…&state=…` for a redirectable failure. */
|
||||
export function errorRedirect(redirectUri: string, state: string | null, error: string, description: string): string {
|
||||
export function errorRedirect(redirectUri: string, state: string | null, error: string, description: string, issuer?: string): string {
|
||||
const url = new URL(redirectUri);
|
||||
url.searchParams.set("error", error);
|
||||
url.searchParams.set("error_description", description);
|
||||
if (state !== null) url.searchParams.set("state", state);
|
||||
// RFC 9207 §2: iss is required on error responses too when the server
|
||||
// advertises authorization_response_iss_parameter_supported.
|
||||
if (issuer !== undefined) url.searchParams.set("iss", issuer);
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ export interface RefreshIssueInput {
|
|||
export interface OidcRefreshStore {
|
||||
issue(input: RefreshIssueInput): Promise<void>;
|
||||
lookupByHash(tokenHash: string): Promise<RefreshTokenRecord | null>;
|
||||
consume(record: RefreshTokenRecord): Promise<void>;
|
||||
consume(record: RefreshTokenRecord): Promise<boolean>;
|
||||
/** Reuse detection: delete the whole chain; returns rows removed. */
|
||||
revokeFamily(workspaceId: string, familyId: string): Promise<number>;
|
||||
/** The minting Hub session must still be live (not revoked, not expired). */
|
||||
|
|
@ -90,10 +90,12 @@ export class PgRefreshStore implements OidcRefreshStore {
|
|||
return row === undefined ? null : fromRow(row);
|
||||
}
|
||||
|
||||
async consume(record: RefreshTokenRecord): Promise<void> {
|
||||
await withTenant(record.workspaceId, (client) =>
|
||||
/** Single-use consumption: true when THIS call won the rotation race. */
|
||||
async consume(record: RefreshTokenRecord): Promise<boolean> {
|
||||
const result = await withTenant(record.workspaceId, (client) =>
|
||||
client.query("UPDATE oidc_refresh_tokens SET consumed_at = now() WHERE id = $1 AND consumed_at IS NULL", [record.id]),
|
||||
);
|
||||
return (result.rowCount ?? 0) === 1;
|
||||
}
|
||||
|
||||
async revokeFamily(workspaceId: string, familyId: string): Promise<number> {
|
||||
|
|
@ -145,8 +147,11 @@ export class MemoryRefreshStore implements OidcRefreshStore {
|
|||
return this.rows.get(tokenHash) ?? null;
|
||||
}
|
||||
|
||||
async consume(record: RefreshTokenRecord): Promise<void> {
|
||||
for (const row of this.rows.values()) if (row.id === record.id) row.consumedAt = new Date();
|
||||
async consume(record: RefreshTokenRecord): Promise<boolean> {
|
||||
const row = this.rows.get(record.id);
|
||||
if (row === undefined || row.consumedAt !== null) return false;
|
||||
row.consumedAt = new Date();
|
||||
return true;
|
||||
}
|
||||
|
||||
async revokeFamily(workspaceId: string, familyId: string): Promise<number> {
|
||||
|
|
|
|||
|
|
@ -223,7 +223,7 @@ export class OidcService implements OnModuleInit {
|
|||
const params = authorizeParamsFrom(query);
|
||||
const result = validateAuthorize(params);
|
||||
if (!result.ok && !result.redirect) throw new BadRequestException(`${result.error}: ${result.description}`);
|
||||
if (!result.ok) return errorRedirect(result.redirectUri, result.state, result.error, result.description);
|
||||
if (!result.ok) return errorRedirect(result.redirectUri, result.state, result.error, result.description, this.config().issuer);
|
||||
const consent = new URL(config.consentUrl);
|
||||
for (const name of AUTHORIZE_PARAM_NAMES) {
|
||||
const value = params[name];
|
||||
|
|
@ -235,9 +235,14 @@ export class OidcService implements OnModuleInit {
|
|||
/** POST /oidc/authorize/consent — the session's user approves (auto, first-party) and gets a code. */
|
||||
async consent(workspaceId: string, userId: string, body: unknown, sid: string | null = null): Promise<{ redirectTo: string }> {
|
||||
const config = this.config();
|
||||
// The session must still be alive: a logged-out (or sid-less) bearer must
|
||||
// never mint codes or downstream tokens, even inside its TTL.
|
||||
if (sid === null || !(await this.deps.refreshStore.sessionAlive(workspaceId, sid))) {
|
||||
throw new UnauthorizedException("the hub session has ended — sign in again");
|
||||
}
|
||||
const result = validateAuthorize(authorizeParamsFrom(body));
|
||||
if (!result.ok && !result.redirect) throw new BadRequestException(`${result.error}: ${result.description}`);
|
||||
if (!result.ok) return { redirectTo: errorRedirect(result.redirectUri, result.state, result.error, result.description) };
|
||||
if (!result.ok) return { redirectTo: errorRedirect(result.redirectUri, result.state, result.error, result.description, this.config().issuer) };
|
||||
const request = result.request;
|
||||
if (!isFirstPartyClient(request.clientId)) {
|
||||
// No consent UI exists yet: anything but a first-party client fails closed.
|
||||
|
|
@ -382,7 +387,12 @@ export class OidcService implements OnModuleInit {
|
|||
throw new OAuthError("invalid_grant", "membership is no longer active");
|
||||
}
|
||||
|
||||
await this.deps.refreshStore.consume(record);
|
||||
if (!(await this.deps.refreshStore.consume(record))) {
|
||||
// A concurrent refresh won the rotation: this presentation is already used,
|
||||
// which is exactly the reuse-detection case — revoke the family.
|
||||
await this.deps.refreshStore.revokeFamily(record.workspaceId, record.familyId);
|
||||
throw new OAuthError("invalid_grant", "refresh token reuse detected; the chain has been revoked", HttpStatus.UNAUTHORIZED);
|
||||
}
|
||||
const scopes = record.scope.split(" ").filter((s) => s !== "");
|
||||
const keyring = await this.keyring();
|
||||
const iat = Math.floor(this.deps.now() / 1000);
|
||||
|
|
|
|||
|
|
@ -79,6 +79,8 @@ function harness(): { service: OidcService; users: FakeUsers; clock: { now: numb
|
|||
const vault = new Vault();
|
||||
const store = new MemoryRefreshStore();
|
||||
const sessions = store.liveSessions;
|
||||
store.liveSessions.add(`${WORKSPACE}:test-sid`);
|
||||
store.liveSessions.add(`${WORKSPACE}:test-sid`);
|
||||
const service = new OidcService({
|
||||
config: () => config,
|
||||
keyring: (c) => OidcKeyring.open(new FileKeyStore(c.keyDir), vault),
|
||||
|
|
@ -91,7 +93,7 @@ function harness(): { service: OidcService; users: FakeUsers; clock: { now: numb
|
|||
}
|
||||
|
||||
async function issueCode(service: OidcService, challenge: string, extra: Record<string, string> = {}): Promise<string> {
|
||||
const { redirectTo } = await service.consent(WORKSPACE, ADA.id, authorizeParams(challenge, extra));
|
||||
const { redirectTo } = await service.consent(WORKSPACE, ADA.id, authorizeParams(challenge, extra), "test-sid");
|
||||
const url = new URL(redirectTo);
|
||||
const code = url.searchParams.get("code");
|
||||
if (code === null) throw new Error(`no code in redirect: ${url.searchParams.get("error") ?? ""}`);
|
||||
|
|
@ -228,7 +230,8 @@ describe("OidcService", () => {
|
|||
it("auto-consents for client_id=web: the session gets a code straight back, no consent screen", async () => {
|
||||
const { service } = harness();
|
||||
const { challenge } = pkce();
|
||||
const { redirectTo } = await service.consent(WORKSPACE, ADA.id, authorizeParams(challenge));
|
||||
await expect(service.consent(WORKSPACE, ADA.id, authorizeParams(challenge))).rejects.toBeInstanceOf(UnauthorizedException); // sid-less bearers cannot mint
|
||||
const { redirectTo } = await service.consent(WORKSPACE, ADA.id, authorizeParams(challenge), "test-sid");
|
||||
const url = new URL(redirectTo);
|
||||
expect(`${url.origin}${url.pathname}`).toBe(WEB_REDIRECT_URI);
|
||||
expect(url.searchParams.get("code")).toMatch(/^[A-Za-z0-9_-]{43}$/);
|
||||
|
|
@ -239,19 +242,19 @@ describe("OidcService", () => {
|
|||
it("only issues codes to active members with a usable, unreserved username", async () => {
|
||||
const { service, users } = harness();
|
||||
const { challenge } = pkce();
|
||||
await expect(service.consent(WORKSPACE, "33333333-3333-4333-8333-333333333333", authorizeParams(challenge))).rejects.toBeInstanceOf(ForbiddenException);
|
||||
await expect(service.consent("44444444-4444-4444-8444-444444444444", ADA.id, authorizeParams(challenge))).rejects.toBeInstanceOf(ForbiddenException);
|
||||
await expect(service.consent(WORKSPACE, "33333333-3333-4333-8333-333333333333", authorizeParams(challenge), "test-sid")).rejects.toBeInstanceOf(ForbiddenException);
|
||||
await expect(service.consent("44444444-4444-4444-8444-444444444444", ADA.id, authorizeParams(challenge), "test-sid")).rejects.toBeInstanceOf(UnauthorizedException); // cross-workspace sid is not live
|
||||
users.members.set(ADA.id, { ...ADA, username: "admin" });
|
||||
await expect(service.consent(WORKSPACE, ADA.id, authorizeParams(challenge))).rejects.toBeInstanceOf(ForbiddenException);
|
||||
await expect(service.consent(WORKSPACE, ADA.id, authorizeParams(challenge), "test-sid")).rejects.toBeInstanceOf(ForbiddenException);
|
||||
users.members.set(ADA.id, { ...ADA, username: "Ada" });
|
||||
await expect(service.consent(WORKSPACE, ADA.id, authorizeParams(challenge))).rejects.toBeInstanceOf(ForbiddenException);
|
||||
await expect(service.consent(WORKSPACE, ADA.id, authorizeParams(challenge), "test-sid")).rejects.toBeInstanceOf(ForbiddenException);
|
||||
});
|
||||
|
||||
it("re-validates relayed params: a forged redirect_uri from the consent page is refused", async () => {
|
||||
const { service } = harness();
|
||||
const { challenge } = pkce();
|
||||
await expect(service.consent(WORKSPACE, ADA.id, authorizeParams(challenge, { redirect_uri: "https://evil.example/cb" }))).rejects.toBeInstanceOf(BadRequestException);
|
||||
const { redirectTo } = await service.consent(WORKSPACE, ADA.id, authorizeParams(challenge, { code_challenge_method: "plain" }));
|
||||
await expect(service.consent(WORKSPACE, ADA.id, authorizeParams(challenge, { redirect_uri: "https://evil.example/cb" }), "test-sid")).rejects.toBeInstanceOf(BadRequestException);
|
||||
const { redirectTo } = await service.consent(WORKSPACE, ADA.id, authorizeParams(challenge, { code_challenge_method: "plain" }), "test-sid");
|
||||
expect(new URL(redirectTo).searchParams.has("code")).toBe(false);
|
||||
});
|
||||
|
||||
|
|
@ -388,11 +391,10 @@ describe("OidcService refresh tokens (phase 2)", () => {
|
|||
expect(rows.filter((row) => row.consumedAt === null)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("a code from a pre-phase2 session (no sid) mints no refresh token", async () => {
|
||||
const h = harness();
|
||||
const { verifier, challenge } = pkce();
|
||||
const tokens = await exchange(h.service, await issueCode(h.service, challenge), verifier);
|
||||
expect(tokens.refresh_token).toBeUndefined();
|
||||
it("a consent without a session id (pre-phase2 client) is rejected outright", async () => {
|
||||
const { service } = harness();
|
||||
const { challenge } = pkce();
|
||||
await expect(service.consent(WORKSPACE, ADA.id, authorizeParams(challenge))).rejects.toBeInstanceOf(UnauthorizedException);
|
||||
});
|
||||
|
||||
it("rejects a revoked (logged-out) session's refresh token and revokes the chain", async () => {
|
||||
|
|
|
|||
141
apps/api/src/outreach/outreach.controller.ts
Normal file
141
apps/api/src/outreach/outreach.controller.ts
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
/**
|
||||
* Outreach contacts (manual DM campaign tracking).
|
||||
*
|
||||
* GET /outreach/contacts?status= outreach.view — list (≤ 2000)
|
||||
* POST /outreach/contacts outreach.manage — { contacts: [...] } bulk add, ≤ 500
|
||||
* PATCH /outreach/contacts/:id outreach.manage — { message?, notes?, status? }
|
||||
* DELETE /outreach/contacts/:id outreach.manage
|
||||
*
|
||||
* No route sends a message: see OutreachService.
|
||||
*/
|
||||
import {
|
||||
BadRequestException,
|
||||
Body,
|
||||
Controller,
|
||||
Delete,
|
||||
Get,
|
||||
HttpCode,
|
||||
Inject,
|
||||
Param,
|
||||
Patch,
|
||||
Post,
|
||||
Query,
|
||||
Req,
|
||||
UseGuards,
|
||||
} from "@nestjs/common";
|
||||
import { requireObject, requireSession } from "../http/validation.js";
|
||||
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import {
|
||||
OUTREACH_MAX_IMPORT,
|
||||
OutreachService,
|
||||
isOutreachStatus,
|
||||
type OutreachContact,
|
||||
type OutreachContactInput,
|
||||
type OutreachContactPatch,
|
||||
type OutreachStatus,
|
||||
} from "./outreach.service.js";
|
||||
|
||||
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
const PLATFORM = /^[a-z0-9][a-z0-9-]{1,31}$/;
|
||||
const LANGUAGE = /^[a-z]{2,3}(-[A-Za-z0-9]{2,8})?$/;
|
||||
|
||||
@Controller("outreach")
|
||||
@UseGuards(RbacGuard)
|
||||
export class OutreachController {
|
||||
constructor(@Inject(OutreachService) private readonly outreach: OutreachService) {}
|
||||
|
||||
@Get("contacts")
|
||||
@RequirePermission("outreach.view")
|
||||
async list(@Req() request: AuthenticatedRequest, @Query("status") status?: string): Promise<OutreachContact[]> {
|
||||
const session = requireSession(request);
|
||||
if (status !== undefined && !isOutreachStatus(status)) throw new BadRequestException("unknown status");
|
||||
return this.outreach.list(session.workspaceId, status as OutreachStatus | undefined);
|
||||
}
|
||||
|
||||
@Post("contacts")
|
||||
@RequirePermission("outreach.manage")
|
||||
async import(@Req() request: AuthenticatedRequest, @Body() body: unknown): Promise<{ added: number; skipped: number }> {
|
||||
const session = requireSession(request);
|
||||
const raw = requireObject(body, "body").contacts;
|
||||
if (!Array.isArray(raw) || raw.length === 0 || raw.length > OUTREACH_MAX_IMPORT) {
|
||||
throw new BadRequestException(`contacts must be an array of 1 to ${OUTREACH_MAX_IMPORT} items`);
|
||||
}
|
||||
const contacts = raw.map((item, index) => parseContact(item, `contacts[${index}]`));
|
||||
return this.outreach.import(session.workspaceId, contacts, { userId: session.userId, role: session.role });
|
||||
}
|
||||
|
||||
@Patch("contacts/:id")
|
||||
@RequirePermission("outreach.manage")
|
||||
async update(@Req() request: AuthenticatedRequest, @Param("id") id: string, @Body() body: unknown): Promise<OutreachContact> {
|
||||
const session = requireSession(request);
|
||||
if (!UUID.test(id)) throw new BadRequestException("id must be a UUID");
|
||||
return this.outreach.update(session.workspaceId, id, parsePatch(body), { userId: session.userId, role: session.role });
|
||||
}
|
||||
|
||||
@Delete("contacts/:id")
|
||||
@HttpCode(204)
|
||||
@RequirePermission("outreach.manage")
|
||||
async remove(@Req() request: AuthenticatedRequest, @Param("id") id: string): Promise<void> {
|
||||
const session = requireSession(request);
|
||||
if (!UUID.test(id)) throw new BadRequestException("id must be a UUID");
|
||||
await this.outreach.remove(session.workspaceId, id, { userId: session.userId, role: session.role });
|
||||
}
|
||||
}
|
||||
|
||||
function text(value: unknown, field: string, max: number, required: boolean): string | undefined {
|
||||
if (value === undefined || value === null || value === "") {
|
||||
if (required) throw new BadRequestException(`${field} is required`);
|
||||
return undefined;
|
||||
}
|
||||
if (typeof value !== "string") throw new BadRequestException(`${field} must be a string`);
|
||||
const trimmed = value.trim();
|
||||
if (required && trimmed === "") throw new BadRequestException(`${field} is required`);
|
||||
if (trimmed.length > max) throw new BadRequestException(`${field} must be at most ${max} characters`);
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
export function parseContact(value: unknown, field: string): OutreachContactInput {
|
||||
const item = requireObject(value, field);
|
||||
const name = text(item.name, `${field}.name`, 200, true) as string;
|
||||
const platform = (text(item.platform, `${field}.platform`, 32, true) as string).toLowerCase();
|
||||
if (!PLATFORM.test(platform)) throw new BadRequestException(`${field}.platform must be a lowercase id`);
|
||||
const handle = text(item.handle, `${field}.handle`, 200, true) as string;
|
||||
const profileUrl = text(item.profileUrl, `${field}.profileUrl`, 500, true) as string;
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(profileUrl);
|
||||
} catch {
|
||||
throw new BadRequestException(`${field}.profileUrl must be a URL`);
|
||||
}
|
||||
if (url.protocol !== "https:" || /\s/.test(profileUrl)) throw new BadRequestException(`${field}.profileUrl must be an https URL`);
|
||||
const language = text(item.language, `${field}.language`, 12, false);
|
||||
if (language !== undefined && !LANGUAGE.test(language)) throw new BadRequestException(`${field}.language must be a language tag such as es or pt-BR`);
|
||||
const contact: OutreachContactInput = { name, platform, handle, profileUrl };
|
||||
const category = text(item.category, `${field}.category`, 100, false);
|
||||
const message = text(item.message, `${field}.message`, 4000, false);
|
||||
const notes = text(item.notes, `${field}.notes`, 2000, false);
|
||||
if (category !== undefined) contact.category = category;
|
||||
if (language !== undefined) contact.language = language;
|
||||
if (message !== undefined) contact.message = message;
|
||||
if (notes !== undefined) contact.notes = notes;
|
||||
return contact;
|
||||
}
|
||||
|
||||
export function parsePatch(value: unknown): OutreachContactPatch {
|
||||
const body = requireObject(value, "body");
|
||||
const patch: OutreachContactPatch = {};
|
||||
if (body.message !== undefined) {
|
||||
if (typeof body.message !== "string" || body.message.length > 4000) throw new BadRequestException("message must be a string of at most 4000 characters");
|
||||
patch.message = body.message;
|
||||
}
|
||||
if (body.notes !== undefined) {
|
||||
if (typeof body.notes !== "string" || body.notes.length > 2000) throw new BadRequestException("notes must be a string of at most 2000 characters");
|
||||
patch.notes = body.notes;
|
||||
}
|
||||
if (body.status !== undefined) {
|
||||
if (!isOutreachStatus(body.status)) throw new BadRequestException("status must be todo, sent, replied or declined");
|
||||
patch.status = body.status;
|
||||
}
|
||||
if (Object.keys(patch).length === 0) throw new BadRequestException("nothing to update");
|
||||
return patch;
|
||||
}
|
||||
13
apps/api/src/outreach/outreach.module.ts
Normal file
13
apps/api/src/outreach/outreach.module.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
import { Module } from "@nestjs/common";
|
||||
import { OutreachController } from "./outreach.controller.js";
|
||||
import { OutreachService } from "./outreach.service.js";
|
||||
|
||||
/**
|
||||
* Outreach contacts: the list of public accounts a collective writes to by
|
||||
* hand, with the drafted message and the sent/replied state per contact.
|
||||
*/
|
||||
@Module({
|
||||
controllers: [OutreachController],
|
||||
providers: [OutreachService],
|
||||
})
|
||||
export class OutreachModule {}
|
||||
229
apps/api/src/outreach/outreach.service.ts
Normal file
229
apps/api/src/outreach/outreach.service.ts
Normal file
|
|
@ -0,0 +1,229 @@
|
|||
/**
|
||||
* Outreach: a workspace's list of public accounts to contact one by one.
|
||||
*
|
||||
* The Hub stores the contact, a drafted message and where the conversation
|
||||
* stands (todo → sent → replied/declined). It never sends anything: the
|
||||
* person copies the text, opens the profile and writes from the platform's
|
||||
* own DM box. Platforms forbid or rate-limit unsolicited bulk DMs, so
|
||||
* sending stays human and the API has no provider calls here.
|
||||
*
|
||||
* Audit entries record status changes and membership of the list, never the
|
||||
* message text.
|
||||
*/
|
||||
import { BadRequestException, Injectable, NotFoundException } from "@nestjs/common";
|
||||
import { appendTenantAudit } from "../audit/audit.service.js";
|
||||
import { withTenant } from "../db/client.js";
|
||||
|
||||
export const OUTREACH_STATUSES = ["todo", "sent", "replied", "declined"] as const;
|
||||
export type OutreachStatus = (typeof OUTREACH_STATUSES)[number];
|
||||
|
||||
/** One import call; larger lists go in several requests. */
|
||||
export const OUTREACH_MAX_IMPORT = 500;
|
||||
export const OUTREACH_MAX_LIST = 2000;
|
||||
|
||||
export interface OutreachContactInput {
|
||||
name: string;
|
||||
platform: string;
|
||||
handle: string;
|
||||
profileUrl: string;
|
||||
category?: string;
|
||||
language?: string;
|
||||
message?: string;
|
||||
notes?: string;
|
||||
}
|
||||
|
||||
export interface OutreachContactPatch {
|
||||
message?: string;
|
||||
notes?: string;
|
||||
status?: OutreachStatus;
|
||||
}
|
||||
|
||||
export interface OutreachContact {
|
||||
id: string;
|
||||
name: string;
|
||||
platform: string;
|
||||
handle: string;
|
||||
profileUrl: string;
|
||||
category: string | null;
|
||||
language: string | null;
|
||||
message: string;
|
||||
notes: string;
|
||||
status: OutreachStatus;
|
||||
sentAt: string | null;
|
||||
repliedAt: string | null;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface OutreachActor {
|
||||
userId: string;
|
||||
role: string;
|
||||
}
|
||||
|
||||
interface OutreachRow {
|
||||
id: string;
|
||||
name: string;
|
||||
platform: string;
|
||||
handle: string;
|
||||
profile_url: string;
|
||||
category: string | null;
|
||||
language: string | null;
|
||||
message: string;
|
||||
notes: string;
|
||||
status: OutreachStatus;
|
||||
sent_at: Date | null;
|
||||
replied_at: Date | null;
|
||||
created_at: Date;
|
||||
updated_at: Date;
|
||||
}
|
||||
|
||||
const COLUMNS = "id, name, platform, handle, profile_url, category, language, message, notes, status, sent_at, replied_at, created_at, updated_at";
|
||||
|
||||
function fromRow(row: OutreachRow): OutreachContact {
|
||||
return {
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
platform: row.platform,
|
||||
handle: row.handle,
|
||||
profileUrl: row.profile_url,
|
||||
category: row.category,
|
||||
language: row.language,
|
||||
message: row.message,
|
||||
notes: row.notes,
|
||||
status: row.status,
|
||||
sentAt: row.sent_at?.toISOString() ?? null,
|
||||
repliedAt: row.replied_at?.toISOString() ?? null,
|
||||
createdAt: row.created_at.toISOString(),
|
||||
updatedAt: row.updated_at.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
export function isOutreachStatus(value: unknown): value is OutreachStatus {
|
||||
return typeof value === "string" && (OUTREACH_STATUSES as readonly string[]).includes(value);
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class OutreachService {
|
||||
async list(workspaceId: string, status?: OutreachStatus): Promise<OutreachContact[]> {
|
||||
return withTenant(workspaceId, async (client) => {
|
||||
const result = await client.query<OutreachRow>(
|
||||
`SELECT ${COLUMNS} FROM outreach_contacts
|
||||
WHERE ($1::text IS NULL OR status = $1)
|
||||
ORDER BY lower(name), platform
|
||||
LIMIT ${OUTREACH_MAX_LIST}`,
|
||||
[status ?? null],
|
||||
);
|
||||
return result.rows.map(fromRow);
|
||||
});
|
||||
}
|
||||
|
||||
/** Bulk add; an account already on the list (same platform + handle) is skipped. */
|
||||
async import(workspaceId: string, contacts: OutreachContactInput[], actor: OutreachActor): Promise<{ added: number; skipped: number }> {
|
||||
if (contacts.length === 0) throw new BadRequestException("contacts must not be empty");
|
||||
if (contacts.length > OUTREACH_MAX_IMPORT) {
|
||||
throw new BadRequestException(`at most ${OUTREACH_MAX_IMPORT} contacts per import`);
|
||||
}
|
||||
const added = await withTenant(workspaceId, async (client) => {
|
||||
const result = await client.query<{ id: string }>(
|
||||
`INSERT INTO outreach_contacts
|
||||
(workspace_id, name, platform, handle, profile_url, category, language, message, notes, created_by)
|
||||
SELECT $1, c.name, c.platform, c.handle, c.profile_url, c.category, c.language,
|
||||
coalesce(c.message, ''), coalesce(c.notes, ''), $2
|
||||
FROM jsonb_to_recordset($3::jsonb) AS c(
|
||||
name text, platform text, handle text, profile_url text,
|
||||
category text, language text, message text, notes text)
|
||||
ON CONFLICT (workspace_id, platform, lower(handle)) DO NOTHING
|
||||
RETURNING id`,
|
||||
[workspaceId, actor.userId, JSON.stringify(contacts.map(toRecord))],
|
||||
);
|
||||
return result.rowCount ?? result.rows.length;
|
||||
});
|
||||
if (added > 0) {
|
||||
await appendTenantAudit(workspaceId, {
|
||||
actorUserId: actor.userId,
|
||||
actorRole: actor.role,
|
||||
action: "outreach.imported",
|
||||
targetType: "outreach_contact",
|
||||
targetId: workspaceId,
|
||||
diff: { added, skipped: contacts.length - added },
|
||||
});
|
||||
}
|
||||
return { added, skipped: contacts.length - added };
|
||||
}
|
||||
|
||||
/**
|
||||
* Edit the draft/notes or move the status. Marking `sent` stamps who and
|
||||
* when; `replied` stamps when; going back to `todo` clears both.
|
||||
*/
|
||||
async update(workspaceId: string, id: string, patch: OutreachContactPatch, actor: OutreachActor): Promise<OutreachContact> {
|
||||
const { contact, previous } = await withTenant(workspaceId, async (client) => {
|
||||
const current = await client.query<{ status: OutreachStatus }>(
|
||||
"SELECT status FROM outreach_contacts WHERE id = $1 FOR UPDATE",
|
||||
[id],
|
||||
);
|
||||
const before = current.rows[0];
|
||||
if (before === undefined) throw new NotFoundException(`outreach contact ${id} not found`);
|
||||
const status = patch.status ?? before.status;
|
||||
const changed = status !== before.status;
|
||||
const result = await client.query<OutreachRow>(
|
||||
`UPDATE outreach_contacts SET
|
||||
message = coalesce($2, message),
|
||||
notes = coalesce($3, notes),
|
||||
status = $4,
|
||||
sent_at = CASE WHEN NOT $5 THEN sent_at WHEN $4 = 'todo' THEN NULL WHEN $4 = 'sent' THEN now() ELSE coalesce(sent_at, now()) END,
|
||||
sent_by = CASE WHEN NOT $5 THEN sent_by WHEN $4 = 'todo' THEN NULL WHEN $4 = 'sent' THEN $6::uuid ELSE coalesce(sent_by, $6::uuid) END,
|
||||
replied_at = CASE WHEN NOT $5 THEN replied_at WHEN $4 = 'replied' THEN now() ELSE NULL END,
|
||||
updated_at = now()
|
||||
WHERE id = $1
|
||||
RETURNING ${COLUMNS}`,
|
||||
[id, patch.message ?? null, patch.notes ?? null, status, changed, actor.userId],
|
||||
);
|
||||
const row = result.rows[0];
|
||||
if (row === undefined) throw new NotFoundException(`outreach contact ${id} not found`);
|
||||
return { contact: fromRow(row), previous: before.status };
|
||||
});
|
||||
if (contact.status !== previous) {
|
||||
await appendTenantAudit(workspaceId, {
|
||||
actorUserId: actor.userId,
|
||||
actorRole: actor.role,
|
||||
action: `outreach.${contact.status}`,
|
||||
targetType: "outreach_contact",
|
||||
targetId: contact.id,
|
||||
diff: { from: previous, to: contact.status, platform: contact.platform, handle: contact.handle },
|
||||
});
|
||||
}
|
||||
return contact;
|
||||
}
|
||||
|
||||
async remove(workspaceId: string, id: string, actor: OutreachActor): Promise<void> {
|
||||
const removed = await withTenant(workspaceId, async (client) => {
|
||||
const result = await client.query<{ platform: string; handle: string }>(
|
||||
"DELETE FROM outreach_contacts WHERE id = $1 RETURNING platform, handle",
|
||||
[id],
|
||||
);
|
||||
return result.rows[0];
|
||||
});
|
||||
if (removed === undefined) throw new NotFoundException(`outreach contact ${id} not found`);
|
||||
await appendTenantAudit(workspaceId, {
|
||||
actorUserId: actor.userId,
|
||||
actorRole: actor.role,
|
||||
action: "outreach.removed",
|
||||
targetType: "outreach_contact",
|
||||
targetId: id,
|
||||
diff: { platform: removed.platform, handle: removed.handle },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function toRecord(contact: OutreachContactInput): Record<string, string | null> {
|
||||
return {
|
||||
name: contact.name,
|
||||
platform: contact.platform,
|
||||
handle: contact.handle,
|
||||
profile_url: contact.profileUrl,
|
||||
category: contact.category ?? null,
|
||||
language: contact.language ?? null,
|
||||
message: contact.message ?? null,
|
||||
notes: contact.notes ?? null,
|
||||
};
|
||||
}
|
||||
136
apps/api/src/outreach/outreach.test.ts
Normal file
136
apps/api/src/outreach/outreach.test.ts
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
import "reflect-metadata";
|
||||
import type { PoolClient } from "pg";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { appendTenantAudit } from "../audit/audit.service.js";
|
||||
import { parseContact, parsePatch } from "./outreach.controller.js";
|
||||
import { OutreachService } from "./outreach.service.js";
|
||||
|
||||
/**
|
||||
* Outreach service + request parsing with the DB faked. RLS and the migration
|
||||
* shape belong to a live-DB suite.
|
||||
*/
|
||||
|
||||
const dbMock = vi.hoisted(() => ({ withTenant: vi.fn() }));
|
||||
vi.mock("../db/client.js", () => ({ withTenant: dbMock.withTenant }));
|
||||
vi.mock("../audit/audit.service.js", () => ({ appendTenantAudit: vi.fn(async () => "audit-id") }));
|
||||
|
||||
const WS = "0d9a1b40-6f0f-4b1e-8f0e-3f3c9e6b4a10";
|
||||
const ID = "11111111-2222-4333-8444-555555555555";
|
||||
const ACTOR = { userId: "8f7f6e50-7b0f-4a2e-9d3e-2f2d1a0c5b21", role: "Editor" };
|
||||
|
||||
const row = (overrides: Record<string, unknown> = {}) => ({
|
||||
id: ID,
|
||||
name: "Climáximo",
|
||||
platform: "bluesky",
|
||||
handle: "climaximopt.bsky.social",
|
||||
profile_url: "https://bsky.app/profile/climaximopt.bsky.social",
|
||||
category: "clima",
|
||||
language: "pt",
|
||||
message: "Olá!",
|
||||
notes: "",
|
||||
status: "todo",
|
||||
sent_at: null,
|
||||
replied_at: null,
|
||||
created_at: new Date("2026-10-15T10:00:00Z"),
|
||||
updated_at: new Date("2026-10-15T10:00:00Z"),
|
||||
...overrides,
|
||||
});
|
||||
|
||||
type Query = (sql: string, values?: unknown[]) => Promise<{ rows: unknown[]; rowCount?: number }>;
|
||||
|
||||
function tenant(query: Query): Array<{ sql: string; values: unknown[] }> {
|
||||
const calls: Array<{ sql: string; values: unknown[] }> = [];
|
||||
dbMock.withTenant.mockImplementation(async (_ws: string, fn: (client: PoolClient) => Promise<unknown>) =>
|
||||
fn({
|
||||
query: async (sql: string, values: unknown[] = []) => {
|
||||
calls.push({ sql, values });
|
||||
return query(sql, values);
|
||||
},
|
||||
} as unknown as PoolClient),
|
||||
);
|
||||
return calls;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
dbMock.withTenant.mockReset();
|
||||
vi.mocked(appendTenantAudit).mockClear();
|
||||
});
|
||||
|
||||
describe("parseContact", () => {
|
||||
const valid = { name: " DAL ", platform: "Instagram-x", handle: "@federationdal", profileUrl: "https://example.org/federationdal", language: "fr" };
|
||||
|
||||
it("trims and lowercases the platform id", () => {
|
||||
expect(parseContact(valid, "c")).toEqual({ name: "DAL", platform: "instagram-x", handle: "@federationdal", profileUrl: "https://example.org/federationdal", language: "fr" });
|
||||
});
|
||||
|
||||
it("rejects non-https profile links and odd platform ids", () => {
|
||||
expect(() => parseContact({ ...valid, profileUrl: "http://example.org/x" }, "c")).toThrow(/https/);
|
||||
expect(() => parseContact({ ...valid, profileUrl: "javascript:alert(1)" }, "c")).toThrow(/https/);
|
||||
expect(() => parseContact({ ...valid, platform: "x y" }, "c")).toThrow(/platform/);
|
||||
expect(() => parseContact({ ...valid, language: "spanish" }, "c")).toThrow(/language/);
|
||||
expect(() => parseContact({ ...valid, name: " " }, "c")).toThrow(/name/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parsePatch", () => {
|
||||
it("accepts the known statuses only and refuses an empty patch", () => {
|
||||
expect(parsePatch({ status: "sent" })).toEqual({ status: "sent" });
|
||||
expect(() => parsePatch({ status: "spam" })).toThrow(/status/);
|
||||
expect(() => parsePatch({})).toThrow(/nothing/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("OutreachService", () => {
|
||||
const service = new OutreachService();
|
||||
|
||||
it("imports in one statement, skips known accounts and audits counts only", async () => {
|
||||
const calls = tenant(async () => ({ rows: [{ id: ID }], rowCount: 1 }));
|
||||
const result = await service.import(WS, [
|
||||
{ name: "A", platform: "bluesky", handle: "a", profileUrl: "https://bsky.app/profile/a", message: "private draft" },
|
||||
{ name: "B", platform: "bluesky", handle: "b", profileUrl: "https://bsky.app/profile/b" },
|
||||
], ACTOR);
|
||||
expect(result).toEqual({ added: 1, skipped: 1 });
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0]?.sql).toContain("ON CONFLICT (workspace_id, platform, lower(handle)) DO NOTHING");
|
||||
expect(JSON.parse(String(calls[0]?.values[2]))).toHaveLength(2);
|
||||
expect(appendTenantAudit).toHaveBeenCalledWith(WS, expect.objectContaining({ action: "outreach.imported", diff: { added: 1, skipped: 1 } }));
|
||||
expect(JSON.stringify(vi.mocked(appendTenantAudit).mock.calls)).not.toContain("private draft");
|
||||
});
|
||||
|
||||
it("does not audit an import that added nothing", async () => {
|
||||
tenant(async () => ({ rows: [], rowCount: 0 }));
|
||||
expect(await service.import(WS, [{ name: "A", platform: "bluesky", handle: "a", profileUrl: "https://bsky.app/profile/a" }], ACTOR)).toEqual({ added: 0, skipped: 1 });
|
||||
expect(appendTenantAudit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("marks a contact sent and audits the transition without the message", async () => {
|
||||
const calls = tenant(async (sql) => {
|
||||
if (sql.startsWith("SELECT status")) return { rows: [{ status: "todo" }] };
|
||||
if (sql.startsWith("UPDATE outreach_contacts")) return { rows: [row({ status: "sent", message: "Olá, secret text", sent_at: new Date("2026-10-15T11:00:00Z") })] };
|
||||
throw new Error(`unexpected query: ${sql}`);
|
||||
});
|
||||
const contact = await service.update(WS, ID, { status: "sent", message: "Olá, secret text" }, ACTOR);
|
||||
expect(contact.status).toBe("sent");
|
||||
expect(contact.sentAt).toBe("2026-10-15T11:00:00.000Z");
|
||||
expect(calls[1]?.values).toEqual([ID, "Olá, secret text", null, "sent", true, ACTOR.userId]);
|
||||
expect(appendTenantAudit).toHaveBeenCalledWith(WS, expect.objectContaining({ action: "outreach.sent", diff: expect.objectContaining({ from: "todo", to: "sent" }) }));
|
||||
expect(JSON.stringify(vi.mocked(appendTenantAudit).mock.calls)).not.toContain("secret text");
|
||||
});
|
||||
|
||||
it("edits the draft without a status change and without an audit entry", async () => {
|
||||
const calls = tenant(async (sql) => {
|
||||
if (sql.startsWith("SELECT status")) return { rows: [{ status: "sent" }] };
|
||||
return { rows: [row({ status: "sent", message: "v2" })] };
|
||||
});
|
||||
await service.update(WS, ID, { message: "v2" }, ACTOR);
|
||||
expect(calls[1]?.values).toEqual([ID, "v2", null, "sent", false, ACTOR.userId]);
|
||||
expect(appendTenantAudit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("404s on an unknown contact", async () => {
|
||||
tenant(async () => ({ rows: [] }));
|
||||
await expect(service.update(WS, ID, { status: "sent" }, ACTOR)).rejects.toThrow(/not found/);
|
||||
await expect(service.remove(WS, ID, ACTOR)).rejects.toThrow(/not found/);
|
||||
expect(appendTenantAudit).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -2,11 +2,11 @@ import { randomUUID } from "node:crypto";
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
import { MemoryRedis } from "./testing/memory-redis.js";
|
||||
import { PreviewService } from "./preview.js";
|
||||
import { keyForHash, sha256, type StorageService } from "../storage/storage.js";
|
||||
import { keyForHash, sha256, directStorageDisabled, type StorageService } from "../storage/storage.js";
|
||||
const WS = randomUUID(), CH = randomUUID(), TAP = randomUUID();
|
||||
function fixture(): { preview: PreviewService; storage: StorageService; redis: MemoryRedis; seed: (file: string, body: string, type?: string) => void; advance: (ms: number) => void } {
|
||||
const redis = new MemoryRedis(), objects = new Map<string, Buffer>(); let now = Date.now();
|
||||
const storage: StorageService = { putContent: async () => { throw new Error("unused"); }, getObject: vi.fn(async key => { await new Promise(r => setTimeout(r, 1)); return objects.get(key) ?? Buffer.alloc(0); }), hasContent: async () => false, presign: async () => { throw new Error("must not redirect to encrypted S3"); } };
|
||||
const storage: StorageService = { putContent: async () => { throw new Error("unused"); }, getObject: vi.fn(async key => { await new Promise(r => setTimeout(r, 1)); return objects.get(key) ?? Buffer.alloc(0); }), hasContent: async () => false, presign: async () => { throw new Error("must not redirect to encrypted S3"); }, ...directStorageDisabled() };
|
||||
const preview = new PreviewService({ redis, storage, secret: "test", now: () => now, assertTap: async () => undefined });
|
||||
const seed = (file: string, body: string, type = "video/mp2t"): void => {
|
||||
const buffer = Buffer.from(body), key = keyForHash(sha256(buffer), type); objects.set(key, buffer);
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
|
|||
import type { PoolClient } from "pg";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { MediaControlService, type IssuedCommand } from "../media-control/media-control.service.js";
|
||||
import { sha256, keyForHash, type StorageService } from "../storage/storage.js";
|
||||
import { sha256, keyForHash, directStorageDisabled, type StorageService } from "../storage/storage.js";
|
||||
import { ProductionService } from "./production.service.js";
|
||||
import type { TenantTransaction } from "./journal.js";
|
||||
const WS = randomUUID(), CH = randomUUID(), SLOT = randomUUID();
|
||||
|
|
@ -29,7 +29,7 @@ function fixture(render?: (input: unknown) => Promise<Buffer>): { production: Pr
|
|||
try { return await fn({ query } as unknown as PoolClient); } finally { unlock(); }
|
||||
};
|
||||
const control = new MediaControlService({ signer: null, assignments: [] }); vi.spyOn(control, "director").mockResolvedValue(ISSUED);
|
||||
const storage: StorageService = { putContent: vi.fn(async (buffer, contentType) => { const hash = sha256(buffer); return { hash, key: keyForHash(hash, contentType), sizeBytes: buffer.length, contentType, deduplicated: false }; }), getObject: async () => Buffer.alloc(0), hasContent: async () => false, presign: async () => "" };
|
||||
const storage: StorageService = { putContent: vi.fn(async (buffer, contentType) => { const hash = sha256(buffer); return { hash, key: keyForHash(hash, contentType), sizeBytes: buffer.length, contentType, deduplicated: false }; }), getObject: async () => Buffer.alloc(0), hasContent: async () => false, presign: async () => "", ...directStorageDisabled() };
|
||||
return { rows, control, storage, production: new ProductionService(control, { transaction, storage, ...(render ? { render } : {}) }) };
|
||||
}
|
||||
describe("revision → render job → content-addressed storage → desired-state pipeline", () => {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import { BadRequestException, ForbiddenException, Inject, Injectable, NotFoundException, Optional, ConflictException } from "@nestjs/common";
|
||||
// eslint-disable-next-line no-restricted-syntax -- Connector composition boundary.
|
||||
import { createInstagramConnector } from "@tactical-media-broadcast-hub/connector-instagram";
|
||||
// eslint-disable-next-line no-restricted-syntax -- Connector composition boundary.
|
||||
import { createMastodonConnector } from "@tactical-media-broadcast-hub/connector-mastodon";
|
||||
import { isPublishingProvider, providerFetch } from "@tactical-media-broadcast-hub/connector-runtime";
|
||||
import { ConnectorRegistry, type CredentialAccessor, type PlatformConnector } from "@tactical-media-broadcast-hub/connector-sdk";
|
||||
|
|
@ -129,6 +131,7 @@ function connectorFor(provider: string): PlatformConnector {
|
|||
};
|
||||
registry.register(createMastodonConnector(options));
|
||||
registry.register(createTelegramConnector(options));
|
||||
registry.register(createInstagramConnector(options));
|
||||
return registry;
|
||||
})();
|
||||
return publishingConnectors.get(provider);
|
||||
|
|
@ -654,7 +657,7 @@ export class PublishingService {
|
|||
async function assertMediaAssetsExist(client: PoolClient, mediaAssetIds: string[]): Promise<void> {
|
||||
if (mediaAssetIds.length === 0) return;
|
||||
const result = await client.query<{ count: string | number }>(
|
||||
"SELECT count(*)::integer AS count FROM media_assets WHERE id = ANY($1::uuid[])",
|
||||
"SELECT count(*)::integer AS count FROM media_assets WHERE id = ANY($1::uuid[]) AND workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid",
|
||||
[mediaAssetIds],
|
||||
);
|
||||
if (Number(result.rows[0]?.count ?? 0) !== mediaAssetIds.length) {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { directStorageDisabled } from "../storage/storage.js";
|
||||
import { BadRequestException, ServiceUnavailableException } from "@nestjs/common";
|
||||
import type { PoolClient } from "pg";
|
||||
import type { Request, Response } from "express";
|
||||
|
|
@ -67,6 +68,7 @@ function fakeStorage(objects: Record<string, Buffer> = {}): {
|
|||
}),
|
||||
hasContent: vi.fn(async () => false),
|
||||
presign: vi.fn(async () => ""),
|
||||
...directStorageDisabled(),
|
||||
};
|
||||
return { storage, puts };
|
||||
}
|
||||
|
|
@ -339,6 +341,7 @@ describe("branding service — uploads (storage reuse, magic-byte sniffing)", ()
|
|||
getObject: vi.fn(async () => Buffer.alloc(0)),
|
||||
hasContent: vi.fn(async () => false),
|
||||
presign: vi.fn(async () => ""),
|
||||
...directStorageDisabled(),
|
||||
};
|
||||
const branding = serviceWith({ storage: broken });
|
||||
await expect(branding.uploadImage(PNG_HEADER, "image/png")).rejects.toBeInstanceOf(ServiceUnavailableException);
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { createHash, generateKeyPairSync, randomBytes } from "node:crypto";
|
||||
|
||||
const s3Calls = vi.hoisted(() => ({ sent: [] as string[] }));
|
||||
const s3Calls = vi.hoisted(() => ({
|
||||
sent: [] as string[],
|
||||
ranges: [] as string[],
|
||||
sealed: null as Buffer | null,
|
||||
}));
|
||||
|
||||
vi.mock("@aws-sdk/client-s3", () => {
|
||||
class FakeCommand {
|
||||
|
|
@ -27,16 +31,40 @@ vi.mock("@aws-sdk/client-s3", () => {
|
|||
s3Calls.sent.push(cmd.marker);
|
||||
if (cmd.marker === "Create") throw new S3ServiceException("Access Denied", 403, "AccessDenied");
|
||||
if (cmd.marker === "Head") throw new S3ServiceException("Not Found", 404, "NotFound");
|
||||
if (cmd.marker === "Get" && s3Calls.sealed !== null) {
|
||||
const range = (cmd.input as { Range: string }).Range;
|
||||
s3Calls.ranges.push(range);
|
||||
const match = /^bytes=(\d+)-(\d+)$/.exec(range)!;
|
||||
const body = s3Calls.sealed.subarray(Number(match[1]), Number(match[2]) + 1);
|
||||
return { Body: { transformToByteArray: async () => body } };
|
||||
}
|
||||
return {};
|
||||
}
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
import { S3Storage, extensionFor, keyForHash, open, seal, sha256 } from "./storage.js";
|
||||
import { DIRECT_SEGMENT_BYTES, S3Storage, directSealedSize, directPlaintextSize, extensionFor, keyForHash, open, seal, sealForDirect, sha256 } from "./storage.js";
|
||||
|
||||
const TEST_KEK = createHash("sha256").update("test-kek").digest();
|
||||
|
||||
describe("SPENC2 sealed size", () => {
|
||||
it.each([
|
||||
[1, 168],
|
||||
[DIRECT_SEGMENT_BYTES - 1, 135 + DIRECT_SEGMENT_BYTES - 1 + 32],
|
||||
[DIRECT_SEGMENT_BYTES, 135 + DIRECT_SEGMENT_BYTES + 32],
|
||||
[DIRECT_SEGMENT_BYTES + 1, 135 + DIRECT_SEGMENT_BYTES + 1 + 64],
|
||||
[2 * 1024 ** 3, 135 + 2 * 1024 ** 3 + 205 * 32],
|
||||
])("computes exact size for %i plaintext bytes", (plain, sealed) => {
|
||||
expect(directSealedSize(plain)).toBe(sealed);
|
||||
expect(directPlaintextSize(sealed)).toBe(plain);
|
||||
});
|
||||
|
||||
it.each([0, -1, 1.5, NaN, Infinity, Number.MAX_SAFE_INTEGER])("rejects invalid plaintext size %s", (size) => {
|
||||
expect(() => directSealedSize(size)).toThrow(RangeError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("storage key scheme (pure)", () => {
|
||||
it("maps content types to extensions, unknown -> bin", () => {
|
||||
expect(extensionFor("image/png")).toBe("png");
|
||||
|
|
@ -116,3 +144,76 @@ describe("S3Storage with object-scoped credentials", () => {
|
|||
expect(s3Calls.sent).toEqual(["Create", "Head", "Put"]);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe("SPENC2 range reads", () => {
|
||||
beforeEach(() => {
|
||||
s3Calls.sent.length = 0;
|
||||
s3Calls.ranges.length = 0;
|
||||
s3Calls.sealed = null;
|
||||
});
|
||||
|
||||
async function fixture(plaintext: Buffer): Promise<S3Storage> {
|
||||
const { privateKey } = generateKeyPairSync("ec", { namedCurve: "P-256" });
|
||||
const storage = new S3Storage({ mediaKek: TEST_KEK, directPrivateKey: privateKey });
|
||||
const jwk = (await storage.directPublicJwk())!;
|
||||
s3Calls.sealed = sealForDirect(plaintext, jwk);
|
||||
expect(directPlaintextSize(s3Calls.sealed.length)).toBe(plaintext.length);
|
||||
return storage;
|
||||
}
|
||||
|
||||
it("recovers plaintext size at full-segment boundaries", () => {
|
||||
for (const size of [1, DIRECT_SEGMENT_BYTES - 1, DIRECT_SEGMENT_BYTES, DIRECT_SEGMENT_BYTES + 1]) {
|
||||
const sealedSize = 135 + size + Math.ceil(size / DIRECT_SEGMENT_BYTES) * 32;
|
||||
expect(directPlaintextSize(sealedSize)).toBe(size);
|
||||
}
|
||||
expect(() => directPlaintextSize(167)).toThrow("invalid SPENC2 sealed size");
|
||||
expect(() => directPlaintextSize(135 + DIRECT_SEGMENT_BYTES + 33)).toThrow("invalid SPENC2 sealed size");
|
||||
});
|
||||
|
||||
it("slices mid-segment bounds across three segments without fetching earlier segments", async () => {
|
||||
const plaintext = randomBytes(4 * DIRECT_SEGMENT_BYTES);
|
||||
const storage = await fixture(plaintext);
|
||||
const start = DIRECT_SEGMENT_BYTES + 321;
|
||||
const end = 3 * DIRECT_SEGMENT_BYTES + 789;
|
||||
const bytes = await storage.readDirectRange("direct/clip.enc", start, end - start);
|
||||
expect(bytes.equals(plaintext.subarray(start, end))).toBe(true);
|
||||
const stride = DIRECT_SEGMENT_BYTES + 32;
|
||||
expect(s3Calls.ranges).toEqual(["bytes=0-134", `bytes=${135 + stride}-${135 + 4 * stride - 1}`]);
|
||||
expect(s3Calls.sent).toEqual(["Get", "Get"]);
|
||||
});
|
||||
|
||||
it("clips the sealed fetch at EOF and slices a final partial segment", async () => {
|
||||
const plaintext = randomBytes(2 * DIRECT_SEGMENT_BYTES + 1234);
|
||||
const storage = await fixture(plaintext);
|
||||
const start = DIRECT_SEGMENT_BYTES + 17;
|
||||
expect((await storage.readDirectRange("direct/clip.enc", start, plaintext.length - start))
|
||||
.equals(plaintext.subarray(start))).toBe(true);
|
||||
expect(s3Calls.ranges).toEqual(["bytes=0-134",
|
||||
`bytes=${135 + DIRECT_SEGMENT_BYTES + 32}-${135 + 3 * (DIRECT_SEGMENT_BYTES + 32) - 1}`]);
|
||||
});
|
||||
|
||||
it("validates the magic on every read before fetching any segments", async () => {
|
||||
const storage = await fixture(Buffer.from("video payload"));
|
||||
await storage.readDirectRange("direct/clip.enc", 0, 3);
|
||||
s3Calls.sealed![0] = 0;
|
||||
await expect(storage.readDirectRange("direct/clip.enc", 0, 3)).rejects.toThrow("magic mismatch");
|
||||
expect(s3Calls.ranges).toEqual(["bytes=0-134", `bytes=135-${135 + DIRECT_SEGMENT_BYTES + 32 - 1}`, "bytes=0-134"]);
|
||||
});
|
||||
|
||||
it("refuses tampered ciphertext and truncated segments", async () => {
|
||||
const storage = await fixture(Buffer.from("authenticated payload"));
|
||||
const last = s3Calls.sealed!.length - 1;
|
||||
s3Calls.sealed![last] = (s3Calls.sealed![last] ?? 0) ^ 1;
|
||||
await expect(storage.readDirectRange("direct/clip.enc", 0, 3)).rejects.toThrow();
|
||||
s3Calls.sealed = s3Calls.sealed!.subarray(0, 140);
|
||||
await expect(storage.readDirectRange("direct/clip.enc", 0, 3)).rejects.toThrow("truncated");
|
||||
});
|
||||
|
||||
it("rejects ranges beyond the segment count or final plaintext length", async () => {
|
||||
const storage = await fixture(Buffer.from("short"));
|
||||
await expect(storage.readDirectRange("direct/clip.enc", DIRECT_SEGMENT_BYTES, 1)).rejects.toThrow(RangeError);
|
||||
expect(s3Calls.ranges).toEqual(["bytes=0-134"]);
|
||||
await expect(storage.readDirectRange("direct/clip.enc", 4, 5)).rejects.toThrow("object length");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,11 +1,16 @@
|
|||
import { createCipheriv, createDecipheriv, createHash, randomBytes } from "node:crypto";
|
||||
import { createCipheriv, createDecipheriv, createHash, createPrivateKey, createPublicKey, diffieHellman, generateKeyPairSync, hkdfSync, randomBytes, randomUUID } from "node:crypto";
|
||||
import { ServiceUnavailableException } from "@nestjs/common";
|
||||
import {
|
||||
AbortMultipartUploadCommand,
|
||||
CompleteMultipartUploadCommand,
|
||||
CreateBucketCommand,
|
||||
CreateMultipartUploadCommand,
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
PutObjectCommand,
|
||||
S3Client,
|
||||
S3ServiceException,
|
||||
UploadPartCommand,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
||||
|
||||
|
|
@ -18,6 +23,13 @@ import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|||
* stays on the PLAINTEXT hash (dedup survives); getObject re-verifies the
|
||||
* hash, so storage-level tampering is detected. SSE-at-rest headers are set
|
||||
* additionally for endpoints that honor them (belt-and-braces).
|
||||
*
|
||||
* SPENC2 (direct-to-R2, 2026-10-16): objects under `direct/` are sealed IN
|
||||
* THE BROWSER (apps/web/src/lib/direct-storage.ts). The server never sees
|
||||
* the payload — it only mints the presigned PUT/GET and unwraps the CEK with
|
||||
* the ECDH share it derives from DIRECT_EC_PRIVATE_KEY. Both frame versions
|
||||
* share `getObject`: the leading magic decides the branch, so a single
|
||||
* reader serves the whole bucket and the SPENC1 path stays byte-identical.
|
||||
*/
|
||||
|
||||
export interface StoredObject {
|
||||
|
|
@ -32,6 +44,59 @@ export interface StorageService {
|
|||
getObject(key: string): Promise<Buffer>;
|
||||
hasContent(hash: string, contentType: string): Promise<boolean>;
|
||||
presign(key: string, ttlSeconds: number): Promise<string>;
|
||||
/** Direct-to-R2: the JWK the browser seals against; null = feature off. */
|
||||
directPublicJwk(): Promise<{ kty: "EC"; crv: "P-256"; x: string; y: string } | null>;
|
||||
/** Direct-to-R2: presigned PUT for the browser to push a SPENC2 frame. */
|
||||
presignPut(key: string, contentType: string, ttlSeconds: number): Promise<string>;
|
||||
createMultipart(key: string, contentType: string): Promise<{ uploadId: string }>;
|
||||
presignPart(key: string, uploadId: string, partNumber: number, ttlSeconds: number): Promise<string>;
|
||||
completeMultipart(key: string, uploadId: string, parts: MultipartPart[]): Promise<void>;
|
||||
abortMultipart(key: string, uploadId: string): Promise<void>;
|
||||
/** Direct-to-R2: existence + exact size, used to gate /files/direct/complete. */
|
||||
headObject(key: string): Promise<{ exists: boolean; contentLength: number | null }>;
|
||||
/** Direct-to-R2: unwrap the frame's CEK (header only) for /files/:id/direct-download. */
|
||||
openDirectHeader(stored: Buffer): Promise<Buffer>;
|
||||
/** Direct-to-R2: fetch only the validated 135-byte SPENC2 header. */
|
||||
openDirectHeaderBlob(key: string): Promise<Buffer>;
|
||||
/** Direct-to-R2: raw still-sealed object bytes — the CEK header lives only in the sealed form. */
|
||||
getSealedObject(key: string): Promise<Buffer>;
|
||||
/** Direct-to-R2: decrypt an arbitrary PLAINTEXT byte range (segment-aligned fetches). */
|
||||
readDirectRange(key: string, start: number, length: number): Promise<Buffer>;
|
||||
}
|
||||
|
||||
export interface MultipartPart {
|
||||
ETag: string;
|
||||
PartNumber: number;
|
||||
}
|
||||
|
||||
/** The direct-only members of StorageService, as one spreadable group. */
|
||||
export type DirectStorageMembers = Pick<
|
||||
StorageService,
|
||||
"directPublicJwk" | "presignPut" | "createMultipart" | "presignPart" | "completeMultipart" | "abortMultipart" | "headObject" | "openDirectHeader" | "openDirectHeaderBlob" | "getSealedObject" | "readDirectRange"
|
||||
>;
|
||||
|
||||
/**
|
||||
* Direct-path stub for in-memory fakes that never serve browser uploads:
|
||||
* reports the feature disabled and refuses any direct operation. Lets a fake
|
||||
* stay a complete StorageService with one spread instead of four stubs.
|
||||
*/
|
||||
export function directStorageDisabled(): DirectStorageMembers {
|
||||
const unavailable = async (): Promise<never> => {
|
||||
throw new Error("direct storage is unavailable");
|
||||
};
|
||||
return {
|
||||
directPublicJwk: async () => null,
|
||||
presignPut: unavailable,
|
||||
createMultipart: unavailable,
|
||||
presignPart: unavailable,
|
||||
completeMultipart: unavailable,
|
||||
abortMultipart: unavailable,
|
||||
headObject: unavailable,
|
||||
openDirectHeader: unavailable,
|
||||
openDirectHeaderBlob: unavailable,
|
||||
getSealedObject: unavailable,
|
||||
readDirectRange: unavailable,
|
||||
};
|
||||
}
|
||||
|
||||
const EXTENSIONS: Record<string, string> = {
|
||||
|
|
@ -89,6 +154,241 @@ export function open(sealed: Buffer, kek: Buffer): Buffer {
|
|||
return Buffer.concat([decipher.update(ciphertext), decipher.final()]);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SPENC2 — browser-sealed direct-to-R2 frames
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Frame magic for browser-sealed objects (spec: "SPENC2" exactly). */
|
||||
export const DIRECT_FRAME_MAGIC = Buffer.from("SPENC2", "ascii");
|
||||
|
||||
/**
|
||||
* Fixed frame layout, all integers big-endian. Apps/web/src/lib/direct-storage.ts
|
||||
* seals the same shape; this reader is the cross-implementation proof target.
|
||||
*
|
||||
* [0..6) magic "SPENC2"
|
||||
* [6..71) ephemeral P-256 public key, uncompressed: 0x04 || X(32) || Y(32)
|
||||
* [71..83) wrapNonce, 12 random bytes
|
||||
* [83..131) wrappedCek: AES-256-GCM(CEK), key = HKDF-SHA256(ECDH, salt=∅,
|
||||
* info="tmbh-direct-cek-wrap", L=32), nonce = wrapNonce → 48 bytes
|
||||
* [131..135) segmentCount, u32
|
||||
* [135..] segments: ctLen u32 | iv 12 | AES-256-GCM ciphertext+tag, key=CEK, no AAD
|
||||
*/
|
||||
const EPHEMERAL_PUBLIC_OFFSET = 6;
|
||||
const WRAP_NONCE_OFFSET = 71;
|
||||
const WRAPPED_CEK_OFFSET = 83;
|
||||
const SEGMENT_COUNT_OFFSET = 131;
|
||||
const SEGMENTS_OFFSET = 135;
|
||||
const WRAPPED_CEK_BYTES = 32 + 16;
|
||||
|
||||
/** HKDF info binding the wrap key to this purpose. ASCII, exact. */
|
||||
const DIRECT_WRAP_INFO = Buffer.from("tmbh-direct-cek-wrap", "ascii");
|
||||
|
||||
/** Segment plaintext ceiling. Mirrors the web side, which seals 10 MiB chunks. */
|
||||
export const DIRECT_SEGMENT_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
/** Exact SPENC2 size: fixed header plus one length, nonce and tag per segment. */
|
||||
export function directSealedSize(plainSizeBytes: number): number {
|
||||
if (!Number.isSafeInteger(plainSizeBytes) || plainSizeBytes < 1) {
|
||||
throw new RangeError("storage: invalid SPENC2 plaintext size");
|
||||
}
|
||||
const size = SEGMENTS_OFFSET + plainSizeBytes + Math.ceil(plainSizeBytes / DIRECT_SEGMENT_BYTES) * 32;
|
||||
if (!Number.isSafeInteger(size)) throw new RangeError("storage: invalid SPENC2 sealed size");
|
||||
return size;
|
||||
}
|
||||
|
||||
/** Direct upload metadata records sealed bytes; HTTP ranges address plaintext. */
|
||||
export function directPlaintextSize(sealedBytes: number): number {
|
||||
const count = Math.ceil((sealedBytes - SEGMENTS_OFFSET) / (DIRECT_SEGMENT_BYTES + 32));
|
||||
const plaintextBytes = sealedBytes - SEGMENTS_OFFSET - count * 32;
|
||||
if (!Number.isSafeInteger(sealedBytes) || count < 1 || plaintextBytes <= (count - 1) * DIRECT_SEGMENT_BYTES) {
|
||||
throw new Error("storage: invalid SPENC2 sealed size");
|
||||
}
|
||||
return plaintextBytes;
|
||||
}
|
||||
|
||||
/** Server direct keypair: P-256 private key from DIRECT_EC_PRIVATE_KEY (base64 DER). */
|
||||
export interface DirectKeyPair {
|
||||
privateKey: ReturnType<typeof createPrivateKey>;
|
||||
/** Compact JWK {kty,crv,x,y} — exactly what the browser's importKey wants. */
|
||||
publicJwk: { kty: "EC"; crv: "P-256"; x: string; y: string };
|
||||
}
|
||||
|
||||
export class DirectKeyError extends Error {}
|
||||
|
||||
/** True when DIRECT_EC_PRIVATE_KEY is present (routes report enabled=false otherwise). */
|
||||
export function directEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
|
||||
return loadDirectPrivateJwk(env) !== null;
|
||||
}
|
||||
|
||||
function loadDirectPrivateJwk(env: NodeJS.ProcessEnv): ReturnType<typeof createPrivateKey> | null {
|
||||
const raw = env.DIRECT_EC_PRIVATE_KEY;
|
||||
if (raw === undefined || raw.trim() === "") return null;
|
||||
let der: Buffer;
|
||||
try {
|
||||
der = Buffer.from(raw.trim(), "base64");
|
||||
} catch {
|
||||
throw new DirectKeyError("DIRECT_EC_PRIVATE_KEY must be base64 DER");
|
||||
}
|
||||
try {
|
||||
return createPrivateKey({ key: der, format: "der", type: "pkcs8" });
|
||||
} catch {
|
||||
throw new DirectKeyError("DIRECT_EC_PRIVATE_KEY is not a PKCS8 P-256 private key");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The direct keypair, deriving the public key from the private one — a
|
||||
* client-supplied public key is never trusted, so the browser can only wrap
|
||||
* CEKs this process can unwrap.
|
||||
*/
|
||||
export function directKeyPairFromEnv(env: NodeJS.ProcessEnv = process.env): DirectKeyPair | null {
|
||||
const privateKey = loadDirectPrivateJwk(env);
|
||||
if (privateKey === null) return null;
|
||||
const jwk = createPublicKey(privateKey).export({ format: "jwk" });
|
||||
if (jwk.kty !== "EC" || jwk.crv !== "P-256" || typeof jwk.x !== "string" || typeof jwk.y !== "string") {
|
||||
throw new DirectKeyError("DIRECT_EC_PRIVATE_KEY is not a P-256 key");
|
||||
}
|
||||
return { privateKey, publicJwk: { kty: "EC", crv: "P-256", x: jwk.x, y: jwk.y } };
|
||||
}
|
||||
|
||||
/** ECDH → HKDF-SHA256(salt empty, info "tmbh-direct-cek-wrap") → 32-byte wrap key. */
|
||||
function directWrapKey(privateKey: ReturnType<typeof createPrivateKey>, ephemeralPoint: Buffer): Buffer {
|
||||
const x = ephemeralPoint.subarray(1, 33).toString("base64url");
|
||||
const y = ephemeralPoint.subarray(33, 65).toString("base64url");
|
||||
let peer: ReturnType<typeof createPublicKey>;
|
||||
try {
|
||||
peer = createPublicKey({ key: { kty: "EC", crv: "P-256", x, y }, format: "jwk" });
|
||||
} catch {
|
||||
throw new Error("storage: SPENC2 frame holds a malformed ephemeral key");
|
||||
}
|
||||
const ikm = Buffer.from(diffieHellman({ privateKey, publicKey: peer }));
|
||||
// RFC 5869: an absent salt defaults to HashLen zero bytes; WebCrypto's empty
|
||||
// salt and Node's zero-filled salt produce the identical PRK (verified).
|
||||
return Buffer.from(hkdfSync("sha256", ikm, Buffer.alloc(0), DIRECT_WRAP_INFO, 32));
|
||||
}
|
||||
|
||||
interface DirectHeader {
|
||||
ephemeralPoint: Buffer;
|
||||
wrapNonce: Buffer;
|
||||
wrappedCek: Buffer;
|
||||
segmentCount: number;
|
||||
}
|
||||
|
||||
/** Validate + slice the fixed header. Throws on any truncation or bad shape. */
|
||||
function parseDirectHeader(stored: Buffer): DirectHeader {
|
||||
if (stored.length < SEGMENTS_OFFSET) {
|
||||
throw new Error("storage: SPENC2 frame is truncated");
|
||||
}
|
||||
if (!stored.subarray(0, DIRECT_FRAME_MAGIC.length).equals(DIRECT_FRAME_MAGIC)) {
|
||||
throw new Error("storage: not a SPENC2 frame (magic mismatch)");
|
||||
}
|
||||
const ephemeralPoint = stored.subarray(EPHEMERAL_PUBLIC_OFFSET, WRAP_NONCE_OFFSET);
|
||||
if (ephemeralPoint[0] !== 0x04) throw new Error("storage: SPENC2 ephemeral key is not an uncompressed point");
|
||||
const wrapNonce = stored.subarray(WRAP_NONCE_OFFSET, WRAPPED_CEK_OFFSET);
|
||||
const wrappedCek = stored.subarray(WRAPPED_CEK_OFFSET, WRAPPED_CEK_OFFSET + WRAPPED_CEK_BYTES);
|
||||
const segmentCount = stored.readUInt32BE(SEGMENT_COUNT_OFFSET);
|
||||
if (segmentCount === 0 || segmentCount > 4_294_967_295 / DIRECT_SEGMENT_BYTES + 1) {
|
||||
throw new Error("storage: SPENC2 segment count is out of range");
|
||||
}
|
||||
return { ephemeralPoint, wrapNonce, wrappedCek, segmentCount };
|
||||
}
|
||||
|
||||
/** Walk the segment table; yields {ciphertext, iv} in order. */
|
||||
function* directSegments(stored: Buffer, segmentCount: number): Generator<{ iv: Buffer; ciphertext: Buffer }> {
|
||||
let offset = SEGMENTS_OFFSET;
|
||||
for (let index = 0; index < segmentCount; index += 1) {
|
||||
if (offset + 4 + 12 > stored.length) throw new Error("storage: SPENC2 segment table is truncated");
|
||||
const ctLen = stored.readUInt32BE(offset);
|
||||
offset += 4;
|
||||
if (ctLen < 16 || offset + 12 + ctLen > stored.length) throw new Error("storage: SPENC2 segment length is out of range");
|
||||
const iv = stored.subarray(offset, offset + 12);
|
||||
offset += 12;
|
||||
const ciphertext = stored.subarray(offset, offset + ctLen);
|
||||
offset += ctLen;
|
||||
yield { iv, ciphertext };
|
||||
}
|
||||
if (offset !== stored.length) throw new Error("storage: SPENC2 frame has trailing bytes");
|
||||
}
|
||||
|
||||
/** Unwrap the CEK (verifies the wrap GCM tag). Exposed for openDirectHeader. */
|
||||
export function unwrapDirectCek(stored: Buffer, privateKey: ReturnType<typeof createPrivateKey>): Buffer {
|
||||
const header = parseDirectHeader(stored);
|
||||
return aesGcmOpen(directWrapKey(privateKey, header.ephemeralPoint), header.wrapNonce, header.wrappedCek);
|
||||
}
|
||||
|
||||
/**
|
||||
* Open a SPENC2 frame: unwrap the CEK, then decrypt every segment. Segment
|
||||
* GCM tags are verified individually, so any flipped byte throws. Empty
|
||||
* plaintexts are rejected (the upload floor is 1 byte).
|
||||
*/
|
||||
export function openDirectFrame(stored: Buffer, privateKey: ReturnType<typeof createPrivateKey>): Buffer {
|
||||
const { segmentCount } = parseDirectHeader(stored);
|
||||
const cek = unwrapDirectCek(stored, privateKey);
|
||||
const parts: Buffer[] = [];
|
||||
for (const segment of directSegments(stored, segmentCount)) {
|
||||
parts.push(aesGcmOpen(cek, segment.iv, segment.ciphertext));
|
||||
}
|
||||
const plaintext = Buffer.concat(parts);
|
||||
if (plaintext.length === 0) throw new Error("storage: SPENC2 frame decrypts to zero bytes");
|
||||
return plaintext;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test helper: seal `plaintext` exactly the way the browser does, so server
|
||||
* readers can be exercised without a DOM. Production uploads come from
|
||||
* apps/web/src/lib/direct-storage.ts — this is the reference implementation.
|
||||
*/
|
||||
export function sealForDirect(plaintext: Buffer, publicJwk: { kty: "EC"; crv: "P-256"; x: string; y: string }): Buffer {
|
||||
if (plaintext.length === 0) throw new Error("sealForDirect: plaintext must not be empty");
|
||||
const cek = randomBytes(32);
|
||||
const { privateKey: ephemeralPrivate, publicKey: ephemeralPublic } = newEphemeralKeyPair();
|
||||
// Wrap to the SERVER's public key — ECDH(ephPriv, serverPub), the same share
|
||||
// the server derives as ECDH(serverPriv, ephPub) in directWrapKey.
|
||||
const wrapKey = directWrapKey(ephemeralPrivate, uncompressedPoint(createPublicKey({ key: { ...publicJwk }, format: "jwk" })));
|
||||
const wrapNonce = randomBytes(12);
|
||||
const wrappedCek = aesGcmSeal(wrapKey, wrapNonce, cek);
|
||||
const segments: Buffer[] = [];
|
||||
for (let offset = 0; offset < plaintext.length; offset += DIRECT_SEGMENT_BYTES) {
|
||||
const iv = randomBytes(12);
|
||||
const ciphertext = aesGcmSeal(cek, iv, plaintext.subarray(offset, offset + DIRECT_SEGMENT_BYTES));
|
||||
segments.push(Buffer.concat([u32be(ciphertext.byteLength), iv, ciphertext]));
|
||||
}
|
||||
return Buffer.concat([
|
||||
DIRECT_FRAME_MAGIC,
|
||||
uncompressedPoint(ephemeralPublic),
|
||||
wrapNonce,
|
||||
wrappedCek,
|
||||
u32be(segments.length),
|
||||
...segments,
|
||||
]);
|
||||
}
|
||||
|
||||
/** Random object key for a browser upload: `direct/<uuid>.enc`. */
|
||||
export function directObjectKey(): string {
|
||||
return `direct/${randomUUID()}.enc`;
|
||||
}
|
||||
|
||||
function u32be(value: number): Buffer {
|
||||
const out = Buffer.alloc(4);
|
||||
out.writeUInt32BE(value, 0);
|
||||
return out;
|
||||
}
|
||||
|
||||
function newEphemeralKeyPair(): { privateKey: ReturnType<typeof createPrivateKey>; publicKey: ReturnType<typeof createPublicKey> } {
|
||||
return generateKeyPairSync("ec", { namedCurve: "P-256" });
|
||||
}
|
||||
|
||||
/** Raised wherever a direct operation needs a key the deployment never supplied. */
|
||||
function missingDirectKey(): ServiceUnavailableException {
|
||||
return new ServiceUnavailableException("DIRECT_EC_PRIVATE_KEY is not configured");
|
||||
}
|
||||
|
||||
function uncompressedPoint(publicKey: ReturnType<typeof createPublicKey>): Buffer {
|
||||
const point = publicKey.export({ type: "spki", format: "der" }).subarray(26);
|
||||
if (point.length !== 65 || point[0] !== 0x04) throw new Error("storage: expected an uncompressed P-256 point");
|
||||
return point;
|
||||
}
|
||||
|
||||
function aesGcmSeal(key: Buffer, iv: Buffer, plaintext: Buffer): Buffer {
|
||||
const cipher = createCipheriv("aes-256-gcm", key, iv);
|
||||
return Buffer.concat([cipher.update(plaintext), cipher.final(), cipher.getAuthTag()]);
|
||||
|
|
@ -122,6 +422,8 @@ export interface S3StorageConfig {
|
|||
bucket?: string;
|
||||
forcePathStyle?: boolean;
|
||||
mediaKek?: Buffer;
|
||||
/** Overrides DIRECT_EC_PRIVATE_KEY (tests). */
|
||||
directPrivateKey?: ReturnType<typeof createPrivateKey>;
|
||||
}
|
||||
|
||||
export function s3ConfigFromEnv(): S3StorageConfig {
|
||||
|
|
@ -139,14 +441,17 @@ export class S3Storage implements StorageService {
|
|||
private readonly client: S3Client;
|
||||
private readonly bucket: string;
|
||||
private readonly kek: Buffer;
|
||||
private readonly directKey: DirectKeyPair | null;
|
||||
private bucketReady = false;
|
||||
|
||||
constructor(config: S3StorageConfig) {
|
||||
this.bucket = config.bucket ?? "tactical-media-broadcast-hub-media";
|
||||
this.kek = config.mediaKek ?? loadMediaKek();
|
||||
this.directKey = config.directPrivateKey !== undefined ? directKeyPairFromPrivateKey(config.directPrivateKey) : directKeyPairFromEnv();
|
||||
this.client = new S3Client({
|
||||
endpoint: config.endpoint,
|
||||
region: config.region ?? "us-east-1",
|
||||
requestChecksumCalculation: "WHEN_REQUIRED",
|
||||
credentials:
|
||||
config.accessKeyId !== undefined && config.secretAccessKey !== undefined
|
||||
? { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }
|
||||
|
|
@ -155,6 +460,11 @@ export class S3Storage implements StorageService {
|
|||
});
|
||||
}
|
||||
|
||||
/** The JWK the browser seals against; null when direct upload is disabled. */
|
||||
async directPublicJwk(): Promise<{ kty: "EC"; crv: "P-256"; x: string; y: string } | null> {
|
||||
return this.directKey === null ? null : { ...this.directKey.publicJwk };
|
||||
}
|
||||
|
||||
private async ensureBucket(): Promise<void> {
|
||||
if (this.bucketReady) return;
|
||||
try {
|
||||
|
|
@ -205,9 +515,21 @@ export class S3Storage implements StorageService {
|
|||
const res = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: key }));
|
||||
if (res.Body === undefined) throw new Error(`empty body for ${key}`);
|
||||
const stored = Buffer.from(await res.Body.transformToByteArray());
|
||||
const plaintext = open(stored, this.kek);
|
||||
const leading = stored.subarray(0, 6).toString("latin1");
|
||||
let plaintext: Buffer;
|
||||
if (leading === "SPENC1") {
|
||||
plaintext = open(stored, this.kek);
|
||||
} else if (leading === "SPENC2") {
|
||||
// Browser-sealed frame: only this process can unwrap the CEK.
|
||||
if (this.directKey === null) throw missingDirectKey();
|
||||
plaintext = openDirectFrame(stored, this.directKey.privateKey);
|
||||
} else {
|
||||
plaintext = open(stored, this.kek); // raises the magic-mismatch error
|
||||
}
|
||||
// Content addressing is on the PLAINTEXT hash, so tampering (or a
|
||||
// mismatched key) is detected here for both frame versions.
|
||||
const hashFromKey = key.split("/")[1]?.split(".")[0];
|
||||
if (hashFromKey !== undefined && sha256(plaintext) !== hashFromKey) {
|
||||
if (hashFromKey !== undefined && hashFromKey.length === 64 && sha256(plaintext) !== hashFromKey) {
|
||||
throw new Error(`storage: integrity failure for ${key} (plaintext hash mismatch)`);
|
||||
}
|
||||
return plaintext;
|
||||
|
|
@ -221,4 +543,123 @@ export class S3Storage implements StorageService {
|
|||
async presign(key: string, ttlSeconds: number): Promise<string> {
|
||||
return getSignedUrl(this.client, new GetObjectCommand({ Bucket: this.bucket, Key: key }), { expiresIn: ttlSeconds });
|
||||
}
|
||||
|
||||
async presignPut(key: string, contentType: string, ttlSeconds: number): Promise<string> {
|
||||
return getSignedUrl(this.client, new PutObjectCommand({ Bucket: this.bucket, Key: key, ContentType: contentType }), { expiresIn: ttlSeconds });
|
||||
}
|
||||
|
||||
async createMultipart(key: string, contentType: string): Promise<{ uploadId: string }> {
|
||||
const result = await this.client.send(new CreateMultipartUploadCommand({
|
||||
Bucket: this.bucket, Key: key, ContentType: contentType,
|
||||
}));
|
||||
if (!result.UploadId) throw new Error("storage: multipart upload has no upload ID");
|
||||
return { uploadId: result.UploadId };
|
||||
}
|
||||
|
||||
async presignPart(key: string, uploadId: string, partNumber: number, ttlSeconds: number): Promise<string> {
|
||||
return getSignedUrl(this.client, new UploadPartCommand({
|
||||
Bucket: this.bucket, Key: key, UploadId: uploadId, PartNumber: partNumber,
|
||||
}), { expiresIn: ttlSeconds });
|
||||
}
|
||||
|
||||
async completeMultipart(key: string, uploadId: string, parts: MultipartPart[]): Promise<void> {
|
||||
await this.client.send(new CompleteMultipartUploadCommand({
|
||||
Bucket: this.bucket, Key: key, UploadId: uploadId, MultipartUpload: { Parts: parts },
|
||||
}));
|
||||
}
|
||||
|
||||
async abortMultipart(key: string, uploadId: string): Promise<void> {
|
||||
await this.client.send(new AbortMultipartUploadCommand({ Bucket: this.bucket, Key: key, UploadId: uploadId }));
|
||||
}
|
||||
|
||||
async headObject(key: string): Promise<{ exists: boolean; contentLength: number | null }> {
|
||||
try {
|
||||
const res = await this.client.send(new HeadObjectCommand({ Bucket: this.bucket, Key: key }));
|
||||
return { exists: true, contentLength: res.ContentLength ?? null };
|
||||
} catch (err) {
|
||||
const status = err instanceof S3ServiceException ? err.$metadata.httpStatusCode : undefined;
|
||||
const name = err instanceof S3ServiceException ? err.name : "";
|
||||
if (status === 404 || name === "NotFound") return { exists: false, contentLength: null };
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async openDirectHeader(stored: Buffer): Promise<Buffer> {
|
||||
if (this.directKey === null) throw missingDirectKey();
|
||||
return unwrapDirectCek(stored, this.directKey.privateKey);
|
||||
}
|
||||
|
||||
async openDirectHeaderBlob(key: string): Promise<Buffer> {
|
||||
const header = await this.getSealedRange(key, 0, SEGMENTS_OFFSET);
|
||||
parseDirectHeader(header);
|
||||
return header;
|
||||
}
|
||||
|
||||
/** S3 uses an inclusive end; callers supply an exclusive end. */
|
||||
private async getSealedRange(key: string, start: number, end: number): Promise<Buffer> {
|
||||
const res = await this.client.send(new GetObjectCommand({
|
||||
Bucket: this.bucket, Key: key, Range: `bytes=${start}-${end - 1}`,
|
||||
}));
|
||||
if (res.Body === undefined) throw new Error(`empty body for ${key}`);
|
||||
return Buffer.from(await res.Body.transformToByteArray());
|
||||
}
|
||||
|
||||
async getSealedObject(key: string): Promise<Buffer> {
|
||||
const res = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: key }));
|
||||
if (res.Body === undefined) throw new Error(`empty body for ${key}`);
|
||||
return Buffer.from(await res.Body.transformToByteArray());
|
||||
}
|
||||
|
||||
/**
|
||||
* Plaintext [start, start+length) of a SPENC2 object WITHOUT reading the
|
||||
* whole frame: the mapping plaintext→sealed offset is deterministic
|
||||
* (header 135 + per segment 4+12+16 with 10 MiB plaintext), so only the
|
||||
* covering segments are fetched (R2 ranged GET) and decrypted.
|
||||
*/
|
||||
async readDirectRange(key: string, start: number, length: number): Promise<Buffer> {
|
||||
if (this.directKey === null) throw missingDirectKey();
|
||||
if (!Number.isSafeInteger(start) || start < 0 || !Number.isSafeInteger(length) || length < 0
|
||||
|| !Number.isSafeInteger(start + length)) {
|
||||
throw new RangeError("storage: invalid plaintext range");
|
||||
}
|
||||
const header = await this.openDirectHeaderBlob(key);
|
||||
const { segmentCount } = parseDirectHeader(header);
|
||||
const cek = unwrapDirectCek(header, this.directKey.privateKey);
|
||||
if (length === 0) return Buffer.alloc(0);
|
||||
const first = Math.floor(start / DIRECT_SEGMENT_BYTES);
|
||||
const last = Math.floor((start + length - 1) / DIRECT_SEGMENT_BYTES);
|
||||
if (last >= segmentCount) throw new RangeError("storage: plaintext range exceeds segment count");
|
||||
const sealedStride = 4 + 12 + DIRECT_SEGMENT_BYTES + 16;
|
||||
// Fetch only covering segments. S3 clips the final partial segment to EOF.
|
||||
const stored = await this.getSealedRange(key,
|
||||
SEGMENTS_OFFSET + first * sealedStride, SEGMENTS_OFFSET + (last + 1) * sealedStride);
|
||||
const parts: Buffer[] = [];
|
||||
let offset = 0;
|
||||
for (let i = first; i <= last; i += 1) {
|
||||
if (offset + 4 + 12 > stored.length) throw new Error("storage: SPENC2 segment table is truncated");
|
||||
const ctLen = stored.readUInt32BE(offset); offset += 4;
|
||||
if (ctLen < 17 || ctLen > DIRECT_SEGMENT_BYTES + 16 || offset + 12 + ctLen > stored.length
|
||||
|| (i < segmentCount - 1 && ctLen !== DIRECT_SEGMENT_BYTES + 16)) {
|
||||
throw new Error("storage: SPENC2 segment length is out of range");
|
||||
}
|
||||
const iv = stored.subarray(offset, offset + 12); offset += 12;
|
||||
const ciphertext = stored.subarray(offset, offset + ctLen); offset += ctLen;
|
||||
const plain = aesGcmOpen(cek, iv, ciphertext);
|
||||
const segStart = i * DIRECT_SEGMENT_BYTES;
|
||||
parts.push(plain.subarray(Math.max(0, start - segStart), Math.min(plain.length, start + length - segStart)));
|
||||
}
|
||||
if (parts.reduce((sum, part) => sum + part.length, 0) !== length) {
|
||||
throw new RangeError("storage: plaintext range exceeds object length");
|
||||
}
|
||||
return Buffer.concat(parts);
|
||||
}
|
||||
}
|
||||
|
||||
/** Build a DirectKeyPair from an already-loaded private key (config + tests). */
|
||||
function directKeyPairFromPrivateKey(privateKey: ReturnType<typeof createPrivateKey>): DirectKeyPair {
|
||||
const jwk = createPublicKey(privateKey).export({ format: "jwk" });
|
||||
if (jwk.kty !== "EC" || jwk.crv !== "P-256" || typeof jwk.x !== "string" || typeof jwk.y !== "string") {
|
||||
throw new DirectKeyError("direct key is not a P-256 key");
|
||||
}
|
||||
return { privateKey, publicJwk: { kty: "EC", crv: "P-256", x: jwk.x, y: jwk.y } };
|
||||
}
|
||||
|
|
|
|||
327
apps/api/src/video/chunked-upload.test.ts
Normal file
327
apps/api/src/video/chunked-upload.test.ts
Normal file
|
|
@ -0,0 +1,327 @@
|
|||
import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
ChunkedUploadService,
|
||||
chunkOffset,
|
||||
type ChunkedUploadSession,
|
||||
} from "./chunked-upload.js";
|
||||
import {
|
||||
CHUNK_SIZE_BYTES,
|
||||
MAX_CHUNKED_UPLOAD_BYTES,
|
||||
chunkBytesFor,
|
||||
totalChunksFor,
|
||||
} from "./upload-limits.js";
|
||||
import { VideoService } from "./video.service.js";
|
||||
import { assetId, serviceWithDatabase, workspaceId, type FakeVideoDatabase } from "./test-fixtures.js";
|
||||
|
||||
const otherWorkspace = "90000000-0000-4000-8000-00000000000f";
|
||||
let directory: string;
|
||||
let service: ChunkedUploadService;
|
||||
let counter = 0;
|
||||
|
||||
beforeEach(async () => {
|
||||
directory = await mkdtemp(join(tmpdir(), "chunked-test-"));
|
||||
counter += 1;
|
||||
service = new ChunkedUploadService(
|
||||
directory,
|
||||
Number.MAX_SAFE_INTEGER,
|
||||
() => `00000000-0000-4000-8000-${String(counter).padStart(12, "0")}`,
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await service.onModuleDestroy();
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function init(totalBytes: number, overrides: Partial<{ filename: string; mimeType: string }> = {}): Promise<ChunkedUploadSession> {
|
||||
return service.init(workspaceId, {
|
||||
filename: overrides.filename ?? "raw-capture.mp4",
|
||||
mimeType: overrides.mimeType ?? "video/mp4",
|
||||
totalBytes,
|
||||
});
|
||||
}
|
||||
|
||||
/** Deterministic pseudo-random chunk bytes; offset-tagged so mis-writes are visible. */
|
||||
function chunkBytes(totalBytes: number, index: number): Buffer {
|
||||
return Buffer.alloc(chunkBytesFor(totalBytes, index), (index % 251) + 1);
|
||||
}
|
||||
|
||||
describe("chunk plan math (pure)", () => {
|
||||
it("splits files into 10 MiB chunks with a short final chunk", () => {
|
||||
expect(CHUNK_SIZE_BYTES).toBe(10 * 1024 * 1024);
|
||||
expect(totalChunksFor(CHUNK_SIZE_BYTES)).toBe(1);
|
||||
expect(totalChunksFor(CHUNK_SIZE_BYTES + 1)).toBe(2);
|
||||
expect(totalChunksFor(3 * CHUNK_SIZE_BYTES)).toBe(3);
|
||||
expect(totalChunksFor(3 * CHUNK_SIZE_BYTES + 123)).toBe(4);
|
||||
// 20 GB target size stays within the 100 GiB ceiling.
|
||||
expect(totalChunksFor(20 * 1024 ** 3)).toBe(2048);
|
||||
expect(MAX_CHUNKED_UPLOAD_BYTES).toBe(100 * 1024 ** 3);
|
||||
});
|
||||
|
||||
it("computes chunk sizes and byte offsets", () => {
|
||||
const total = 2 * CHUNK_SIZE_BYTES + 5;
|
||||
expect(chunkBytesFor(total, 0)).toBe(CHUNK_SIZE_BYTES);
|
||||
expect(chunkBytesFor(total, 1)).toBe(CHUNK_SIZE_BYTES);
|
||||
expect(chunkBytesFor(total, 2)).toBe(5);
|
||||
expect(chunkOffset(0)).toBe(0);
|
||||
expect(chunkOffset(1)).toBe(CHUNK_SIZE_BYTES);
|
||||
expect(chunkOffset(2047)).toBe(2047 * CHUNK_SIZE_BYTES);
|
||||
expect(() => chunkBytesFor(total, 3)).toThrow("outside the 3-chunk upload");
|
||||
expect(() => chunkOffset(-1)).toThrow("non-negative");
|
||||
});
|
||||
});
|
||||
|
||||
describe("init", () => {
|
||||
it("creates a 0600 temp file under /tmp/uploads/<uploadId>.tmp and tracks the session", async () => {
|
||||
const session = await init(2 * CHUNK_SIZE_BYTES);
|
||||
expect(session.uploadId).toMatch(/^[0-9a-f-]{36}$/);
|
||||
expect(session.totalChunks).toBe(2);
|
||||
expect(session.tempPath).toBe(join(directory, `${session.uploadId}.tmp`));
|
||||
expect((await stat(session.tempPath)).size).toBe(0);
|
||||
expect((await stat(session.tempPath)).mode & 0o777).toBe(0o600);
|
||||
expect(service.activeSessions.get(session.uploadId)).toBe(session);
|
||||
expect(session.workspaceId).toBe(workspaceId);
|
||||
});
|
||||
|
||||
it("rejects non-video MIME types, bad filenames, and bad byte counts", async () => {
|
||||
await expect(init(1, { mimeType: "image/png" })).rejects.toMatchObject({ status: 400 });
|
||||
await expect(init(1, { filename: " " })).rejects.toMatchObject({ status: 400 });
|
||||
await expect(init(0)).rejects.toMatchObject({ status: 400, message: expect.stringContaining("positive safe integer") });
|
||||
await expect(init(1.5)).rejects.toMatchObject({ status: 400 });
|
||||
await expect(init(Number.MAX_SAFE_INTEGER + 1)).rejects.toMatchObject({ status: 400 });
|
||||
// Spec ceiling: totalBytes ≤ 100 GiB.
|
||||
await expect(init(MAX_CHUNKED_UPLOAD_BYTES + 1)).rejects.toMatchObject({
|
||||
status: 400,
|
||||
message: expect.stringContaining("100 GiB"),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("appendChunk — correct offset writes", () => {
|
||||
it("writes chunks at index × 10 MiB regardless of arrival order (sparse middle-out)", async () => {
|
||||
const total = 3 * CHUNK_SIZE_BYTES;
|
||||
await init(total);
|
||||
const uploadId = firstUploadId();
|
||||
// Deliberately out of order: last, first, middle.
|
||||
const third = chunkBytes(total, 2);
|
||||
await service.appendChunk(uploadId, workspaceId, 2, third);
|
||||
await service.appendChunk(uploadId, workspaceId, 0, chunkBytes(total, 0));
|
||||
const receipt = await service.appendChunk(uploadId, workspaceId, 1, chunkBytes(total, 1));
|
||||
expect(receipt).toEqual({ received: true, bytesSoFar: total });
|
||||
const assembled = await readFile(join(directory, `${uploadId}.tmp`));
|
||||
expect(assembled.length).toBe(total);
|
||||
expect(assembled.subarray(0, CHUNK_SIZE_BYTES)).toEqual(chunkBytes(total, 0));
|
||||
expect(assembled.subarray(CHUNK_SIZE_BYTES, 2 * CHUNK_SIZE_BYTES)).toEqual(chunkBytes(total, 1));
|
||||
expect(assembled.subarray(2 * CHUNK_SIZE_BYTES)).toEqual(third);
|
||||
});
|
||||
|
||||
it("fills holes with zero bytes before their chunk arrives, and reports progressive bytesSoFar", async () => {
|
||||
const total = 3 * CHUNK_SIZE_BYTES;
|
||||
await init(total);
|
||||
const uploadId = firstUploadId();
|
||||
const afterFirst = await service.appendChunk(uploadId, workspaceId, 0, chunkBytes(total, 0));
|
||||
expect(afterFirst.bytesSoFar).toBe(CHUNK_SIZE_BYTES);
|
||||
const afterLast = await service.appendChunk(uploadId, workspaceId, 2, chunkBytes(total, 2));
|
||||
expect(afterLast.bytesSoFar).toBe(2 * CHUNK_SIZE_BYTES);
|
||||
// Chunk 1 has not arrived: its range must read as zeros, not garbage.
|
||||
const partial = await readFile(join(directory, `${uploadId}.tmp`));
|
||||
expect(partial.subarray(CHUNK_SIZE_BYTES, 2 * CHUNK_SIZE_BYTES)).toEqual(Buffer.alloc(CHUNK_SIZE_BYTES, 0));
|
||||
// The file is already full length (sparse).
|
||||
expect(partial.length).toBe(total);
|
||||
});
|
||||
|
||||
it("is idempotent for a retransmitted chunk and rejects wrong-sized bodies", async () => {
|
||||
const total = 2 * CHUNK_SIZE_BYTES;
|
||||
await init(total);
|
||||
const uploadId = firstUploadId();
|
||||
await service.appendChunk(uploadId, workspaceId, 0, chunkBytes(total, 0));
|
||||
await service.appendChunk(uploadId, workspaceId, 0, chunkBytes(total, 0));
|
||||
expect((await service.verifyComplete(uploadId, workspaceId)).bytesSoFar).toBe(CHUNK_SIZE_BYTES);
|
||||
// Too short.
|
||||
await expect(service.appendChunk(uploadId, workspaceId, 1, Buffer.alloc(CHUNK_SIZE_BYTES - 1, 7)))
|
||||
.rejects.toMatchObject({ status: 400, message: expect.stringContaining("exactly") });
|
||||
// Too long.
|
||||
await expect(service.appendChunk(uploadId, workspaceId, 1, Buffer.alloc(CHUNK_SIZE_BYTES + 1, 7)))
|
||||
.rejects.toMatchObject({ status: 400 });
|
||||
// Out of range index.
|
||||
await expect(service.appendChunk(uploadId, workspaceId, 2, Buffer.alloc(1)))
|
||||
.rejects.toMatchObject({ status: 400, message: expect.stringContaining("between 0 and 1") });
|
||||
await expect(service.appendChunk(uploadId, workspaceId, -1, Buffer.alloc(1))).rejects.toMatchObject({ status: 400 });
|
||||
});
|
||||
|
||||
it("enforces tenant isolation and rejects unknown/finished uploads", async () => {
|
||||
const session = await init(CHUNK_SIZE_BYTES);
|
||||
await expect(service.appendChunk(session.uploadId, otherWorkspace, 0, Buffer.alloc(CHUNK_SIZE_BYTES)))
|
||||
.rejects.toMatchObject({ status: 404 });
|
||||
await expect(service.appendChunk("not-a-uuid", workspaceId, 0, Buffer.alloc(1))).rejects.toMatchObject({ status: 400 });
|
||||
await expect(service.appendChunk("11111111-1111-4111-8111-111111111111", workspaceId, 0, Buffer.alloc(1)))
|
||||
.rejects.toMatchObject({ status: 404 });
|
||||
await service.appendChunk(session.uploadId, workspaceId, 0, Buffer.alloc(CHUNK_SIZE_BYTES));
|
||||
service.complete(session.uploadId);
|
||||
await expect(service.appendChunk(session.uploadId, workspaceId, 0, Buffer.alloc(CHUNK_SIZE_BYTES)))
|
||||
.rejects.toMatchObject({ status: 409 });
|
||||
expect(service.activeSessions.has(session.uploadId)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("verifyComplete / assertAssembled — missing chunk + size verification", () => {
|
||||
it("reports missing chunks and under-length files until every chunk lands", async () => {
|
||||
const total = 3 * CHUNK_SIZE_BYTES;
|
||||
await init(total);
|
||||
const uploadId = firstUploadId();
|
||||
const first = await service.verifyComplete(uploadId, workspaceId);
|
||||
expect(first.missing).toEqual([0, 1, 2]);
|
||||
expect(first.bytesSoFar).toBe(0);
|
||||
// Sparse file is already totalBytes long on disk, so size alone would lie —
|
||||
// the chunk-count check is what catches the gap.
|
||||
expect(first.fileBytes).toBe(0); // nothing written yet
|
||||
expect(() => service.assertAssembled(first)).toThrow("missing 3 chunk(s)");
|
||||
|
||||
await service.appendChunk(uploadId, workspaceId, 1, chunkBytes(total, 1));
|
||||
const middle = await service.verifyComplete(uploadId, workspaceId);
|
||||
expect(middle.missing).toEqual([0, 2]);
|
||||
expect(() => service.assertAssembled(middle)).toThrow("missing 2 chunk(s): 0, 2");
|
||||
|
||||
await service.appendChunk(uploadId, workspaceId, 0, chunkBytes(total, 0));
|
||||
await service.appendChunk(uploadId, workspaceId, 2, chunkBytes(total, 2));
|
||||
const done = await service.verifyComplete(uploadId, workspaceId);
|
||||
expect(done.missing).toEqual([]);
|
||||
expect(done.bytesSoFar).toBe(total);
|
||||
expect(done.fileBytes).toBe(total);
|
||||
expect(() => service.assertAssembled(done)).not.toThrow();
|
||||
});
|
||||
|
||||
it("detects a declared size that disagrees with the assembled file", async () => {
|
||||
const total = 2 * CHUNK_SIZE_BYTES;
|
||||
const session = await init(total);
|
||||
await service.appendChunk(session.uploadId, workspaceId, 0, chunkBytes(total, 0));
|
||||
await service.appendChunk(session.uploadId, workspaceId, 1, chunkBytes(total, 1));
|
||||
// Simulate a truncated/oversized file behind the assembler's back.
|
||||
await writeFile(session.tempPath, Buffer.alloc(total - 1, 9));
|
||||
const check = await service.verifyComplete(session.uploadId, workspaceId);
|
||||
expect(check.missing).toEqual([]);
|
||||
expect(check.fileBytes).toBe(total - 1);
|
||||
expect(() => service.assertAssembled(check)).toThrow("bytes on disk");
|
||||
expect(() => service.assertAssembled(check)).toThrow(`expected ${total}`);
|
||||
});
|
||||
|
||||
it("detects byte-accounting drift when a received set over/under-counts totalBytes", async () => {
|
||||
const total = 2 * CHUNK_SIZE_BYTES;
|
||||
const session = await init(total);
|
||||
session.received.add(0);
|
||||
session.received.add(1);
|
||||
session.received.add(1); // Set: no duplicate effect, but simulate accounting drift below.
|
||||
const check = await service.verifyComplete(session.uploadId, workspaceId);
|
||||
expect(check.bytesSoFar).toBe(2 * CHUNK_SIZE_BYTES);
|
||||
// Forcing a mismatched declaration is caught by the service-level check.
|
||||
const drifted: typeof check = { ...check, session: { ...session, totalBytes: total + 5 } };
|
||||
expect(() => service.assertAssembled(drifted)).toThrow(/received \d+ bytes but declared/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("abort, cleanup, and expiry sweeper", () => {
|
||||
it("abort removes the temp file and session, and is idempotent per-uploadId", async () => {
|
||||
const session = await init(CHUNK_SIZE_BYTES);
|
||||
await service.appendChunk(session.uploadId, workspaceId, 0, Buffer.alloc(CHUNK_SIZE_BYTES));
|
||||
await service.abort(session.uploadId, workspaceId);
|
||||
await expect(stat(session.tempPath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(service.activeSessions.has(session.uploadId)).toBe(false);
|
||||
await expect(service.abort(session.uploadId, workspaceId)).rejects.toMatchObject({ status: 404 });
|
||||
await expect(service.verifyComplete(session.uploadId, workspaceId)).rejects.toMatchObject({ status: 404 });
|
||||
});
|
||||
|
||||
it("cleanup drops the entry after complete; failure paths can still see the file", async () => {
|
||||
const session = await init(CHUNK_SIZE_BYTES);
|
||||
await service.appendChunk(session.uploadId, workspaceId, 0, Buffer.alloc(CHUNK_SIZE_BYTES));
|
||||
service.complete(session.uploadId);
|
||||
await service.cleanup(session.uploadId);
|
||||
await expect(stat(session.tempPath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(service.activeSessions.has(session.uploadId)).toBe(false);
|
||||
await expect(service.cleanup(session.uploadId)).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("sweeps sessions idle past the 24h TTL along with their temp files", async () => {
|
||||
const session = await init(CHUNK_SIZE_BYTES);
|
||||
await service.appendChunk(session.uploadId, workspaceId, 0, Buffer.alloc(CHUNK_SIZE_BYTES));
|
||||
const now = Date.now();
|
||||
// Fresh session is retained.
|
||||
expect(await service.sweepExpired(now)).toEqual([]);
|
||||
// 24h minus a second: still retained.
|
||||
expect(await service.sweepExpired(now + 24 * 60 * 60 * 1000 - 1000)).toEqual([]);
|
||||
expect(service.activeSessions.has(session.uploadId)).toBe(true);
|
||||
// TTL reached: swept.
|
||||
const swept = await service.sweepExpired(now + 24 * 60 * 60 * 1000 + 1);
|
||||
expect(swept).toEqual([session.uploadId]);
|
||||
await expect(stat(session.tempPath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(service.activeSessions.has(session.uploadId)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("end-to-end assemble → seal → register → probe (VideoService tail end)", () => {
|
||||
it("uploadVideoFromPath stores, probes, and returns the same VideoUploadResult shape as POST /video/upload", async () => {
|
||||
const database: FakeVideoDatabase = {};
|
||||
const bytes = Buffer.concat([Buffer.from("MP4DATA-"), Buffer.alloc(512, 3)]);
|
||||
const probe = vi.fn(async (path: string) => {
|
||||
expect(await readFile(path)).toEqual(bytes);
|
||||
expect(database.job?.state).toBe("probing");
|
||||
return { width: 1920, height: 1080, durationMs: 90_000, frameRate: 24, codec: "hevc" };
|
||||
});
|
||||
const { service: video, storage, appendAudit } = serviceWithDatabase(database, probe);
|
||||
|
||||
// Assemble through the real chunk pipeline (small single-chunk file).
|
||||
const session = await service.init(workspaceId, {
|
||||
filename: "capture.mp4",
|
||||
mimeType: "video/mp4",
|
||||
totalBytes: bytes.length,
|
||||
});
|
||||
await service.appendChunk(session.uploadId, workspaceId, 0, bytes);
|
||||
const check = await service.verifyComplete(session.uploadId, workspaceId);
|
||||
expect(() => service.assertAssembled(check)).not.toThrow();
|
||||
|
||||
const uploaded = await video.uploadVideoFromPath(workspaceId, session.tempPath, "video/mp4", bytes.length);
|
||||
expect(storage.putContent).toHaveBeenCalledWith(bytes, "video/mp4");
|
||||
expect(uploaded.videoAsset).toMatchObject({
|
||||
storageKey: expect.stringMatching(/^sha256\/[a-f0-9]{64}\.mp4$/),
|
||||
sizeBytes: bytes.length,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
durationMs: 90_000,
|
||||
codec: "hevc",
|
||||
});
|
||||
// Identical shape/outcome to the single-shot path: a completed probe job.
|
||||
expect(uploaded.processingJob).toMatchObject({ state: "complete", progressPercent: 100, outputKey: uploaded.videoAsset.storageKey });
|
||||
expect(database.transitions).toEqual(["queued", "probing", "transcoding", "complete"]);
|
||||
expect(appendAudit).toHaveBeenCalledWith(workspaceId, expect.objectContaining({ action: "video.uploaded", targetId: assetId }));
|
||||
// Cleanup is the caller's (controller's) job, verified separately.
|
||||
expect(probe).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("rejects a declared size that disagrees with the file on disk", async () => {
|
||||
const { service: video } = serviceWithDatabase({}, async () => ({}));
|
||||
const path = join(directory, "mismatch.tmp");
|
||||
await writeFile(path, Buffer.alloc(16, 1));
|
||||
await expect(video.uploadVideoFromPath(workspaceId, path, "video/mp4", 15))
|
||||
.rejects.toMatchObject({ status: 400, message: expect.stringContaining("16 bytes on disk but declared 15") });
|
||||
await expect(video.uploadVideoFromPath(workspaceId, join(directory, "missing.tmp"), "video/mp4", 16))
|
||||
.rejects.toMatchObject({ status: 400, message: expect.stringContaining("missing on disk") });
|
||||
});
|
||||
|
||||
it("refuses non-video MIME types and empty files", async () => {
|
||||
const { service: video } = serviceWithDatabase({}, async () => ({}));
|
||||
const path = join(directory, "notvideo.tmp");
|
||||
await writeFile(path, Buffer.alloc(4, 1));
|
||||
await expect(video.uploadVideoFromPath(workspaceId, path, "image/png", 4))
|
||||
.rejects.toMatchObject({ status: 400, message: expect.stringContaining("video media type") });
|
||||
await expect(video.uploadVideoFromPath(workspaceId, path, "video/mp4", 0))
|
||||
.rejects.toMatchObject({ status: 400, message: expect.stringContaining("positive safe integer") });
|
||||
});
|
||||
});
|
||||
|
||||
function firstUploadId(): string {
|
||||
const ids = [...service.activeSessions.keys()];
|
||||
if (ids.length !== 1) throw new Error(`expected exactly one session, found ${ids.length}`);
|
||||
return ids[0] as string;
|
||||
}
|
||||
299
apps/api/src/video/chunked-upload.ts
Normal file
299
apps/api/src/video/chunked-upload.ts
Normal file
|
|
@ -0,0 +1,299 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { mkdir, open, rm, stat } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { BadRequestException, ConflictException, Injectable, NotFoundException, OnModuleDestroy, Optional } from "@nestjs/common";
|
||||
import {
|
||||
CHUNK_SIZE_BYTES,
|
||||
CHUNKED_UPLOAD_TTL_MS,
|
||||
MAX_CHUNKED_UPLOAD_BYTES,
|
||||
chunkBytesFor,
|
||||
totalChunksFor,
|
||||
} from "./upload-limits.js";
|
||||
|
||||
/**
|
||||
* Chunked (resumable-style) large-file ingest.
|
||||
*
|
||||
* Cloudflare's free plan rejects proxied request bodies above 100 MB, so a 20 GB
|
||||
* camera original can never arrive in one POST /video/upload. The client instead
|
||||
* slices the file into CHUNK_SIZE_BYTES (10 MiB) chunks — each far under the
|
||||
* limit — and PUTs them one at a time. Each chunk is written at its fixed offset
|
||||
* (`index × 10 MiB`) into a sparse temp file, so chunks may arrive in ANY order
|
||||
* and may be retransmitted after a network failure without corrupting the file.
|
||||
*
|
||||
* On complete, the assembled file goes through the SAME SPENC1 seal → S3
|
||||
* putContent path as every other upload.
|
||||
*
|
||||
* ── MEMORY REQUIREMENT (read this) ────────────────────────────────────────────
|
||||
* The seal function (apps/api/src/storage/storage.ts) takes a whole Buffer, and
|
||||
* S3 PutObject is likewise handed one body. Assembling therefore peaks at
|
||||
* roughly 2× file size of RAM: one Buffer of plaintext for the seal plus the
|
||||
* sealed copy the S3 client transmits (plus ~1× transiently for
|
||||
* Buffer.concat in seal). There is no way to bound that without streaming
|
||||
* encryption + multipart S3, which would change the storage contract for every
|
||||
* other caller — out of scope here.
|
||||
*
|
||||
* Practical sizing: a 20 GB upload needs a host with ≥ 48 GB RAM
|
||||
* (MAX_OLD_SPACE_SIZE raised, or --max_old-space-size=32768+, plus headroom for
|
||||
* the sealed copy). Up to ~2 GiB the ordinary multipart path is fine and cheaper.
|
||||
* Disk also matters: /tmp must hold the assembled file (see SP_CHUNK_UPLOAD_DIR,
|
||||
* default /tmp/uploads) on top of whatever S3 buffers.
|
||||
* ─────────────────────────────────────────────────────────────────────────────
|
||||
*/
|
||||
|
||||
export interface ChunkedUploadSession {
|
||||
uploadId: string;
|
||||
/** Owning workspace; chunks and complete are rejected for other tenants. */
|
||||
workspaceId: string;
|
||||
filename: string;
|
||||
mimeType: string;
|
||||
totalBytes: number;
|
||||
totalChunks: number;
|
||||
/** Indices of chunks whose bytes are durably on disk. */
|
||||
received: Set<number>;
|
||||
createdAt: number;
|
||||
updatedAt: number;
|
||||
/** Set once /complete has been accepted; the entry is dropped afterwards. */
|
||||
completed?: boolean;
|
||||
tempPath: string;
|
||||
}
|
||||
|
||||
export interface InitChunkedUploadRequest {
|
||||
filename: string;
|
||||
mimeType: string;
|
||||
totalBytes: number;
|
||||
}
|
||||
|
||||
export interface ChunkReceipt {
|
||||
received: true;
|
||||
bytesSoFar: number;
|
||||
}
|
||||
|
||||
export interface CompletionCheck {
|
||||
session: ChunkedUploadSession;
|
||||
/** Missing chunk indices, ascending — empty when the upload is complete. */
|
||||
missing: number[];
|
||||
/** Bytes durably written so far (sum of received chunk sizes). */
|
||||
bytesSoFar: number;
|
||||
/** Size of the assembled file on disk (sparse holes read as zero bytes). */
|
||||
fileBytes: number;
|
||||
}
|
||||
|
||||
/** A pure function that makes the on-disk layout testable without touching a disk. */
|
||||
export function chunkOffset(index: number, chunkSize: number = CHUNK_SIZE_BYTES): number {
|
||||
if (!Number.isSafeInteger(index) || index < 0) throw new Error("index must be a non-negative integer");
|
||||
return index * chunkSize;
|
||||
}
|
||||
|
||||
/** Max chunk byte length a client may PUT for the given session. */
|
||||
export function maxChunkBytesFor(totalBytes: number, index: number, chunkSize: number = CHUNK_SIZE_BYTES): number {
|
||||
return chunkBytesFor(totalBytes, index, chunkSize);
|
||||
}
|
||||
|
||||
const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
|
||||
@Injectable()
|
||||
export class ChunkedUploadService implements OnModuleDestroy {
|
||||
/** uploadId → session. Module-level by design: one assembler per API process. */
|
||||
private readonly sessions = new Map<string, ChunkedUploadSession>();
|
||||
private sweeper: NodeJS.Timeout | undefined;
|
||||
|
||||
constructor(
|
||||
@Optional() private readonly directory: string = process.env.SP_CHUNK_UPLOAD_DIR ?? join(tmpdir(), "uploads"),
|
||||
@Optional() private readonly sweeperIntervalMs: number = 30 * 60 * 1000,
|
||||
@Optional() private readonly generateId: () => string = randomUUID,
|
||||
) {
|
||||
// Only arm the sweeper when running under a live event loop (not unit tests
|
||||
// that construct the service directly and would hold the process open).
|
||||
if (process.env.VITEST === undefined) {
|
||||
this.sweeper = setInterval(() => void this.sweepExpired(), this.sweeperIntervalMs);
|
||||
this.sweeper.unref?.();
|
||||
}
|
||||
}
|
||||
|
||||
/** Test seam: the live sessions map. */
|
||||
get activeSessions(): Map<string, ChunkedUploadSession> {
|
||||
return this.sessions;
|
||||
}
|
||||
|
||||
async init(workspaceId: string, request: InitChunkedUploadRequest): Promise<ChunkedUploadSession> {
|
||||
const filename = request.filename.trim();
|
||||
if (filename === "" || filename.length > 512) {
|
||||
throw new BadRequestException("filename must be a non-empty string of at most 512 characters");
|
||||
}
|
||||
const mimeType = request.mimeType.trim();
|
||||
if (!mimeType.toLowerCase().startsWith("video/")) {
|
||||
throw new BadRequestException("mimeType must be a video media type");
|
||||
}
|
||||
if (!Number.isSafeInteger(request.totalBytes) || request.totalBytes < 1) {
|
||||
throw new BadRequestException("totalBytes must be a positive safe integer");
|
||||
}
|
||||
if (request.totalBytes > MAX_CHUNKED_UPLOAD_BYTES) {
|
||||
throw new BadRequestException("totalBytes exceeds the 100 GiB chunked upload limit");
|
||||
}
|
||||
const totalChunks = totalChunksFor(request.totalBytes);
|
||||
const uploadId = this.generateId();
|
||||
await mkdir(this.directory, { recursive: true });
|
||||
const tempPath = join(this.directory, `${uploadId}.tmp`);
|
||||
// Create the file eagerly (and exclusively per uploadId) so a later chunk
|
||||
// never races a first-time create, and so the path is known to be writable.
|
||||
const handle = await open(tempPath, "wx", 0o600).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "EEXIST") throw new ConflictException("temp upload path already exists; retry init");
|
||||
throw error;
|
||||
});
|
||||
await handle.close();
|
||||
const now = Date.now();
|
||||
const session: ChunkedUploadSession = {
|
||||
uploadId,
|
||||
workspaceId,
|
||||
filename,
|
||||
mimeType,
|
||||
totalBytes: request.totalBytes,
|
||||
totalChunks,
|
||||
received: new Set<number>(),
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
tempPath,
|
||||
};
|
||||
this.sessions.set(uploadId, session);
|
||||
return session;
|
||||
}
|
||||
|
||||
async appendChunk(uploadId: string, workspaceId: string, index: number, bytes: Buffer): Promise<ChunkReceipt> {
|
||||
const session = this.requireSession(uploadId, workspaceId);
|
||||
if (!Number.isSafeInteger(index) || index < 0 || index >= session.totalChunks) {
|
||||
throw new BadRequestException(`index must be an integer between 0 and ${session.totalChunks - 1}`);
|
||||
}
|
||||
const expected = chunkBytesFor(session.totalBytes, index);
|
||||
if (bytes.length !== expected) {
|
||||
throw new BadRequestException(
|
||||
`chunk ${index} must be exactly ${expected} bytes (got ${bytes.length}); the final chunk carries the remainder`,
|
||||
);
|
||||
}
|
||||
// Positional write at index × 10 MiB: arrival order does not matter and a
|
||||
// retransmitted chunk is idempotent (same bytes, same offset).
|
||||
const handle = await open(session.tempPath, "r+").catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "ENOENT") {
|
||||
throw new NotFoundException(`upload ${uploadId} temp file is gone; start a new upload`);
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
try {
|
||||
const { bytesWritten } = await handle.write(bytes, 0, bytes.length, chunkOffset(index));
|
||||
if (bytesWritten !== expected) {
|
||||
throw new Error(`short write for chunk ${index}: ${bytesWritten} of ${expected} bytes`);
|
||||
}
|
||||
// Durability before acknowledgement: a chunk reported received must
|
||||
// actually be on disk if the API restarts mid-upload.
|
||||
await handle.datasync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
session.received.add(index);
|
||||
session.updatedAt = Date.now();
|
||||
return { received: true, bytesSoFar: this.bytesSoFar(session) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies every chunk landed before ingest: exact chunk-count check against
|
||||
* the session, byte accounting against the assembled file on disk, and — the
|
||||
* authoritative check — that the file size equals the declared totalBytes.
|
||||
*/
|
||||
async verifyComplete(uploadId: string, workspaceId: string): Promise<CompletionCheck> {
|
||||
const session = this.requireSession(uploadId, workspaceId);
|
||||
const missing: number[] = [];
|
||||
for (let index = 0; index < session.totalChunks; index += 1) {
|
||||
if (!session.received.has(index)) missing.push(index);
|
||||
}
|
||||
const fileBytes = await stat(session.tempPath).then((info) => info.size).catch(() => -1);
|
||||
return { session, missing, bytesSoFar: this.bytesSoFar(session), fileBytes };
|
||||
}
|
||||
|
||||
/** Size verification for tests / callers: the file must be exactly totalBytes. */
|
||||
assertAssembled(check: CompletionCheck): void {
|
||||
if (check.missing.length > 0) {
|
||||
throw new BadRequestException(
|
||||
`upload ${check.session.uploadId} is missing ${check.missing.length} chunk(s): ${describeIndices(check.missing)}`,
|
||||
);
|
||||
}
|
||||
if (check.bytesSoFar !== check.session.totalBytes) {
|
||||
throw new BadRequestException(
|
||||
`upload ${check.session.uploadId} received ${check.bytesSoFar} bytes but declared ${check.session.totalBytes}`,
|
||||
);
|
||||
}
|
||||
if (check.fileBytes !== check.session.totalBytes) {
|
||||
throw new BadRequestException(
|
||||
`assembled file for ${check.session.uploadId} is ${check.fileBytes} bytes on disk; expected ${check.session.totalBytes}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Marks the session consumed so no further chunk can be appended. */
|
||||
complete(uploadId: string): void {
|
||||
const session = this.sessions.get(uploadId);
|
||||
if (session === undefined) throw new NotFoundException(`upload ${uploadId} not found`);
|
||||
session.completed = true;
|
||||
}
|
||||
|
||||
/** Removes the temp file and the session entry. Idempotent (abort + sweep). */
|
||||
async abort(uploadId: string, workspaceId: string): Promise<void> {
|
||||
const session = this.requireSession(uploadId, workspaceId);
|
||||
this.sessions.delete(uploadId);
|
||||
await rm(session.tempPath, { force: true });
|
||||
}
|
||||
|
||||
/** Drops the temp file and entry after complete; keeps failures visible for retry. */
|
||||
async cleanup(uploadId: string): Promise<void> {
|
||||
const session = this.sessions.get(uploadId);
|
||||
if (session === undefined) return;
|
||||
this.sessions.delete(uploadId);
|
||||
await rm(session.tempPath, { force: true });
|
||||
}
|
||||
|
||||
/** Removes sessions idle past the TTL along with their temp files. */
|
||||
async sweepExpired(now: number = Date.now()): Promise<string[]> {
|
||||
const expired: string[] = [];
|
||||
for (const [uploadId, session] of this.sessions) {
|
||||
if (now - session.updatedAt <= CHUNKED_UPLOAD_TTL_MS) continue;
|
||||
expired.push(uploadId);
|
||||
this.sessions.delete(uploadId);
|
||||
await rm(session.tempPath, { force: true });
|
||||
}
|
||||
return expired;
|
||||
}
|
||||
|
||||
async onModuleDestroy(): Promise<void> {
|
||||
if (this.sweeper !== undefined) clearInterval(this.sweeper);
|
||||
// In-flight sessions are resumable in principle, but the process is going
|
||||
// away: drop the entries so nothing can half-append later.
|
||||
for (const [uploadId, session] of this.sessions) {
|
||||
this.sessions.delete(uploadId);
|
||||
await rm(session.tempPath, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
private requireSession(uploadId: string, workspaceId: string): ChunkedUploadSession {
|
||||
if (!UUID_PATTERN.test(uploadId)) throw new BadRequestException("uploadId must be a UUID");
|
||||
const session = this.sessions.get(uploadId);
|
||||
if (session === undefined) throw new NotFoundException(`upload ${uploadId} not found or expired`);
|
||||
if (session.workspaceId !== workspaceId) {
|
||||
// Do not leak existence across tenants.
|
||||
throw new NotFoundException(`upload ${uploadId} not found or expired`);
|
||||
}
|
||||
if (session.completed === true) {
|
||||
throw new ConflictException(`upload ${uploadId} has already been completed`);
|
||||
}
|
||||
return session;
|
||||
}
|
||||
|
||||
private bytesSoFar(session: ChunkedUploadSession): number {
|
||||
let total = 0;
|
||||
for (const index of session.received) total += chunkBytesFor(session.totalBytes, index);
|
||||
return total;
|
||||
}
|
||||
}
|
||||
|
||||
function describeIndices(indices: number[]): string {
|
||||
const shown = indices.slice(0, 8).map(String).join(", ");
|
||||
return indices.length > 8 ? `${shown}, … (+${indices.length - 8} more)` : shown;
|
||||
}
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
import { Readable } from "node:stream";
|
||||
import { BadRequestException, ServiceUnavailableException } from "@nestjs/common";
|
||||
import { OcisNotFoundError, OcisWebdavClient, ocisConfigFromEnv } from "../group-files/ocis-webdav.js";
|
||||
/** Imports buffer the whole file in memory; 256 MiB is the v1 cap (streaming is a follow-up). */
|
||||
export const IMPORT_MAX_BYTES = 256 * 1024 * 1024;
|
||||
|
||||
|
|
@ -34,19 +33,6 @@ export async function bufferWithCap(stream: Readable, capBytes: number): Promise
|
|||
});
|
||||
}
|
||||
|
||||
let ocisClient: OcisWebdavClient | null = null;
|
||||
|
||||
/** Drive import: pull one oCIS resource through the service-account WebDAV client. */
|
||||
export async function loadDriveBytes(resourceId: string): Promise<{ bytes: Buffer; mime: string }> {
|
||||
ocisClient ??= new OcisWebdavClient(ocisConfigFromEnv());
|
||||
const download = await ocisClient.download(resourceId).catch((error: unknown) => {
|
||||
if (error instanceof OcisNotFoundError) throw new BadRequestException("no such Drive resource");
|
||||
throw new ServiceUnavailableException("Drive is unreachable — try again later");
|
||||
});
|
||||
const bytes = await bufferWithCap(download.stream, IMPORT_MAX_BYTES);
|
||||
return { bytes, mime: download.contentType ?? "application/octet-stream" };
|
||||
}
|
||||
|
||||
export function matrixMediaConfig(): { homeserver: string; token: string } | null {
|
||||
const token = process.env.MATRIX_MEDIA_TOKEN?.trim() ?? "";
|
||||
if (token === "") return null;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { keyForHash, sha256, type StorageService } from "../storage/storage.js";
|
||||
import { keyForHash, sha256, type StorageService, directStorageDisabled } from "../storage/storage.js";
|
||||
import type { PoolClient, QueryResult } from "pg";
|
||||
import { vi } from "vitest";
|
||||
import {
|
||||
|
|
@ -16,6 +16,8 @@ export interface FakeVideoDatabase {
|
|||
transitions?: string[];
|
||||
/** Raw INSERT statements seen, for asserting what intake wrote. */
|
||||
statements?: Array<{ sql: string; values: readonly unknown[] }>;
|
||||
/** Image storage keys this workspace holds (logo overlay ownership). */
|
||||
ownedImageKeys?: string[];
|
||||
}
|
||||
|
||||
function result(rows: Array<Record<string, unknown>>): QueryResult<Record<string, unknown>> {
|
||||
|
|
@ -43,6 +45,7 @@ export function serviceWithDatabase(
|
|||
}),
|
||||
hasContent: async (hash, type) => objects.has(keyForHash(hash, type)),
|
||||
presign: async () => { throw new Error("Encrypted source must be decrypted before probing"); },
|
||||
...directStorageDisabled(),
|
||||
};
|
||||
database.transitions = [];
|
||||
database.statements = [];
|
||||
|
|
@ -88,6 +91,9 @@ export function serviceWithDatabase(
|
|||
if (sql === "SELECT * FROM video_assets WHERE id = $1") {
|
||||
return result(database.asset === undefined ? [] : [database.asset]);
|
||||
}
|
||||
if (sql.includes("branding->>'logoAssetKey'")) {
|
||||
return result(database.ownedImageKeys?.includes(String(values[0])) === true ? [{}] : []);
|
||||
}
|
||||
if (sql.includes("SET state = 'probing'")) {
|
||||
if (database.job?.state !== "queued") return result([]);
|
||||
database.job.state = "probing";
|
||||
|
|
|
|||
|
|
@ -1,2 +1,41 @@
|
|||
// StorageService seals whole Buffers; bound multipart memory use per request.
|
||||
export const MAX_VIDEO_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Chunked ingest: POST /video/upload/init → PUT /video/upload/:uploadId/chunk/:index
|
||||
* → POST /video/upload/:uploadId/complete (see chunked-upload.ts).
|
||||
*
|
||||
* Cloudflare's free plan rejects proxied request bodies above 100 MB, so the
|
||||
* client slices any file into CHUNK_SIZE_BYTES chunks — each comfortably under
|
||||
* the limit — and the server reassembles them on disk before the normal
|
||||
* SPENC1 seal → S3 putContent path. This removes the practical size limit:
|
||||
* only MAX_CHUNKED_UPLOAD_BYTES (and server RAM, documented in chunked-upload.ts)
|
||||
* bounds an upload.
|
||||
*/
|
||||
export const CHUNK_SIZE_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
/** Hard ceiling for a single chunked upload (spec: validate totalBytes ≤ 100 GiB). */
|
||||
export const MAX_CHUNKED_UPLOAD_BYTES = 100 * 1024 * 1024 * 1024;
|
||||
|
||||
/** An upload session that receives nothing for this long is swept (file + entry). */
|
||||
export const CHUNKED_UPLOAD_TTL_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
/** Number of 10 MiB chunks a client must send for a file of totalBytes. */
|
||||
export function totalChunksFor(totalBytes: number, chunkSize: number = CHUNK_SIZE_BYTES): number {
|
||||
if (!Number.isSafeInteger(totalBytes) || totalBytes < 1) {
|
||||
throw new Error("totalBytes must be a positive safe integer");
|
||||
}
|
||||
return Math.ceil(totalBytes / chunkSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* Exact byte length of chunk `index`: every chunk is a full `chunkSize` except
|
||||
* the last, which carries the remainder. Enforced on the server so the
|
||||
* assembled file can never be short or over-long.
|
||||
*/
|
||||
export function chunkBytesFor(totalBytes: number, index: number, chunkSize: number = CHUNK_SIZE_BYTES): number {
|
||||
if (!Number.isSafeInteger(index) || index < 0) throw new Error("index must be a non-negative integer");
|
||||
const totalChunks = totalChunksFor(totalBytes, chunkSize);
|
||||
if (index >= totalChunks) throw new Error(`index ${index} is outside the ${totalChunks}-chunk upload`);
|
||||
return Math.min(chunkSize, totalBytes - index * chunkSize);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,9 @@ import {
|
|||
Get,
|
||||
Inject,
|
||||
Param,
|
||||
ParseIntPipe,
|
||||
Post,
|
||||
Put,
|
||||
Query,
|
||||
Req,
|
||||
UseGuards,
|
||||
|
|
@ -19,14 +21,28 @@ import { parseId, parseLimit, requireObject, requireSession, requireString } fro
|
|||
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
|
||||
import {
|
||||
VideoService,
|
||||
type DirectUploadCompleteInput,
|
||||
type DirectUploadConfig,
|
||||
type DirectUploadInit,
|
||||
type ProduceRequestResult,
|
||||
type VideoLibraryPage,
|
||||
type VideoStatus,
|
||||
type VideoUploadResult,
|
||||
} from "./video.service.js";
|
||||
import { parseProduceRequest } from "./produce-request.js";
|
||||
import { MAX_VIDEO_UPLOAD_BYTES } from "./upload-limits.js";
|
||||
import { loadDriveBytes, loadMatrixBytes, parseMxcUri } from "./import-sources.js";
|
||||
import {
|
||||
CHUNK_SIZE_BYTES,
|
||||
MAX_CHUNKED_UPLOAD_BYTES,
|
||||
MAX_VIDEO_UPLOAD_BYTES,
|
||||
totalChunksFor,
|
||||
} from "./upload-limits.js";
|
||||
import {
|
||||
ChunkedUploadService,
|
||||
type ChunkReceipt,
|
||||
type InitChunkedUploadRequest,
|
||||
} from "./chunked-upload.js";
|
||||
import { loadMatrixBytes, parseMxcUri } from "./import-sources.js";
|
||||
import { isUuid } from "../http/validation.js";
|
||||
|
||||
interface VideoFile {
|
||||
buffer: Buffer;
|
||||
|
|
@ -42,7 +58,116 @@ interface UploadVideoDto {
|
|||
@Controller("video")
|
||||
@UseGuards(RbacGuard)
|
||||
export class VideoController {
|
||||
constructor(@Inject(VideoService) private readonly video: VideoService) {}
|
||||
constructor(
|
||||
@Inject(VideoService) private readonly video: VideoService,
|
||||
private readonly chunked: ChunkedUploadService,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Chunked ingest for files that cannot clear Cloudflare's 100 MB proxied-body
|
||||
* cap (or the 2 GiB whole-body limit). The client slices the file into 10 MiB
|
||||
* chunks and PUTs them sequentially; the server reassembles on disk, then runs
|
||||
* the SAME SPENC1 seal → S3 putContent path as POST /video/upload.
|
||||
*
|
||||
* MEMORY: reading the assembled file for the seal peaks around 2× file size of
|
||||
* RAM (see chunked-upload.ts) — a 20 GB upload needs a ≥ 48 GB host.
|
||||
*/
|
||||
/**
|
||||
* Direct-to-R2 ingest: the BROWSER seals (SPENC2) and PUTs straight to object
|
||||
* storage, so this box never touches video bytes. The api only mints the
|
||||
* presigned PUT (init), HEAD-checks + registers + queues a probe job for the
|
||||
* media-worker (complete). Same ceiling as the chunked path, without the RAM.
|
||||
*
|
||||
* GET /video/direct/config capability + the JWK the browser seals against
|
||||
* POST /video/direct/init presigned PUT for a browser-sealed frame
|
||||
* POST /video/direct/complete HEAD-checked register + queued probe job
|
||||
*/
|
||||
@Get("direct/config")
|
||||
@RequirePermission("media.view")
|
||||
async directConfig(): Promise<DirectUploadConfig> {
|
||||
return this.video.directUploadConfig();
|
||||
}
|
||||
|
||||
@Post("direct/init")
|
||||
@RequirePermission("media.manage")
|
||||
async initDirectUpload(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Body() body: unknown,
|
||||
): Promise<DirectUploadInit> {
|
||||
const session = requireSession(request);
|
||||
return this.video.initDirectUpload(session.workspaceId, requireDirectInit(body));
|
||||
}
|
||||
|
||||
@Post("direct/complete")
|
||||
@RequirePermission("media.manage")
|
||||
async completeDirectUpload(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Body() body: unknown,
|
||||
): Promise<VideoStatus> {
|
||||
const session = requireSession(request);
|
||||
return this.video.completeDirectUpload(session.workspaceId, parseDirectComplete(body));
|
||||
}
|
||||
|
||||
@Post("upload/init")
|
||||
@RequirePermission("media.manage")
|
||||
async initChunkedUpload(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Body() body: unknown,
|
||||
): Promise<{ uploadId: string; chunkSizeBytes: number; totalChunks: number }> {
|
||||
const session = requireSession(request);
|
||||
const dto = parseInitUpload(body);
|
||||
const created = await this.chunked.init(session.workspaceId, dto);
|
||||
return {
|
||||
uploadId: created.uploadId,
|
||||
chunkSizeBytes: CHUNK_SIZE_BYTES,
|
||||
totalChunks: created.totalChunks,
|
||||
};
|
||||
}
|
||||
|
||||
@Put("upload/:uploadId/chunk/:index")
|
||||
@RequirePermission("media.manage")
|
||||
async uploadChunk(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Param("uploadId") uploadId: string,
|
||||
@Param("index", ParseIntPipe) index: number,
|
||||
): Promise<ChunkReceipt> {
|
||||
const session = requireSession(request);
|
||||
const bytes = await readRawBody(request);
|
||||
return this.chunked.appendChunk(uploadId, session.workspaceId, index, bytes);
|
||||
}
|
||||
|
||||
@Post("upload/:uploadId/complete")
|
||||
@RequirePermission("media.manage")
|
||||
async completeChunkedUpload(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Param("uploadId") uploadId: string,
|
||||
): Promise<VideoUploadResult> {
|
||||
const session = requireSession(request);
|
||||
const check = await this.chunked.verifyComplete(uploadId, session.workspaceId);
|
||||
this.chunked.assertAssembled(check);
|
||||
this.chunked.complete(uploadId);
|
||||
try {
|
||||
return await this.video.uploadVideoFromPath(
|
||||
session.workspaceId,
|
||||
check.session.tempPath,
|
||||
check.session.mimeType,
|
||||
check.session.totalBytes,
|
||||
);
|
||||
} finally {
|
||||
// Never leave an assembled multi-GB temp file behind — success or failure.
|
||||
await this.chunked.cleanup(uploadId);
|
||||
}
|
||||
}
|
||||
|
||||
@Delete("upload/:uploadId")
|
||||
@RequirePermission("media.manage")
|
||||
async abortChunkedUpload(
|
||||
@Req() request: Request & AuthenticatedRequest,
|
||||
@Param("uploadId") uploadId: string,
|
||||
): Promise<void> {
|
||||
const session = requireSession(request);
|
||||
await this.chunked.abort(uploadId, session.workspaceId);
|
||||
}
|
||||
|
||||
@Post("upload")
|
||||
@RequirePermission("media.manage")
|
||||
|
|
@ -63,16 +188,6 @@ export class VideoController {
|
|||
return this.video.uploadVideo(session.workspaceId, dto.storageKey, dto.mimeType, dto.sizeBytes);
|
||||
}
|
||||
|
||||
@Post("import-drive")
|
||||
@RequirePermission("media.manage")
|
||||
async importDrive(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise<VideoUploadResult> {
|
||||
const session = requireSession(request);
|
||||
const input = requireObject(body, "body");
|
||||
const resourceId = requireString(input.resourceId, "resourceId");
|
||||
const { bytes, mime } = await loadDriveBytes(resourceId);
|
||||
return this.video.importVideoBytes(session.workspaceId, bytes, mime, "drive", resourceId);
|
||||
}
|
||||
|
||||
@Post("import-chat")
|
||||
@RequirePermission("media.manage")
|
||||
async importChat(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise<VideoUploadResult> {
|
||||
|
|
@ -126,6 +241,41 @@ export class VideoController {
|
|||
}
|
||||
}
|
||||
|
||||
/** POST /video/direct/init body: the browser's pick, bounded before any bytes are sealed. */
|
||||
function requireDirectInit(value: unknown): { filename: string; mimeType: string; sizeBytes: number } {
|
||||
const body = requireObject(value, "body");
|
||||
const sizeBytes = body.sizeBytes;
|
||||
if (typeof sizeBytes !== "number" || !Number.isSafeInteger(sizeBytes) || sizeBytes < 1) {
|
||||
throw new BadRequestException("sizeBytes must be a positive safe integer");
|
||||
}
|
||||
if (sizeBytes > MAX_CHUNKED_UPLOAD_BYTES) {
|
||||
throw new BadRequestException("sizeBytes exceeds the 100 GiB chunked upload limit");
|
||||
}
|
||||
return {
|
||||
filename: requireString(body.filename, "filename"),
|
||||
mimeType: requireString(body.mimeType, "mimeType"),
|
||||
sizeBytes,
|
||||
};
|
||||
}
|
||||
|
||||
/** POST /video/direct/complete body: the commit half of the direct handshake. */
|
||||
function parseDirectComplete(value: unknown): DirectUploadCompleteInput {
|
||||
const body = requireObject(value, "body");
|
||||
const uploadId = requireString(body.uploadId, "uploadId");
|
||||
if (!isUuid(uploadId)) throw new BadRequestException("uploadId must be a UUID");
|
||||
const sizeBytes = body.sizeBytes;
|
||||
if (typeof sizeBytes !== "number" || !Number.isSafeInteger(sizeBytes) || sizeBytes < 1) {
|
||||
throw new BadRequestException("sizeBytes must be a positive safe integer");
|
||||
}
|
||||
return {
|
||||
uploadId,
|
||||
objectKey: requireString(body.objectKey, "objectKey"),
|
||||
sizeBytes,
|
||||
mimeType: requireString(body.mimeType, "mimeType"),
|
||||
filename: requireString(body.filename, "filename"),
|
||||
};
|
||||
}
|
||||
|
||||
function parseUploadVideo(value: unknown): UploadVideoDto {
|
||||
const body = requireObject(value, "body");
|
||||
const sizeBytes = body.sizeBytes;
|
||||
|
|
@ -138,3 +288,52 @@ function parseUploadVideo(value: unknown): UploadVideoDto {
|
|||
sizeBytes,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Raw octet-stream body reader for chunk PUTs. Express leaves a non-multipart
|
||||
* body unread until something consumes it; read exactly Content-Length bytes so
|
||||
* a truncated client upload is rejected instead of silently padding the file.
|
||||
*/
|
||||
async function readRawBody(request: Request): Promise<Buffer> {
|
||||
const declared = Number(request.headers["content-length"] ?? NaN);
|
||||
const declaredBytes = Number.isSafeInteger(declared) ? declared : undefined;
|
||||
const chunks: Buffer[] = [];
|
||||
let received = 0;
|
||||
for await (const part of request) {
|
||||
const chunk = Buffer.isBuffer(part) ? part : Buffer.from(String(part));
|
||||
received += chunk.length;
|
||||
if (declaredBytes !== undefined && received > declaredBytes) {
|
||||
throw new BadRequestException("request body exceeded its Content-Length");
|
||||
}
|
||||
if (received > CHUNK_SIZE_BYTES) {
|
||||
throw new BadRequestException(`chunk exceeds the ${CHUNK_SIZE_BYTES} byte chunk size`);
|
||||
}
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const body = Buffer.concat(chunks);
|
||||
if (declaredBytes !== undefined && body.length !== declaredBytes) {
|
||||
throw new BadRequestException("request body was shorter than its Content-Length");
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
function parseInitUpload(value: unknown): InitChunkedUploadRequest {
|
||||
const body = requireObject(value, "body");
|
||||
const totalBytes = body.totalBytes;
|
||||
if (typeof totalBytes !== "number" || !Number.isSafeInteger(totalBytes) || totalBytes < 1) {
|
||||
throw new BadRequestException("totalBytes must be a positive safe integer");
|
||||
}
|
||||
if (totalBytes > MAX_CHUNKED_UPLOAD_BYTES) {
|
||||
throw new BadRequestException("totalBytes exceeds the 100 GiB chunked upload limit");
|
||||
}
|
||||
return {
|
||||
filename: requireString(body.filename, "filename"),
|
||||
mimeType: requireString(body.mimeType, "mimeType"),
|
||||
totalBytes,
|
||||
};
|
||||
}
|
||||
|
||||
/** Client-facing chunk math (kept server-side so a stale client cannot lie). */
|
||||
export function chunkPlan(totalBytes: number): { chunkSizeBytes: number; totalChunks: number } {
|
||||
return { chunkSizeBytes: CHUNK_SIZE_BYTES, totalChunks: totalChunksFor(totalBytes) };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ describe("video multipart HTTP routes", () => {
|
|||
const response = await upload(videoForm());
|
||||
expect(response.status).toBe(201);
|
||||
const result = await response.json() as VideoStatus;
|
||||
expect(result.processingJob.state).toBe("complete");
|
||||
expect(result.processingJob!.state).toBe("complete");
|
||||
expect(result.videoAsset).toMatchObject({ durationMs: 1000, width: 320, height: 180 });
|
||||
const status = await injectHttp(app?.getHttpAdapter().getInstance(), `http://api.test/video/${result.videoAsset.id}/status`, {
|
||||
headers: { authorization: "Bearer test-session" },
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { join } from "node:path";
|
|||
import { Pool, type PoolClient } from "pg";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { runMigrations } from "../db/migrate.js";
|
||||
import { sha256, keyForHash, type StorageService } from "../storage/storage.js";
|
||||
import { sha256, keyForHash, type StorageService, directStorageDisabled } from "../storage/storage.js";
|
||||
import { VideoService, type VideoServiceDependencies } from "./video.service.js";
|
||||
|
||||
const dbAvailable = process.env.DATABASE_URL !== undefined;
|
||||
|
|
@ -45,6 +45,7 @@ describe.skipIf(!dbAvailable)("video library (live DB)", () => {
|
|||
},
|
||||
hasContent: async (hash, type) => objects.has(keyForHash(hash, type)),
|
||||
presign: async () => "unused",
|
||||
...directStorageDisabled(),
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
|
|
@ -91,8 +92,8 @@ describe.skipIf(!dbAvailable)("video library (live DB)", () => {
|
|||
expect(page.items[0]?.processingState).toBe("complete");
|
||||
|
||||
const status = await service.getVideoStatus(uploaded.videoAsset.id, workspaceId);
|
||||
expect(status.processingJob.state).toBe("complete");
|
||||
expect(status.processingJob.processingNote).toContain("Transcode skipped");
|
||||
expect(status.processingJob!.state).toBe("complete");
|
||||
expect(status.processingJob!.processingNote).toContain("Transcode skipped");
|
||||
expect(status.videoAsset).toMatchObject({ width: 640, height: 360, durationMs: 1000, codec: "h264" });
|
||||
await expect(service.getVideoStatus(uploaded.videoAsset.id, randomUUID())).rejects.toThrow(/not found/);
|
||||
|
||||
|
|
@ -122,7 +123,7 @@ describe.skipIf(!dbAvailable)("video library (live DB)", () => {
|
|||
expect(second.videoAsset.id).not.toBe(first.videoAsset.id);
|
||||
expect(second.videoAsset.storageKey).toBe(first.videoAsset.storageKey);
|
||||
await service.deleteVideo(first.videoAsset.id, workspaceId);
|
||||
expect((await service.getVideoStatus(second.videoAsset.id, workspaceId)).processingJob.state).toBe("complete");
|
||||
expect((await service.getVideoStatus(second.videoAsset.id, workspaceId)).processingJob!.state).toBe("complete");
|
||||
expect(await storage.getObject(second.videoAsset.storageKey)).toEqual(bytes);
|
||||
await service.deleteVideo(second.videoAsset.id, workspaceId);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { readFile } from "node:fs/promises";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { keyForHash, sha256, seal, open, type StorageService } from "../storage/storage.js";
|
||||
import { keyForHash, sha256, seal, open, type StorageService, directStorageDisabled } from "../storage/storage.js";
|
||||
import { probeVideoFile } from "./ffprobe.js";
|
||||
import { serviceWithDatabase, workspaceId } from "./test-fixtures.js";
|
||||
|
||||
|
|
@ -30,11 +30,12 @@ describe.skipIf(fixture === undefined)("real ffprobe roundtrip (encrypted in-mem
|
|||
},
|
||||
hasContent: async (hash, contentType) => objects.has(keyForHash(hash, contentType)),
|
||||
presign: async () => { throw new Error("must decrypt, not presign"); },
|
||||
...directStorageDisabled(),
|
||||
};
|
||||
const { service } = serviceWithDatabase({}, probeVideoFile, storage);
|
||||
const uploaded = await service.uploadVideoBytes(workspaceId, bytes, "video/mp4");
|
||||
const status = await service.getVideoStatus(uploaded.videoAsset.id, workspaceId);
|
||||
expect(status.processingJob.state).toBe("complete");
|
||||
expect(status.processingJob!.state).toBe("complete");
|
||||
expect(status.videoAsset).toMatchObject({ durationMs: 1000, width: 320, height: 180, frameRate: 25 });
|
||||
expect(status.videoAsset.codec).toBeDefined();
|
||||
expect(objects.get(status.videoAsset.storageKey)?.equals(bytes)).toBe(false);
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ describe("video byte ingest and processing-job state machine", () => {
|
|||
expect(storage.getObject).toHaveBeenCalledWith(uploaded.videoAsset.storageKey);
|
||||
expect(uploaded.videoAsset).toMatchObject({ sizeBytes: bytes.length, width: 1280, height: 720, durationMs: 42_000, codec: "h264" });
|
||||
expect(uploaded.processingJob).toMatchObject({ state: "complete", progressPercent: 100, outputKey: uploaded.videoAsset.storageKey });
|
||||
expect(uploaded.processingJob.processingNote).toContain("Transcode skipped");
|
||||
expect(uploaded.processingJob!.processingNote).toContain("Transcode skipped");
|
||||
// ...and the produce INSERT is the only thing that followed it.
|
||||
expect(database.transitions).toEqual(["queued", "probing", "transcoding", "complete"]);
|
||||
await expect(stat(inputPath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
|
|
@ -46,7 +46,7 @@ describe("video byte ingest and processing-job state machine", () => {
|
|||
const uploaded = await service.uploadVideoBytes(workspaceId, Buffer.from("broken"), "video/mp4");
|
||||
expect(uploaded.processingJob).toMatchObject({ state: "failed", error: "unreadable media" });
|
||||
expect(database.transitions).toEqual(["queued", "probing", "failed"]);
|
||||
expect((await service.getVideoStatus(assetId, workspaceId)).processingJob.error).toBe("unreadable media");
|
||||
expect((await service.getVideoStatus(assetId, workspaceId)).processingJob!.error).toBe("unreadable media");
|
||||
await expect(stat(inputPath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
|
||||
|
|
@ -85,7 +85,7 @@ describe("video byte ingest and processing-job state machine", () => {
|
|||
await expect(service.uploadVideo(workspaceId, stored.key, "video/mp4", 1)).rejects.toThrow("sizeBytes does not match");
|
||||
await expect(service.uploadVideo(workspaceId, "sha256/missing.mp4", "video/mp4", 5)).rejects.toThrow("existing readable encrypted object");
|
||||
const uploaded = await service.uploadVideo(workspaceId, stored.key, "video/mp4", stored.sizeBytes);
|
||||
expect(uploaded.processingJob.state).toBe("complete");
|
||||
expect(uploaded.processingJob!.state).toBe("complete");
|
||||
expect(uploaded.videoAsset.width).toBe(640);
|
||||
});
|
||||
|
||||
|
|
@ -148,6 +148,19 @@ describe("video byte ingest and processing-job state machine", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("accepts a logo overlay only when the image belongs to the workspace", async () => {
|
||||
const owned = `sha256/${"a".repeat(64)}.png`;
|
||||
const foreign = `sha256/${"b".repeat(64)}.png`;
|
||||
const database: FakeVideoDatabase = { ownedImageKeys: [owned] };
|
||||
const { service } = serviceWithDatabase(database, async () => ({}));
|
||||
await service.uploadVideoBytes(workspaceId, Buffer.from("source"), "video/mp4");
|
||||
const logo = (key: string): ProduceRequest => ({ overlays: [{ kind: "logo", logoStorageKey: key, position: { x: 90, y: 5 } }] });
|
||||
await expect(service.produceVideo(assetId, workspaceId, logo(foreign), "u")).rejects.toMatchObject({ status: 400 });
|
||||
expect(database.statements?.some((s) => s.sql.includes("INSERT INTO video_processing_jobs") && String(s.values[3]).includes(foreign))).toBe(false);
|
||||
const produced = await service.produceVideo(assetId, workspaceId, logo(owned), "u");
|
||||
expect(produced.job.state).toBe("queued");
|
||||
});
|
||||
|
||||
it("refuses to produce from an unknown asset", async () => {
|
||||
const { service } = serviceWithDatabase({}, async () => ({}));
|
||||
await expect(service.produceVideo("30000000-0000-4000-8000-000000000005", workspaceId, {}, "u")).rejects.toMatchObject({ status: 404 });
|
||||
|
|
@ -157,7 +170,7 @@ describe("video byte ingest and processing-job state machine", () => {
|
|||
const { service, storage } = serviceWithDatabase({}, async () => ({}));
|
||||
await expect(service.uploadVideoBytes(workspaceId, Buffer.alloc(0), "video/mp4")).rejects.toMatchObject({ status: 400 });
|
||||
await expect(service.uploadVideoBytes(workspaceId, Buffer.from("image"), "image/png")).rejects.toMatchObject({ status: 400 });
|
||||
await expect(service.uploadVideoBytes(workspaceId, Buffer.allocUnsafe(MAX_VIDEO_UPLOAD_BYTES + 1), "video/mp4")).rejects.toThrow("100 MiB upload limit");
|
||||
await expect(service.uploadVideoBytes(workspaceId, Buffer.allocUnsafe(MAX_VIDEO_UPLOAD_BYTES + 1), "video/mp4")).rejects.toThrow("video exceeds the chunked upload limit");
|
||||
expect(storage.putContent).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { BadRequestException, Inject, Injectable, NotFoundException, Optional, ServiceUnavailableException } from "@nestjs/common";
|
||||
|
|
@ -6,9 +7,12 @@ import type { PoolClient } from "pg";
|
|||
import { appendTenantAudit, type AuditEvent } from "../audit/audit.service.js";
|
||||
import { withTenant } from "../db/client.js";
|
||||
import { probeVideoFile, type VideoProbeResult } from "./ffprobe.js";
|
||||
import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js";
|
||||
import {
|
||||
S3Storage, s3ConfigFromEnv, type StorageService,
|
||||
directEnabled, directObjectKey,
|
||||
} from "../storage/storage.js";
|
||||
import { type ProduceRequest } from "./produce-request.js";
|
||||
import { MAX_VIDEO_UPLOAD_BYTES } from "./upload-limits.js";
|
||||
import { MAX_CHUNKED_UPLOAD_BYTES, MAX_VIDEO_UPLOAD_BYTES } from "./upload-limits.js";
|
||||
|
||||
const NO_OP_TRANSCODE_NOTE = "Transcode skipped: original stored object retained; rendering belongs to the media-worker compositor.";
|
||||
|
||||
|
|
@ -78,11 +82,39 @@ export interface VideoLibraryPage {
|
|||
|
||||
export interface VideoStatus {
|
||||
videoAsset: VideoAssetRecord;
|
||||
processingJob: VideoProcessingJobRecord;
|
||||
processingJob: VideoProcessingJobRecord | null;
|
||||
}
|
||||
|
||||
export type VideoUploadResult = VideoStatus;
|
||||
|
||||
/** GET /video/direct/config — the browser's capability check before it seals. */
|
||||
export interface DirectUploadConfig {
|
||||
enabled: boolean;
|
||||
publicKeyJwk: { kty: "EC"; crv: "P-256"; x: string; y: string } | null;
|
||||
maxBytes: number;
|
||||
}
|
||||
|
||||
/** POST /video/direct/init response. Nothing is persisted until complete. */
|
||||
export interface DirectUploadInit {
|
||||
uploadId: string;
|
||||
objectKey: string;
|
||||
putUrl: string;
|
||||
}
|
||||
|
||||
/** POST /video/direct/complete request (HEAD-checked, then registered + queued). */
|
||||
export interface DirectUploadCompleteInput {
|
||||
uploadId: string;
|
||||
objectKey: string;
|
||||
sizeBytes: number;
|
||||
mimeType: string;
|
||||
filename: string;
|
||||
}
|
||||
|
||||
/** Presigned PUT lifetime for a browser upload (spec: 900 s). */
|
||||
export const DIRECT_PUT_TTL_SECONDS = 900;
|
||||
const DIRECT_KEY_PREFIX = "direct/";
|
||||
const DIRECT_OBJECT_SUFFIX = ".enc";
|
||||
|
||||
type TenantTransaction = <T>(workspaceId: string, fn: (client: PoolClient) => Promise<T>) => Promise<T>;
|
||||
type AuditAppender = (workspaceId: string, event: AuditEvent) => Promise<string>;
|
||||
|
||||
|
|
@ -139,6 +171,135 @@ export class VideoService {
|
|||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunked-ingest tail end: seal + store + probe an already-assembled file on
|
||||
* local disk (see chunked-upload.ts). The bytes never sat in a request body,
|
||||
* so the 2 GiB whole-body cap does not apply — MAX_CHUNKED_UPLOAD_BYTES does.
|
||||
*
|
||||
* MEMORY: reading the assembled file into one Buffer peaks around 2× file
|
||||
* size of RAM (seal takes a Buffer and S3 takes the sealed copy). A 20 GB
|
||||
* upload therefore needs a host with ≥ 48 GB RAM; /tmp must also hold the
|
||||
* file itself.
|
||||
*/
|
||||
async uploadVideoFromPath(
|
||||
workspaceId: string,
|
||||
inputPath: string,
|
||||
mimeType: string,
|
||||
sizeBytes: number,
|
||||
): Promise<VideoUploadResult> {
|
||||
validateUpload("pending", mimeType, sizeBytes);
|
||||
if (sizeBytes > MAX_CHUNKED_UPLOAD_BYTES) {
|
||||
throw new BadRequestException("video exceeds the 100 GiB chunked upload limit");
|
||||
}
|
||||
const onDisk = await stat(inputPath).catch(() => {
|
||||
throw new BadRequestException("assembled upload file is missing on disk");
|
||||
});
|
||||
if (onDisk.size !== sizeBytes) {
|
||||
throw new BadRequestException(
|
||||
`assembled upload is ${onDisk.size} bytes on disk but declared ${sizeBytes}`,
|
||||
);
|
||||
}
|
||||
const storage = this.requireStorage();
|
||||
const bytes = await readFile(inputPath).catch(() => {
|
||||
throw new ServiceUnavailableException("could not read the assembled upload from disk");
|
||||
});
|
||||
const stored = await storage.putContent(bytes, mimeType).catch(() => {
|
||||
throw new ServiceUnavailableException("Video storage unavailable: could not store encrypted upload in S3/MinIO.");
|
||||
});
|
||||
return this.registerAndProbe(workspaceId, stored.key, stored.contentType, stored.sizeBytes);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Direct-to-R2 (browser-sealed SPENC2): the api mints the presign and the
|
||||
// rows; the payload goes browser → R2 → media-worker. No video bytes ever
|
||||
// touch this process, so the chunked/whole-body memory ceilings do not apply.
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
/** Whether this deployment can unwrap browser-sealed frames (DIRECT_EC_PRIVATE_KEY). */
|
||||
async directUploadConfig(): Promise<DirectUploadConfig> {
|
||||
const storage = this.requireStorage();
|
||||
const publicKeyJwk = await storage.directPublicJwk();
|
||||
return {
|
||||
enabled: directEnabled(),
|
||||
publicKeyJwk,
|
||||
maxBytes: MAX_CHUNKED_UPLOAD_BYTES,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Mint the presigned PUT for a browser upload. The size is bounded up front
|
||||
* so an oversized pick fails before any bytes are sealed; nothing is
|
||||
* persisted — the asset row only appears on complete, so an abandoned PUT
|
||||
* leaves an orphan object (no row), never a dangling row without an object.
|
||||
*/
|
||||
async initDirectUpload(
|
||||
workspaceId: string,
|
||||
input: { filename: string; mimeType: string; sizeBytes: number },
|
||||
): Promise<DirectUploadInit> {
|
||||
validateDirectUpload(input);
|
||||
if ((await this.requireStorage().directPublicJwk()) === null) {
|
||||
throw new ServiceUnavailableException("DIRECT_EC_PRIVATE_KEY is not configured");
|
||||
}
|
||||
// ONE uuid: the object key binds to the upload id so /complete can pin it.
|
||||
const uploadId = randomUUID();
|
||||
const objectKey = `direct/${uploadId}.enc`;
|
||||
const putUrl = await this.requireStorage()
|
||||
.presignPut(objectKey, input.mimeType, DIRECT_PUT_TTL_SECONDS)
|
||||
.catch(() => { throw new ServiceUnavailableException("Video storage is unavailable: could not presign the direct PUT"); });
|
||||
return { uploadId, objectKey, putUrl };
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a completed direct upload. The object is HEAD-checked (exists,
|
||||
* exact declared size) BEFORE any row is written, so metadata can never point
|
||||
* at a missing or partial PUT — the same gate /files/direct/complete uses.
|
||||
*
|
||||
* The probe deliberately does NOT run here: the insert queues kind 'probe'
|
||||
* for the media-worker, which downloads the sealed object off R2, decrypts it
|
||||
* and runs ffprobe itself. This box never touches video bytes on this path.
|
||||
*/
|
||||
async completeDirectUpload(workspaceId: string, input: DirectUploadCompleteInput): Promise<VideoStatus> {
|
||||
if (!isUuid(input.uploadId)) throw new BadRequestException("uploadId must be a UUID");
|
||||
if (input.objectKey !== `${DIRECT_KEY_PREFIX}${input.uploadId}${DIRECT_OBJECT_SUFFIX}`) {
|
||||
throw new BadRequestException("objectKey must be direct/<uploadId>.enc");
|
||||
}
|
||||
validateDirectUpload(input);
|
||||
|
||||
const head = await this.requireStorage().headObject(input.objectKey).catch(() => {
|
||||
throw new ServiceUnavailableException("Video storage is unavailable: could not verify the uploaded object");
|
||||
});
|
||||
if (!head.exists || head.contentLength !== input.sizeBytes) {
|
||||
throw new BadRequestException("uploaded object is missing or its size does not match sizeBytes");
|
||||
}
|
||||
|
||||
const registered = await this.dependencies.tenantTransaction(workspaceId, async (client) => {
|
||||
const assetResult = await client.query(
|
||||
`INSERT INTO video_assets (workspace_id, storage_key, mime_type, size_bytes)
|
||||
VALUES ($1, $2, $3, $4) RETURNING *`,
|
||||
[workspaceId, input.objectKey, input.mimeType, input.sizeBytes],
|
||||
);
|
||||
const videoAsset = assetFromRow(requiredRow(assetResult.rows[0], "created video asset"));
|
||||
// kind 'probe' is 039's default: the media-worker claims it and runs ffprobe.
|
||||
const jobResult = await client.query(
|
||||
`INSERT INTO video_processing_jobs (workspace_id, video_asset_id, state, kind)
|
||||
VALUES ($1, $2, 'queued', 'probe') RETURNING *`,
|
||||
[workspaceId, videoAsset.id],
|
||||
);
|
||||
return {
|
||||
videoAsset,
|
||||
processingJob: jobFromRow(requiredRow(jobResult.rows[0], "created video processing job")),
|
||||
};
|
||||
});
|
||||
|
||||
await this.audit(workspaceId, "video.uploaded", registered.videoAsset.id, {
|
||||
mimeType: input.mimeType,
|
||||
sizeBytes: input.sizeBytes,
|
||||
uploadPath: "direct",
|
||||
});
|
||||
// The worker probes; status reports the queued job the drawer polls on.
|
||||
return this.getVideoStatus(registered.videoAsset.id, workspaceId);
|
||||
}
|
||||
|
||||
async uploadVideo(
|
||||
workspaceId: string,
|
||||
storageKey: string,
|
||||
|
|
@ -180,7 +341,7 @@ export class VideoService {
|
|||
} catch (error) {
|
||||
// Probe errors are recorded on the job; return its ID and failure in status.
|
||||
const status = await this.getVideoStatus(uploaded.videoAsset.id, workspaceId);
|
||||
if (status.processingJob.state !== "failed") throw error;
|
||||
if (status.processingJob === null || status.processingJob.state !== "failed") throw error;
|
||||
}
|
||||
await this.audit(workspaceId, "video.uploaded", uploaded.videoAsset.id, {
|
||||
mimeType,
|
||||
|
|
@ -208,8 +369,10 @@ export class VideoService {
|
|||
);
|
||||
if (result.rowCount !== 1) throw new NotFoundException(`video ${assetId} not found`);
|
||||
const row = requiredRow(result.rows[0], "video status");
|
||||
// A jobless asset is a legitimate state (pre-probe direct registrations,
|
||||
// legacy rows): surface it instead of 500ing the library click.
|
||||
if (row.job_id === null || row.job_id === undefined) {
|
||||
throw new Error(`video ${assetId} has no processing job`);
|
||||
return { videoAsset: assetFromRow(row), processingJob: null };
|
||||
}
|
||||
return { videoAsset: assetFromRow(row), processingJob: joinedJobFromRow(row) };
|
||||
});
|
||||
|
|
@ -381,6 +544,7 @@ export class VideoService {
|
|||
const created = await this.dependencies.tenantTransaction(workspaceId, async (client) => {
|
||||
const source = await client.query("SELECT * FROM video_assets WHERE id = $1", [assetId]);
|
||||
if (source.rowCount !== 1) throw new NotFoundException(`video ${assetId} not found`);
|
||||
await assertLogoKeysOwned(client, workspaceId, request);
|
||||
const jobResult = await client.query(
|
||||
`INSERT INTO video_processing_jobs (workspace_id, video_asset_id, state, kind, input_json)
|
||||
VALUES ($1, $2, 'queued', $3, $4::jsonb) RETURNING *`,
|
||||
|
|
@ -445,6 +609,22 @@ export class VideoService {
|
|||
}
|
||||
}
|
||||
|
||||
/** Shared validation for the direct init/complete payloads (no sniffing: the declared type stands). */
|
||||
function validateDirectUpload(input: { filename?: string; mimeType: string; sizeBytes: number }): void {
|
||||
if (input.filename !== undefined && (input.filename.trim() === "" || input.filename.length > 255)) {
|
||||
throw new BadRequestException("filename must be 1-255 characters");
|
||||
}
|
||||
if (!input.mimeType.toLowerCase().startsWith("video/")) {
|
||||
throw new BadRequestException("mimeType must be a video media type");
|
||||
}
|
||||
if (!Number.isSafeInteger(input.sizeBytes) || input.sizeBytes < 1) {
|
||||
throw new BadRequestException("sizeBytes must be a positive safe integer");
|
||||
}
|
||||
if (input.sizeBytes > MAX_CHUNKED_UPLOAD_BYTES) {
|
||||
throw new BadRequestException("video exceeds the 100 GiB chunked upload limit");
|
||||
}
|
||||
}
|
||||
|
||||
function validateUpload(storageKey: string, mimeType: string, sizeBytes: number): void {
|
||||
if (storageKey.trim() === "") throw new BadRequestException("storageKey must be a non-empty string");
|
||||
if (!mimeType.toLowerCase().startsWith("video/")) {
|
||||
|
|
@ -453,7 +633,7 @@ function validateUpload(storageKey: string, mimeType: string, sizeBytes: number)
|
|||
if (!Number.isSafeInteger(sizeBytes) || sizeBytes < 1) {
|
||||
throw new BadRequestException("sizeBytes must be a positive safe integer");
|
||||
}
|
||||
if (sizeBytes > MAX_VIDEO_UPLOAD_BYTES) throw new BadRequestException("video exceeds the 100 MiB upload limit");
|
||||
if (sizeBytes > MAX_CHUNKED_UPLOAD_BYTES) throw new BadRequestException("video exceeds the chunked upload limit");
|
||||
}
|
||||
|
||||
function requiredRow(row: unknown, description: string): Record<string, unknown> {
|
||||
|
|
@ -595,3 +775,25 @@ function decodeCursor(cursor: string): VideoCursor {
|
|||
function isUuid(value: string): boolean {
|
||||
return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Storage keys are content-addressed in one shared bucket, so a well-formed key
|
||||
* alone proves nothing about who owns the object. A logo overlay may only use
|
||||
* an image this workspace holds: an attachment, a group asset or its branding.
|
||||
*/
|
||||
async function assertLogoKeysOwned(
|
||||
client: { query: (sql: string, values: unknown[]) => Promise<{ rowCount: number | null }> },
|
||||
workspaceId: string,
|
||||
request: ProduceRequest,
|
||||
): Promise<void> {
|
||||
const keys = [...new Set((request.overlays ?? []).flatMap((overlay) => (overlay.logoStorageKey === undefined ? [] : [overlay.logoStorageKey])))];
|
||||
for (const key of keys) {
|
||||
const owned = await client.query(
|
||||
`SELECT 1 WHERE EXISTS (SELECT 1 FROM media_assets WHERE storage_key = $1 AND workspace_id = $2)
|
||||
OR EXISTS (SELECT 1 FROM group_assets WHERE asset_id = $1 AND workspace_id = $2)
|
||||
OR EXISTS (SELECT 1 FROM workspaces WHERE id = $2 AND branding->>'logoAssetKey' = $1)`,
|
||||
[key, workspaceId],
|
||||
);
|
||||
if (owned.rowCount !== 1) throw new BadRequestException("logoStorageKey is not an image of this collective");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -75,8 +75,8 @@ describe("GET /video/:id/status", () => {
|
|||
const before = await service.getVideoStatus(assetId, workspaceId);
|
||||
const response = await status();
|
||||
const body = await response.json() as VideoStatus;
|
||||
expect(body.processingJob.state).toBe(before.processingJob.state);
|
||||
expect(body.processingJob.state).not.toBe("complete");
|
||||
expect(body.processingJob!.state).toBe(before.processingJob!.state);
|
||||
expect(body.processingJob!.state).not.toBe("complete");
|
||||
});
|
||||
|
||||
it("returns 404 for an unknown asset in the workspace", async () => {
|
||||
|
|
|
|||
|
|
@ -15,7 +15,8 @@
|
|||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.1146.0",
|
||||
"pg": "^8.23.1",
|
||||
"pino": "^9.3.2"
|
||||
"pino": "^9.3.2",
|
||||
"sharp": "^0.35.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.5.0",
|
||||
|
|
|
|||
61
apps/media-worker/scripts/install-windows-worker.ps1
Normal file
61
apps/media-worker/scripts/install-windows-worker.ps1
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
# Run in native Windows PowerShell as the user who will transcode on moxexleyhp1.
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$JobRoot = 'C:\transcode-jobs',
|
||||
[string]$PythonExe
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
if ($env:OS -ne 'Windows_NT') {
|
||||
throw 'Run this installer on Windows directly, not inside WSL.'
|
||||
}
|
||||
|
||||
$WorkerScript = (Resolve-Path (Join-Path $PSScriptRoot 'windows-transcode-worker.py')).Path
|
||||
if (-not $PythonExe) {
|
||||
$PythonCommand = Get-Command python.exe -ErrorAction SilentlyContinue
|
||||
if (-not $PythonCommand) {
|
||||
$PythonCommand = Get-Command py.exe -ErrorAction SilentlyContinue
|
||||
}
|
||||
if (-not $PythonCommand) {
|
||||
throw 'Install Python 3.9+ for Windows, then rerun with -PythonExe C:\path\to\python.exe.'
|
||||
}
|
||||
$PythonExe = $PythonCommand.Source
|
||||
}
|
||||
# Resolve py.exe (if used) to the real interpreter for the scheduled task.
|
||||
$ResolvedPython = & $PythonExe -c 'import sys; assert sys.version_info >= (3, 9); print(sys.executable)'
|
||||
if ($LASTEXITCODE -ne 0 -or -not $ResolvedPython) {
|
||||
throw 'Could not run Python 3.9+. Pass -PythonExe with a working Windows interpreter.'
|
||||
}
|
||||
$PythonExe = ($ResolvedPython | Select-Object -Last 1).Trim()
|
||||
$FfmpegExe = (Get-Command ffmpeg.exe -ErrorAction Stop).Source
|
||||
$JobRoot = [System.IO.Path]::GetFullPath($JobRoot)
|
||||
$Inbox = Join-Path $JobRoot 'inbox'
|
||||
foreach ($Name in @('inbox', 'done', 'failed')) {
|
||||
New-Item -ItemType Directory -Force -Path (Join-Path $JobRoot $Name) | Out-Null
|
||||
}
|
||||
|
||||
$TaskName = 'TMBH-TranscodeWorker'
|
||||
$UserId = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name
|
||||
$Arguments = '-u "{0}" --inbox "{1}" --ffmpeg "{2}"' -f $WorkerScript, $Inbox, $FfmpegExe
|
||||
$Action = New-ScheduledTaskAction -Execute $PythonExe -Argument $Arguments -WorkingDirectory $PSScriptRoot
|
||||
$AtLogon = New-ScheduledTaskTrigger -AtLogOn -User $UserId
|
||||
# No repetition duration means this trigger repeats indefinitely.
|
||||
$EveryFiveMinutes = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) -RepetitionInterval (New-TimeSpan -Minutes 5)
|
||||
$Principal = New-ScheduledTaskPrincipal -UserId $UserId -LogonType Interactive -RunLevel Limited
|
||||
$Settings = New-ScheduledTaskSettingsSet -MultipleInstances IgnoreNew -ExecutionTimeLimit ([TimeSpan]::Zero) -StartWhenAvailable -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries
|
||||
|
||||
Register-ScheduledTask -TaskName $TaskName -Action $Action -Trigger @($AtLogon, $EveryFiveMinutes) -Principal $Principal -Settings $Settings -Description 'Poll JSON jobs and transcode with native Windows NVENC or Intel QSV.' -Force | Out-Null
|
||||
Start-ScheduledTask -TaskName $TaskName
|
||||
|
||||
Write-Host "Installed and started: $TaskName (logon + every 5 minutes; duplicate starts ignored)."
|
||||
Write-Host "Python: $PythonExe"
|
||||
Write-Host "FFmpeg: $FfmpegExe"
|
||||
Write-Host "Worker: $WorkerScript"
|
||||
Write-Host "Inbox: $Inbox"
|
||||
Write-Host "Done: $(Join-Path $JobRoot 'done'); failed: $(Join-Path $JobRoot 'failed') with <job>.json.err diagnostics."
|
||||
Write-Host 'Keep this Windows account logged on; the task uses its interactive GPU session.'
|
||||
Write-Host 'Keep the repository at this location, or rerun this installer after moving it.'
|
||||
Write-Host 'Publish jobs by writing a .tmp file, closing it, then renaming it to .json.'
|
||||
Write-Host "Status: Get-ScheduledTask -TaskName '$TaskName'"
|
||||
Write-Host "Stop: Stop-ScheduledTask -TaskName '$TaskName'"
|
||||
Write-Host "Manual tests: $(Join-Path $PSScriptRoot 'test-windows-worker.md')"
|
||||
116
apps/media-worker/scripts/test-windows-worker.md
Normal file
116
apps/media-worker/scripts/test-windows-worker.md
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
# Native Windows transcode worker: manual checks on moxexleyhp1
|
||||
|
||||
Run these commands in **Windows PowerShell**, using Windows Python 3.9+ and
|
||||
the existing WinGet ffmpeg on PATH. Keep this Windows account logged on.
|
||||
|
||||
## Install
|
||||
|
||||
1. Open PowerShell in the Windows checkout of this repository.
|
||||
2. Run:
|
||||
|
||||
```powershell
|
||||
powershell.exe -NoProfile -ExecutionPolicy Bypass -File .\apps\media-worker\scripts\install-windows-worker.ps1
|
||||
Get-ScheduledTask -TaskName 'TMBH-TranscodeWorker'
|
||||
```
|
||||
|
||||
If Python is not found, add `-PythonExe 'C:\path\to\python.exe'` to the
|
||||
installer command. The optional `-JobRoot` changes all three queue folders.
|
||||
The task starts immediately, polls every 5 seconds, starts at logon, and
|
||||
is triggered every 5 minutes to recover if it exits. Duplicate starts are
|
||||
ignored; a per-inbox file lock also prevents competing manual workers.
|
||||
The installer can be rerun to update the task.
|
||||
|
||||
3. Verify `inbox`, `done`, and `failed` exist under `C:\transcode-jobs`.
|
||||
Generate a short input with video and audio:
|
||||
|
||||
```powershell
|
||||
ffmpeg -hide_banner -n -f lavfi -i 'testsrc2=size=1280x720:rate=30' -f lavfi -i 'sine=frequency=440:sample_rate=48000' -t 10 -c:v mpeg4 -q:v 3 -c:a aac 'C:\transcode-jobs\sample.mp4'
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Sample generation failed' }
|
||||
```
|
||||
|
||||
Use fresh output/job names when repeating these checks. Existing output
|
||||
files are rejected and preserved. Relative input/output paths resolve
|
||||
against the inbox; absolute Windows paths are recommended. Outputs use
|
||||
their extension to choose the container. Jobs run sequentially.
|
||||
|
||||
## NVENC H.264 on the RTX 3090
|
||||
|
||||
- [ ] Publish a job atomically, so polling never reads a partly written JSON:
|
||||
|
||||
```powershell
|
||||
@{ input = 'C:\transcode-jobs\sample.mp4'; output = 'C:\transcode-jobs\nvenc.mp4'; preset = 'nvenc_h264'; crf = 23 } | ConvertTo-Json | Set-Content -Encoding UTF8 'C:\transcode-jobs\inbox\nvenc.tmp'
|
||||
Move-Item 'C:\transcode-jobs\inbox\nvenc.tmp' 'C:\transcode-jobs\inbox\nvenc.json'
|
||||
```
|
||||
|
||||
- [ ] Within a poll interval plus encode time, `done\nvenc.json` and a
|
||||
playable, nonempty `C:\transcode-jobs\nvenc.mp4` exist; the inbox job is gone.
|
||||
- [ ] Inspect the output:
|
||||
|
||||
```powershell
|
||||
ffprobe -v error -show_entries stream=codec_name,codec_type -of json 'C:\transcode-jobs\nvenc.mp4'
|
||||
```
|
||||
|
||||
Expect H.264 video and AAC audio. While encoding, observe NVIDIA's **Video
|
||||
Encode** graph in Task Manager (use a longer input if needed).
|
||||
- [ ] Repeat with `preset = 'nvenc_hevc'`, a new job name, and output
|
||||
`nvenc-hevc.mp4`; expect HEVC video and AAC audio. If the installed ffmpeg
|
||||
lacks `hevc_nvenc`, expect a failed job with encoder diagnostics instead.
|
||||
|
||||
## Intel QuickSync H.264
|
||||
|
||||
- [ ] Publish a QSV job, omitting `crf` to exercise the default quality of 23:
|
||||
|
||||
```powershell
|
||||
@{ input = 'C:\transcode-jobs\sample.mp4'; output = 'C:\transcode-jobs\qsv.mp4'; preset = 'qsv_h264' } | ConvertTo-Json | Set-Content -Encoding UTF8 'C:\transcode-jobs\inbox\qsv.tmp'
|
||||
Move-Item 'C:\transcode-jobs\inbox\qsv.tmp' 'C:\transcode-jobs\inbox\qsv.json'
|
||||
```
|
||||
|
||||
- [ ] Verify `done\qsv.json`, playable `qsv.mp4`, and H.264/AAC streams using
|
||||
the ffprobe command above with the QSV output path.
|
||||
- [ ] Observe the Intel GPU's **Video Encode** graph during the job. Encoder
|
||||
initialization failures belong in `failed` and must include ffmpeg's error;
|
||||
the worker never retries with a software video encoder.
|
||||
|
||||
## Failure and continued polling
|
||||
|
||||
- [ ] Queue a missing input:
|
||||
|
||||
```powershell
|
||||
@{ input = 'C:\transcode-jobs\does-not-exist.mp4'; output = 'C:\transcode-jobs\must-not-exist.mp4'; preset = 'nvenc_h264' } | ConvertTo-Json | Set-Content -Encoding UTF8 'C:\transcode-jobs\inbox\missing.tmp'
|
||||
Move-Item 'C:\transcode-jobs\inbox\missing.tmp' 'C:\transcode-jobs\inbox\missing.json'
|
||||
```
|
||||
|
||||
- [ ] Expect `failed\missing.json` and `failed\missing.json.err`, mentioning
|
||||
the missing input. No output is created and the inbox job is gone.
|
||||
- [ ] Queue a text file as input with a valid preset. Expect a failed JSON
|
||||
and `.json.err` containing ffmpeg's nonzero exit code and stderr. No final
|
||||
output or `.part.mp4` file should remain.
|
||||
- [ ] Queue malformed JSON, an unknown preset, and a string `crf` in separate
|
||||
jobs. Each must move to `failed` with a useful `.err` file.
|
||||
- [ ] Queue an existing output path (e.g. the sample). Expect failure without
|
||||
changing the existing file.
|
||||
- [ ] Queue another valid job with a fresh output name; verify completion
|
||||
after the failed jobs. Reuse a failed job filename and verify both archived
|
||||
jobs and their errors survive (a UUID is added on name collisions).
|
||||
|
||||
## Task lifecycle and foreground diagnostics
|
||||
|
||||
- [ ] Log off and back on; verify the task runs. Stop the task, then verify
|
||||
the repeating trigger starts it again within 5 minutes.
|
||||
- [ ] For foreground diagnostics or a single pass, stop **and disable** the
|
||||
task first so its repeating trigger cannot race the manual run:
|
||||
|
||||
```powershell
|
||||
Disable-ScheduledTask -TaskName 'TMBH-TranscodeWorker'
|
||||
Stop-ScheduledTask -TaskName 'TMBH-TranscodeWorker'
|
||||
python .\apps\media-worker\scripts\windows-transcode-worker.py --once
|
||||
# Exit status is 1 if any job in this pass failed; otherwise 0.
|
||||
Enable-ScheduledTask -TaskName 'TMBH-TranscodeWorker'
|
||||
Start-ScheduledTask -TaskName 'TMBH-TranscodeWorker'
|
||||
```
|
||||
|
||||
Omit `--once` for continuous foreground polling; press Ctrl+C to stop.
|
||||
A second worker on the same inbox must exit without executing jobs.
|
||||
Force-stopping during an encode can leave a hidden `.part` output; remove
|
||||
it after confirming no ffmpeg process is using it. Its job stays in the
|
||||
inbox for retry. Normal encode failures clean up the temporary output.
|
||||
185
apps/media-worker/scripts/windows-transcode-worker.py
Normal file
185
apps/media-worker/scripts/windows-transcode-worker.py
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Native Windows hardware transcode worker; Python standard library only."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
|
||||
|
||||
PRESETS = {
|
||||
"nvenc_h264": ["-c:v", "h264_nvenc", "-preset", "p4", "-rc", "vbr"],
|
||||
"qsv_h264": ["-c:v", "h264_qsv"],
|
||||
"nvenc_hevc": ["-c:v", "hevc_nvenc", "-preset", "p4", "-rc", "vbr"],
|
||||
}
|
||||
|
||||
|
||||
class WorkerLock:
|
||||
"""Windows releases this byte-range lock even if the worker crashes."""
|
||||
|
||||
def __init__(self, inbox):
|
||||
self.path = inbox / ".worker.lock"
|
||||
self.file = None
|
||||
|
||||
def __enter__(self):
|
||||
import msvcrt
|
||||
|
||||
self.file = self.path.open("a+b")
|
||||
self.file.seek(0, os.SEEK_END)
|
||||
if self.file.tell() == 0:
|
||||
self.file.write(b"\0")
|
||||
self.file.flush()
|
||||
self.file.seek(0)
|
||||
try:
|
||||
msvcrt.locking(self.file.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
except OSError:
|
||||
self.file.close()
|
||||
self.file = None
|
||||
raise
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
# Closing the handle releases the lock; keep the file to avoid races.
|
||||
self.file.close()
|
||||
|
||||
|
||||
def load_job(job_path):
|
||||
# Windows PowerShell 5.1's UTF8 output includes a BOM.
|
||||
with job_path.open(encoding="utf-8-sig") as source:
|
||||
job = json.load(source)
|
||||
if not isinstance(job, dict):
|
||||
raise ValueError("Job must be a JSON object")
|
||||
for key in ("input", "output", "preset"):
|
||||
if not isinstance(job.get(key), str) or not job[key].strip():
|
||||
raise ValueError("Job requires a nonempty string: " + key)
|
||||
if job["preset"] not in PRESETS:
|
||||
raise ValueError("Unknown preset: " + job["preset"])
|
||||
quality = job.get("crf", 23)
|
||||
if not isinstance(quality, int) or isinstance(quality, bool):
|
||||
raise ValueError("crf must be an integer")
|
||||
|
||||
def job_file(value):
|
||||
path = Path(value).expanduser()
|
||||
return (job_path.parent / path).resolve()
|
||||
|
||||
input_path = job_file(job["input"])
|
||||
output_path = job_file(job["output"])
|
||||
if not input_path.is_file():
|
||||
raise ValueError("Input file does not exist: " + str(input_path))
|
||||
if input_path == output_path:
|
||||
raise ValueError("Input and output must be different files")
|
||||
if output_path.exists():
|
||||
raise ValueError("Output already exists: " + str(output_path))
|
||||
if not output_path.suffix:
|
||||
raise ValueError("Output needs a container extension, such as .mp4")
|
||||
return input_path, output_path, job["preset"], quality
|
||||
|
||||
|
||||
def archive_job(job_path, directory, error=None):
|
||||
target = directory / job_path.name
|
||||
while target.exists() or target.with_suffix(target.suffix + ".err").exists():
|
||||
target = directory / (job_path.stem + "-" + uuid.uuid4().hex + ".json")
|
||||
if error is not None:
|
||||
target.with_suffix(target.suffix + ".err").write_text(error, encoding="utf-8")
|
||||
job_path.rename(target)
|
||||
return target
|
||||
|
||||
|
||||
def process_job(job_path, done, failed, ffmpeg):
|
||||
temporary_output = None
|
||||
try:
|
||||
input_path, output_path, preset, quality = load_job(job_path)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Keep the container suffix so ffmpeg can infer the output format.
|
||||
temporary_output = output_path.with_name(
|
||||
"." + output_path.stem + "-" + uuid.uuid4().hex + ".part" + output_path.suffix
|
||||
)
|
||||
video_args = list(PRESETS[preset])
|
||||
if preset == "qsv_h264":
|
||||
video_args += ["-global_quality", str(quality)]
|
||||
else:
|
||||
video_args += ["-cq", str(quality), "-b:v", "0"]
|
||||
command = [
|
||||
ffmpeg, "-hide_banner", "-nostdin", "-n", "-i", str(input_path),
|
||||
"-map", "0:v:0", "-map", "0:a:0?",
|
||||
*video_args, "-pix_fmt", "nv12", "-c:a", "aac", "-b:a", "192k",
|
||||
str(temporary_output),
|
||||
]
|
||||
logging.info("Job %s: %s", job_path.name, subprocess.list2cmdline(command))
|
||||
with tempfile.TemporaryFile() as stderr:
|
||||
result = subprocess.run(command, stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL, stderr=stderr, check=False)
|
||||
if result.returncode != 0:
|
||||
# Bound diagnostic size even for a long-running failed encode.
|
||||
stderr.seek(0, os.SEEK_END)
|
||||
stderr.seek(max(0, stderr.tell() - 65536))
|
||||
detail = stderr.read().decode("utf-8", errors="replace")
|
||||
raise RuntimeError(
|
||||
"ffmpeg exited with code {}\nCommand: {}\n\n{}".format(
|
||||
result.returncode, subprocess.list2cmdline(command), detail
|
||||
)
|
||||
)
|
||||
if not temporary_output.is_file() or temporary_output.stat().st_size == 0:
|
||||
raise RuntimeError("ffmpeg returned success without a nonempty output")
|
||||
# On Windows rename refuses to replace an existing destination.
|
||||
temporary_output.rename(output_path)
|
||||
archive_job(job_path, done)
|
||||
logging.info("Completed %s -> %s", job_path.name, output_path)
|
||||
return True
|
||||
except Exception as error:
|
||||
logging.error("Failed %s: %s", job_path.name, error)
|
||||
archive_job(job_path, failed, "{}: {}\n".format(type(error).__name__, error))
|
||||
return False
|
||||
finally:
|
||||
if temporary_output is not None:
|
||||
temporary_output.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--inbox", type=Path, default=Path(r"C:\transcode-jobs\inbox"),
|
||||
help="JSON inbox; done/failed are sibling directories")
|
||||
parser.add_argument("--ffmpeg", default="ffmpeg", help="ffmpeg executable (default: PATH)")
|
||||
parser.add_argument("--poll-interval", type=float, default=5, help="Poll interval in seconds")
|
||||
parser.add_argument("--once", action="store_true", help="Process the current inbox and exit")
|
||||
args = parser.parse_args()
|
||||
if os.name != "nt":
|
||||
parser.error("Run this worker with Windows Python directly, not inside WSL")
|
||||
if not math.isfinite(args.poll_interval) or args.poll_interval <= 0:
|
||||
parser.error("--poll-interval must be a finite positive number")
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
inbox = args.inbox.resolve()
|
||||
done, failed = inbox.parent / "done", inbox.parent / "failed"
|
||||
for directory in (inbox, done, failed):
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
lock = WorkerLock(inbox)
|
||||
try:
|
||||
lock.__enter__()
|
||||
except OSError as error:
|
||||
logging.error("Cannot acquire worker lock (another worker may be running): %s", error)
|
||||
return 1
|
||||
try:
|
||||
logging.info("Polling %s", inbox)
|
||||
had_failure = False
|
||||
while True:
|
||||
for job_path in sorted(inbox.glob("*.json")):
|
||||
if job_path.is_file():
|
||||
had_failure = not process_job(job_path, done, failed, args.ffmpeg) or had_failure
|
||||
if args.once:
|
||||
return int(had_failure)
|
||||
time.sleep(args.poll_interval)
|
||||
except KeyboardInterrupt:
|
||||
logging.info("Worker stopped")
|
||||
return 0
|
||||
finally:
|
||||
lock.__exit__()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
305
apps/media-worker/src/producer/assemble-builder.test.ts
Normal file
305
apps/media-worker/src/producer/assemble-builder.test.ts
Normal file
|
|
@ -0,0 +1,305 @@
|
|||
/**
|
||||
* Assemble filter-graph builder (editor-v2 L1): pure-function tests for the
|
||||
* adopted plan §2 graph — per-clip chains per fit/aspect, card segments, the
|
||||
* concat, overlay timing windows, the VO duck path, and the gates — exactly the
|
||||
* argv-shape discipline of produce/command-builder.test.ts. No ffmpeg run here
|
||||
* (the runner suite owns the encode); every assertion is on built strings.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import {
|
||||
AssembleBuildError, ASSEMBLE_PRESET, buildAssembleArgs, buildAssembleFilter, clipVideoChain,
|
||||
itemDurationMs, itemStartMs, secondsOf, timelineTotalMs, validateTrims,
|
||||
type AssemblePlan, type AssembleClipSource, type AssembleInputIndex,
|
||||
} from "./assemble-builder";
|
||||
import { EDL_ASPECTS, parseAssembleInput, type AssembleInput, type EdlProject } from "./edl-plan";
|
||||
|
||||
const SCRATCH = "/var/tmp/media-worker/scratch/assemble";
|
||||
const H1 = "1".repeat(64);
|
||||
const H2 = "2".repeat(64);
|
||||
const CARD1 = "3".repeat(64);
|
||||
const CARD2 = "4".repeat(64);
|
||||
const OVL1 = "5".repeat(64);
|
||||
const OVL2 = "6".repeat(64);
|
||||
const VO = "7".repeat(64);
|
||||
const ASSET = "00000000-0000-4000-8000-000000000001";
|
||||
const ASSET2 = "00000000-0000-4000-8000-000000000002";
|
||||
const VOICEOVER_ASSET = "00000000-0000-4000-8000-000000000005";
|
||||
const EDL_ID = "00000000-0000-4000-8000-000000000010";
|
||||
|
||||
/** The canonical fixture project: two clips, a card between, one overlay, a VO. */
|
||||
function project(overrides: Partial<EdlProject> = {}): EdlProject {
|
||||
return {
|
||||
version: 1,
|
||||
aspect: "16:9",
|
||||
fit: "cover",
|
||||
fps: 30,
|
||||
items: [
|
||||
{ kind: "clip", assetId: ASSET, trimStartMs: 1_000, trimEndMs: 4_000, rotate: 0, volumeDb: 0 },
|
||||
{ kind: "card", text: "Chapter One", durationMs: 2_000, style: "title" },
|
||||
{ kind: "clip", assetId: ASSET2, trimStartMs: 0, trimEndMs: 3_000, rotate: 90, volumeDb: -6 },
|
||||
],
|
||||
overlays: [
|
||||
{ overlayType: "lower_third", fields: { title: "Ada Lovelace" }, startMs: 1_000, endMs: 3_500 },
|
||||
],
|
||||
voiceover: { audioAssetId: VOICEOVER_ASSET, gainDb: -6, duckOriginal: true, offsetMs: 500 },
|
||||
outputLabel: "assembly v1",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function input(overrides: Partial<AssembleInput> = {}): AssembleInput {
|
||||
return parseAssembleInput({ edlProjectId: EDL_ID, project: project(overrides.project as unknown as EdlProject), ...overrides } as unknown as AssembleInput);
|
||||
}
|
||||
|
||||
function plan(overrides: Partial<AssemblePlan> = {}, projectOverrides: Partial<EdlProject> = {}): AssemblePlan {
|
||||
const p = project(projectOverrides);
|
||||
const built = input({ project: p });
|
||||
const clips: AssembleClipSource[] = [];
|
||||
for (const entry of p.items) {
|
||||
if (entry.kind !== "clip") continue;
|
||||
clips.push({ clip: entry, path: `${SCRATCH}/${entry.assetId === ASSET ? H1 : H2}.mp4`, durationMs: 30_000, hasAudio: true });
|
||||
}
|
||||
const cards = p.items.flatMap((entry, i) => entry.kind === "card" ? [{ itemIndex: i, path: `${SCRATCH}/${entry === p.items[1] ? CARD1 : CARD2}.png`, durationMs: entry.durationMs }] : []);
|
||||
const overlays = p.overlays.map((overlay, j) => ({ overlay, path: `${SCRATCH}/${j === 0 ? OVL1 : OVL2}.png` }));
|
||||
return {
|
||||
input: built,
|
||||
clips, cards, overlays,
|
||||
voiceover: p.voiceover === null ? null : { path: `${SCRATCH}/${VO}.m4a`, gainDb: p.voiceover.gainDb, duckOriginal: p.voiceover.duckOriginal, offsetMs: p.voiceover.offsetMs },
|
||||
outputPath: `${SCRATCH}/output.mp4`,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function index(p: AssemblePlan): AssembleInputIndex {
|
||||
return { timeline: p.input.project.items.map((_, i) => i), overlays: p.overlays.map((_, j) => p.input.project.items.length + j), voiceover: p.voiceover === null ? null : p.input.project.items.length + p.overlays.length };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Timeline math: absolute overlay windows depend on it.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test("timeline math: item durations accumulate into absolute starts", () => {
|
||||
const p = project();
|
||||
assert.equal(itemDurationMs(p.items[0]!), 3_000);
|
||||
assert.equal(itemDurationMs(p.items[1]!), 2_000);
|
||||
assert.equal(itemStartMs(p.items, 0), 0);
|
||||
assert.equal(itemStartMs(p.items, 1), 3_000);
|
||||
assert.equal(itemStartMs(p.items, 2), 5_000);
|
||||
assert.equal(timelineTotalMs(p.items), 8_000);
|
||||
assert.equal(secondsOf(1_234), "1.234");
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Per-clip chains: fit and rotation.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test("cover fit center-crops: scale increase + crop to the canvas", () => {
|
||||
const chain = clipVideoChain(0, "cover", "16:9");
|
||||
const { w, h } = EDL_ASPECTS["16:9"];
|
||||
assert.equal(chain, `scale=${w}:${h}:force_original_aspect_ratio=increase:force_divisible_by=2,crop=${w}:${h},setsar=1,fps=30,format=yuv420p`);
|
||||
assert.ok(!chain.includes("pad="), "cover must never letterbox");
|
||||
});
|
||||
|
||||
test("contain fit letterboxes: scale decrease + centered pad", () => {
|
||||
const chain = clipVideoChain(0, "contain", "16:9");
|
||||
const { w, h } = EDL_ASPECTS["16:9"];
|
||||
assert.equal(chain, `scale=${w}:${h}:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=${w}:${h}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=30,format=yuv420p`);
|
||||
assert.ok(!chain.includes("crop="), "contain must never crop");
|
||||
});
|
||||
|
||||
test("rotation is applied BEFORE the fit chain, per closed enum", () => {
|
||||
assert.ok(clipVideoChain(90, "contain", "16:9").startsWith("transpose=1,scale="));
|
||||
assert.ok(clipVideoChain(270, "cover", "9:16").startsWith("transpose=2,scale="));
|
||||
assert.ok(clipVideoChain(180, "contain", "16:9").startsWith("transpose=1,transpose=1,scale="));
|
||||
assert.ok(!clipVideoChain(0, "contain", "16:9").includes("transpose"), "0° adds no transpose");
|
||||
});
|
||||
|
||||
test("portrait canvas geometry comes from the aspect table, not the produce preset", () => {
|
||||
const chain = clipVideoChain(0, "cover", "9:16");
|
||||
const { w, h } = EDL_ASPECTS["9:16"];
|
||||
assert.ok(chain.includes(`scale=${w}:${h}`) && chain.includes(`crop=${w}:${h}`));
|
||||
assert.equal(w, 1080);
|
||||
assert.equal(h, 1920);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// The graph: clips, cards, concat, overlays, VO.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test("per-item chains feed one concat v=1:a=1 in timeline order", () => {
|
||||
const p = plan();
|
||||
const g = buildAssembleFilter(p, index(p));
|
||||
const chains = g.split(";");
|
||||
// clip0 video, clip0 audio, card video, card audio, clip1 video, clip1 audio, concat
|
||||
assert.match(chains[0]!, /^\[0:v\]scale=1280:720[^;]*\[v0\]$/);
|
||||
assert.match(chains[1]!, /^\[0:a\]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo\[a0\]$/);
|
||||
assert.match(chains[2]!, /^\[1:v\]scale=1280:720[^;]*\[v1\]$/);
|
||||
assert.equal(chains[3]!, `anullsrc=r=${ASSEMBLE_PRESET.sampleRate}:cl=stereo,atrim=0:2.000,asetpts=PTS-STARTPTS,aformat=sample_fmts=fltp:sample_rates=${ASSEMBLE_PRESET.sampleRate}:channel_layouts=stereo[a1]`);
|
||||
// The rotated clip transposes before its fit chain and applies its volume.
|
||||
assert.ok(chains[4]!.startsWith("[2:v]transpose=1,scale="));
|
||||
assert.ok(chains[5]!.includes("volume=-6dB"));
|
||||
// concat consumes every item label, video then audio, in order.
|
||||
assert.equal(chains[6]!, "[v0][a0][v1][a1][v2][a2]concat=n=3:v=1:a=1[vcat][acat]");
|
||||
});
|
||||
|
||||
test("a card segment is the looped PNG + anullsrc, never drawtext", () => {
|
||||
const p = plan();
|
||||
const g = buildAssembleFilter(p, index(p));
|
||||
assert.ok(g.includes("anullsrc="), "card audio is silence");
|
||||
const args = buildAssembleArgs(p);
|
||||
const loopAt = args.indexOf("-loop");
|
||||
assert.ok(loopAt > 0 && args[loopAt + 1] === "1", "the card input loops one PNG");
|
||||
const tAt = args.indexOf("-t", loopAt);
|
||||
assert.equal(args[tAt + 1], "2.000", "the card input is bounded by its duration");
|
||||
assert.ok(!g.includes("drawtext"), "NO drawtext anywhere in an assembly (injection-safe by construction)");
|
||||
assert.ok(!args.some((a) => a.includes("drawtext")));
|
||||
});
|
||||
|
||||
test("overlays gate on ABSOLUTE timeline seconds via enable='between(t,s,e)'", () => {
|
||||
const p = plan();
|
||||
const g = buildAssembleFilter(p, index(p));
|
||||
// The overlay window 1000..3500 is timeline-absolute: the card starts at 3000,
|
||||
// so a source-relative time would be wrong by the first clip's 3000 ms.
|
||||
const overlayChain = g.split(";").find((c) => c.includes("overlay="))!;
|
||||
assert.ok(overlayChain.includes(`enable='between(t,1.000,3.500)'`), overlayChain);
|
||||
assert.ok(overlayChain.startsWith("[vcat][3:v]overlay="), "the overlay chain reads the concatenated base");
|
||||
assert.ok(overlayChain.includes("eof_action=repeat"), "a single-frame PNG is held by eof_action=repeat");
|
||||
});
|
||||
|
||||
test("multiple overlays stack in request order (z-order), the last one closing the chain", () => {
|
||||
const p = plan({}, {
|
||||
overlays: [
|
||||
{ overlayType: "lower_third", fields: { title: "one" }, startMs: 0, endMs: 2_000 },
|
||||
{ overlayType: "chyron", fields: { title: "two" }, startMs: 1_000, endMs: 3_000 },
|
||||
],
|
||||
});
|
||||
const g = buildAssembleFilter(p, index(p));
|
||||
const chains = g.split(";");
|
||||
const first = chains.find((c) => c.startsWith("[vcat]"))!;
|
||||
const second = chains.find((c) => c.startsWith("[ov0]"))!;
|
||||
assert.ok(first.endsWith("[ov0]"));
|
||||
assert.ok(second.endsWith("[vout]"), "the last overlay emits the final video label");
|
||||
assert.ok(g.includes("between(t,0.000,2.000)") && g.includes("between(t,1.000,3.000)"));
|
||||
});
|
||||
|
||||
test("voiceover with duckOriginal runs adelay + sidechaincompress keyed by the delayed VO", () => {
|
||||
const p = plan();
|
||||
const g = buildAssembleFilter(p, index(p));
|
||||
const chains = g.split(";");
|
||||
const voChain = chains.find((c) => c.startsWith("[4:a]"))!;
|
||||
assert.ok(voChain.includes("adelay=500:all=1"), "offsetMs becomes an adelay on all channels");
|
||||
assert.ok(voChain.includes("volume=-6dB"), "gainDb applies to the VO");
|
||||
assert.ok(voChain.includes("asplit=2[voMix][voKey]"), "the delayed VO feeds both the key and the mix");
|
||||
const sidechain = chains.find((c) => c.includes("sidechaincompress"))!;
|
||||
assert.ok(sidechain.startsWith("[acat][voKey]"), "the compressor ducks the assembled bed");
|
||||
assert.ok(g.includes("[duck][voMix]amix=inputs=2:duration=first:dropout_transition=2:normalize=0[aout]"));
|
||||
});
|
||||
|
||||
test("voiceover without ducking is a plain amix over the bed", () => {
|
||||
const p = plan({}, { voiceover: { audioAssetId: VOICEOVER_ASSET, gainDb: 0, duckOriginal: false, offsetMs: 0 } });
|
||||
const g = buildAssembleFilter(p, index(p));
|
||||
assert.ok(!g.includes("sidechaincompress"));
|
||||
const voFeed = g.split(";").find((c) => c.includes("[voGain]"))!;
|
||||
assert.ok(voFeed.startsWith("[4:a]"), "the VO chain reads its input pad");
|
||||
assert.ok(!voFeed.includes("adelay"), "a zero offset emits no adelay");
|
||||
assert.ok(!voFeed.includes("volume="), "a zero gain emits no volume filter");
|
||||
assert.ok(g.includes("[acat][voGain]amix=inputs=2:duration=first:dropout_transition=2:normalize=0[aout]"));
|
||||
});
|
||||
|
||||
test("no voiceover: the concatenated bed passes through as the audio out", () => {
|
||||
const p = plan({}, { voiceover: null, overlays: [] });
|
||||
const g = buildAssembleFilter(p, index(p));
|
||||
assert.ok(g.endsWith("[vcat]null[vout];[acat]anull[aout]"), "a passthrough keeps both labels well-formed");
|
||||
assert.ok(!g.includes("amix") && !g.includes("sidechaincompress"));
|
||||
});
|
||||
|
||||
test("an audio-less clip contributes an anullsrc audio segment, never [i:a]", () => {
|
||||
const p = plan();
|
||||
p.clips[1]!.hasAudio = false;
|
||||
const g = buildAssembleFilter(p, index(p));
|
||||
const chains = g.split(";");
|
||||
assert.ok(!chains[5]!.includes("[2:a]"), "the audio-less clip must not reference its input pad");
|
||||
assert.ok(chains[5]!.startsWith("anullsrc="));
|
||||
// The audio-bearing clip keeps its input pad.
|
||||
assert.ok(chains[1]!.startsWith("[0:a]"));
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pre-spawn trim gate.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test("the trim gate rejects an out-point beyond the probe duration BEFORE any spawn", () => {
|
||||
const p = plan();
|
||||
p.clips[0]!.durationMs = 3_500; // trimEnd 4000 > 3500
|
||||
assert.throws(() => validateTrims(p.input.project.items, p.clips), /exceeds the source's probed 3500ms/);
|
||||
assert.throws(() => buildAssembleArgs(p), /exceeds the source's probed 3500ms/);
|
||||
});
|
||||
|
||||
test("the trim gate rejects inverted trims and demands a probe duration", () => {
|
||||
const p = plan();
|
||||
const inverted = { ...p.clips[0]!.clip, trimStartMs: 4_000, trimEndMs: 4_000 };
|
||||
p.input.project.items[0] = inverted;
|
||||
p.clips[0]!.clip = inverted;
|
||||
assert.throws(() => validateTrims(p.input.project.items, p.clips), /trimStartMs must be before trimEndMs/);
|
||||
const noProbe = plan();
|
||||
noProbe.clips[0]!.durationMs = 0;
|
||||
assert.throws(() => validateTrims(noProbe.input.project.items, noProbe.clips), /exceeds the source's probed 0ms/);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// argv shape.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test("argv: input seeks before each -i, encoder is medium/crf 20/aac 192k/faststart", () => {
|
||||
const p = plan();
|
||||
const args = buildAssembleArgs(p);
|
||||
// clip 0: -ss 1.000 -t 3.000 before its input
|
||||
const ss0 = args.indexOf("-ss");
|
||||
assert.equal(args[ss0 + 1], "1.000");
|
||||
assert.equal(args[args.indexOf("-t", ss0) + 1], "3.000");
|
||||
const inputs = args.flatMap((a, i) => (a === "-i" ? [args[i + 1]] : []));
|
||||
assert.deepEqual(inputs, [
|
||||
`file:${SCRATCH}/${H1}.mp4`,
|
||||
`file:${SCRATCH}/${CARD1}.png`,
|
||||
`file:${SCRATCH}/${H2}.mp4`,
|
||||
`file:${SCRATCH}/${OVL1}.png`,
|
||||
`file:${SCRATCH}/${VO}.m4a`,
|
||||
]);
|
||||
for (const [token, value] of [
|
||||
["-c:v", "libx264"], ["-preset", "medium"], ["-crf", "20"],
|
||||
["-c:a", "aac"], ["-b:a", "192k"], ["-movflags", "+faststart"],
|
||||
["-f", "mp4"],
|
||||
] as const) {
|
||||
const at = args.indexOf(token);
|
||||
assert.ok(at > 0 && args[at + 1] === value, `argv must carry ${token} ${value}`);
|
||||
}
|
||||
const maps = args.flatMap((a, i) => (a === "-map" ? [args[i + 1]] : []));
|
||||
assert.deepEqual(maps, ["[vout]", "[aout]"], "argv maps the final video and audio labels");
|
||||
// A global -t bounds the encode at the timeline length (the looping card input cannot run away).
|
||||
const globalT = args.lastIndexOf("-t");
|
||||
assert.equal(args[globalT + 1], "8.000");
|
||||
assert.ok(!args.some((a) => a.includes("$(") || a.startsWith(";") || a.includes("`")));
|
||||
});
|
||||
|
||||
test("a portrait project renders 9:16 geometry end to end", () => {
|
||||
const p = plan({}, { aspect: "9:16", fit: "contain" });
|
||||
const args = buildAssembleArgs(p);
|
||||
assert.ok(args.join(" ").includes("scale=1080:1920"));
|
||||
assert.ok(args.join(" ").includes("pad=1080:1920"));
|
||||
});
|
||||
|
||||
test("plans with unsafe paths or missing sources are refused before any spawn", () => {
|
||||
const bad = plan();
|
||||
bad.outputPath = `${SCRATCH}/../escape.mp4`;
|
||||
assert.throws(() => buildAssembleArgs(bad), AssembleBuildError);
|
||||
const missing = plan();
|
||||
missing.clips = missing.clips.slice(0, 1); // two clips in the timeline, one downloaded
|
||||
assert.throws(() => buildAssembleArgs(missing), /no downloaded source for the clip/);
|
||||
const unsynced = plan({}, { voiceover: null });
|
||||
unsynced.voiceover = { path: `${SCRATCH}/${VO}.m4a`, gainDb: 0, duckOriginal: false, offsetMs: 0 };
|
||||
assert.throws(() => buildAssembleArgs(unsynced), /voiceover the project does not name/);
|
||||
const noPng = plan();
|
||||
noPng.cards[0]!.path = `${SCRATCH}/card.png`;
|
||||
assert.throws(() => buildAssembleArgs(noPng), /rendered <sha256>\.png/);
|
||||
});
|
||||
388
apps/media-worker/src/producer/assemble-builder.ts
Normal file
388
apps/media-worker/src/producer/assemble-builder.ts
Normal file
|
|
@ -0,0 +1,388 @@
|
|||
/**
|
||||
* Assemble command builder (editor-v2 L1; adopted plan §2 graph, BINDING).
|
||||
*
|
||||
* Typed plan in, argv out — the same discipline as produce/command-builder.ts:
|
||||
* nothing here takes a free-form filter string, and NO operator text ever
|
||||
* reaches a filter. That is the plan's injection stance made concrete:
|
||||
*
|
||||
* clips → the asset's scratch path (named by content hash) + numeric -ss/-t
|
||||
* cards → a PNG whose pixels are the api-side renderer's (rasterized off the
|
||||
* filtergraph entirely — "no drawtext at all" per the verdict)
|
||||
* overlays → PNGs too: an EdlOverlay names the api's overlay templates, and the
|
||||
* SAME mechanism the compose path uses for card/overlay pixels —
|
||||
* production/render.ts's overlaySvg → sharp worker → PNG — produces
|
||||
* them (see overlay-png.ts); ffmpeg only ever composites the PNG
|
||||
* text → never a filter argument anywhere in this module
|
||||
*
|
||||
* Graph (one process, one filter_complex, one encode):
|
||||
* per clip i [i:v] transpose → scale/crop-or-pad (fit) → setsar=1 → fps=30 → format [vK]
|
||||
* [i:a] aformat… , volume=dB [aK]
|
||||
* per card k (-loop 1 -t dur PNG) [c:v] fps=30,format [vK] + anullsrc → [aK]
|
||||
* concat n=N:v=1:a=1 → [vcat][acat]
|
||||
* per overlay [vcat][oJ] overlay=enable='between(t,s,e)' [vJ] (absolute timeline ms)
|
||||
* voiceover adelay=offsetMs → volume=gain; duckOriginal → sidechaincompress
|
||||
* keyed by the VO (the produce path's duck chain), else amix duration=first
|
||||
* out libx264 preset medium crf 20, aac 192k, +faststart
|
||||
*/
|
||||
import { EDL_ASPECTS, type AssembleInput, type EdlClip, type EdlItem, type EdlOverlay, type EdlRotate } from "./edl-plan";
|
||||
import { PRODUCE_PRESET } from "./command-builder";
|
||||
|
||||
export class AssembleBuildError extends Error {
|
||||
constructor(message: string) { super(message); this.name = "AssembleBuildError"; }
|
||||
}
|
||||
function fail(message: string): never { throw new AssembleBuildError(message); }
|
||||
|
||||
/** Output encoder settings: plan §2 (medium/CRF 20/aac 192k), sample rate from the produce preset. */
|
||||
export const ASSEMBLE_PRESET = {
|
||||
videoCodec: "libx264", preset: "medium", crf: 20,
|
||||
audioCodec: "aac", audioBitrateKbps: 192,
|
||||
sampleRate: PRODUCE_PRESET.sampleRate, fps: PRODUCE_PRESET.fps,
|
||||
/** 15-min hard cap on one assemble (the gate the runner enforces too). */
|
||||
maxDurationMs: 15 * 60_000,
|
||||
} as const;
|
||||
|
||||
/** Scratch paths are absolute, depth-bounded, and every component is name-safe. */
|
||||
const SAFE_NAME = /^[0-9a-zA-Z][0-9a-zA-Z._-]{0,127}$/;
|
||||
function scratchPath(path: string, field: string): string {
|
||||
if (!path.startsWith("/") || path.includes("..")) fail(`${field} must be an absolute scratch path`);
|
||||
const parts = path.slice(1).split("/");
|
||||
if (parts.length > 8) fail(`${field} is too deep`);
|
||||
for (const part of parts) if (!SAFE_NAME.test(part)) fail(`${field} has an unsafe component`);
|
||||
return path;
|
||||
}
|
||||
|
||||
/** A downloaded source/overlay/VO file: a bare 64-hex name + known extension. */
|
||||
const SCRATCH_FILE = /^[0-9a-f]{64}\.(mp4|png|webm|m4a|aac|mp3|wav|bin)$/;;
|
||||
|
||||
export interface AssembleClipSource {
|
||||
/** The EdlClip this input renders (its trim/rotate/volume ride along). */
|
||||
clip: EdlClip;
|
||||
/** Scratch path of the downloaded asset bytes. */
|
||||
path: string;
|
||||
/** The probe row's durationMs — the pre-spawn trim gate is checked against it. */
|
||||
durationMs: number;
|
||||
/** hasAudio===false → the clip's audio is a matching anullsrc segment, never [i:a]. */
|
||||
hasAudio: boolean;
|
||||
}
|
||||
|
||||
export interface AssembleCardSource {
|
||||
itemIndex: number;
|
||||
/** Scratch path of the card's full-frame PNG (renderer output, the target canvas size). */
|
||||
path: string;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
export interface AssembleOverlaySource {
|
||||
overlay: EdlOverlay;
|
||||
/** Scratch path of the rendered overlay PNG. */
|
||||
path: string;
|
||||
}
|
||||
|
||||
export interface AssembleVoiceoverSource {
|
||||
path: string;
|
||||
gainDb: number;
|
||||
duckOriginal: boolean;
|
||||
offsetMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* The validated, downloaded, pre-gated plan. Built by the runner once every
|
||||
* input exists on disk and every trim has been checked against its probe
|
||||
* duration; the builder only re-validates shapes from here on.
|
||||
*/
|
||||
export interface AssemblePlan {
|
||||
input: AssembleInput;
|
||||
/** Timeline inputs in ORDER (concat order), each already carrying its scratch path. */
|
||||
clips: AssembleClipSource[];
|
||||
cards: AssembleCardSource[];
|
||||
overlays: AssembleOverlaySource[];
|
||||
voiceover: AssembleVoiceoverSource | null;
|
||||
/** Scratch path of the mp4 written. */
|
||||
outputPath: string;
|
||||
}
|
||||
|
||||
/** Where each downloaded file landed in the argv input order (0-based `-i` indexes). */
|
||||
export interface AssembleInputIndex {
|
||||
/** One index per timeline item, in timeline order. */
|
||||
timeline: number[];
|
||||
/** One index per overlay PNG, in overlay order. */
|
||||
overlays: number[];
|
||||
voiceover: number | null;
|
||||
}
|
||||
|
||||
/** Transpose filter per closed rotation enum (0 → passthrough keeps the chain uniform). */
|
||||
export function transposeFor(rotate: EdlRotate): string {
|
||||
switch (rotate) {
|
||||
case 0: return "transpose=nosuch"; // replaced by the caller; unreachable
|
||||
case 90: return "transpose=1";
|
||||
case 180: return "transpose=1,transpose=1";
|
||||
case 270: return "transpose=2";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The per-clip video fit chain, EXPORTED for the unit tests and for reasoning
|
||||
* about the two fit modes:
|
||||
*
|
||||
* cover = center-crop: scale so the canvas is fully covered, then crop the excess
|
||||
* contain = letterbox: scale to fit INSIDE the canvas, then pad the remainder with
|
||||
* the identity black (#060403 lives in the renderer's pixels, not ffmpeg's
|
||||
* pad color, so ffmpeg pads plain black)
|
||||
*
|
||||
* Rotation is applied BEFORE fitting (a 90° phone clip of a vertical action shot
|
||||
* must fit the portrait canvas, not the pre-rotation rectangle).
|
||||
*/
|
||||
export function clipVideoChain(rotate: EdlRotate, fit: "cover" | "contain", aspect: keyof typeof EDL_ASPECTS): string {
|
||||
const { w: W, h: H } = EDL_ASPECTS[aspect];
|
||||
const rotated = rotate !== 0 ? `${transposeFor(rotate)},` : "";
|
||||
if (fit === "cover") {
|
||||
return `${rotated}scale=${W}:${H}:force_original_aspect_ratio=increase:force_divisible_by=2,crop=${W}:${H},setsar=1,fps=${ASSEMBLE_PRESET.fps},format=yuv420p`;
|
||||
}
|
||||
return `${rotated}scale=${W}:${H}:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=${W}:${H}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=${ASSEMBLE_PRESET.fps},format=yuv420p`;
|
||||
}
|
||||
|
||||
function clipAudioChain(volumeDb: number, hasAudio: boolean, inputIndex: number, trimSpanSec: string): string {
|
||||
const aformat = `aformat=sample_fmts=fltp:sample_rates=${ASSEMBLE_PRESET.sampleRate}:channel_layouts=stereo`;
|
||||
if (!hasAudio) {
|
||||
// anullsrc is an INFINITE source — unbounded, concat never finishes the
|
||||
// segment and the encode deadlocks (2026-10-16 production hang). atrim pins
|
||||
// the bed to the clip's trim span; asetpts restarts timestamps.
|
||||
return `anullsrc=r=${ASSEMBLE_PRESET.sampleRate}:cl=stereo,atrim=0:${trimSpanSec},asetpts=PTS-STARTPTS,aformat=sample_fmts=fltp:sample_rates=${ASSEMBLE_PRESET.sampleRate}:channel_layouts=stereo`;
|
||||
}
|
||||
return `[${inputIndex}:a]${aformat}${volumeDb === 0 ? "" : `,volume=${volumeDb}dB`}`;
|
||||
}
|
||||
|
||||
/** Absolute timeline ms → seconds for `between(t,…)`; 3 decimals keeps ms honest. */
|
||||
export function secondsOf(ms: number): string { return (ms / 1000).toFixed(3); }
|
||||
|
||||
/** An item's timeline length in ms: a clip's trim span, a card's duration. */
|
||||
export function itemDurationMs(item: EdlItem): number {
|
||||
return item.kind === "clip" ? item.trimEndMs - item.trimStartMs : item.durationMs;
|
||||
}
|
||||
|
||||
/** Timeline-relative start of item i (item durations accumulate, in order). */
|
||||
export function itemStartMs(items: readonly EdlItem[], index: number): number {
|
||||
let start = 0;
|
||||
for (let i = 0; i < index; i += 1) start += itemDurationMs(items[i]!);
|
||||
return start;
|
||||
}
|
||||
|
||||
/** Timeline length in ms (concat order is items order). */
|
||||
export function timelineTotalMs(items: readonly EdlItem[]): number {
|
||||
return items.reduce((total, item) => total + itemDurationMs(item), 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* The whole `-filter_complex` string. Ordering is load-bearing:
|
||||
* 1. per-item chains (timeline order) — clip chains consume their input pads,
|
||||
* card video chains consume the looped PNG pads, card audio is anullsrc
|
||||
* 2. concat v=1:a=1 over every item in timeline order
|
||||
* 3. overlay chain over [vcat], each gated by enable='between(t,s,e)' with
|
||||
* ABSOLUTE timeline seconds (the overlays track is timeline-absolute per plan d3)
|
||||
* 4. audio: [acat] alone, + adelay'd VO mixed with amix duration=first, or
|
||||
* ducked through sidechaincompress keyed by the (delayed) VO
|
||||
*/
|
||||
export function buildAssembleFilter(plan: AssemblePlan, index: AssembleInputIndex): string {
|
||||
const project = plan.input.project;
|
||||
const items = project.items;
|
||||
if (index.timeline.length !== items.length) fail("one input index per timeline item is required");
|
||||
if (index.overlays.length !== plan.overlays.length) fail("one input index per overlay is required");
|
||||
if (plan.clips.length + plan.cards.length !== items.length) fail("clips + cards must cover every timeline item");
|
||||
if (project.voiceover !== null && plan.voiceover === null) fail("the project names a voiceover the plan did not download");
|
||||
if (project.voiceover === null && plan.voiceover !== null) fail("the plan carries a voiceover the project does not name");
|
||||
|
||||
const chains: string[] = [];
|
||||
let clipCursor = 0;
|
||||
let cardCursor = 0;
|
||||
|
||||
items.forEach((entry, i) => {
|
||||
const v = `v${i}`;
|
||||
const a = `a${i}`;
|
||||
if (entry.kind === "clip") {
|
||||
// Match by VALUE (assetId + trim + rotate + volume): the runner builds the
|
||||
// plan from parsed items, and a re-parse produces equal-but-distinct objects.
|
||||
const source = plan.clips[clipCursor];
|
||||
if (source === undefined || clipEquals(source.clip, entry) === false) fail(`clip ${i} is missing its downloaded source`);
|
||||
const at = index.timeline[i];
|
||||
if (at === undefined) fail(`clip ${i} has no input index`);
|
||||
chains.push(`[${at}:v]${clipVideoChain(entry.rotate, project.fit, project.aspect)}[${v}]`);
|
||||
chains.push(`${clipAudioChain(entry.volumeDb, source.hasAudio, at, ((entry.trimEndMs - entry.trimStartMs) / 1000).toFixed(3))}[${a}]`);
|
||||
clipCursor += 1;
|
||||
} else {
|
||||
const source = plan.cards[cardCursor];
|
||||
if (source === undefined || source.itemIndex !== i) fail(`card ${i} is missing its rendered PNG`);
|
||||
const at = index.timeline[i];
|
||||
if (at === undefined) fail(`card ${i} has no input index`);
|
||||
// -loop 1 -t {dur} supplies the frames; the chain only normalizes to the canvas.
|
||||
chains.push(`[${at}:v]scale=${EDL_ASPECTS[project.aspect].w}:${EDL_ASPECTS[project.aspect].h},setsar=1,fps=${ASSEMBLE_PRESET.fps},format=yuv420p[${v}]`);
|
||||
chains.push(`anullsrc=r=${ASSEMBLE_PRESET.sampleRate}:cl=stereo,atrim=0:${(source.durationMs / 1000).toFixed(3)},asetpts=PTS-STARTPTS,aformat=sample_fmts=fltp:sample_rates=${ASSEMBLE_PRESET.sampleRate}:channel_layouts=stereo[${a}]`);
|
||||
cardCursor += 1;
|
||||
}
|
||||
});
|
||||
if (clipCursor !== plan.clips.length || cardCursor !== plan.cards.length) fail("clips/cards disagree with the timeline");
|
||||
|
||||
// concat v=1:a=1 consumes SEGMENT-INTERLEAVED pairs ([v0][a0][v1][a1]…) —
|
||||
// grouped v-then-a order is a runtime ffmpeg failure.
|
||||
const concatIn = items.map((_, i) => `[v${i}][a${i}]`).join("");
|
||||
chains.push(`${concatIn}concat=n=${items.length}:v=1:a=1[vcat][acat]`);
|
||||
|
||||
// Overlays: absolute-timeline windows over the concatenated base. PNG inputs are
|
||||
// single-frame loops held by eof_action=repeat (the compositor's asset rule).
|
||||
let cur = "vcat";
|
||||
plan.overlays.forEach((source, j) => {
|
||||
const next = j === plan.overlays.length - 1 ? "vout" : `ov${j}`;
|
||||
const at = index.overlays[j];
|
||||
if (at === undefined) fail(`overlay ${j} has no input index`);
|
||||
const s = secondsOf(source.overlay.startMs);
|
||||
const e = secondsOf(source.overlay.endMs);
|
||||
chains.push(`[${cur}][${at}:v]overlay=x=0:y=0:eof_action=repeat:enable='between(t,${s},${e})'[${next}]`);
|
||||
cur = next;
|
||||
});
|
||||
// With no overlays the concatenated base IS the video out; a null passthrough
|
||||
// keeps [vout] a real, mapped label.
|
||||
if (plan.overlays.length === 0) chains.push(`[vcat]null[vout]`);
|
||||
|
||||
// Audio: the concatenated bed, plus the voiceover.
|
||||
const vo = plan.voiceover;
|
||||
if (vo === null) {
|
||||
chains.push(`[acat]anull[aout]`);
|
||||
return chains.join(";");
|
||||
}
|
||||
const voIndex = index.voiceover ?? fail("voiceover input index required");
|
||||
const aformat = `aformat=sample_fmts=fltp:sample_rates=${ASSEMBLE_PRESET.sampleRate}:channel_layouts=stereo`;
|
||||
const delay = vo.offsetMs > 0 ? `adelay=${vo.offsetMs}:all=1` : "";
|
||||
const gain = vo.gainDb !== 0 ? `volume=${vo.gainDb}dB` : "";
|
||||
const voChain = `[${voIndex}:a]${aformat}${delay ? `,${delay}` : ""}${gain ? `,${gain}` : ""}`;
|
||||
if (vo.duckOriginal) {
|
||||
// The duck path: the delayed VO keys a compressor on the bed, and the same
|
||||
// delayed VO is what gets mixed in — the sidechain "hears" the VO exactly
|
||||
// where the audience does, so the duck opens in time with the offset.
|
||||
chains.push(`${voChain},asplit=2[voMix][voKey]`);
|
||||
chains.push(`[acat][voKey]sidechaincompress=threshold=0.03:ratio=8:attack=20:release=300:makeup=1:link=average[duck]`);
|
||||
chains.push(`[duck][voMix]amix=inputs=2:duration=first:dropout_transition=2:normalize=0[aout]`);
|
||||
} else {
|
||||
chains.push(`${voChain}[voGain]`);
|
||||
chains.push(`[acat][voGain]amix=inputs=2:duration=first:dropout_transition=2:normalize=0[aout]`);
|
||||
}
|
||||
return chains.join(";");
|
||||
}
|
||||
|
||||
/** Value equality for a clip (parse identity is not stable across a re-parse). */
|
||||
function clipEquals(a: EdlClip, b: EdlClip): boolean {
|
||||
return a.assetId === b.assetId && a.trimStartMs === b.trimStartMs && a.trimEndMs === b.trimEndMs
|
||||
&& a.rotate === b.rotate && a.volumeDb === b.volumeDb;
|
||||
}
|
||||
|
||||
/** Pre-spawn trim gate: every clip's out-point must sit inside the probed source. */
|
||||
export function validateTrims(items: readonly EdlItem[], sources: readonly AssembleClipSource[]): void {
|
||||
items.forEach((entry, i) => {
|
||||
if (entry.kind !== "clip") return;
|
||||
const source = sources.find((candidate) => clipEquals(candidate.clip, entry));
|
||||
if (source === undefined) fail(`items[${i}]: no downloaded source for the clip`);
|
||||
if (entry.trimStartMs >= entry.trimEndMs) fail(`items[${i}]: trimStartMs must be before trimEndMs`);
|
||||
if (entry.trimEndMs > source.durationMs) {
|
||||
// The plan's late-failure guard: an out-point past the real duration only
|
||||
// surfaces as ffmpeg's "Invalid data found while processing input" — fail
|
||||
// BEFORE the spawn, with a message a crew can act on.
|
||||
fail(`items[${i}]: trim ${entry.trimEndMs}ms exceeds the source's probed ${source.durationMs}ms`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function validatePlan(plan: AssemblePlan): void {
|
||||
scratchPath(plan.outputPath, "outputPath");
|
||||
for (const source of plan.clips) {
|
||||
scratchPath(source.path, "clip.path");
|
||||
if (!SCRATCH_FILE.test(source.path.split("/").at(-1) ?? "")) fail("clip.path must be a downloaded <sha256>.<ext> scratch file");
|
||||
if (!(source.durationMs > 0)) fail("clip.durationMs must be positive");
|
||||
}
|
||||
for (const card of plan.cards) {
|
||||
scratchPath(card.path, "card.path");
|
||||
if (!SCRATCH_FILE.test(card.path.split("/").at(-1) ?? "")) fail("card.path must be a rendered <sha256>.png scratch file");
|
||||
}
|
||||
for (const source of plan.overlays) {
|
||||
scratchPath(source.path, "overlay.path");
|
||||
if (!SCRATCH_FILE.test(source.path.split("/").at(-1) ?? "")) fail("overlay.path must be a rendered <sha256>.png scratch file");
|
||||
}
|
||||
if (plan.voiceover !== null) {
|
||||
scratchPath(plan.voiceover.path, "voiceover.path");
|
||||
if (!SCRATCH_FILE.test(plan.voiceover.path.split("/").at(-1) ?? "")) fail("voiceover.path must be a downloaded scratch file");
|
||||
}
|
||||
validateTrims(plan.input.project.items, plan.clips);
|
||||
}
|
||||
|
||||
/**
|
||||
* Full assemble argv. Inputs in strict order: one per timeline item, then one
|
||||
* per overlay PNG, then the voiceover; every timeline clip carries its own
|
||||
* `-ss/-t` BEFORE its `-i` (input seek: accurate under re-encode), cards carry
|
||||
* `-loop 1 -t dur`. A global `-t` bounds the encode at the timeline length so a
|
||||
* looping card input can never run away.
|
||||
*/
|
||||
export function buildAssembleArgs(plan: AssemblePlan): string[] {
|
||||
validatePlan(plan);
|
||||
const project = plan.input.project;
|
||||
const { w: W, h: H } = EDL_ASPECTS[project.aspect];
|
||||
const args = ["-hide_banner", "-nostdin", "-loglevel", "error", "-y"];
|
||||
const index: AssembleInputIndex = { timeline: [], overlays: [], voiceover: null };
|
||||
let next = 0;
|
||||
|
||||
project.items.forEach((entry) => {
|
||||
if (entry.kind === "clip") {
|
||||
const source = plan.clips.find((source) => clipEquals(source.clip, entry));
|
||||
if (source === undefined) fail("clip source missing");
|
||||
const durationSec = (entry.trimEndMs - entry.trimStartMs) / 1000;
|
||||
args.push("-ss", (entry.trimStartMs / 1000).toFixed(3), "-t", durationSec.toFixed(3), "-i", `file:${source.path}`);
|
||||
} else {
|
||||
const source = plan.cards.find((card) => card.itemIndex === project.items.indexOf(entry));
|
||||
if (source === undefined) fail("card source missing");
|
||||
args.push("-loop", "1", "-t", (entry.durationMs / 1000).toFixed(3), "-i", `file:${source.path}`);
|
||||
}
|
||||
index.timeline.push(next);
|
||||
next += 1;
|
||||
});
|
||||
|
||||
for (const source of plan.overlays) {
|
||||
args.push("-i", `file:${source.path}`);
|
||||
index.overlays.push(next);
|
||||
next += 1;
|
||||
}
|
||||
|
||||
if (plan.voiceover !== null) {
|
||||
args.push("-i", `file:${plan.voiceover.path}`);
|
||||
index.voiceover = next;
|
||||
next += 1;
|
||||
}
|
||||
|
||||
args.push("-filter_complex", buildAssembleFilter(plan, index));
|
||||
const totalSec = timelineTotalMs(project.items) / 1000;
|
||||
args.push("-t", totalSec.toFixed(3));
|
||||
args.push(
|
||||
"-map", "[vout]", "-map", "[aout]",
|
||||
"-c:v", ASSEMBLE_PRESET.videoCodec, "-preset", ASSEMBLE_PRESET.preset, "-crf", String(ASSEMBLE_PRESET.crf),
|
||||
"-pix_fmt", "yuv420p",
|
||||
"-g", String(ASSEMBLE_PRESET.fps * 2), "-keyint_min", String(ASSEMBLE_PRESET.fps * 2), "-sc_threshold", "0",
|
||||
"-c:a", ASSEMBLE_PRESET.audioCodec, "-b:a", `${ASSEMBLE_PRESET.audioBitrateKbps}k`,
|
||||
"-ar", String(ASSEMBLE_PRESET.sampleRate),
|
||||
"-movflags", "+faststart",
|
||||
"-metadata", `width=${W}`, "-metadata", `height=${H}`,
|
||||
"-f", "mp4", `file:${plan.outputPath}`,
|
||||
);
|
||||
return args;
|
||||
}
|
||||
|
||||
/** Failure reasons the runner maps onto the job row; none carries operator text or a path. */
|
||||
export type AssembleFailure =
|
||||
| "invalid request"
|
||||
| "asset unavailable"
|
||||
| "render unavailable"
|
||||
| "download failed"
|
||||
| "encode timed out"
|
||||
| "encode failed"
|
||||
| "publish failed";
|
||||
|
||||
export function assembleError(failure: AssembleFailure, detail?: string): Error {
|
||||
const error = new Error(`assemble: ${failure}`);
|
||||
error.name = failure;
|
||||
if (detail !== undefined) (error as Error & { detail?: string }).detail = detail;
|
||||
return error;
|
||||
}
|
||||
|
|
@ -169,6 +169,31 @@ test("a source without audio mixes the voiceover over a synthetic silence bed",
|
|||
assert.ok(g.includes("duration=longest"));
|
||||
});
|
||||
|
||||
test("an audio-less source never references [0:a]: the bed is a valid source-filter chain", () => {
|
||||
// No voiceover: the synthetic bed alone is the output's audio.
|
||||
const bare = graph({ input: { path: `${SCRATCH}/${HASH}.mp4`, durationSec: 2, audioBed: "none" } });
|
||||
const bareChains = bare.split(";");
|
||||
// anullsrc is a source filter, so aformat follows a COMMA — the missing comma
|
||||
// produced `anullsrc=...aformat=...`, which ffmpeg rejects with "Option
|
||||
// 'sample_rates' not found".
|
||||
assert.equal(bareChains.at(-1), `anullsrc=r=${48000}:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[aout]`);
|
||||
assert.ok(!bare.includes("[0:a]"), "an audio-less source must never be referenced as [0:a]");
|
||||
|
||||
// Ducking over silence: bedRaw gets the same comma-separated form.
|
||||
const duck = graph({ input: { path: `${SCRATCH}/${HASH}.mp4`, durationSec: 2, audioBed: "none" },
|
||||
voiceover: { path: `${SCRATCH}/${VO_HASH}.m4a`, gainDb: -6, duck: true } });
|
||||
const duckChains = duck.split(";");
|
||||
const bedRaw = duckChains.find((c) => c.endsWith("[bedRaw]"));
|
||||
assert.ok(bedRaw !== undefined, "ducking path must still produce [bedRaw]");
|
||||
assert.ok(bedRaw.startsWith(`anullsrc=r=${48000}:cl=stereo,aformat=`), "synthetic bed + aformat must be comma-separated");
|
||||
assert.ok(duck.includes("duration=longest"), "a synthetic bed mixes to longest, bounded by -t");
|
||||
assert.ok(!duck.includes("[0:a]"));
|
||||
|
||||
// An audio-bearing source keeps the input-pad form (no comma after [0:a]).
|
||||
const withAudio = graph({ input: { path: `${SCRATCH}/${HASH}.mp4`, durationSec: 2, audioBed: "source" } });
|
||||
assert.ok(withAudio.includes("[0:a]aformat=sample_fmts=fltp"), "the source bed stays an input pad chain");
|
||||
});
|
||||
|
||||
test("a synthetic bed must be bounded by the source duration", () => {
|
||||
assert.throws(() => graph({
|
||||
input: { path: `${SCRATCH}/${HASH}.mp4`, durationSec: null, audioBed: "none" },
|
||||
|
|
|
|||
|
|
@ -27,7 +27,13 @@ import { createHash } from "node:crypto";
|
|||
export const PRODUCE_PRESET = { width: 1280, height: 720, fps: 30, videoBitrateKbps: 3000, audioBitrateKbps: 128, sampleRate: 48000 } as const;
|
||||
|
||||
export type Lane = "local" | "remote";
|
||||
export type ProduceJobKind = "compose" | "mix";
|
||||
/**
|
||||
* Ingest probe jobs (039's default kind) are claimed by the worker too: the
|
||||
* direct-to-R2 path queues one and the worker ffprobes the sealed object off
|
||||
* R2, so the api box never touches video bytes. parseProduceRequest still only
|
||||
* accepts compose/mix payloads — a probe row carries no input_json at all.
|
||||
*/
|
||||
export type ProduceJobKind = "probe" | "compose" | "mix" | "assemble";
|
||||
|
||||
/** Named bottom-row presets the web posts; the API also accepts raw `{x,y}` percent. */
|
||||
export type OverlayPositionPreset = "bottom-left" | "bottom-center" | "bottom-right";
|
||||
|
|
@ -186,7 +192,7 @@ function gainDbChecked(value: unknown): number {
|
|||
}
|
||||
|
||||
/** Strict allowlist parse of the API's persisted `input_json`; the worker re-validates it. */
|
||||
export function parseProduceRequest(value: unknown, kind: ProduceJobKind): ProduceRequest {
|
||||
export function parseProduceRequest(value: unknown, kind: Exclude<ProduceJobKind, "probe">): ProduceRequest {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) fail("input_json must be an object");
|
||||
onlyKeys(value, ["overlays", "voiceover", "outputLabel", "lane", "requestedBy"], "input_json");
|
||||
const raw = value as Record<string, unknown>;
|
||||
|
|
@ -265,12 +271,33 @@ export interface ProducePlan {
|
|||
/** ffmpeg input indexes: source 0, then each logo, then the voiceover — argv order. */
|
||||
export interface ProduceInputIndex { logos: number[]; voiceover: number | null }
|
||||
|
||||
/** `aformat` as a filter step (a real input's chain: comma-separated, after a comma). */
|
||||
function audioFormat(): string {
|
||||
return `aformat=sample_fmts=fltp:sample_rates=${PRODUCE_PRESET.sampleRate}:channel_layouts=stereo`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The synthetic silence bed for an audio-less source. anullsrc is a source
|
||||
* filter, so it cannot be followed by `aformat` — it is configured directly with
|
||||
* the output preset's rate and stereo layout, and the encoder's `-ar`/channel
|
||||
* handling downstream already guarantees the format.
|
||||
*/
|
||||
function silenceBed(): string {
|
||||
return `anullsrc=r=${PRODUCE_PRESET.sampleRate}:cl=stereo`;
|
||||
}
|
||||
|
||||
function gain(gainDb: number): string { return `volume=${gainDb}dB`; }
|
||||
|
||||
/**
|
||||
* One bed chain: `[0:a]` is an input pad, so `aformat` follows it directly, but
|
||||
* the synthetic bed is a source filter, so it needs the `,` separator before
|
||||
* `aformat` (the missing comma is what made the audio-less path emit
|
||||
* `anullsrc=...aformat=...`, which ffmpeg rejects outright).
|
||||
*/
|
||||
function bedChain(bed: string, label: string): string {
|
||||
return `${bed}${bed.startsWith("[") ? "" : ","}${audioFormat()}[${label}]`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The `-filter_complex` string: video chain (scale → overlays in request order) and
|
||||
* the audio mix. `min(px,W-tw)`-style expressions are single-quoted because `min`
|
||||
|
|
@ -312,10 +339,10 @@ export function buildProduceFilter(plan: ProducePlan, index: ProduceInputIndex):
|
|||
// on the bed. `duration=first` ends with the source bed; a synthetic bed is
|
||||
// unbounded, so it mixes to `longest` and only the source's `-t` can end the encode.
|
||||
const vo = plan.voiceover;
|
||||
const bed = plan.input.audioBed === "source" ? "[0:a]" : `anullsrc=r=${PRODUCE_PRESET.sampleRate}:cl=stereo`;
|
||||
const bed = plan.input.audioBed === "source" ? "[0:a]" : silenceBed();
|
||||
|
||||
if (vo === null) {
|
||||
chains.push(`${bed}${audioFormat()}[aout]`);
|
||||
chains.push(bedChain(bed, "aout"));
|
||||
return chains.join(";");
|
||||
}
|
||||
const voIndex = index.voiceover ?? fail("voiceover input index required");
|
||||
|
|
@ -324,13 +351,13 @@ export function buildProduceFilter(plan: ProducePlan, index: ProduceInputIndex):
|
|||
if (vo.duck) {
|
||||
// asplit feeds the same voiceover to the compressor key and to the mix.
|
||||
chains.push(`${voIn}${audioFormat()},asplit=2[voMix][voKey]`);
|
||||
chains.push(`${bed}${audioFormat()}[bedRaw]`);
|
||||
chains.push(bedChain(bed, "bedRaw"));
|
||||
chains.push(`[bedRaw][voKey]sidechaincompress=threshold=0.03:ratio=8:attack=20:release=300:makeup=1:link=average[duck]`);
|
||||
if (vo.gainDb !== null && vo.gainDb !== 0) chains.push(`[voMix]${gain(vo.gainDb)}[voGain]`);
|
||||
const mixIn = vo.gainDb !== null && vo.gainDb !== 0 ? "voGain" : "voMix";
|
||||
chains.push(`[duck][${mixIn}]amix=inputs=2:duration=${amixDuration}:dropout_transition=2:normalize=0[aout]`);
|
||||
} else {
|
||||
chains.push(`${bed}${audioFormat()}[bedFmt]`);
|
||||
chains.push(bedChain(bed, "bedFmt"));
|
||||
chains.push(`${voIn}${audioFormat()}${vo.gainDb === null || vo.gainDb === 0 ? "" : `,${gain(vo.gainDb)}`}[voGain]`);
|
||||
chains.push(`[bedFmt][voGain]amix=inputs=2:duration=${amixDuration}:dropout_transition=2:normalize=0[aout]`);
|
||||
}
|
||||
|
|
|
|||
157
apps/media-worker/src/producer/edl-plan.test.ts
Normal file
157
apps/media-worker/src/producer/edl-plan.test.ts
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
/**
|
||||
* EdlPlan — the worker's mirror of the api's EDL types (apps/api/src/media-control/
|
||||
* edl.ts), parsed LENIENTLY: the api enforces the strict contract before
|
||||
* enqueueing; the worker re-checks structure and bounds and fails the JOB (never
|
||||
* the process) on anything this build does not understand. The api side owns the
|
||||
* mirror-assert test that keeps the two files in sync.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import {
|
||||
AssembleInput, EDL_ASPECTS, MAX_ITEMS, MAX_TOTAL_MS, assembleScratchName, gainDbOk, parseAssembleInput, timelineDurationMs,
|
||||
} from "./edl-plan";
|
||||
|
||||
const ASSET = "00000000-0000-4000-8000-000000000001";
|
||||
const VOICEOVER = "00000000-0000-4000-8000-000000000005";
|
||||
const EDL_ID = "00000000-0000-4000-8000-000000000010";
|
||||
|
||||
function project(overrides: Record<string, unknown> = {}): Record<string, unknown> {
|
||||
return {
|
||||
version: 1,
|
||||
aspect: "16:9",
|
||||
fit: "cover",
|
||||
fps: 30,
|
||||
items: [
|
||||
{ kind: "clip", assetId: ASSET, trimStartMs: 0, trimEndMs: 3_000, rotate: 0, volumeDb: 0 },
|
||||
{ kind: "card", text: "Chapter One", durationMs: 2_000, style: "title" },
|
||||
],
|
||||
overlays: [],
|
||||
voiceover: null,
|
||||
outputLabel: "assembly",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function envelope(projectValue: Record<string, unknown>, overrides: Record<string, unknown> = {}): Record<string, unknown> {
|
||||
return { edlProjectId: EDL_ID, project: projectValue, ...overrides };
|
||||
}
|
||||
|
||||
test("the canonical envelope parses: envelope fields + full project", () => {
|
||||
const parsed: AssembleInput = parseAssembleInput(envelope(project(), { lane: "remote", requestedBy: ASSET }));
|
||||
assert.equal(parsed.edlProjectId, EDL_ID);
|
||||
assert.equal(parsed.lane, "remote");
|
||||
assert.equal(parsed.requestedBy, ASSET);
|
||||
assert.equal(parsed.project.aspect, "16:9");
|
||||
assert.equal(parsed.project.fit, "cover");
|
||||
assert.equal(parsed.project.items.length, 2);
|
||||
assert.equal(parsed.project.items[0]!.kind, "clip");
|
||||
assert.equal(parsed.project.items[1]!.kind, "card");
|
||||
assert.equal(parsed.project.voiceover, null);
|
||||
// lane/requestedBy are optional on the envelope (the api writes lane for routing).
|
||||
const bare = parseAssembleInput(envelope(project()));
|
||||
assert.equal(bare.lane, null);
|
||||
assert.equal(bare.requestedBy, null);
|
||||
});
|
||||
|
||||
test("timeline math: clips by trim span, cards by duration", () => {
|
||||
const parsed = parseAssembleInput(envelope(project({
|
||||
items: [
|
||||
{ kind: "clip", assetId: ASSET, trimStartMs: 1_000, trimEndMs: 4_000, rotate: 0, volumeDb: 0 },
|
||||
{ kind: "card", text: "x", durationMs: 2_500, style: "chapter" },
|
||||
],
|
||||
})));
|
||||
assert.equal(timelineDurationMs(parsed.project), 5_500);
|
||||
});
|
||||
|
||||
test("version is the closed discriminator: anything but 1 is an unknown shape", () => {
|
||||
assert.throws(() => parseAssembleInput(envelope(project({ version: 2 }))), /version must be 1/);
|
||||
assert.throws(() => parseAssembleInput(envelope({ ...project(), version: "1" })), /version must be 1/);
|
||||
});
|
||||
|
||||
test("closed enums: aspect, fit, rotation, card style, overlay type", () => {
|
||||
assert.throws(() => parseAssembleInput(envelope(project({ aspect: "4:3" }))), /aspect/);
|
||||
assert.throws(() => parseAssembleInput(envelope(project({ fit: "fill" }))), /fit/);
|
||||
assert.throws(() => parseAssembleInput(envelope(project({
|
||||
items: [{ kind: "clip", assetId: ASSET, trimStartMs: 0, trimEndMs: 1_000, rotate: 45, volumeDb: 0 }],
|
||||
}))), /rotate/);
|
||||
assert.throws(() => parseAssembleInput(envelope(project({
|
||||
items: [{ kind: "card", text: "x", durationMs: 2_000, style: "epigraph" }],
|
||||
}))), /style/);
|
||||
assert.throws(() => parseAssembleInput(envelope(project({
|
||||
overlays: [{ overlayType: "clock", fields: { title: "x" }, startMs: 0, endMs: 2_000 }],
|
||||
}))), /overlayType/);
|
||||
});
|
||||
|
||||
test("gains sit on the 3 dB ladder within -36..+6 (the api's GAIN_DB_STEP rule)", () => {
|
||||
assert.equal(gainDbOk(0), true);
|
||||
assert.equal(gainDbOk(-36), true);
|
||||
assert.equal(gainDbOk(6), true);
|
||||
assert.equal(gainDbOk(-5), false);
|
||||
assert.equal(gainDbOk(9), false);
|
||||
assert.throws(() => parseAssembleInput(envelope(project({
|
||||
items: [{ kind: "clip", assetId: ASSET, trimStartMs: 0, trimEndMs: 1_000, rotate: 0, volumeDb: -5 }],
|
||||
}))), /multiple of 3 dB/);
|
||||
assert.throws(() => parseAssembleInput(envelope(project({
|
||||
voiceover: { audioAssetId: VOICEOVER, gainDb: 9, duckOriginal: false, offsetMs: 0 },
|
||||
}))), /multiple of 3 dB/);
|
||||
});
|
||||
|
||||
test("bounds: card duration, overlay window, item count, total length", () => {
|
||||
assert.throws(() => parseAssembleInput(envelope(project({
|
||||
items: [{ kind: "card", text: "x", durationMs: 500, style: "title" }],
|
||||
}))), /durationMs must be between/);
|
||||
assert.throws(() => parseAssembleInput(envelope(project({
|
||||
items: [{ kind: "card", text: "x", durationMs: 20_000, style: "title" }],
|
||||
}))), /durationMs must be between/);
|
||||
assert.throws(() => parseAssembleInput(envelope(project({
|
||||
overlays: [{ overlayType: "lower_third", fields: { title: "x" }, startMs: 0, endMs: 500 }],
|
||||
}))), /window must be between/);
|
||||
const many = Array.from({ length: MAX_ITEMS + 1 }, () => ({ kind: "card", text: "x", durationMs: 1_000, style: "chapter" }));
|
||||
assert.throws(() => parseAssembleInput(envelope(project({ items: many }))), /at most 50/);
|
||||
// 15-min total cap: valid clips (trims are probe-gated at runtime, not parse)
|
||||
// with spans summing past MAX_TOTAL_MS fail here.
|
||||
const long = Array.from({ length: 3 }, () => ({ kind: "clip", assetId: ASSET, trimStartMs: 0, trimEndMs: 400_000, rotate: 0, volumeDb: 0 }));
|
||||
assert.throws(() => parseAssembleInput(envelope(project({ items: long }))), /at most 900000 ms/);
|
||||
assert.ok(MAX_TOTAL_MS === 15 * 60_000);
|
||||
});
|
||||
|
||||
test("an empty timeline, or one an overlay outlives, fails", () => {
|
||||
assert.throws(() => parseAssembleInput(envelope(project({ items: [] }))), /must not be empty/);
|
||||
assert.throws(() => parseAssembleInput(envelope(project({
|
||||
overlays: [{ overlayType: "lower_third", fields: { title: "x" }, startMs: 10_000, endMs: 12_000 }],
|
||||
}))), /beyond the timeline/);
|
||||
});
|
||||
|
||||
test("unknown keys are refused everywhere — nothing is silently ignored", () => {
|
||||
assert.throws(() => parseAssembleInput({ edlProjectId: EDL_ID, project: project(), extra: 1 }), /input_json\.extra/);
|
||||
assert.throws(() => parseAssembleInput(envelope(project({ transition: "dissolve" }))), /project\.transition/);
|
||||
assert.throws(() => parseAssembleInput(envelope(project({
|
||||
items: [{ kind: "clip", assetId: ASSET, trimStartMs: 0, trimEndMs: 1_000, rotate: 0, volumeDb: 0, speed: 2 }],
|
||||
}))), /speed/);
|
||||
});
|
||||
|
||||
test("the voiceover block is closed and fully typed", () => {
|
||||
const vo = { audioAssetId: VOICEOVER, gainDb: -3, duckOriginal: true, offsetMs: 250 };
|
||||
const parsed = parseAssembleInput(envelope(project({ voiceover: vo })));
|
||||
assert.deepEqual(parsed.project.voiceover, { ...vo });
|
||||
assert.throws(() => parseAssembleInput(envelope(project({ voiceover: { ...vo, audioAssetId: "not-a-uuid" } }))), /audioAssetId/);
|
||||
assert.throws(() => parseAssembleInput(envelope(project({ voiceover: { ...vo, offsetMs: -1 } }))), /offsetMs/);
|
||||
assert.throws(() => parseAssembleInput(envelope(project({ voiceover: { ...vo, duckOriginal: "yes" } }))), /duckOriginal/);
|
||||
});
|
||||
|
||||
test("the envelope's edlProjectId must be a uuid", () => {
|
||||
assert.throws(() => parseAssembleInput({ edlProjectId: "edl-1", project: project() }), /edlProjectId must be a UUID/);
|
||||
});
|
||||
|
||||
test("the scratch name is the project's hash: same project → same dir, no ids in paths", () => {
|
||||
const a = parseAssembleInput(envelope(project()));
|
||||
const b = parseAssembleInput(envelope(project()));
|
||||
const c = parseAssembleInput(envelope(project({ outputLabel: "changed" })));
|
||||
assert.equal(assembleScratchName(a), assembleScratchName(b));
|
||||
assert.notEqual(assembleScratchName(a), assembleScratchName(c));
|
||||
assert.match(assembleScratchName(a), /^[0-9a-f]{64}$/);
|
||||
});
|
||||
|
||||
test("the aspect table is the plan's two canvases", () => {
|
||||
assert.deepEqual(EDL_ASPECTS, { "16:9": { w: 1280, h: 720 }, "9:16": { w: 1080, h: 1920 } });
|
||||
});
|
||||
278
apps/media-worker/src/producer/edl-plan.ts
Normal file
278
apps/media-worker/src/producer/edl-plan.ts
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
/**
|
||||
* EdlProject — the editor-v2 timeline contract, LENIENTLY validated worker-side.
|
||||
*
|
||||
* mirror of apps/api/src/media-control/edl.ts — keep in sync (mirror-assert lives in the api)
|
||||
*
|
||||
* The api module owns the strict validator crews post against (closed enums,
|
||||
* basis-point geometry, GAIN_DB_STEP multiples, <=50 items, <=15 min total) and a
|
||||
* mirror-assert test that fails when the two files drift. This copy exists so the
|
||||
* worker package never imports the api module graph (the filemap's boundary rule,
|
||||
* same standing as ./ffprobe.ts being a verbatim port of the api's ffprobe).
|
||||
*
|
||||
* Why LENIENT here: input_json is a queued row. The api already rejected every
|
||||
* bad shape before enqueueing; the worker only needs to (a) trust nothing it
|
||||
* would feed to a shell — nothing here ever is, argv only — and (b) fail the JOB,
|
||||
* not the process, when a row is not the shape this build understands (an
|
||||
* upgraded api ahead of the worker, a tampered row, an unknown `version`). So
|
||||
* this parser checks structure + bounds, ignores nothing silently, and throws a
|
||||
* single error type with a field path — the runner maps that to a failed row.
|
||||
*
|
||||
* v1 is closed (the plan's anti-NLE-creep rule): version !== 1 is an unknown
|
||||
* shape, and the mirroring api test is what enforces field-for-field sync.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
/** The two canvases v1 renders, per the adopted plan §1 (landscape 720p, portrait 1080x1920). */
|
||||
export const EDL_ASPECTS = { "16:9": { w: 1280, h: 720 }, "9:16": { w: 1080, h: 1920 } } as const;
|
||||
export type EdlAspect = keyof typeof EDL_ASPECTS;
|
||||
export const EDL_FIT = ["cover", "contain"] as const;
|
||||
export type EdlFit = (typeof EDL_FIT)[number];
|
||||
export const CARD_STYLES = ["title", "chapter"] as const;
|
||||
export type EdlCardStyle = (typeof CARD_STYLES)[number];
|
||||
/** EdlOverlay.overlayType: the production overlay templates the api renderer knows. */
|
||||
export const OVERLAY_TYPES = ["lower_third", "chyron", "logo"] as const;
|
||||
export type EdlOverlayType = (typeof OVERLAY_TYPES)[number];
|
||||
export const ROTATIONS = [0, 90, 180, 270] as const;
|
||||
export type EdlRotate = (typeof ROTATIONS)[number];
|
||||
|
||||
/** Closed 3 dB ladder for per-clip and voiceover gain (GAIN_DB_STEP, -36..+6). */
|
||||
export const GAIN_DB_STEP = 3;
|
||||
export const GAIN_DB_MIN = -36;
|
||||
export const GAIN_DB_MAX = 6;
|
||||
/** 3 dB-multiple inside the ladder's span; 0 is a multiple of the step. */
|
||||
export function gainDbOk(value: number): boolean {
|
||||
return Number.isInteger(value / GAIN_DB_STEP) && value >= GAIN_DB_MIN && value <= GAIN_DB_MAX;
|
||||
}
|
||||
|
||||
/** Plan §1 caps, enforced leniently here (the api enforces them strictly). */
|
||||
export const MAX_ITEMS = 50;
|
||||
export const MAX_OVERLAYS = 16;
|
||||
export const MAX_TOTAL_MS = 15 * 60_000;
|
||||
export const MAX_CARD_TEXT = 120;
|
||||
export const CARD_DURATION_MS = { min: 1_000, max: 10_000 } as const;
|
||||
export const OVERLAY_WINDOW_MS = { min: 1_000, max: 10_000 } as const;
|
||||
export const MAX_OUTPUT_LABEL = 80;
|
||||
|
||||
export interface EdlClip {
|
||||
kind: "clip";
|
||||
assetId: string;
|
||||
trimStartMs: number;
|
||||
trimEndMs: number;
|
||||
rotate: EdlRotate;
|
||||
volumeDb: number;
|
||||
}
|
||||
|
||||
export interface EdlCard {
|
||||
kind: "card";
|
||||
text: string;
|
||||
durationMs: number;
|
||||
style: EdlCardStyle;
|
||||
}
|
||||
|
||||
export type EdlItem = EdlClip | EdlCard;
|
||||
|
||||
export interface EdlOverlay {
|
||||
overlayType: EdlOverlayType;
|
||||
fields: Record<string, string>;
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
}
|
||||
|
||||
export interface EdlVoiceover {
|
||||
audioAssetId: string;
|
||||
gainDb: number;
|
||||
duckOriginal: boolean;
|
||||
offsetMs: number;
|
||||
}
|
||||
|
||||
export interface EdlProject {
|
||||
version: 1;
|
||||
aspect: EdlAspect;
|
||||
fit: EdlFit;
|
||||
fps: 30;
|
||||
items: EdlItem[];
|
||||
overlays: EdlOverlay[];
|
||||
voiceover: EdlVoiceover | null;
|
||||
outputLabel: string;
|
||||
}
|
||||
|
||||
/** The `video_processing_jobs.input_json` envelope of an assemble row (043). */
|
||||
export interface AssembleInput {
|
||||
edlProjectId: string;
|
||||
project: EdlProject;
|
||||
lane: "local" | "remote" | null;
|
||||
requestedBy: string | null;
|
||||
}
|
||||
|
||||
export class EdlParseError extends Error {
|
||||
constructor(message: string) { super(message); this.name = "EdlParseError"; }
|
||||
}
|
||||
function fail(message: string): never { throw new EdlParseError(message); }
|
||||
|
||||
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
||||
|
||||
function record(value: unknown, field: string, allowed: readonly string[]): Record<string, unknown> {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${field} must be an object`);
|
||||
for (const key of Object.keys(value)) if (!allowed.includes(key)) fail(`${field}.${key} is not allowed`);
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function stringField(value: unknown, field: string, max: number): string {
|
||||
if (typeof value !== "string") fail(`${field} must be a string`);
|
||||
if (value.length > max) fail(`${field} must be at most ${max} characters`);
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Integer ms >= 0 (the plan's "times derived, no floats" rule). */
|
||||
function msField(value: unknown, field: string): number {
|
||||
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) fail(`${field} must be integer milliseconds >= 0`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function intInSet(value: unknown, field: string, set: readonly number[]): number {
|
||||
if (typeof value !== "number" || !Number.isInteger(value) || !set.includes(value)) fail(`${field} must be one of ${set.join("|")}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function enumIn<T extends string>(value: unknown, field: string, set: readonly T[]): T {
|
||||
if (typeof value !== "string" || !set.includes(value as T)) fail(`${field} must be one of ${set.join("|")}`);
|
||||
return value as T;
|
||||
}
|
||||
|
||||
function uuidField(value: unknown, field: string): string {
|
||||
if (typeof value !== "string" || !UUID.test(value)) fail(`${field} must be a UUID`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function gainDb(value: unknown, field: string): number {
|
||||
const n = typeof value === "number" && Number.isFinite(value) ? Math.round(value * 100) / 100 : NaN;
|
||||
if (!Number.isFinite(n) || !gainDbOk(n)) fail(`${field} must be a multiple of ${GAIN_DB_STEP} dB between ${GAIN_DB_MIN} and ${GAIN_DB_MAX}`);
|
||||
return n;
|
||||
}
|
||||
|
||||
/**
|
||||
* Card/overlay text: the api's overlay-text character class (render.ts's
|
||||
* `text()`): letters, numbers, punctuation, spaces, marks, symbols — no control
|
||||
* characters, no filtergraph metacharacters. The card never reaches a filter
|
||||
* (it becomes a PNG), so this is a belt-and-braces re-check, not the injection
|
||||
* defense; the defense is that text only ever becomes pixels.
|
||||
*/
|
||||
function overlayTextField(value: unknown, field: string, max: number): string {
|
||||
const text = stringField(value, field, max);
|
||||
if (!/^[\p{L}\p{N}\p{P}\p{Zs}\p{M}\p{S}]*$/u.test(text)) fail(`${field} must not contain control characters`);
|
||||
return text;
|
||||
}
|
||||
|
||||
function item(value: unknown, index: number): EdlItem {
|
||||
const field = `project.items[${index}]`;
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${field} must be an object`);
|
||||
const kind = (value as Record<string, unknown>).kind;
|
||||
if (kind === "clip") {
|
||||
const c = record(value, field, ["kind", "assetId", "trimStartMs", "trimEndMs", "rotate", "volumeDb"]);
|
||||
return {
|
||||
kind: "clip",
|
||||
assetId: uuidField(c.assetId, `${field}.assetId`),
|
||||
trimStartMs: msField(c.trimStartMs, `${field}.trimStartMs`),
|
||||
trimEndMs: msField(c.trimEndMs, `${field}.trimEndMs`),
|
||||
rotate: intInSet(c.rotate, `${field}.rotate`, ROTATIONS) as EdlRotate,
|
||||
volumeDb: gainDb(c.volumeDb, `${field}.volumeDb`),
|
||||
};
|
||||
}
|
||||
if (kind === "card") {
|
||||
const c = record(value, field, ["kind", "text", "durationMs", "style"]);
|
||||
const durationMs = msField(c.durationMs, `${field}.durationMs`);
|
||||
if (durationMs < CARD_DURATION_MS.min || durationMs > CARD_DURATION_MS.max) {
|
||||
fail(`${field}.durationMs must be between ${CARD_DURATION_MS.min} and ${CARD_DURATION_MS.max}`);
|
||||
}
|
||||
return {
|
||||
kind: "card",
|
||||
text: overlayTextField(c.text, `${field}.text`, MAX_CARD_TEXT),
|
||||
durationMs,
|
||||
style: enumIn(c.style, `${field}.style`, CARD_STYLES),
|
||||
};
|
||||
}
|
||||
fail(`${field}.kind must be 'clip' or 'card'`);
|
||||
}
|
||||
|
||||
function overlay(value: unknown, index: number): EdlOverlay {
|
||||
const field = `project.overlays[${index}]`;
|
||||
const o = record(value, field, ["overlayType", "fields", "startMs", "endMs"]);
|
||||
const overlayType = enumIn(o.overlayType, `${field}.overlayType`, OVERLAY_TYPES);
|
||||
const fieldsRaw = o.fields === undefined || o.fields === null ? {} : o.fields;
|
||||
if (typeof fieldsRaw !== "object" || Array.isArray(fieldsRaw)) fail(`${field}.fields must be an object`);
|
||||
const fields: Record<string, string> = {};
|
||||
for (const [key, entry] of Object.entries(fieldsRaw as Record<string, unknown>)) {
|
||||
if (key.length > 64) fail(`${field}.fields has an over-long key`);
|
||||
fields[key] = overlayTextField(entry, `${field}.fields.${key}`, 200);
|
||||
}
|
||||
const startMs = msField(o.startMs, `${field}.startMs`);
|
||||
const endMs = msField(o.endMs, `${field}.endMs`);
|
||||
if (endMs - startMs < OVERLAY_WINDOW_MS.min || endMs - startMs > OVERLAY_WINDOW_MS.max) {
|
||||
fail(`${field} window must be between ${OVERLAY_WINDOW_MS.min} and ${OVERLAY_WINDOW_MS.max} ms`);
|
||||
}
|
||||
if (startMs < 0) fail(`${field}.startMs must be >= 0`);
|
||||
if (overlayType !== "logo" && fields.title === undefined) fail(`${field}.fields.title is required for ${overlayType}`);
|
||||
return { overlayType, fields, startMs, endMs };
|
||||
}
|
||||
|
||||
/** Total timeline length in ms (clips by trim span, cards by their duration). */
|
||||
export function timelineDurationMs(project: EdlProject): number {
|
||||
return project.items.reduce((total, entry) =>
|
||||
total + (entry.kind === "clip" ? entry.trimEndMs - entry.trimStartMs : entry.durationMs), 0);
|
||||
}
|
||||
|
||||
/** Parse `input_json` of an assemble row: { edlProjectId, project, lane?, requestedBy? }. */
|
||||
export function parseAssembleInput(value: unknown): AssembleInput {
|
||||
const env = record(value, "input_json", ["edlProjectId", "project", "lane", "requestedBy"]);
|
||||
const edlProjectId = uuidField(env.edlProjectId, "input_json.edlProjectId");
|
||||
const lane = env.lane === undefined || env.lane === null ? null
|
||||
: enumIn(env.lane, "input_json.lane", ["local", "remote"] as const);
|
||||
const requestedBy = env.requestedBy === undefined || env.requestedBy === null ? null
|
||||
: uuidField(env.requestedBy, "input_json.requestedBy");
|
||||
|
||||
const p = record(env.project, "project", ["version", "aspect", "fit", "fps", "items", "overlays", "voiceover", "outputLabel"]);
|
||||
if (p.version !== 1) fail("project.version must be 1 (this build renders v1 timelines only)");
|
||||
const aspect = enumIn(p.aspect, "project.aspect", Object.keys(EDL_ASPECTS) as EdlAspect[]);
|
||||
const fit = enumIn(p.fit, "project.fit", EDL_FIT);
|
||||
const fps = intInSet(p.fps, "project.fps", [30]);
|
||||
const itemsRaw = p.items === undefined || p.items === null ? [] : p.items;
|
||||
if (!Array.isArray(itemsRaw)) fail("project.items must be an array");
|
||||
if (itemsRaw.length === 0) fail("project.items must not be empty");
|
||||
if (itemsRaw.length > MAX_ITEMS) fail(`project.items allows at most ${MAX_ITEMS} entries`);
|
||||
const items = itemsRaw.map(item);
|
||||
const overlaysRaw = p.overlays === undefined || p.overlays === null ? [] : p.overlays;
|
||||
if (!Array.isArray(overlaysRaw)) fail("project.overlays must be an array");
|
||||
if (overlaysRaw.length > MAX_OVERLAYS) fail(`project.overlays allows at most ${MAX_OVERLAYS} entries`);
|
||||
const overlays = overlaysRaw.map(overlay);
|
||||
|
||||
let voiceover: EdlVoiceover | null = null;
|
||||
if (p.voiceover !== undefined && p.voiceover !== null) {
|
||||
const v = record(p.voiceover, "project.voiceover", ["audioAssetId", "gainDb", "duckOriginal", "offsetMs"]);
|
||||
voiceover = {
|
||||
audioAssetId: uuidField(v.audioAssetId, "project.voiceover.audioAssetId"),
|
||||
gainDb: gainDb(v.gainDb, "project.voiceover.gainDb"),
|
||||
duckOriginal: v.duckOriginal === undefined ? false : typeof v.duckOriginal === "boolean" ? v.duckOriginal : fail("project.voiceover.duckOriginal must be a boolean"),
|
||||
offsetMs: msField(v.offsetMs, "project.voiceover.offsetMs"),
|
||||
};
|
||||
}
|
||||
|
||||
const outputLabel = p.outputLabel === undefined || p.outputLabel === null ? ""
|
||||
: stringField(p.outputLabel, "project.outputLabel", MAX_OUTPUT_LABEL);
|
||||
const project: EdlProject = { version: 1, aspect, fit, fps: 30, items, overlays, voiceover, outputLabel };
|
||||
const total = timelineDurationMs(project);
|
||||
if (total <= 0) fail("project timeline must be longer than 0 ms");
|
||||
if (total > MAX_TOTAL_MS) fail(`project timeline must be at most ${MAX_TOTAL_MS} ms`);
|
||||
// An overlay that starts after the timeline can never show; the api rejects it, we fail it.
|
||||
for (const o of overlays) if (o.startMs >= total) fail("project.overlays start beyond the timeline");
|
||||
return { edlProjectId, project, lane, requestedBy };
|
||||
}
|
||||
|
||||
/**
|
||||
* The job's scratch identity: the same project renders into the same sha-named
|
||||
* dir, so a re-render of an unchanged project reuses fetched assets (the plan's
|
||||
* idempotency rule). Input ids never reach a path — only the hash does.
|
||||
*/
|
||||
export function assembleScratchName(input: AssembleInput): string {
|
||||
const canonical = JSON.stringify({ edlProjectId: input.edlProjectId, project: input.project });
|
||||
return createHash("sha256").update(canonical).digest("hex");
|
||||
}
|
||||
166
apps/media-worker/src/producer/ffprobe.ts
Normal file
166
apps/media-worker/src/producer/ffprobe.ts
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
/**
|
||||
* ffprobe for the worker probe lane — PORTED VERBATIM from
|
||||
* apps/api/src/video/ffprobe.ts (buildFfprobeArgs / probeVideoFile /
|
||||
* parseFfprobeOutput and the helpers beneath). The api keeps its own copy: the
|
||||
* worker package must not depend on the api module graph, and the two parsers
|
||||
* must agree bit for bit on the metadata written to video_assets.
|
||||
*/
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
const FFPROBE_TIMEOUT_MS = 30_000;
|
||||
const MAX_OUTPUT_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
export interface VideoProbeResult {
|
||||
width?: number;
|
||||
height?: number;
|
||||
durationMs?: number;
|
||||
frameRate?: number;
|
||||
codec?: string;
|
||||
audioCodec?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep the input as a discrete argv value. `-i` ensures even a path beginning
|
||||
* with a dash is treated as input rather than another ffprobe option.
|
||||
*/
|
||||
export function buildFfprobeArgs(inputPath: string): string[] {
|
||||
return [
|
||||
"-v",
|
||||
"error",
|
||||
"-print_format",
|
||||
"json",
|
||||
"-show_format",
|
||||
"-show_streams",
|
||||
"-i",
|
||||
inputPath,
|
||||
];
|
||||
}
|
||||
|
||||
export async function probeVideoFile(
|
||||
inputPath: string,
|
||||
executable = process.env.FFPROBE_PATH ?? "ffprobe",
|
||||
): Promise<VideoProbeResult> {
|
||||
if (inputPath.length === 0) throw new Error("ffprobe input path must not be empty");
|
||||
|
||||
const output = await new Promise<string>((resolve, reject) => {
|
||||
const child = spawn(executable, buildFfprobeArgs(inputPath), {
|
||||
shell: false,
|
||||
windowsHide: true,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const stdout: Buffer[] = [];
|
||||
const stderr: Buffer[] = [];
|
||||
let outputBytes = 0;
|
||||
let settled = false;
|
||||
|
||||
const finish = (error: Error | undefined, value?: string): void => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timeout);
|
||||
if (error === undefined) resolve(value ?? "");
|
||||
else reject(error);
|
||||
};
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
child.kill("SIGKILL");
|
||||
finish(new Error(`ffprobe timed out after ${FFPROBE_TIMEOUT_MS}ms`));
|
||||
}, FFPROBE_TIMEOUT_MS);
|
||||
|
||||
child.stdout.on("data", (chunk: Buffer) => {
|
||||
outputBytes += chunk.length;
|
||||
if (outputBytes > MAX_OUTPUT_BYTES) {
|
||||
child.kill("SIGKILL");
|
||||
finish(new Error("ffprobe output exceeded the 10 MiB limit"));
|
||||
return;
|
||||
}
|
||||
stdout.push(chunk);
|
||||
});
|
||||
child.stderr.on("data", (chunk: Buffer) => {
|
||||
if (Buffer.concat(stderr).length < 16_384) stderr.push(chunk);
|
||||
});
|
||||
child.once("error", (error) => finish(error));
|
||||
child.once("close", (code, signal) => {
|
||||
if (code !== 0) {
|
||||
const detail = Buffer.concat(stderr).toString("utf8").trim();
|
||||
finish(
|
||||
new Error(
|
||||
`ffprobe failed (${signal === null ? `exit ${String(code)}` : `signal ${signal}`})${
|
||||
detail === "" ? "" : `: ${detail}`
|
||||
}`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
finish(undefined, Buffer.concat(stdout).toString("utf8"));
|
||||
});
|
||||
});
|
||||
|
||||
return parseFfprobeOutput(output);
|
||||
}
|
||||
|
||||
export function parseFfprobeOutput(output: string): VideoProbeResult {
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(output) as unknown;
|
||||
} catch {
|
||||
throw new Error("ffprobe returned invalid JSON");
|
||||
}
|
||||
const root = asRecord(parsed);
|
||||
if (root === undefined) throw new Error("ffprobe returned an invalid result");
|
||||
const streams = Array.isArray(root.streams)
|
||||
? root.streams.map(asRecord).filter((stream): stream is Record<string, unknown> => stream !== undefined)
|
||||
: [];
|
||||
const format = asRecord(root.format);
|
||||
const video = streams.find((stream) => stream.codec_type === "video");
|
||||
if (video === undefined) throw new Error("ffprobe found no video stream");
|
||||
const audio = streams.find((stream) => stream.codec_type === "audio");
|
||||
const durationSeconds = finiteNumber(format?.duration) ?? finiteNumber(video?.duration);
|
||||
const frameRate = rationalNumber(video?.avg_frame_rate) ?? rationalNumber(video?.r_frame_rate);
|
||||
|
||||
return {
|
||||
...optionalNumber("width", finiteInteger(video?.width)),
|
||||
...optionalNumber("height", finiteInteger(video?.height)),
|
||||
...optionalNumber(
|
||||
"durationMs",
|
||||
durationSeconds === undefined ? undefined : Math.max(0, Math.round(durationSeconds * 1000)),
|
||||
),
|
||||
...optionalNumber("frameRate", frameRate),
|
||||
...optionalString("codec", video?.codec_name),
|
||||
...optionalString("audioCodec", audio?.codec_name),
|
||||
};
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | undefined {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined;
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function finiteNumber(value: unknown): number | undefined {
|
||||
const number = typeof value === "number" ? value : typeof value === "string" ? Number(value) : NaN;
|
||||
return Number.isFinite(number) ? number : undefined;
|
||||
}
|
||||
|
||||
function finiteInteger(value: unknown): number | undefined {
|
||||
const number = finiteNumber(value);
|
||||
return number !== undefined && Number.isInteger(number) ? number : undefined;
|
||||
}
|
||||
|
||||
function rationalNumber(value: unknown): number | undefined {
|
||||
if (typeof value !== "string") return finiteNumber(value);
|
||||
const [numeratorText, denominatorText, extra] = value.split("/");
|
||||
if (extra !== undefined || numeratorText === undefined || denominatorText === undefined) {
|
||||
return finiteNumber(value);
|
||||
}
|
||||
const numerator = Number(numeratorText);
|
||||
const denominator = Number(denominatorText);
|
||||
if (!Number.isFinite(numerator) || !Number.isFinite(denominator) || denominator === 0) return undefined;
|
||||
return numerator / denominator;
|
||||
}
|
||||
|
||||
function optionalNumber<K extends string>(key: K, value: number | undefined): Partial<Record<K, number>> {
|
||||
return value === undefined ? {} : { [key]: value } as Record<K, number>;
|
||||
}
|
||||
|
||||
function optionalString<K extends string>(key: K, value: unknown): Partial<Record<K, string>> {
|
||||
return typeof value !== "string" || value === "" ? {} : { [key]: value } as Record<K, string>;
|
||||
}
|
||||
99
apps/media-worker/src/producer/overlay-png.test.ts
Normal file
99
apps/media-worker/src/producer/overlay-png.test.ts
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
/**
|
||||
* Overlay/card PNG renderer for the assemble lane: the api renderer's mechanism
|
||||
* (overlaySvg template → isolated sharp worker → PNG), relocated to the worker so
|
||||
* text becomes PIXELS offline — never a drawtext argument. The SVG-shape tests
|
||||
* are pure functions; the rasterization tests run only when sharp resolves
|
||||
* (SHARP_MODULE_PATH or an ambient install), the same guard the api's render
|
||||
* suite leans on.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { createRequire } from "node:module";
|
||||
import { cardSvg, overlayTemplateSvg, renderAssemblyArtwork } from "./overlay-png";
|
||||
import { parseAssembleInput, type EdlProject } from "./edl-plan";
|
||||
|
||||
const ASSET = "00000000-0000-4000-8000-000000000001";
|
||||
const EDL_ID = "00000000-0000-4000-8000-000000000010";
|
||||
|
||||
function project(overrides: Partial<EdlProject> = {}): EdlProject {
|
||||
return parseAssembleInput({
|
||||
edlProjectId: EDL_ID,
|
||||
project: {
|
||||
version: 1, aspect: "16:9", fit: "cover", fps: 30,
|
||||
items: [
|
||||
{ kind: "clip", assetId: ASSET, trimStartMs: 0, trimEndMs: 2_000, rotate: 0, volumeDb: 0 },
|
||||
{ kind: "card", text: "Solidarity Forever", durationMs: 2_000, style: "title" },
|
||||
],
|
||||
overlays: [{ overlayType: "lower_third", fields: { title: "Ada <Lovelace> & Sons" }, startMs: 0, endMs: 2_000 }],
|
||||
voiceover: null, outputLabel: "x",
|
||||
...overrides,
|
||||
},
|
||||
}).project;
|
||||
}
|
||||
|
||||
/** sharp resolution mirrors the module's own resolver (env override first). */
|
||||
function hasSharp(): boolean {
|
||||
const require = createRequire(__filename);
|
||||
try {
|
||||
require.resolve(process.env.SHARP_MODULE_PATH ?? "sharp");
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const PNG_MAGIC = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]);
|
||||
|
||||
test("overlay templates keep the api renderer's shape: rect + red accent + escaped text", () => {
|
||||
const { svg, width, height } = overlayTemplateSvg(project(), "lower_third", { title: 'Ada <Lovelace> & "Crew"' });
|
||||
assert.ok(svg.includes("<rect") && svg.includes("#FF1610") && svg.includes('fill="#171717"'));
|
||||
assert.ok(svg.includes("<Lovelace>"), "SVG-significant characters are escaped");
|
||||
assert.ok(svg.includes("& \"Crew\"".replace("& \"Crew\"", "& "Crew"")), "quotes escape too");
|
||||
assert.ok(!svg.includes("<Lovelace>"), "no raw markup can survive into the SVG");
|
||||
assert.ok(width < 1280 && height < 720, "a lower third is a bar, not the frame");
|
||||
});
|
||||
|
||||
test("a chyron is a full-width strip; style red swaps the fill", () => {
|
||||
const wide = overlayTemplateSvg(project(), "chyron", { title: "march" });
|
||||
assert.equal(wide.width, 1280);
|
||||
const red = overlayTemplateSvg(project(), "lower_third", { title: "x", style: "red" });
|
||||
assert.ok(red.svg.includes('fill="#FF1610"'));
|
||||
});
|
||||
|
||||
test("a card owns the whole canvas: #060403 ground + wrapped text, per style", () => {
|
||||
const title = cardSvg(project(), "Solidarity Forever", "title");
|
||||
assert.equal(title.width, 1280);
|
||||
assert.equal(title.height, 720);
|
||||
assert.ok(title.svg.includes('fill="#060403"'), "the identity page color is the ground");
|
||||
assert.ok(title.svg.includes("Solidarity Forever"));
|
||||
const portrait = cardSvg(project({ aspect: "9:16" }), "x", "chapter");
|
||||
assert.equal(portrait.width, 1080);
|
||||
assert.equal(portrait.height, 1920);
|
||||
// Long text wraps at a character budget instead of overflowing the frame.
|
||||
const wrapped = cardSvg(project(), "a".repeat(60), "title");
|
||||
assert.ok((wrapped.svg.match(/<text /g) ?? []).length > 1, "60 chars on a title card must wrap to multiple lines");
|
||||
});
|
||||
|
||||
test("rasterized artwork is real PNGs at the canvas geometry", { skip: !hasSharp() ? "needs sharp (SHARP_MODULE_PATH or ambient install)" : false }, async () => {
|
||||
const { cards, overlays } = await renderAssemblyArtwork(parseAssembleInput({
|
||||
edlProjectId: EDL_ID,
|
||||
project: {
|
||||
version: 1, aspect: "9:16", fit: "cover", fps: 30,
|
||||
items: [
|
||||
{ kind: "clip", assetId: ASSET, trimStartMs: 0, trimEndMs: 1_000, rotate: 0, volumeDb: 0 },
|
||||
{ kind: "card", text: "Chapter", durationMs: 1_500, style: "chapter" },
|
||||
],
|
||||
overlays: [{ overlayType: "chyron", fields: { title: "BREAKING" }, startMs: 0, endMs: 1_000 }],
|
||||
voiceover: null, outputLabel: "x",
|
||||
},
|
||||
}));
|
||||
assert.equal(cards.length, 1);
|
||||
assert.equal(overlays.length, 1);
|
||||
for (const png of [...cards, ...overlays]) {
|
||||
assert.ok(png.bytes.subarray(0, 8).equals(PNG_MAGIC), "renderer output is a PNG");
|
||||
assert.match(png.name, /^[0-9a-f]{64}\.png$/, "scratch names are content hashes");
|
||||
}
|
||||
// The card PNG is the full portrait canvas (the renderer wrote it at 1080x1920).
|
||||
assert.equal(cards[0]!.bytes.readUInt32BE(16), 1080);
|
||||
assert.equal(cards[0]!.bytes.readUInt32BE(20), 1920);
|
||||
});
|
||||
193
apps/media-worker/src/producer/overlay-png.ts
Normal file
193
apps/media-worker/src/producer/overlay-png.ts
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
/**
|
||||
* Card + overlay PNG rasterization for the assemble lane (editor-v2 L1).
|
||||
*
|
||||
* THE MECHANISM IS THE API'S, VERBATIM IN SHAPE: apps/api/src/production/render.ts
|
||||
* (overlaySvg → an isolated sharp worker thread → PNG). That is exactly how
|
||||
* compose-path overlay pixels are produced today: production.service.ts renders
|
||||
* an overlay through `renderOverlay`, seals the PNG into object storage, and
|
||||
* ffmpeg only ever sees the sha-named asset (`overlay_revisions.asset_id`).
|
||||
*
|
||||
* The worker needs the same pixels OFFLINE (it has no api to call, and the plan
|
||||
* forbids drawtext for user text outright), so it runs the renderer's exact SVG
|
||||
* templates in its own isolated sharp worker — same template markup, same
|
||||
* geometry, same escape function, same resource limits — and writes the PNG to
|
||||
* the job's scratch dir under a content-hash name.
|
||||
*
|
||||
* sharp itself is resolved like the api resolves it (SHARP_MODULE_PATH, else the
|
||||
* ambient install): the worker image carries the same binary the api does, and a
|
||||
* box without it fails the JOB as `render unavailable` — never a fallback to
|
||||
* drawtext, because a crew must never ship text that was not renderer-rasterized.
|
||||
*/
|
||||
import { Worker } from "node:worker_threads";
|
||||
import { createHash } from "node:crypto";
|
||||
import { createRequire } from "node:module";
|
||||
import { EDL_ASPECTS, type AssembleInput, type EdlProject } from "./edl-plan";
|
||||
|
||||
export const OVERLAY_RENDER_TIMEOUT_MS = 5_000;
|
||||
const MAX_PNG_BYTES = 8_000_000;
|
||||
|
||||
/** The api renderer's escape (render.ts): the only five SVG-significant characters. */
|
||||
function escapeXml(value: string): string {
|
||||
return value.replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[c] ?? c);
|
||||
}
|
||||
|
||||
/** The api's overlay-text charset re-check (belt and braces; edl-plan checked it too). */
|
||||
function overlayText(value: string): string {
|
||||
if (!/^[\p{L}\p{N}\p{P}\p{Zs}\p{M}\p{S}]*$/u.test(value)) throw new Error("invalid overlay text");
|
||||
return value;
|
||||
}
|
||||
|
||||
export interface RenderedPng { bytes: Buffer; /** Content-hash scratch filename: <64hex>.png */ name: string }
|
||||
|
||||
/**
|
||||
* The api's lower-third/chyron template, scaled to the assemble canvas: the same
|
||||
* rect + red accent + DejaVu text markup render.ts emits, at the target canvas
|
||||
* geometry so a 9:16 timeline gets a 9:16-wide bar. A logo overlay names a
|
||||
* sanitized geometric SVG (fields.svg), which the api path rasterizes through
|
||||
* the very same worker call — so it is accepted here identically.
|
||||
*/
|
||||
export function overlayTemplateSvg(project: EdlProject, overlayType: string, fields: Record<string, string>): { svg: string; width: number; height: number } {
|
||||
const { w, h } = EDL_ASPECTS[project.aspect];
|
||||
if (overlayType === "logo") {
|
||||
const svg = fields.svg ?? "";
|
||||
if (svg.length > 64_000) throw new Error("unsafe SVG");
|
||||
return { svg, width: Math.min(512, w), height: Math.min(512, h) };
|
||||
}
|
||||
// Chyron is a full-width strip; a lower third is a bar in the lower third.
|
||||
const width = overlayType === "chyron" ? w : Math.round(w * 0.9);
|
||||
const height = overlayType === "chyron" ? Math.max(96, Math.round(h / 12)) : Math.max(120, Math.round(h / 6));
|
||||
const fill = fields.style === "red" ? "#FF1610" : "#171717";
|
||||
const titleSize = Math.round(height * 0.38);
|
||||
const subtitleSize = Math.round(height * 0.24);
|
||||
const title = escapeXml(overlayText(fields.title ?? ""));
|
||||
const subtitle = escapeXml(overlayText(fields.subtitle ?? ""));
|
||||
return {
|
||||
width, height,
|
||||
svg: `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}"><rect x="0" y="0" width="${width}" height="${height}" rx="12" fill="${fill}"/><rect width="14" height="${height}" fill="#FF1610"/><text x="48" y="${Math.round(height * 0.42)}" font-family="DejaVu Sans" font-size="${titleSize}" fill="white">${title}</text><text x="48" y="${Math.round(height * 0.72)}" font-family="DejaVu Sans" font-size="${subtitleSize}" fill="white">${subtitle}</text></svg>`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A card owns the FULL FRAME for its duration (plan d2), so its PNG is the whole
|
||||
* canvas: #060403 ground, red accent bar, the card text set in the renderer's
|
||||
* font stack. `title` is the hero line; `chapter` is the quieter section mark.
|
||||
*/
|
||||
export function cardSvg(project: EdlProject, text: string, style: "title" | "chapter"): { svg: string; width: number; height: number } {
|
||||
const { w, h } = EDL_ASPECTS[project.aspect];
|
||||
const titleSize = Math.round(w * (style === "title" ? 0.072 : 0.045));
|
||||
const lineHeight = Math.round(titleSize * 1.25);
|
||||
const lines = wrapLines(text, style === "title" ? 22 : 30);
|
||||
const blockHeight = lines.length * lineHeight;
|
||||
const startY = Math.round(h / 2 - blockHeight / 2 + titleSize);
|
||||
const body = lines.map((line, i) =>
|
||||
`<text x="${Math.round(w * 0.09)}" y="${startY + i * lineHeight}" font-family="DejaVu Sans" font-size="${titleSize}" fill="white">${escapeXml(line)}</text>`).join("");
|
||||
const accentHeight = style === "title" ? Math.round(h * 0.5) : Math.round(h * 0.16);
|
||||
const accentY = Math.round((h - accentHeight) / 2);
|
||||
const svg = `<svg xmlns="http://www.w3.org/2000/svg" width="${w}" height="${h}"><rect width="${w}" height="${h}" fill="#060403"/>${style === "title" ? `<rect x="0" y="${accentY}" width="${Math.round(w * 0.012)}" height="${accentHeight}" fill="#FF1610"/>` : `<rect x="${Math.round(w * 0.09)}" y="${startY - titleSize - Math.round(titleSize * 0.6)}" width="${Math.round(w * 0.08)}" height="4" fill="#FF1610"/>`}${body}</svg>`;
|
||||
return { svg, width: w, height: h };
|
||||
}
|
||||
|
||||
/** Simple word wrap at a character budget (the card text cap is 120 chars total). */
|
||||
function wrapLines(text: string, maxChars: number): string[] {
|
||||
// Unbreakable words longer than the budget (hashtags, links) hard-split —
|
||||
// a 60-char wall of text must never overflow the card frame.
|
||||
const words: string[] = [];
|
||||
for (const raw of text.split(/\s+/)) {
|
||||
if (raw.length <= maxChars) { words.push(raw); continue; }
|
||||
for (let i = 0; i < raw.length; i += maxChars) words.push(raw.slice(i, i + maxChars));
|
||||
}
|
||||
if (words.length === 0) return [""];
|
||||
const lines: string[] = [];
|
||||
let current = "";
|
||||
for (const word of words) {
|
||||
if (current === "") current = word;
|
||||
else if ((current + " " + word).length <= maxChars) current += " " + word;
|
||||
else { lines.push(current); current = word; }
|
||||
}
|
||||
if (current !== "") lines.push(current);
|
||||
return lines.slice(0, 5);
|
||||
}
|
||||
|
||||
function sharpPath(): string {
|
||||
// eslint-disable-next-line no-restricted-syntax -- createRequire is how the api resolves sharp too.
|
||||
const require = createRequire(__filename);
|
||||
if (process.env.SHARP_MODULE_PATH) return require.resolve(process.env.SHARP_MODULE_PATH);
|
||||
try { return require.resolve("sharp"); } catch {
|
||||
throw new Error("sharp not found — install it or point SHARP_MODULE_PATH at a sharp build");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The api's RENDER_SCRIPT, character-for-character in behavior: an isolated
|
||||
* worker thread, sharp's cache off and concurrency pinned to 1, contain-fit
|
||||
* resize onto a transparent ground, PNG out, bounded wall time, no credentials
|
||||
* in the thread's environment (render.ts's exact env allowlist).
|
||||
*/
|
||||
const RENDER_SCRIPT = "const {parentPort,workerData}=require('node:worker_threads');const sharp=require(workerData.modulePath);sharp.cache(false);sharp.concurrency(1);sharp(Buffer.from(workerData.svg),{limitInputPixels:16777216}).resize(workerData.width,workerData.height,{fit:'contain',background:{r:0,g:0,b:0,alpha:0}}).png().toBuffer().then(png=>parentPort.postMessage(png)).catch(()=>{throw new Error('PNG rasterization failed')})";
|
||||
|
||||
let activeJobs = 0;
|
||||
const MAX_CONCURRENT_RENDERS = 4;
|
||||
|
||||
async function rasterize(svg: string, width: number, height: number): Promise<Buffer> {
|
||||
if (activeJobs >= MAX_CONCURRENT_RENDERS) throw new Error("overlay render capacity exceeded");
|
||||
activeJobs += 1;
|
||||
try {
|
||||
return await new Promise<Buffer>((resolve, reject) => {
|
||||
const worker = new Worker(RENDER_SCRIPT, {
|
||||
eval: true,
|
||||
workerData: { svg, width, height, modulePath: sharpPath() },
|
||||
resourceLimits: { maxOldGenerationSizeMb: 128, maxYoungGenerationSizeMb: 16, stackSizeMb: 4 },
|
||||
env: { PATH: process.env.PATH, HOME: process.env.HOME, LANG: "C.UTF-8" },
|
||||
});
|
||||
let settled = false;
|
||||
const finish = (error: Error | null, body?: Buffer): void => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
void worker.terminate();
|
||||
if (error !== null || body === undefined) reject(error ?? new Error("overlay render failed"));
|
||||
else resolve(body);
|
||||
};
|
||||
const timer = setTimeout(() => finish(new Error("overlay render timed out")), OVERLAY_RENDER_TIMEOUT_MS);
|
||||
worker.once("message", (message: unknown) => {
|
||||
if (!(message instanceof Uint8Array) || message.byteLength > MAX_PNG_BYTES || message.byteLength < 8) finish(new Error("invalid overlay render output"));
|
||||
else finish(null, Buffer.from(message));
|
||||
});
|
||||
worker.once("error", (error: Error) => finish(error));
|
||||
worker.once("exit", () => finish(new Error("overlay render exited before producing PNG")));
|
||||
});
|
||||
} finally {
|
||||
activeJobs -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
async function render(svg: string, width: number, height: number): Promise<RenderedPng> {
|
||||
const bytes = await rasterize(svg, width, height);
|
||||
return { bytes, name: `${createHash("sha256").update(bytes).digest("hex")}.png` };
|
||||
}
|
||||
|
||||
/** Render every card of a project to a full-frame PNG (in timeline item order). */
|
||||
export async function renderCards(project: EdlProject): Promise<RenderedPng[]> {
|
||||
const out: RenderedPng[] = [];
|
||||
for (const entry of project.items) {
|
||||
if (entry.kind !== "card") continue;
|
||||
const { svg, width, height } = cardSvg(project, entry.text, entry.style);
|
||||
out.push(await render(svg, width, height));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Render every timed overlay to its PNG (in overlay order). */
|
||||
export async function renderOverlays(project: EdlProject): Promise<RenderedPng[]> {
|
||||
const out: RenderedPng[] = [];
|
||||
for (const overlay of project.overlays) {
|
||||
const { svg, width, height } = overlayTemplateSvg(project, overlay.overlayType, overlay.fields);
|
||||
out.push(await render(svg, width, height));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Convenience: cards + overlays in one pass (the runner's call). */
|
||||
export async function renderAssemblyArtwork(input: AssembleInput): Promise<{ cards: RenderedPng[]; overlays: RenderedPng[] }> {
|
||||
return { cards: await renderCards(input.project), overlays: await renderOverlays(input.project) };
|
||||
}
|
||||
|
|
@ -107,6 +107,15 @@ function claimRow(overrides: Record<string, unknown> = {}): Record<string, unkno
|
|||
};
|
||||
}
|
||||
|
||||
/** 039's ingest row: kind 'probe', input_json NULL — the direct-to-R2 queue. */
|
||||
function probeRow(overrides: Record<string, unknown> = {}): Record<string, unknown> {
|
||||
return {
|
||||
id: JOB, workspace_id: WS, kind: "probe", input_json: null,
|
||||
video_asset_id: ASSET,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function assetRow(): Record<string, unknown> {
|
||||
return {
|
||||
id: ASSET, storage_key: "sha256/" + "a".repeat(64) + ".mp4", mime_type: "video/mp4",
|
||||
|
|
@ -137,7 +146,34 @@ test("claim goes through the definer function with typed lane + lease params", a
|
|||
assert.equal(job!.workspaceId, WS);
|
||||
assert.equal(job!.source.storageKey.startsWith("sha256/"), true);
|
||||
assert.equal(job!.source.hasAudio, true);
|
||||
assert.equal(job!.request.overlays[0]!.kind, "lower_third");
|
||||
assert.equal(job!.request!.overlays[0]!.kind, "lower_third");
|
||||
});
|
||||
|
||||
test("a probed audio-less asset reports hasAudio:false so the runner uses a silent bed", async () => {
|
||||
// The probe writes duration_ms and audio_codec together: audio_codec NULL with
|
||||
// a duration means probed-and-silent (muted screen recording, GIF conversion).
|
||||
// Before the fix this mapped to hasAudio:null → audioBed "source" → ffmpeg
|
||||
// "Stream specifier ':a' matches no streams".
|
||||
const pool = new FakePool();
|
||||
pool.expectClient({ rows: [{ ...assetRow(), audio_codec: null }] });
|
||||
pool.reply([claimRow()]);
|
||||
|
||||
const job = await storeOf(pool).claim("local");
|
||||
|
||||
assert.ok(job !== null);
|
||||
assert.equal(job.source.hasAudio, false, "probed + audio_codec NULL must be hasAudio:false");
|
||||
// The runner turns hasAudio===false into audioBed "none" (synthetic silence).
|
||||
});
|
||||
|
||||
test("an unprobed asset (NULL duration AND NULL codec) stays hasAudio:null", async () => {
|
||||
const pool = new FakePool();
|
||||
pool.expectClient({ rows: [{ ...assetRow(), audio_codec: null, duration_ms: null }] });
|
||||
pool.reply([claimRow()]);
|
||||
|
||||
const job = await storeOf(pool).claim("local");
|
||||
|
||||
assert.ok(job !== null);
|
||||
assert.equal(job.source.hasAudio, null, "unprobed is unknown, not audio-less — the source bed stays");
|
||||
});
|
||||
|
||||
test("claim passes the lane through: a remote worker binds lane='remote'", async () => {
|
||||
|
|
@ -279,6 +315,109 @@ test("a mix row's voiceover request survives the claim", async () => {
|
|||
})]);
|
||||
const job: ClaimedProduceJob | null = await storeOf(pool).claim("local");
|
||||
assert.equal(job!.kind, "mix");
|
||||
assert.equal(job!.request.voiceover!.assetId, VOICEOVER);
|
||||
assert.equal(job!.request.voiceover!.duckOriginal, true);
|
||||
assert.equal(job!.request!.voiceover!.assetId, VOICEOVER);
|
||||
assert.equal(job!.request!.voiceover!.duckOriginal, true);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Probe lane (direct-to-R2 ingest): the worker claims kind 'probe' rows and
|
||||
// ffprobes the sealed source itself, so the api box never touches video bytes.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test("claim returns a queued probe row with request null and its source asset", async () => {
|
||||
const pool = new FakePool();
|
||||
pool.expectClient({ rows: [assetRow()] });
|
||||
pool.reply([probeRow()]);
|
||||
|
||||
const job = await storeOf(pool).claim("local");
|
||||
|
||||
assert.ok(job !== null, "a probe row is claimable, not skipped");
|
||||
assert.equal(job.kind, "probe");
|
||||
assert.equal(job.request, null, "a probe row carries no input_json to parse");
|
||||
assert.equal(job.jobId, JOB);
|
||||
assert.equal(job.workspaceId, WS);
|
||||
assert.equal(job.source.storageKey, assetRow().storage_key);
|
||||
});
|
||||
|
||||
test("a probe row that somehow carries input_json is failed, never stranded", async () => {
|
||||
const pool = new FakePool();
|
||||
pool.reply([probeRow({ input_json: { overlays: [] } })]);
|
||||
const client = pool.expectClient();
|
||||
|
||||
const job = await storeOf(pool).claim("local");
|
||||
|
||||
assert.equal(job, null);
|
||||
const failing = client.calls.find((c) => /state = 'failed'/.test(c.sql))!;
|
||||
assert.ok(failing !== undefined);
|
||||
assert.match(failing.params[1] as string, /probe row carries input_json/);
|
||||
});
|
||||
|
||||
test("a truly unknown kind is still failed as unknown", async () => {
|
||||
const pool = new FakePool();
|
||||
pool.reply([claimRow({ kind: "transmogrify" })]);
|
||||
const client = pool.expectClient();
|
||||
|
||||
const job = await storeOf(pool).claim("local");
|
||||
|
||||
assert.equal(job, null);
|
||||
const failing = client.calls.find((c) => /state = 'failed'/.test(c.sql))!;
|
||||
assert.match(failing.params[1] as string, /unknown kind/);
|
||||
});
|
||||
|
||||
test("completeProbe writes the ffprobe columns on the source's own row, then completes the job", async () => {
|
||||
const pool = new FakePool();
|
||||
pool.reply([{ workspace_id: WS }]); // produce_job_workspace
|
||||
const client = pool.expectClient(
|
||||
{ rows: [{ storage_key: "direct/" + JOB + ".enc" }] }, // UPDATE video_assets
|
||||
{ rows: [{ id: JOB }] }, // UPDATE video_processing_jobs
|
||||
);
|
||||
|
||||
await storeOf(pool).completeProbe(JOB, ASSET, {
|
||||
width: 1920, height: 1080, durationMs: 61_000, frameRate: 29.97, codec: "h264", audioCodec: "aac",
|
||||
});
|
||||
|
||||
const setLocal = client.calls.find((c) => /set_config\('app\.workspace_id'/.test(c.sql))!;
|
||||
assert.equal(setLocal.params[0], WS, "the metadata write is workspace-scoped");
|
||||
const assetUpdate = client.calls.find((c) => /UPDATE video_assets/.test(c.sql))!;
|
||||
assert.deepEqual(
|
||||
assetUpdate.params.slice(0, 7),
|
||||
[ASSET, 1920, 1080, 61_000, 29.97, "h264", "aac"],
|
||||
"039's column set, in the api's probeVideo order",
|
||||
);
|
||||
assert.match(assetUpdate.sql, /SET width = \$2, height = \$3, duration_ms = \$4/);
|
||||
const jobUpdate = client.calls.find((c) => /UPDATE video_processing_jobs/.test(c.sql))!;
|
||||
assert.equal(jobUpdate.params[0], JOB);
|
||||
assert.equal(jobUpdate.params[1], "direct/" + JOB + ".enc", "output_key points at the retained object");
|
||||
assert.match(jobUpdate.sql, /state = 'complete', progress_percent = 100/);
|
||||
assert.match(client.calls.at(-1)!.sql, /COMMIT/);
|
||||
assert.equal(client.released(), 1);
|
||||
});
|
||||
|
||||
test("completeProbe throws when the source asset is gone (tenant-deleted mid-probe)", async () => {
|
||||
const pool = new FakePool();
|
||||
pool.reply([{ workspace_id: WS }]);
|
||||
pool.expectClient({ rows: [] }); // UPDATE video_assets matches nothing
|
||||
|
||||
await assert.rejects(
|
||||
() => storeOf(pool).completeProbe(JOB, ASSET, {
|
||||
width: null, height: null, durationMs: null, frameRate: null, codec: null, audioCodec: null,
|
||||
}),
|
||||
/source asset .* not found/,
|
||||
);
|
||||
});
|
||||
|
||||
test("completeProbe throws when the job lost its claim", async () => {
|
||||
const pool = new FakePool();
|
||||
pool.reply([{ workspace_id: WS }]);
|
||||
pool.expectClient(
|
||||
{ rows: [{ storage_key: "direct/" + JOB + ".enc" }] },
|
||||
{ rows: [] }, // the job UPDATE matches nothing: the lease was reclaimed
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
() => storeOf(pool).completeProbe(JOB, ASSET, {
|
||||
width: null, height: null, durationMs: null, frameRate: null, codec: null, audioCodec: null,
|
||||
}),
|
||||
/lost its claim/,
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -18,9 +18,10 @@
|
|||
import type { Pool, PoolClient } from "pg";
|
||||
import type { Lane, ProduceJobKind } from "./command-builder";
|
||||
import { parseProduceRequest } from "./command-builder";
|
||||
import type { ClaimedProduceJob, ProduceJobStore, ProduceOutput, ProduceSourceAsset } from "./store";
|
||||
import { parseAssembleInput, type AssembleInput } from "./edl-plan";
|
||||
import type { ClaimedProduceJob, ProduceJobStore, ProduceOutput, ProduceSourceAsset, ProbeResult } from "./store";
|
||||
|
||||
const KINDS: readonly ProduceJobKind[] = ["compose", "mix"];
|
||||
const KINDS: readonly ProduceJobKind[] = ["probe", "compose", "mix", "assemble"];
|
||||
|
||||
/**
|
||||
* A claimed `transcoding` row older than this is assumed ownerless (the worker
|
||||
|
|
@ -47,6 +48,11 @@ function uuid(value: unknown, field: string): string {
|
|||
return text;
|
||||
}
|
||||
|
||||
/** True once 009's probe job has run: the probe writes duration + codecs together, so duration≠NULL means probed. */
|
||||
function probed(row: Record<string, unknown>): boolean {
|
||||
return row.duration_ms !== null && row.duration_ms !== undefined;
|
||||
}
|
||||
|
||||
function nullableNumber(value: unknown): number | null {
|
||||
if (value === null || value === undefined) return null;
|
||||
const n = typeof value === "number" ? value : Number(value);
|
||||
|
|
@ -63,9 +69,11 @@ function sourceFromRow(row: Record<string, unknown>): ProduceSourceAsset {
|
|||
height: nullableNumber(row.height),
|
||||
durationMs: row.duration_ms === null || row.duration_ms === undefined ? null : Math.round(Number(row.duration_ms)),
|
||||
frameRate: nullableNumber(row.frame_rate),
|
||||
// Probed "no audio track" (009 stores an empty codec) → a silent bed, so the
|
||||
// output always has an audio stream.
|
||||
hasAudio: row.audio_codec === null || row.audio_codec === undefined ? null : String(row.audio_codec) !== "",
|
||||
// An unprobed asset has NULL codec AND NULL duration (009's probe fills all
|
||||
// of them at once); a probed one always has a duration. So the probed
|
||||
// audio-less case is duration≠NULL + audio_codec NULL/'' — which must become
|
||||
// a silent bed, or the graph references [0:a] on a file with no audio track.
|
||||
hasAudio: probed(row) ? String(row.audio_codec ?? "") !== "" : null,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -102,24 +110,66 @@ export class PostgresProduceJobStore implements ProduceJobStore {
|
|||
if (claimed.rowCount === 0) return null;
|
||||
const row = claimed.rows[0]!;
|
||||
const workspaceId = uuid(row.workspace_id, "workspace id");
|
||||
const kind = row.kind === "mix" ? "mix" : row.kind === "compose" ? "compose" : null;
|
||||
const kind = row.kind === "mix" ? "mix" : row.kind === "compose" ? "compose"
|
||||
: row.kind === "probe" ? "probe" : row.kind === "assemble" ? "assemble" : null;
|
||||
let request = null;
|
||||
if (kind !== null) {
|
||||
let assemble: AssembleInput | null = null;
|
||||
if (kind === "compose" || kind === "mix") {
|
||||
try {
|
||||
request = parseProduceRequest(row.input_json, kind);
|
||||
} catch {
|
||||
request = null;
|
||||
}
|
||||
if (request === null) {
|
||||
// A row the worker cannot honour must FAIL here — never strand in `transcoding`.
|
||||
// A null return (not a throw) commits the failure instead of rolling it back.
|
||||
await this.tenant(workspaceId, (client) =>
|
||||
client.query(
|
||||
`UPDATE video_processing_jobs
|
||||
SET state = 'failed', progress_percent = 100, error = $2, updated_at = now()
|
||||
WHERE id = $1 AND state = 'transcoding'`,
|
||||
[row.id, "produce invalid request"],
|
||||
));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
if (kind === null || request === null) {
|
||||
// A row the worker cannot honour must FAIL here — never strand in `transcoding`.
|
||||
// A null return (not a throw) commits the failure instead of rolling it back.
|
||||
if (kind === "assemble") {
|
||||
// The assemble lane's payload is the EDL envelope (043); parseAssembleInput is
|
||||
// deliberately lenient — unknown shapes (a version the api knows and this
|
||||
// build does not) fail the row HERE with a bounded reason, exactly like an
|
||||
// undecodable compose request: never a strand, never a crash.
|
||||
try {
|
||||
assemble = parseAssembleInput(row.input_json);
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.message : "undecodable input_json";
|
||||
await this.tenant(workspaceId, (client) =>
|
||||
client.query(
|
||||
`UPDATE video_processing_jobs
|
||||
SET state = 'failed', progress_percent = 100, error = $2, updated_at = now()
|
||||
WHERE id = $1 AND state = 'transcoding'`,
|
||||
[row.id, `assemble invalid request: ${reason.slice(0, 160)}`],
|
||||
));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
if (kind === null) {
|
||||
await this.tenant(workspaceId, (client) =>
|
||||
client.query(
|
||||
`UPDATE video_processing_jobs
|
||||
SET state = 'failed', progress_percent = 100, error = $2, updated_at = now()
|
||||
WHERE id = $1 AND state = 'transcoding'`,
|
||||
[row.id, kind === null ? "produce job store: unknown kind" : "produce invalid request"],
|
||||
[row.id, "produce job store: unknown kind"],
|
||||
));
|
||||
return null;
|
||||
}
|
||||
if (kind === "probe" && row.input_json !== null) {
|
||||
// Probe rows carry no request payload; input_json means upstream wrote it wrong.
|
||||
await this.tenant(workspaceId, (client) =>
|
||||
client.query(
|
||||
`UPDATE video_processing_jobs
|
||||
SET state = 'failed', progress_percent = 100, error = $2, updated_at = now()
|
||||
WHERE id = $1 AND state = 'transcoding'`,
|
||||
[row.id, "produce job store: probe row carries input_json"],
|
||||
));
|
||||
return null;
|
||||
}
|
||||
|
|
@ -145,6 +195,7 @@ export class PostgresProduceJobStore implements ProduceJobStore {
|
|||
workspaceId,
|
||||
kind,
|
||||
request,
|
||||
assemble,
|
||||
source: sourceFromRow(assets.rows[0] as Record<string, unknown>),
|
||||
claimedAt: this.now().toISOString(),
|
||||
};
|
||||
|
|
@ -185,6 +236,36 @@ export class PostgresProduceJobStore implements ProduceJobStore {
|
|||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* A probe job's terminal write: the ffprobe metadata onto the source's OWN
|
||||
* video_assets row (039's column set — the api's probeVideo writes exactly
|
||||
* these), then the job to complete 100 with the retained object as its
|
||||
* output. Runs inside the job's tenant transaction like every post-claim
|
||||
* write, so FORCE RLS still binds it as sp_app.
|
||||
*/
|
||||
async completeProbe(jobId: string, sourceAssetId: string, result: ProbeResult): Promise<void> {
|
||||
const workspaceId = await this.workspaceOf(jobId);
|
||||
await this.tenant(workspaceId, async (client) => {
|
||||
const updated = await client.query(
|
||||
`UPDATE video_assets
|
||||
SET width = $2, height = $3, duration_ms = $4, frame_rate = $5,
|
||||
codec = $6, audio_codec = $7
|
||||
WHERE id = $1 RETURNING storage_key`,
|
||||
[sourceAssetId, result.width, result.height, result.durationMs, result.frameRate, result.codec, result.audioCodec],
|
||||
);
|
||||
if (updated.rowCount === 0) throw new Error(`produce job store: source asset ${sourceAssetId} not found`);
|
||||
const completed = await client.query(
|
||||
`UPDATE video_processing_jobs
|
||||
SET state = 'complete', progress_percent = 100,
|
||||
output_key = $2, error = NULL, updated_at = now()
|
||||
WHERE id = $1 AND state = 'transcoding'
|
||||
RETURNING id`,
|
||||
[jobId, updated.rows[0]?.storage_key],
|
||||
);
|
||||
if (completed.rowCount === 0) throw new Error(`produce job store: job ${jobId} lost its claim`);
|
||||
});
|
||||
}
|
||||
|
||||
async fail(jobId: string, reason: string): Promise<void> {
|
||||
await this.tenant(await this.workspaceOf(jobId), async (client) => {
|
||||
await client.query(
|
||||
|
|
@ -207,6 +288,23 @@ export class PostgresProduceJobStore implements ProduceJobStore {
|
|||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The assemble lane's asset read: the FULL row, because the runner needs the
|
||||
* probe metadata (durationMs is the pre-spawn trim gate's authority, audio_codec
|
||||
* decides the clip's audio bed). Same tenant scope as every post-claim read.
|
||||
*/
|
||||
async assembleAsset(workspaceId: string, assetId: string): Promise<ProduceSourceAsset | null> {
|
||||
return this.tenant(workspaceId, async (client) => {
|
||||
const rows = await client.query(
|
||||
`SELECT id, storage_key, mime_type, size_bytes, width, height, duration_ms, frame_rate, audio_codec
|
||||
FROM video_assets WHERE id = $1`,
|
||||
[assetId],
|
||||
);
|
||||
if (rows.rowCount === 0) return null;
|
||||
return sourceFromRow(rows.rows[0] as Record<string, unknown>);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* A claimed job's workspace, for the tenant-scoped follow-up writes: the 040
|
||||
* definer lookup (012's live_monitor_source_workspace shape), because an
|
||||
|
|
|
|||
|
|
@ -43,6 +43,17 @@ function makeClip(path: string): void {
|
|||
|
||||
function key(bytes: Buffer): string { return `sha256/${createHash("sha256").update(bytes).digest("hex")}.mp4`; }
|
||||
|
||||
/**
|
||||
* The audio-less class from the bug report: video only, no lavfi sine, no audio
|
||||
* track at all (muted screen recording / GIF conversion / synthetic test clip).
|
||||
*/
|
||||
function makeSilentClip(path: string): void {
|
||||
const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y",
|
||||
"-f", "lavfi", "-i", "testsrc2=s=320x180:r=30:d=2",
|
||||
"-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", "-movflags", "+faststart", path]);
|
||||
if (r.status !== 0) throw new Error(`silent fixture encode failed: ${r.stderr?.toString() ?? ""}`);
|
||||
}
|
||||
|
||||
function doubles() {
|
||||
const events: Array<{ type: string; jobId?: string; detail?: string }> = [];
|
||||
const jobs: Array<{ jobId: string; state: string; reason?: string; output?: { storageKey: string; producedAssetId: string } }> = [];
|
||||
|
|
@ -76,8 +87,19 @@ function doubles() {
|
|||
const storageKey = assetKeys.get(assetId);
|
||||
return storageKey === undefined ? null : { storageKey, mimeType: "audio/mp4" };
|
||||
},
|
||||
async completeProbe() { /* probe completions are asserted in the pg-store suite */ },
|
||||
async assembleAsset(_workspaceId, assetId) {
|
||||
const storageKey = assetKeys.get(assetId);
|
||||
return storageKey === undefined ? null
|
||||
: { id: assetId, storageKey, mimeType: "video/mp4", sizeBytes: 1, width: 1280, height: 720, durationMs: 30_000, frameRate: 30, hasAudio: true };
|
||||
},
|
||||
};
|
||||
const storage: ProduceObjectStorage = {
|
||||
async getSealedObject(k) {
|
||||
const body = objects.get(k);
|
||||
if (body === undefined) throw new Error("not found");
|
||||
return body;
|
||||
},
|
||||
async getObject(k) {
|
||||
const body = objects.get(k);
|
||||
if (body === undefined) throw new Error("not found");
|
||||
|
|
@ -94,6 +116,7 @@ function job(input: string, kind: "compose" | "mix", request: unknown, source: P
|
|||
workspaceId: WS,
|
||||
kind,
|
||||
request: parseProduceRequest(request, kind),
|
||||
assemble: null,
|
||||
source: {
|
||||
id: UUID(1), storageKey: input, mimeType: "video/mp4", sizeBytes: 1,
|
||||
width: 320, height: 180, durationMs: 2000, frameRate: 30, hasAudio: true, ...source,
|
||||
|
|
@ -132,6 +155,32 @@ test("compose job: encode, publish, derived asset, complete", { skip: !hasFfmpeg
|
|||
assert.ok(body !== undefined && body.subarray(4, 8).toString("latin1") === "ftyp");
|
||||
});
|
||||
|
||||
test("an audio-less source produces output WITH audio (synthetic bed), not a [0:a] crash", { skip: !hasFfmpeg() || !hasFont() ? "needs ffmpeg + a ttf font" : false }, async () => {
|
||||
const d = doubles();
|
||||
const clip = join(SCRATCH, "source-silent.mp4");
|
||||
makeSilentClip(clip);
|
||||
const bytes = await import("node:fs/promises").then((fs) => fs.readFile(clip));
|
||||
const inputKey = key(bytes);
|
||||
d.objects.set(inputKey, bytes);
|
||||
// hasAudio:false is what the pg store reports for a probed audio-less asset.
|
||||
d.setQueue([job(inputKey, "compose",
|
||||
{ overlays: [{ kind: "lower_third", text: "Ada Lovelace", position: { x: 8, y: 85 } }], lane: "local" },
|
||||
{ hasAudio: false, durationMs: 2000 })]);
|
||||
|
||||
await runner(d).pollOnce();
|
||||
|
||||
assert.equal(d.jobs[0]?.state, "complete", d.jobs[0]?.reason);
|
||||
assert.ok(d.jobs[0]?.output?.storageKey.startsWith("sha256/"));
|
||||
// The produced mp4 must actually carry an audio stream — the whole point of the bed.
|
||||
const body = d.objects.get(d.published()!.key);
|
||||
assert.ok(body !== undefined && body.subarray(4, 8).toString("latin1") === "ftyp");
|
||||
const probe = spawnSync("ffprobe", ["-v", "error", "-print_format", "json", "-show_streams", "-i", `pipe:0`],
|
||||
{ input: body, encoding: "buffer" });
|
||||
const streams = JSON.parse(probe.stdout.toString("utf8")).streams as Array<{ codec_type: string }>;
|
||||
assert.ok(streams.some((s) => s.codec_type === "audio"), "output must have an audio stream");
|
||||
assert.ok(streams.some((s) => s.codec_type === "video"), "output must keep its video stream");
|
||||
});
|
||||
|
||||
test("a mix job with a voiceover runs the ducking path end to end", { skip: !hasFfmpeg() || !hasFont() ? "needs ffmpeg + a ttf font" : false }, async () => {
|
||||
const d = doubles();
|
||||
const clip = join(SCRATCH, "source-mix.mp4");
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
/**
|
||||
* Produce runner: the compose/mix job executor.
|
||||
* Produce runner: the compose/mix job executor, the ingest probe lane, and the
|
||||
* editor-v2 assemble lane.
|
||||
*
|
||||
* Poll loop in the supervisor's shape (one claim per tick, retry with backoff on
|
||||
* store errors, stop flag honoured between ticks), one ffmpeg per job through
|
||||
|
|
@ -21,7 +22,11 @@ import {
|
|||
buildProduceArgs, produceError, PRODUCE_PRESET, scratchAssetName,
|
||||
type Lane, type ProducePlan, type ProduceFailure,
|
||||
} from "./command-builder";
|
||||
import type { ClaimedProduceJob, ProduceJobStore, ProduceObjectStorage } from "./store";
|
||||
import { assembleError, buildAssembleArgs, validateTrims, type AssemblePlan, type AssembleClipSource } from "./assemble-builder";
|
||||
import { renderAssemblyArtwork } from "./overlay-png";
|
||||
import { assembleScratchName, EDL_ASPECTS } from "./edl-plan";
|
||||
import type { ClaimedProduceJob, ProduceJobStore, ProduceObjectStorage, ProbeResult, ProduceSourceAsset } from "./store";
|
||||
import { probeVideoFile, type VideoProbeResult } from "./ffprobe";
|
||||
|
||||
export interface ProduceRunnerDeps {
|
||||
jobs: ProduceJobStore;
|
||||
|
|
@ -39,13 +44,25 @@ export interface ProduceRunnerDeps {
|
|||
encodeTimeoutMs?: number;
|
||||
/** Cap on one download. */
|
||||
downloadTimeoutMs?: number;
|
||||
/** Hard cap on one probe-lane ffprobe run. */
|
||||
probeTimeoutMs?: number;
|
||||
/** Hard cap on the card+overlay PNG render pass of an assemble job. */
|
||||
renderArtworkTimeoutMs?: number;
|
||||
/** Injection for tests; `which ffmpeg`. */
|
||||
ffmpegBinary?: string;
|
||||
ffprobeBinary?: string;
|
||||
scheduler?: { setTimeout(ms: number, action: () => void): { cancel(): void } };
|
||||
/** The probe lane's ffprobe result; defaults to a real ffprobe run. */
|
||||
probeInput?: (inputPath: string) => Promise<VideoProbeResult>;
|
||||
/**
|
||||
* Card + overlay PNG rasterization (the api renderer's sharp-worker mechanism,
|
||||
* overlay-png.ts). Injectable so tests double it; a box without sharp fails the
|
||||
* JOB as `assemble render unavailable` — there is no drawtext fallback, ever.
|
||||
*/
|
||||
renderArtwork?: (project: import("./edl-plan").EdlProject) => Promise<{ cards: { bytes: Buffer; name: string }[]; overlays: { bytes: Buffer; name: string }[] }>;
|
||||
}
|
||||
|
||||
const DEFAULTS = { pollIntervalMs: 2_000, encodeTimeoutMs: 15 * 60_000, downloadTimeoutMs: 120_000 };
|
||||
const DEFAULTS = { pollIntervalMs: 2_000, encodeTimeoutMs: 15 * 60_000, downloadTimeoutMs: 120_000, probeTimeoutMs: 30_000 };
|
||||
const defaultScheduler = {
|
||||
setTimeout(ms: number, action: () => void): { cancel(): void } {
|
||||
const timer = setTimeout(action, ms);
|
||||
|
|
@ -56,12 +73,23 @@ const defaultScheduler = {
|
|||
/** ffmpeg's own progress line is not parsed; these are the checkpoints a job walks through. */
|
||||
const STAGES: ReadonlyArray<[label: string, percent: number]> = [["claimed", 5], ["downloaded", 20], ["encoded", 80], ["published", 95]];
|
||||
|
||||
/**
|
||||
* The assemble lane walks more checkpoints than a single-source produce (artwork,
|
||||
* per-input fetches), so it reports its own ladder; same rule — every write bumps
|
||||
* `updated_at`, which is the lease's liveness signal for 044's stale sweep.
|
||||
*/
|
||||
const ASSEMBLE_STAGES: ReadonlyArray<[label: string, percent: number]> = [
|
||||
["claimed", 5], ["artwork", 15], ["sources", 30], ["encoded", 80], ["published", 95],
|
||||
];
|
||||
|
||||
export class ProduceRunner {
|
||||
private stopped = false;
|
||||
private timer: { cancel(): void } | null = null;
|
||||
private readonly pollIntervalMs: number;
|
||||
private readonly encodeTimeoutMs: number;
|
||||
private readonly downloadTimeoutMs: number;
|
||||
private readonly probeTimeoutMs: number;
|
||||
private readonly renderArtworkTimeoutMs: number;
|
||||
private readonly scheduler: NonNullable<ProduceRunnerDeps["scheduler"]>;
|
||||
private running: Promise<unknown> | null = null;
|
||||
|
||||
|
|
@ -69,6 +97,8 @@ export class ProduceRunner {
|
|||
this.pollIntervalMs = deps.pollIntervalMs ?? DEFAULTS.pollIntervalMs;
|
||||
this.encodeTimeoutMs = deps.encodeTimeoutMs ?? DEFAULTS.encodeTimeoutMs;
|
||||
this.downloadTimeoutMs = deps.downloadTimeoutMs ?? DEFAULTS.downloadTimeoutMs;
|
||||
this.probeTimeoutMs = deps.probeTimeoutMs ?? DEFAULTS.probeTimeoutMs;
|
||||
this.renderArtworkTimeoutMs = deps.renderArtworkTimeoutMs ?? 60_000;
|
||||
this.scheduler = deps.scheduler ?? defaultScheduler;
|
||||
}
|
||||
|
||||
|
|
@ -122,6 +152,12 @@ export class ProduceRunner {
|
|||
|
||||
private async run(job: ClaimedProduceJob): Promise<void> {
|
||||
this.deps.onEvent?.({ type: "claimed", jobId: job.jobId });
|
||||
// The probe lane never touches ffmpeg's encoder: it downloads the sealed
|
||||
// source, opens it (SPENC1 or SPENC2 by magic), ffprobes a scratch copy and
|
||||
// writes the metadata back — the api's probeVideo, relocated off the api box
|
||||
// so the direct-to-R2 path never moves video bytes through a request.
|
||||
if (job.kind === "probe") return this.runProbe(job);
|
||||
if (job.kind === "assemble") return this.runAssemble(job);
|
||||
const directory = await mkdtemp(join(this.deps.scratchRoot, "produce-"));
|
||||
try {
|
||||
const plan = await this.prepare(job, directory);
|
||||
|
|
@ -146,12 +182,271 @@ export class ProduceRunner {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The assemble lane (editor-v2 L1): an EDL timeline → ONE ffmpeg process.
|
||||
*
|
||||
* 1. resolve every clip asset through the store (probe metadata + storage key)
|
||||
* 2. render cards + overlays to PNGs — the api renderer's sharp-worker
|
||||
* mechanism, so text becomes pixels and never a filter argument
|
||||
* 3. download every input to a sha-named scratch file
|
||||
* 4. PRE-SPAWN trim gate: every trimEndMs checked against the probe
|
||||
* durationMs (fail with a human reason, not ffmpeg's "Invalid data found")
|
||||
* 5. one argv (buildAssembleArgs) → encode → publish → derived asset row +
|
||||
* job complete, exactly the compose lane's registerAndProbe-equivalent
|
||||
*
|
||||
* Scratch is the job's own mkdtemp dir, removed in `finally`; stderr redacted;
|
||||
* 15-min hard timeout through the same encode path the compose lane uses.
|
||||
*/
|
||||
private async runAssemble(job: ClaimedProduceJob): Promise<void> {
|
||||
const input = job.assemble;
|
||||
if (input === null) {
|
||||
// The claim already fails undecodable rows; reaching here means a lane mixup.
|
||||
const reason = "assemble invalid request: no EDL payload";
|
||||
await this.deps.jobs.fail(job.jobId, reason).catch(() => undefined);
|
||||
this.deps.onEvent?.({ type: "failed", jobId: job.jobId, detail: reason });
|
||||
return;
|
||||
}
|
||||
const directory = await mkdtemp(join(this.deps.scratchRoot, "assemble-"));
|
||||
try {
|
||||
const project = input.project;
|
||||
const render = this.deps.renderArtwork ?? ((p: import("./edl-plan").EdlProject) => renderAssemblyArtwork({ edlProjectId: input.edlProjectId, project: p, lane: input.lane, requestedBy: input.requestedBy }));
|
||||
const artwork = await withTimeout(render(project), this.renderArtworkTimeoutMs, "render unavailable");
|
||||
await this.deps.jobs.progress(job.jobId, ASSEMBLE_STAGES[1]![1]);
|
||||
|
||||
// Every clip resolves to its asset row; duplicates of one asset are fetched once.
|
||||
const assetIds = [...new Set(project.items.filter((e): e is import("./edl-plan").EdlClip => e.kind === "clip").map((e) => e.assetId))];
|
||||
const assets = new Map<string, ProduceSourceAsset>();
|
||||
for (const assetId of assetIds) {
|
||||
const asset = await this.deps.jobs.assembleAsset(job.workspaceId, assetId);
|
||||
if (asset === null) throw assembleError("asset unavailable", `clip ${assetId}`);
|
||||
assets.set(assetId, asset);
|
||||
}
|
||||
let voiceoverAsset: ProduceSourceAsset | null = null;
|
||||
if (project.voiceover !== null) {
|
||||
voiceoverAsset = await this.deps.jobs.assembleAsset(job.workspaceId, project.voiceover.audioAssetId);
|
||||
if (voiceoverAsset === null) throw assembleError("asset unavailable", "voiceover");
|
||||
}
|
||||
await this.deps.jobs.progress(job.jobId, ASSEMBLE_STAGES[2]![1] / 2);
|
||||
|
||||
// Download: one sha-named scratch file per unique object (cards/overlays are
|
||||
// already rendered; write them under their PNG content hash).
|
||||
const files = new Map<string, string>();
|
||||
const download = async (storageKey: string): Promise<string> => {
|
||||
const existing = files.get(storageKey);
|
||||
if (existing !== undefined) return existing;
|
||||
const path = await this.download(storageKey, directory);
|
||||
files.set(storageKey, path);
|
||||
return path;
|
||||
};
|
||||
const clips: AssembleClipSource[] = [];
|
||||
let cardCursor = 0;
|
||||
for (const entry of project.items) {
|
||||
if (entry.kind !== "clip") continue;
|
||||
const asset = assets.get(entry.assetId)!;
|
||||
clips.push({
|
||||
clip: entry,
|
||||
path: await download(asset.storageKey),
|
||||
durationMs: asset.durationMs ?? 0,
|
||||
hasAudio: asset.hasAudio !== false,
|
||||
});
|
||||
}
|
||||
const cardFiles: { itemIndex: number; path: string; durationMs: number }[] = [];
|
||||
project.items.forEach((entry, i) => {
|
||||
if (entry.kind !== "card") return;
|
||||
const png = artwork.cards[cardCursor];
|
||||
cardCursor += 1;
|
||||
if (png === undefined) throw assembleError("render unavailable", "card artwork missing");
|
||||
cardFiles.push({ itemIndex: i, path: join(directory, png.name), durationMs: entry.durationMs });
|
||||
});
|
||||
if (cardCursor !== artwork.cards.length) throw assembleError("render unavailable", "card artwork mismatch");
|
||||
const overlayFiles: { overlay: import("./edl-plan").EdlOverlay; path: string }[] = [];
|
||||
for (let j = 0; j < project.overlays.length; j += 1) {
|
||||
const png = artwork.overlays[j];
|
||||
if (png === undefined) throw assembleError("render unavailable", "overlay artwork missing");
|
||||
overlayFiles.push({ overlay: project.overlays[j]!, path: join(directory, png.name) });
|
||||
}
|
||||
for (const png of [...artwork.cards, ...artwork.overlays]) {
|
||||
await writeFile(join(directory, png.name), png.bytes, { mode: 0o600 });
|
||||
}
|
||||
let voiceover: { path: string; gainDb: number; duckOriginal: boolean; offsetMs: number } | null = null;
|
||||
if (project.voiceover !== null && voiceoverAsset !== null) {
|
||||
voiceover = {
|
||||
path: await download(voiceoverAsset.storageKey),
|
||||
gainDb: project.voiceover.gainDb,
|
||||
duckOriginal: project.voiceover.duckOriginal,
|
||||
offsetMs: project.voiceover.offsetMs,
|
||||
};
|
||||
}
|
||||
await this.deps.jobs.progress(job.jobId, ASSEMBLE_STAGES[2]![1]);
|
||||
|
||||
const plan: AssemblePlan = {
|
||||
input,
|
||||
clips,
|
||||
cards: cardFiles,
|
||||
overlays: overlayFiles,
|
||||
voiceover,
|
||||
outputPath: join(directory, "output.mp4"),
|
||||
};
|
||||
// Pre-spawn gate (plan §2 risk #4): trims vs probe durations, before any spawn.
|
||||
validateTrims(project.items, clips);
|
||||
|
||||
await this.encodeAssemble(job, plan);
|
||||
await this.deps.jobs.progress(job.jobId, ASSEMBLE_STAGES[3]![1]);
|
||||
const output = await this.publishAssemble(job, plan);
|
||||
await this.deps.jobs.progress(job.jobId, ASSEMBLE_STAGES[4]![1]);
|
||||
// derived_from = the project's FIRST source clip (043's convention; full
|
||||
// provenance lives in the edl_projects row the api owns).
|
||||
const firstSourceId = project.items.find((e): e is import("./edl-plan").EdlClip => e.kind === "clip")?.assetId ?? job.source.id;
|
||||
const producedAssetId = await this.deps.jobs.complete(job.jobId, output, firstSourceId);
|
||||
this.deps.onEvent?.({ type: "complete", jobId: job.jobId, detail: `${output.storageKey} → asset ${producedAssetId} (assemble ${assembleScratchName(input).slice(0, 12)})` });
|
||||
} catch (error: unknown) {
|
||||
const reason = this.assembleReason(error);
|
||||
this.deps.onStderr(`assemble job ${job.jobId}: ${reason}`);
|
||||
await this.deps.jobs.fail(job.jobId, reason).catch((failError: unknown) => {
|
||||
this.deps.onEvent?.({ type: "store-error", jobId: job.jobId, detail: errorMessage(failError) });
|
||||
});
|
||||
this.deps.onEvent?.({ type: "failed", jobId: job.jobId, detail: reason });
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true }).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
/** Bounded, secret-free assemble failure reason (known names, never stderr/paths). */
|
||||
private assembleReason(error: unknown): string {
|
||||
const name = error instanceof Error ? error.name : "";
|
||||
const known = ["invalid request", "asset unavailable", "render unavailable", "download failed", "encode timed out", "encode failed", "publish failed"] as const;
|
||||
if ((known as readonly string[]).includes(name)) return `assemble ${name}`;
|
||||
return `assemble failed: ${this.deps.redactor.redact(String(error)).slice(0, 200)}`;
|
||||
}
|
||||
|
||||
/** The assemble encode: same spawn discipline as the compose lane (argv only, redacted stderr, hard timeout). */
|
||||
private async encodeAssemble(job: ClaimedProduceJob, plan: AssemblePlan): Promise<void> {
|
||||
const args = buildAssembleArgs(plan);
|
||||
const redactor = this.deps.redactor;
|
||||
const managed = spawnManaged(this.deps.ffmpegBinary ?? "ffmpeg", args, ["ignore", "ignore", "pipe"], redactor,
|
||||
(message) => this.deps.onStderr(`job ${job.jobId}: ${message}`));
|
||||
const timer = this.scheduler.setTimeout(this.encodeTimeoutMs, () => {
|
||||
void managed.terminate(2_000).then(() => undefined, () => undefined);
|
||||
});
|
||||
try {
|
||||
const exited = await managed.exited;
|
||||
if (this.stopped) throw assembleError("encode failed", "worker stopped");
|
||||
if (exited.code !== 0) throw assembleError(exited.code === null ? "encode timed out" : "encode failed", `code=${exited.code} signal=${exited.signal ?? "-"}`);
|
||||
const info = await stat(plan.outputPath).catch(() => null);
|
||||
if (info === null || info.size === 0) throw assembleError("encode failed", "no output written");
|
||||
} finally {
|
||||
timer.cancel();
|
||||
await managed.terminate(0).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
/** The assemble publish: real duration via ffprobe, real canvas geometry, then the shared complete(). */
|
||||
private async publishAssemble(job: ClaimedProduceJob, plan: AssemblePlan): Promise<{ storageKey: string; sizeBytes: number; durationMs: number | null; width: number; height: number; frameRate: number }> {
|
||||
let body: Buffer;
|
||||
try {
|
||||
body = await readFile(plan.outputPath);
|
||||
} catch {
|
||||
throw assembleError("encode failed", "output unreadable");
|
||||
}
|
||||
let key: string;
|
||||
try {
|
||||
const stored = await withTimeout(this.deps.storage.putContent(body, "video/mp4"), this.downloadTimeoutMs, "publish failed");
|
||||
key = stored.key;
|
||||
} catch {
|
||||
throw assembleError("publish failed");
|
||||
}
|
||||
const durationMs = await this.probeDurationMs(plan.outputPath);
|
||||
const canvas = EDL_ASPECTS[plan.input.project.aspect];
|
||||
return {
|
||||
storageKey: key,
|
||||
sizeBytes: body.byteLength,
|
||||
durationMs,
|
||||
width: canvas.w,
|
||||
height: canvas.h,
|
||||
frameRate: 30,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The probe lane: sealed source → scratch file → ffprobe → metadata onto the
|
||||
* source's own video_assets row → job complete 100. Failures mark the job
|
||||
* failed with the ffprobe error, mirroring the api's probeVideo failure path.
|
||||
*/
|
||||
private async runProbe(job: ClaimedProduceJob): Promise<void> {
|
||||
const directory = await mkdtemp(join(this.deps.scratchRoot, "probe-"));
|
||||
try {
|
||||
const bytes = await this.downloadSource(job.source.storageKey);
|
||||
const inputPath = join(directory, scratchAssetName(job.source.storageKey, "bin"));
|
||||
await writeFile(inputPath, bytes, { mode: 0o600 });
|
||||
await this.deps.jobs.progress(job.jobId, 50);
|
||||
const metadata = await this.probeInput(inputPath);
|
||||
await this.deps.jobs.completeProbe(job.jobId, job.source.id, {
|
||||
width: metadata.width ?? null,
|
||||
height: metadata.height ?? null,
|
||||
durationMs: metadata.durationMs ?? null,
|
||||
frameRate: metadata.frameRate ?? null,
|
||||
codec: metadata.codec ?? null,
|
||||
audioCodec: metadata.audioCodec ?? null,
|
||||
});
|
||||
this.deps.onEvent?.({ type: "complete", jobId: job.jobId, detail: `probe → ${job.source.id}` });
|
||||
} catch (error: unknown) {
|
||||
const reason = this.probeReason(error);
|
||||
this.deps.onStderr(`probe job ${job.jobId}: ${reason}`);
|
||||
await this.deps.jobs.fail(job.jobId, reason).catch((failError: unknown) => {
|
||||
// A failed `fail()` leaves the row in `transcoding`; the claim's
|
||||
// stale-lease sweep requeues it, so say what actually went wrong.
|
||||
this.deps.onEvent?.({ type: "store-error", jobId: job.jobId, detail: errorMessage(failError) });
|
||||
});
|
||||
this.deps.onEvent?.({ type: "failed", jobId: job.jobId, detail: reason });
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true }).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the source PLAINTEXT: getObject picks the frame branch by magic, so
|
||||
* SPENC1 (api-sealed) and SPENC2 (browser-sealed direct upload) sources open
|
||||
* identically before ffprobe. getSealedObject stays for CEK-header reads.
|
||||
*/
|
||||
private async downloadSource(storageKey: string): Promise<Buffer> {
|
||||
try {
|
||||
return await withTimeout(this.deps.storage.getObject(storageKey), this.downloadTimeoutMs, "download failed");
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof Error && error.name === "download failed") throw error;
|
||||
throw produceError("download failed");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The probe lane's ffprobe: a real ffprobe run unless the deps inject one.
|
||||
* Reuses the api's arg list + parser (apps/api/src/video/ffprobe.ts ported
|
||||
* verbatim into ./ffprobe.ts) so the metadata matches the api's bit for bit.
|
||||
*/
|
||||
private async probeInput(inputPath: string): Promise<VideoProbeResult> {
|
||||
const probe = this.deps.probeInput ?? ((path: string) => probeVideoFile(path, this.deps.ffprobeBinary));
|
||||
return await new Promise<VideoProbeResult>((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(probeTimeoutError()), this.probeTimeoutMs);
|
||||
probe(inputPath).then(
|
||||
(value) => { clearTimeout(timer); resolve(value); },
|
||||
(error: unknown) => { clearTimeout(timer); reject(error); },
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/** Bounded, secret-free probe failure reason: a known name, never stderr or a path. */
|
||||
private probeReason(error: unknown): string {
|
||||
const name = error instanceof Error ? error.name : "";
|
||||
if ((["download failed", "probe timed out"] as readonly string[]).includes(name)) return `probe ${name}`;
|
||||
return `probe failed: ${this.deps.redactor.redact(String(error)).slice(0, 200)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Download every input the validated request names, and build the plan: the
|
||||
* source bytes plus one file per logo overlay (keyed by content hash) and the
|
||||
* voiceover, all straight into this job's scratch dir.
|
||||
*/
|
||||
private async prepare(job: ClaimedProduceJob, directory: string): Promise<ProducePlan> {
|
||||
if (job.request === null) throw produceError("invalid request", "probe row on the compose lane");
|
||||
const source = await this.download(job.source.storageKey, directory);
|
||||
const logos: { path: string }[] = [];
|
||||
for (const overlay of job.request.overlays) {
|
||||
|
|
@ -171,7 +466,12 @@ export class ProduceRunner {
|
|||
};
|
||||
}
|
||||
const durationSec = job.source.durationMs === null ? null : job.source.durationMs / 1000;
|
||||
// A probed asset without an audio track gets a silent bed, so the output always has audio.
|
||||
// An audio-less source (muted screen recording, GIF conversion, synthetic
|
||||
// test clip) must not feed [0:a] into the graph — ffmpeg dies with "Stream
|
||||
// specifier ':a' matches no streams". hasAudio===false means probed and
|
||||
// audio-less; null means the probe never ran, where the source bed stays
|
||||
// (the [0:a] reference then fails loudly rather than silently dropping audio
|
||||
// from a file that may well have a track).
|
||||
const audioBed = job.source.hasAudio === false ? "none" : "source";
|
||||
return {
|
||||
input: { path: source, durationSec, audioBed },
|
||||
|
|
@ -277,9 +577,18 @@ function errorMessage(error: unknown): string {
|
|||
return error instanceof Error && error.name ? error.name : "unknown error";
|
||||
}
|
||||
|
||||
function withTimeout<T>(promise: Promise<T>, ms: number, name: ProduceFailure): Promise<T> {
|
||||
/** The probe lane's own timeout error (name is what probeReason matches on). */
|
||||
function probeTimeoutError(): Error {
|
||||
const error = new Error("probe: probe timed out");
|
||||
error.name = "probe timed out";
|
||||
return error;
|
||||
}
|
||||
|
||||
function withTimeout<T>(promise: Promise<T>, ms: number, name: ProduceFailure | import("./assemble-builder").AssembleFailure): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(produceError(name)), ms);
|
||||
const timer = setTimeout(() => reject(
|
||||
name === "render unavailable" ? assembleError(name) : produceError(name as ProduceFailure),
|
||||
), ms);
|
||||
promise.then((value) => { clearTimeout(timer); resolve(value); },
|
||||
(error: unknown) => { clearTimeout(timer); reject(error); });
|
||||
});
|
||||
|
|
|
|||
|
|
@ -3,18 +3,31 @@
|
|||
* `apps/api/src/storage/storage.ts` uses (SPENC1 frames, KEK from S3_KEK), so a
|
||||
* `getObject` here opens bytes a `putContent` there sealed — and vice versa.
|
||||
*
|
||||
* SPENC2 (direct-to-R2): objects under `direct/` are sealed IN THE BROWSER and
|
||||
* PUT straight to R2. The worker opens them with the ECDH share it derives from
|
||||
* DIRECT_EC_PRIVATE_KEY, so the API box never has to touch video bytes at all.
|
||||
* loadDirectPrivateJwk / directWrapKey / unwrapDirectCek / openDirectFrame below
|
||||
* are PORTED VERBATIM from apps/api/src/storage/storage.ts (same env var, same
|
||||
* PKCS8 base64 DER, same HKDF info, same frame offsets) — the two readers must
|
||||
* stay byte-compatible; a divergence breaks the whole direct path silently.
|
||||
*
|
||||
* The worker must never reimplement the API's policy (dedup HEAD, presign, bucket
|
||||
* provisioning); it needs only the two operations a produce job uses: download
|
||||
* source/logo/voiceover bytes, publish the produced mp4. Reusing the seal/open
|
||||
* frame format (rather than routing through the API) keeps the worker off the
|
||||
* API's request path, which is the point of a queue.
|
||||
*/
|
||||
import { createCipheriv, createDecipheriv, createHash, randomBytes } from "node:crypto";
|
||||
import { createCipheriv, createDecipheriv, createHash, createPrivateKey, createPublicKey, diffieHellman, hkdfSync, randomBytes } from "node:crypto";
|
||||
import { GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
|
||||
import type { ProduceObjectStorage } from "./store";
|
||||
|
||||
const FRAME_MAGIC = Buffer.from("SPENC1");
|
||||
|
||||
/** True when DIRECT_EC_PRIVATE_KEY is present (the produce worker reports it). */
|
||||
export function workerDirectEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
|
||||
return directKeyPairFromEnv(env) !== null;
|
||||
}
|
||||
|
||||
export interface WorkerStorageConfig {
|
||||
endpoint?: string;
|
||||
region?: string;
|
||||
|
|
@ -24,6 +37,8 @@ export interface WorkerStorageConfig {
|
|||
forcePathStyle?: boolean;
|
||||
/** 32-byte KEK; when omitted, the dev mirror of the API's loadMediaKek(). */
|
||||
mediaKek?: Buffer;
|
||||
/** Overrides DIRECT_EC_PRIVATE_KEY (tests). */
|
||||
directPrivateKey?: ReturnType<typeof createPrivateKey>;
|
||||
}
|
||||
|
||||
function loadMediaKek(): Buffer {
|
||||
|
|
@ -94,6 +109,164 @@ function openFrame(sealed: Buffer, kek: Buffer): Buffer {
|
|||
|
||||
function keyForHash(hash: string): string { return `sha256/${hash}.mp4`; }
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SPENC2 — browser-sealed direct-to-R2 frames
|
||||
// (ported verbatim from apps/api/src/storage/storage.ts; see the file header)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Frame magic for browser-sealed objects (spec: "SPENC2" exactly). */
|
||||
export const DIRECT_FRAME_MAGIC = Buffer.from("SPENC2", "ascii");
|
||||
|
||||
/**
|
||||
* Fixed frame layout, all integers big-endian. Apps/web/src/lib/direct-storage.ts
|
||||
* seals the same shape; the api's reader is the cross-implementation proof target.
|
||||
*
|
||||
* [0..6) magic "SPENC2"
|
||||
* [6..71) ephemeral P-256 public key, uncompressed: 0x04 || X(32) || Y(32)
|
||||
* [71..83) wrapNonce, 12 random bytes
|
||||
* [83..131) wrappedCek: AES-256-GCM(CEK), key = HKDF-SHA256(ECDH, salt=∅,
|
||||
* info="tmbh-direct-cek-wrap", L=32), nonce = wrapNonce → 48 bytes
|
||||
* [131..135) segmentCount, u32
|
||||
* [135..] segments: ctLen u32 | iv 12 | AES-256-GCM ciphertext+tag, no AAD
|
||||
*/
|
||||
const EPHEMERAL_PUBLIC_OFFSET = 6;
|
||||
const WRAP_NONCE_OFFSET = 71;
|
||||
const WRAPPED_CEK_OFFSET = 83;
|
||||
const SEGMENT_COUNT_OFFSET = 131;
|
||||
const SEGMENTS_OFFSET = 135;
|
||||
const WRAPPED_CEK_BYTES = 32 + 16;
|
||||
|
||||
/** HKDF info binding the wrap key to this purpose. ASCII, exact. */
|
||||
const DIRECT_WRAP_INFO = Buffer.from("tmbh-direct-cek-wrap", "ascii");
|
||||
|
||||
/** Segment plaintext ceiling. Mirrors the web side, which seals 10 MiB chunks. */
|
||||
export const DIRECT_SEGMENT_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
export class DirectKeyError extends Error {}
|
||||
|
||||
/** Server direct keypair: P-256 private key from DIRECT_EC_PRIVATE_KEY (base64 DER). */
|
||||
export interface DirectKeyPair {
|
||||
privateKey: ReturnType<typeof createPrivateKey>;
|
||||
/** Compact JWK {kty,crv,x,y} — exactly what the browser's importKey wants. */
|
||||
publicJwk: { kty: "EC"; crv: "P-256"; x: string; y: string };
|
||||
}
|
||||
|
||||
function loadDirectPrivateJwk(env: NodeJS.ProcessEnv): ReturnType<typeof createPrivateKey> | null {
|
||||
const raw = env.DIRECT_EC_PRIVATE_KEY;
|
||||
if (raw === undefined || raw.trim() === "") return null;
|
||||
let der: Buffer;
|
||||
try {
|
||||
der = Buffer.from(raw.trim(), "base64");
|
||||
} catch {
|
||||
throw new DirectKeyError("DIRECT_EC_PRIVATE_KEY must be base64 DER");
|
||||
}
|
||||
try {
|
||||
return createPrivateKey({ key: der, format: "der", type: "pkcs8" });
|
||||
} catch {
|
||||
throw new DirectKeyError("DIRECT_EC_PRIVATE_KEY is not a PKCS8 P-256 private key");
|
||||
}
|
||||
}
|
||||
|
||||
/** Build a DirectKeyPair from an already-loaded private key (config + tests). */
|
||||
function directKeyPairFromPrivateKey(privateKey: ReturnType<typeof createPrivateKey>): DirectKeyPair {
|
||||
const jwk = createPublicKey(privateKey).export({ format: "jwk" });
|
||||
if (jwk.kty !== "EC" || jwk.crv !== "P-256" || typeof jwk.x !== "string" || typeof jwk.y !== "string") {
|
||||
throw new DirectKeyError("direct key is not a P-256 key");
|
||||
}
|
||||
return { privateKey, publicJwk: { kty: "EC", crv: "P-256", x: jwk.x, y: jwk.y } };
|
||||
}
|
||||
|
||||
/** The direct keypair from env, deriving the public key from the private one. */
|
||||
export function directKeyPairFromEnv(env: NodeJS.ProcessEnv = process.env): DirectKeyPair | null {
|
||||
const privateKey = loadDirectPrivateJwk(env);
|
||||
if (privateKey === null) return null;
|
||||
return directKeyPairFromPrivateKey(privateKey);
|
||||
}
|
||||
|
||||
/** ECDH → HKDF-SHA256(salt empty, info "tmbh-direct-cek-wrap") → 32-byte wrap key. */
|
||||
function directWrapKey(privateKey: ReturnType<typeof createPrivateKey>, ephemeralPoint: Buffer): Buffer {
|
||||
const x = ephemeralPoint.subarray(1, 33).toString("base64url");
|
||||
const y = ephemeralPoint.subarray(33, 65).toString("base64url");
|
||||
let peer: ReturnType<typeof createPublicKey>;
|
||||
try {
|
||||
peer = createPublicKey({ key: { kty: "EC", crv: "P-256", x, y }, format: "jwk" });
|
||||
} catch {
|
||||
throw new Error("storage: SPENC2 frame holds a malformed ephemeral key");
|
||||
}
|
||||
const ikm = Buffer.from(diffieHellman({ privateKey, publicKey: peer }));
|
||||
// RFC 5869: an absent salt defaults to HashLen zero bytes; WebCrypto's empty
|
||||
// salt and Node's zero-filled salt produce the identical PRK (verified).
|
||||
return Buffer.from(hkdfSync("sha256", ikm, Buffer.alloc(0), DIRECT_WRAP_INFO, 32));
|
||||
}
|
||||
|
||||
interface DirectHeader {
|
||||
ephemeralPoint: Buffer;
|
||||
wrapNonce: Buffer;
|
||||
wrappedCek: Buffer;
|
||||
segmentCount: number;
|
||||
}
|
||||
|
||||
/** Validate + slice the fixed header. Throws on any truncation or bad shape. */
|
||||
function parseDirectHeader(stored: Buffer): DirectHeader {
|
||||
if (stored.length < SEGMENTS_OFFSET) {
|
||||
throw new Error("storage: SPENC2 frame is truncated");
|
||||
}
|
||||
const ephemeralPoint = stored.subarray(EPHEMERAL_PUBLIC_OFFSET, WRAP_NONCE_OFFSET);
|
||||
if (ephemeralPoint[0] !== 0x04) throw new Error("storage: SPENC2 ephemeral key is not an uncompressed point");
|
||||
const wrapNonce = stored.subarray(WRAP_NONCE_OFFSET, WRAPPED_CEK_OFFSET);
|
||||
const wrappedCek = stored.subarray(WRAPPED_CEK_OFFSET, WRAPPED_CEK_OFFSET + WRAPPED_CEK_BYTES);
|
||||
const segmentCount = stored.readUInt32BE(SEGMENT_COUNT_OFFSET);
|
||||
if (segmentCount === 0 || segmentCount > 4_294_967_295 / DIRECT_SEGMENT_BYTES + 1) {
|
||||
throw new Error("storage: SPENC2 segment count is out of range");
|
||||
}
|
||||
return { ephemeralPoint, wrapNonce, wrappedCek, segmentCount };
|
||||
}
|
||||
|
||||
/** Walk the segment table; yields {ciphertext, iv} in order. */
|
||||
function* directSegments(stored: Buffer, segmentCount: number): Generator<{ iv: Buffer; ciphertext: Buffer }> {
|
||||
let offset = SEGMENTS_OFFSET;
|
||||
for (let index = 0; index < segmentCount; index += 1) {
|
||||
if (offset + 4 + 12 > stored.length) throw new Error("storage: SPENC2 segment table is truncated");
|
||||
const ctLen = stored.readUInt32BE(offset);
|
||||
offset += 4;
|
||||
if (ctLen < 16 || offset + 12 + ctLen > stored.length) throw new Error("storage: SPENC2 segment length is out of range");
|
||||
const iv = stored.subarray(offset, offset + 12);
|
||||
offset += 12;
|
||||
const ciphertext = stored.subarray(offset, offset + ctLen);
|
||||
offset += ctLen;
|
||||
yield { iv, ciphertext };
|
||||
}
|
||||
if (offset !== stored.length) throw new Error("storage: SPENC2 frame has trailing bytes");
|
||||
}
|
||||
|
||||
/** Unwrap the CEK (verifies the wrap GCM tag). */
|
||||
function unwrapDirectCek(stored: Buffer, privateKey: ReturnType<typeof createPrivateKey>): Buffer {
|
||||
const header = parseDirectHeader(stored);
|
||||
return aesGcmOpen(directWrapKey(privateKey, header.ephemeralPoint), header.wrapNonce, header.wrappedCek);
|
||||
}
|
||||
|
||||
/**
|
||||
* Open a SPENC2 frame: unwrap the CEK, then decrypt every segment. Segment
|
||||
* GCM tags are verified individually, so any flipped byte throws. Empty
|
||||
* plaintexts are rejected (the upload floor is 1 byte).
|
||||
*/
|
||||
export function openDirectFrame(stored: Buffer, privateKey: ReturnType<typeof createPrivateKey>): Buffer {
|
||||
const { segmentCount } = parseDirectHeader(stored);
|
||||
const cek = unwrapDirectCek(stored, privateKey);
|
||||
const parts: Buffer[] = [];
|
||||
for (const segment of directSegments(stored, segmentCount)) {
|
||||
parts.push(aesGcmOpen(cek, segment.iv, segment.ciphertext));
|
||||
}
|
||||
const plaintext = Buffer.concat(parts);
|
||||
if (plaintext.length === 0) throw new Error("storage: SPENC2 frame decrypts to zero bytes");
|
||||
return plaintext;
|
||||
}
|
||||
|
||||
/** True when the leading magic says the object is browser-sealed. */
|
||||
function isDirectFrame(stored: Buffer): boolean {
|
||||
return stored.subarray(0, DIRECT_FRAME_MAGIC.length).equals(DIRECT_FRAME_MAGIC);
|
||||
}
|
||||
|
||||
/**
|
||||
* Envelope-encrypted S3 storage for the produce worker. Object bodies are sealed
|
||||
* exactly like the API's `S3Storage` (magic | iv | wrapped CEK | body+tag), so
|
||||
|
|
@ -103,10 +276,12 @@ export class WorkerS3Storage implements ProduceObjectStorage {
|
|||
private readonly client: S3Client;
|
||||
private readonly bucket: string;
|
||||
private readonly kek: Buffer;
|
||||
private readonly directKey: DirectKeyPair | null;
|
||||
|
||||
constructor(config: WorkerStorageConfig) {
|
||||
this.bucket = config.bucket ?? "tactical-media-broadcast-hub-media";
|
||||
this.kek = config.mediaKek ?? loadMediaKek();
|
||||
this.directKey = config.directPrivateKey !== undefined ? directKeyPairFromPrivateKey(config.directPrivateKey) : directKeyPairFromEnv();
|
||||
this.client = new S3Client({
|
||||
endpoint: config.endpoint,
|
||||
region: config.region ?? "us-east-1",
|
||||
|
|
@ -118,11 +293,29 @@ export class WorkerS3Storage implements ProduceObjectStorage {
|
|||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Raw still-sealed object bytes — no decrypt, no hash check. The direct path
|
||||
* keeps the CEK header only in the sealed form (the api's getSealedObject
|
||||
* twin), so a caller that needs the frame verbatim reads it here.
|
||||
*/
|
||||
async getSealedObject(key: string): Promise<Buffer> {
|
||||
const res = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: key }));
|
||||
if (res.Body === undefined) throw new Error(`empty body for ${key}`);
|
||||
return Buffer.from(await res.Body.transformToByteArray());
|
||||
}
|
||||
|
||||
/**
|
||||
* Open an object by its leading magic, mirroring the api's getObject: SPENC1
|
||||
* opens under the KEK, SPENC2 (browser-sealed direct upload) opens under the
|
||||
* ECDH share from DIRECT_EC_PRIVATE_KEY. One reader serves the whole bucket.
|
||||
*/
|
||||
async getObject(key: string): Promise<Buffer> {
|
||||
const res = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: key }));
|
||||
if (res.Body === undefined) throw new Error(`empty body for ${key}`);
|
||||
const stored = Buffer.from(await res.Body.transformToByteArray());
|
||||
const plaintext = openFrame(stored, this.kek);
|
||||
const plaintext = isDirectFrame(stored)
|
||||
? this.openDirectStored(stored)
|
||||
: openFrame(stored, this.kek); // raises the magic-mismatch error for junk
|
||||
// Content addressing: the key names the plaintext hash, so tampering is detected.
|
||||
const hashFromKey = key.split("/")[1]?.split(".")[0];
|
||||
if (hashFromKey !== undefined && /^[0-9a-f]{64}$/.test(hashFromKey)) {
|
||||
|
|
@ -133,6 +326,12 @@ export class WorkerS3Storage implements ProduceObjectStorage {
|
|||
return plaintext;
|
||||
}
|
||||
|
||||
/** SPENC2 branch: only a process holding the direct private key can unwrap. */
|
||||
private openDirectStored(stored: Buffer): Buffer {
|
||||
if (this.directKey === null) throw new Error("DIRECT_EC_PRIVATE_KEY is not configured");
|
||||
return openDirectFrame(stored, this.directKey.privateKey);
|
||||
}
|
||||
|
||||
async putContent(body: Buffer, contentType: "video/mp4"): Promise<{ key: string }> {
|
||||
if (contentType !== "video/mp4") throw new Error("produce storage publishes video/mp4 only");
|
||||
const hash = createHash("sha256").update(body).digest("hex");
|
||||
|
|
|
|||
|
|
@ -8,8 +8,10 @@
|
|||
* the same standing as `ObservedStore`, `AssetStore` and `PreviewObjectStore`.
|
||||
*/
|
||||
import type { Lane, ProduceJobKind, ProduceRequest } from "./command-builder";
|
||||
import type { AssembleInput } from "./edl-plan";
|
||||
|
||||
export type { Lane, ProduceJobKind, ProduceRequest } from "./command-builder";
|
||||
export type { AssembleInput } from "./edl-plan";
|
||||
|
||||
/** The source `video_assets` row a claimed job renders from. */
|
||||
export interface ProduceSourceAsset {
|
||||
|
|
@ -29,7 +31,15 @@ export interface ClaimedProduceJob {
|
|||
jobId: string;
|
||||
workspaceId: string;
|
||||
kind: ProduceJobKind;
|
||||
request: ProduceRequest;
|
||||
/** The validated producer request; null for a probe row (no input_json). */
|
||||
request: ProduceRequest | null;
|
||||
/**
|
||||
* The parsed assemble payload ({ edlProjectId, project }); null on every other
|
||||
* kind. An assemble row's `video_asset_id` points at the project's first clip
|
||||
* (the row's source asset), while the project itself may name many more — the
|
||||
* runner resolves each through `asset()`.
|
||||
*/
|
||||
assemble: AssembleInput | null;
|
||||
source: ProduceSourceAsset;
|
||||
claimedAt: string;
|
||||
}
|
||||
|
|
@ -44,10 +54,20 @@ export interface ProduceOutput {
|
|||
frameRate: number;
|
||||
}
|
||||
|
||||
/** ffprobe's write-back for a probe job (039's column set, all nullable). */
|
||||
export interface ProbeResult {
|
||||
width: number | null;
|
||||
height: number | null;
|
||||
durationMs: number | null;
|
||||
frameRate: number | null;
|
||||
codec: string | null;
|
||||
audioCodec: string | null;
|
||||
}
|
||||
|
||||
export interface ProduceJobStore {
|
||||
/**
|
||||
* Claim exactly one queued compose/mix row for this worker's lane, with lease
|
||||
* semantics: the `queued → transcoding` state bump inside
|
||||
* Claim exactly one queued probe/compose/mix row for this worker's lane, with
|
||||
* lease semantics: the `queued → transcoding` state bump inside
|
||||
* `FOR UPDATE SKIP LOCKED` is the lease, so two workers can never take the
|
||||
* same row and a row stays claimed until this worker finishes it.
|
||||
*
|
||||
|
|
@ -62,10 +82,18 @@ export interface ProduceJobStore {
|
|||
progress(jobId: string, percent: number): Promise<void>;
|
||||
/** Insert the derived asset and complete the job pointing at it. */
|
||||
complete(jobId: string, output: ProduceOutput, derivedFrom: string): Promise<string>;
|
||||
/** Write a probe job's ffprobe metadata onto its own video_assets row, then complete. */
|
||||
completeProbe(jobId: string, sourceAssetId: string, result: ProbeResult): Promise<void>;
|
||||
/** Fail a claimed job; `reason` must already be redacted/bounded. */
|
||||
fail(jobId: string, reason: string): Promise<void>;
|
||||
/** Resolve a voiceover asset id to its storage key. */
|
||||
/** Resolve a voiceover/clip asset id to its storage key. */
|
||||
asset(workspaceId: string, assetId: string): Promise<{ storageKey: string; mimeType: string } | null>;
|
||||
/**
|
||||
* Resolve an asset id to its FULL row (the assemble lane needs probe metadata:
|
||||
* durationMs for the pre-spawn trim gate, hasAudio for the audio bed, and the
|
||||
* storage key to download). Returns null when the row is gone (tenant-deleted).
|
||||
*/
|
||||
assembleAsset(workspaceId: string, assetId: string): Promise<ProduceSourceAsset | null>;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -74,6 +102,9 @@ export interface ProduceJobStore {
|
|||
* through exactly this pair. The adapter lives in `s3-storage.ts`.
|
||||
*/
|
||||
export interface ProduceObjectStorage {
|
||||
/** Opens SPENC1 (KEK) and SPENC2 (direct ECDH) frames by leading magic. */
|
||||
getObject(key: string): Promise<Buffer>;
|
||||
/** Raw still-sealed bytes (the api's getSealedObject twin): the direct frame verbatim. */
|
||||
getSealedObject(key: string): Promise<Buffer>;
|
||||
putContent(body: Buffer, contentType: "video/mp4"): Promise<{ key: string }>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,26 +11,27 @@
|
|||
"test:integration": "node scripts/wp3-integration.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@temporalio/client": "1.24.0",
|
||||
"@temporalio/worker": "1.24.0",
|
||||
"@temporalio/workflow": "1.24.0",
|
||||
"pino": "^9.3.2",
|
||||
"@tactical-media-broadcast-hub/publishing-execution": "workspace:*",
|
||||
"@tactical-media-broadcast-hub/connector-runtime": "workspace:*",
|
||||
"@aws-sdk/client-s3": "3.1146.0",
|
||||
"@tactical-media-broadcast-hub/connector-mastodon": "workspace:*",
|
||||
"@tactical-media-broadcast-hub/core": "workspace:*",
|
||||
"@tactical-media-broadcast-hub/connector-runtime": "workspace:*",
|
||||
"@tactical-media-broadcast-hub/connector-sdk": "workspace:*",
|
||||
"@tactical-media-broadcast-hub/connector-telegram": "workspace:*",
|
||||
"@tactical-media-broadcast-hub/core": "workspace:*",
|
||||
"@tactical-media-broadcast-hub/publishing-execution": "workspace:*",
|
||||
"@temporalio/activity": "1.24.0",
|
||||
"@temporalio/client": "1.24.0",
|
||||
"@temporalio/common": "1.24.0",
|
||||
"pg": "^8.23.1"
|
||||
"@temporalio/worker": "1.24.0",
|
||||
"@temporalio/workflow": "1.24.0",
|
||||
"pg": "^8.23.1",
|
||||
"pino": "^9.3.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.5.0",
|
||||
"tsx": "^4.17.0",
|
||||
"vitest": "^2.0.5",
|
||||
"@types/pg": "^8.23.1",
|
||||
"typescript": "^5.5.4"
|
||||
"tsx": "^4.17.0",
|
||||
"typescript": "^5.5.4",
|
||||
"vitest": "^2.0.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22.13.0"
|
||||
|
|
|
|||
|
|
@ -23,7 +23,20 @@ function connectionOf(job: ClaimedJob): ConnectorConnection {
|
|||
function postOf(job: ClaimedJob): NormalizedPost {
|
||||
const visibility = job.post.visibility;
|
||||
if (!["public", "unlisted", "private", "followers", "provider_default"].includes(visibility)) throw new Error("INVALID_POST_VISIBILITY");
|
||||
return { ...job.post, visibility: visibility as NormalizedPost["visibility"] };
|
||||
const post = job.post;
|
||||
// Positional alt texts (aligned with mediaAssetIds at intake) become the
|
||||
// connector's asset-id → description map; the Mastodon upload reads it.
|
||||
const providerMetadata: Record<string, unknown> = { ...post.providerMetadata };
|
||||
const altTexts = post.altTexts;
|
||||
if (Array.isArray(altTexts) && altTexts.length > 0) {
|
||||
const altText: Record<string, string> = {};
|
||||
post.mediaAssetIds.forEach((id, index) => {
|
||||
const text = altTexts[index];
|
||||
if (typeof text === "string" && text.trim() !== "") altText[id] = text;
|
||||
});
|
||||
if (Object.keys(altText).length > 0) providerMetadata.altText = altText;
|
||||
}
|
||||
return { ...post, visibility: visibility as NormalizedPost["visibility"], providerMetadata };
|
||||
}
|
||||
/** Persist only documented remote evidence; unknown fields never enter DB/history. */
|
||||
export function safeMetadata(metadata: Record<string, unknown>): Record<string, unknown> {
|
||||
|
|
|
|||
80
apps/scheduler/src/media-resolver.ts
Normal file
80
apps/scheduler/src/media-resolver.ts
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
/**
|
||||
* Scheduler-side implementation of the publishing media seam: resolves a
|
||||
* workspace-scoped media asset id to its row (via RLS tenant context) and its
|
||||
* plaintext bytes (S3 download + SPENC1 open, identical to the api's
|
||||
* S3Storage.getObject so either side reads what the other wrote).
|
||||
*/
|
||||
import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3";
|
||||
import type { Pool } from "pg";
|
||||
import type { MediaAsset } from "@tactical-media-broadcast-hub/core";
|
||||
import { openMediaFrame, mediaKekFromEnv, type PublishingMediaHooks } from "@tactical-media-broadcast-hub/connector-runtime";
|
||||
|
||||
interface MediaAssetRow {
|
||||
id: string;
|
||||
workspace_id: string;
|
||||
storage_key: string;
|
||||
mime: string;
|
||||
size_bytes: string | number;
|
||||
created_at: Date;
|
||||
}
|
||||
|
||||
export class PublishingMediaResolver implements PublishingMediaHooks {
|
||||
private readonly pool: Pool;
|
||||
private readonly client: S3Client;
|
||||
private readonly bucket: string;
|
||||
private readonly kek: Buffer;
|
||||
|
||||
constructor(pool: Pool) {
|
||||
this.pool = pool;
|
||||
this.bucket = process.env.S3_BUCKET ?? "tactical-media-broadcast-hub-media";
|
||||
this.kek = mediaKekFromEnv(process.env);
|
||||
this.client = new S3Client({
|
||||
endpoint: process.env.S3_ENDPOINT,
|
||||
region: process.env.S3_REGION ?? "us-east-1",
|
||||
credentials:
|
||||
process.env.S3_ACCESS_KEY !== undefined && process.env.S3_SECRET_KEY !== undefined
|
||||
? { accessKeyId: process.env.S3_ACCESS_KEY, secretAccessKey: process.env.S3_SECRET_KEY }
|
||||
: undefined,
|
||||
forcePathStyle: process.env.S3_FORCE_PATH_STYLE !== "false",
|
||||
});
|
||||
}
|
||||
|
||||
private async loadRow(workspaceId: string, assetId: string): Promise<MediaAssetRow> {
|
||||
const client = await this.pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]);
|
||||
const result = await client.query<MediaAssetRow>(
|
||||
"SELECT id, workspace_id, storage_key, mime, size_bytes, created_at FROM media_assets WHERE id = $1",
|
||||
[assetId],
|
||||
);
|
||||
await client.query("COMMIT");
|
||||
const row = result.rows[0];
|
||||
if (row === undefined) throw new Error(`media asset ${assetId} not found in workspace ${workspaceId}`);
|
||||
return row;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async loadAsset(workspaceId: string, assetId: string): Promise<MediaAsset> {
|
||||
const row = await this.loadRow(workspaceId, assetId);
|
||||
return {
|
||||
id: row.id,
|
||||
workspaceId: row.workspace_id,
|
||||
storageKey: row.storage_key,
|
||||
mimeType: row.mime,
|
||||
sizeBytes: Number(row.size_bytes),
|
||||
createdAt: new Date(row.created_at),
|
||||
};
|
||||
}
|
||||
|
||||
async openAsset(workspaceId: string, asset: MediaAsset): Promise<Uint8Array> {
|
||||
await this.loadRow(workspaceId, asset.id);
|
||||
const res = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: asset.storageKey }));
|
||||
if (res.Body === undefined) throw new Error(`empty object body for ${asset.storageKey}`);
|
||||
const stored = Buffer.from(await res.Body.transformToByteArray());
|
||||
const plaintext = openMediaFrame(stored, this.kek);
|
||||
return new Uint8Array(plaintext);
|
||||
}
|
||||
}
|
||||
|
|
@ -9,6 +9,7 @@ import pino from "pino";
|
|||
import { createPublishingRegistry } from "@tactical-media-broadcast-hub/connector-runtime";
|
||||
import { PublishingExecutionRepository, assertPublishingSchema, type AttemptRef } from "@tactical-media-broadcast-hub/publishing-execution";
|
||||
import { PublishActivity } from "./activities/publish.activity.js";
|
||||
import { PublishingMediaResolver } from "./media-resolver.js";
|
||||
import { DirectorActivity } from "./director/director.activity.js";
|
||||
import { DirectorHttpClient } from "./director/http.client.js";
|
||||
import { HttpDirectorStatePort, HttpDesiredStatePort } from "./director/adapters.js";
|
||||
|
|
@ -61,7 +62,7 @@ export async function main(): Promise<void> {
|
|||
try {
|
||||
await assertPublishingSchema(pool);
|
||||
const repository = new PublishingExecutionRepository(pool);
|
||||
const registry = createPublishingRegistry(pool);
|
||||
const registry = createPublishingRegistry(pool, new PublishingMediaResolver(pool));
|
||||
native = await NativeConnection.connect({ address });
|
||||
connection = await Connection.connect({ address });
|
||||
const temporal = new Client({ connection, namespace });
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@
|
|||
"lint": "eslint src",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test:manual": "node --require ./scripts/control-room-register.cjs --test src/lib/participation-manual.test.ts",
|
||||
"test:register": "node --require ./scripts/control-room-register.cjs --test src/lib/register-form.test.ts"
|
||||
"test:register": "node --require ./scripts/control-room-register.cjs --test src/lib/register-form.test.ts",
|
||||
"test:outreach": "node --require ./scripts/control-room-register.cjs --test src/lib/outreach-types.test.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@tactical-media-broadcast-hub/connector-sdk": "workspace:^",
|
||||
|
|
|
|||
9
apps/web/src/app/(files-host)/files-app/page.tsx
Normal file
9
apps/web/src/app/(files-host)/files-app/page.tsx
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
import { FilesExplorer } from "../../../components/files/explorer/files-explorer";
|
||||
|
||||
/**
|
||||
* Full-screen files experience. files.<domain>/ is rewritten here by
|
||||
* src/middleware.ts; the route also works directly on any host.
|
||||
*/
|
||||
export default function FilesAppPage(): React.ReactElement {
|
||||
return <FilesExplorer />;
|
||||
}
|
||||
22
apps/web/src/app/(files-host)/layout.tsx
Normal file
22
apps/web/src/app/(files-host)/layout.tsx
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
"use client";
|
||||
|
||||
import type { ReactNode } from "react";
|
||||
import { AuthProvider } from "../../lib/auth-store";
|
||||
import { AuthGate } from "../../components/auth/auth-gate";
|
||||
import { FilesLangProvider } from "../../components/files/i18n";
|
||||
|
||||
/**
|
||||
* Shell for the dedicated files.* host. It sits OUTSIDE the (main) group on
|
||||
* purpose: no hub sidebar, bottom nav or command palette — the explorer owns
|
||||
* the whole viewport. Auth and the Files EN/ES preference are provided here
|
||||
* the same way (main)/layout.tsx and (main)/files/layout.tsx do.
|
||||
*/
|
||||
export default function FilesHostLayout({ children }: { children: ReactNode }): React.ReactElement {
|
||||
return (
|
||||
<AuthProvider>
|
||||
<FilesLangProvider>
|
||||
<AuthGate>{children}</AuthGate>
|
||||
</FilesLangProvider>
|
||||
</AuthProvider>
|
||||
);
|
||||
}
|
||||
|
|
@ -274,7 +274,10 @@ export default function ComposePage(): React.ReactElement {
|
|||
className="w-52 py-1 text-xs"
|
||||
/>
|
||||
</span>
|
||||
<div className="ml-auto flex gap-2">
|
||||
<div className="ml-auto flex items-center gap-2">
|
||||
{targets.length === 0 && !over && (
|
||||
<span className="text-xs text-warn">Connect a channel to publish — open Channels in the sidebar.</span>
|
||||
)}
|
||||
<Button variant="outline" disabled={busy || mediaBusy || chars === 0} onClick={() => void saveDraft()}>Save draft</Button>
|
||||
<Button variant="primary" disabled={busy || mediaBusy || targets.length === 0 || chars === 0 || over} onClick={() => void publish()}>
|
||||
{scheduledAt !== "" ? "Schedule" : approval === "review" ? "Submit" : "Publish"}
|
||||
|
|
@ -292,7 +295,8 @@ export default function ComposePage(): React.ReactElement {
|
|||
)}
|
||||
<div className="fixed left-0 right-0 z-30 border-t border-border bg-surface px-4 py-2 lg:hidden" style={{ bottom: `calc(var(--mobile-tab-height) + var(--mobile-safe-bottom) + ${keyboardOffset}px)` }}>
|
||||
<p className="mb-2 truncate text-2xs text-text-faint">{approval === "review" || !can("approval.grant") ? "→ Editors approve" : "→ Ready to publish"} · {scheduledAt ? `publishes ${new Date(scheduledAt).toLocaleString()}` : "post now"} · {selected.length} accounts</p>
|
||||
<div className="flex gap-2"><Button variant="ghost" className="min-h-11" onClick={() => router.back()}>Cancel</Button><Button variant="outline" className="min-h-11 flex-1" disabled={busy || mediaBusy || chars === 0} onClick={() => void saveDraft()}>Draft</Button><Button variant="primary" className="min-h-11 flex-1" disabled={busy || selected.length === 0 || chars === 0 || over || mediaBusy} onClick={() => void publish()}>{scheduledAt ? "Schedule" : approval === "review" || !can("approval.grant") ? "Submit" : "Post"}</Button></div>
|
||||
{selected.length === 0 && <p className="mb-1 text-2xs text-warn">Connect a channel to publish — open Channels in the sidebar.</p>}
|
||||
<div className="flex gap-2"><Button variant="ghost" className="min-h-11" onClick={() => router.back()}>Cancel</Button><Button variant="outline" className="min-h-11 flex-1" disabled={busy || mediaBusy || chars === 0} onClick={() => void saveDraft()}>Draft</Button><Button variant="primary" className="min-h-11 flex-1" disabled={busy || selected.length === 0 || chars === 0 || over || mediaBusy} onClick={() => void publish()}>{scheduledAt ? "Schedule" : approval === "review" || !can("approval.grant") ? "Submit" : "Publish"}</Button></div>
|
||||
</div>
|
||||
<Sheet open={scheduleOpen} onClose={() => setScheduleOpen(false)} title="Schedule post" footer={<Button variant="primary" className="min-h-11 w-full justify-center" onClick={() => setScheduleOpen(false)}>Done</Button>}><label className="block text-xs text-text-faint" htmlFor="mobile-schedule">Date and time</label><Input id="mobile-schedule" type="datetime-local" value={scheduledAt} onChange={(event) => setScheduledAt(event.target.value)} className="mt-2 min-h-12" /><div className="mt-4 flex gap-2">{[1, 24, 48].map((hours) => <button key={hours} type="button" className="min-h-11 rounded-md border border-border px-3 text-xs" onClick={() => { const date = new Date(Date.now() + hours * 3600000); setScheduledAt(`${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, "0")}-${String(date.getDate()).padStart(2, "0")}T${String(date.getHours()).padStart(2, "0")}:${String(date.getMinutes()).padStart(2, "0")}`); }}>{hours === 1 ? "In 1 hour" : hours === 24 ? "Tomorrow" : "In 2 days"}</button>)}</div><button type="button" onClick={() => setScheduledAt("")} className="mt-4 min-h-11 text-xs text-text-faint">Clear schedule</button></Sheet>
|
||||
<Sheet open={previewOpen} onClose={() => setPreviewOpen(false)} title="Target previews">{selected.map((target) => <div key={target.key} className="mb-3 rounded-md border border-border p-3"><p className="text-xs text-text-faint">{target.label}</p><p className="mt-2 whitespace-pre-wrap text-sm">{editor?.getText() || "Your post preview appears here…"}</p></div>)}</Sheet>
|
||||
|
|
|
|||
|
|
@ -1,79 +0,0 @@
|
|||
"use client";
|
||||
|
||||
import { EmbeddedApp } from "../../../../components/embed/embedded-app";
|
||||
import { useFilesLang } from "../../../../components/files/i18n";
|
||||
import { useApiResource } from "../../../../lib/use-api";
|
||||
import { useIsMobile } from "../../../../hooks/use-is-mobile";
|
||||
|
||||
/**
|
||||
* Drive — the oCIS web UI (personal space, shares, office documents) embedded
|
||||
* full height under the Files tabs. The group Library tab keeps the Hub-owned
|
||||
* features (upload into a collective, share with a collective, retention).
|
||||
*
|
||||
* Sign-in follows GET /drive/session's authMode: "interactive" means oCIS's
|
||||
* own identity provider — the user signs in once inside the frame and oCIS
|
||||
* web keeps its tokens (tokenStorageLocal) on the files origin, which this
|
||||
* frame shares because app.* and files.* are the same site. "oidc" means oCIS
|
||||
* and the Hub share an issuer and the frame signs in silently.
|
||||
*/
|
||||
|
||||
/** Same host FILES_PUBLIC_BASE defaults to; used only while the API is offline. */
|
||||
const FALLBACK_DRIVE_URL = "https://files.occupymedia.cc";
|
||||
|
||||
interface DriveSession {
|
||||
driveUrl: string | null;
|
||||
authMode: "interactive" | "oidc";
|
||||
}
|
||||
|
||||
const driveView = (session: DriveSession): DriveSession | null => session;
|
||||
|
||||
export default function DrivePage(): React.ReactElement {
|
||||
const { t } = useFilesLang();
|
||||
const drive = useApiResource<DriveSession, DriveSession | null>("/drive/session", driveView, null);
|
||||
const isMobile = useIsMobile();
|
||||
|
||||
const driveUrl = drive.source === "sample" ? FALLBACK_DRIVE_URL : drive.data?.driveUrl ?? null;
|
||||
const notConfigured = drive.source === "live" && drive.error === null && driveUrl === null;
|
||||
const error =
|
||||
drive.error ??
|
||||
(notConfigured ? t({ en: "The drive is not configured on this server (FILES_DRIVE_URL).", es: "La unidad no está configurada en este servidor (FILES_DRIVE_URL)." }) : null);
|
||||
const notice =
|
||||
drive.data?.authMode === "oidc"
|
||||
? t({ en: "Collective files live in the Library tab.", es: "Los archivos de los colectivos están en la pestaña Biblioteca." })
|
||||
: t({
|
||||
en: "Sign in once inside the drive — it stays signed in. Collective files live in the Library tab.",
|
||||
es: "Inicia sesión una vez dentro de la unidad y quedará abierta. Los archivos de los colectivos están en la pestaña Biblioteca.",
|
||||
});
|
||||
|
||||
if (isMobile) {
|
||||
// Mobile: the oCIS file browser under the bottom tab bar breaks (nested
|
||||
// scrolling, double chrome) — hand off to the standalone drive instead
|
||||
// (panel consensus: gateway pattern). The Library tab is fully native.
|
||||
return (
|
||||
<div className="flex h-full flex-col justify-center px-6 py-10 text-center">
|
||||
<h1 className="text-lg font-bold">{t({ en: "Drive", es: "Unidad" })}</h1>
|
||||
<p className="mx-auto mt-2 max-w-xs text-xs leading-relaxed text-text-dim">{error ?? notice}</p>
|
||||
<div className="mx-auto mt-8 w-full max-w-xs">
|
||||
<a
|
||||
href={`${driveUrl ?? FALLBACK_DRIVE_URL}/`}
|
||||
className="flex min-h-12 items-center justify-center rounded-xl bg-accent px-4 text-sm font-semibold text-white transition-opacity hover:opacity-90"
|
||||
>
|
||||
{t({ en: "Open drive ↗", es: "Abrir unidad ↗" })}
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<EmbeddedApp
|
||||
title={t({ en: "Drive", es: "Unidad" })}
|
||||
src={drive.source === "loading" || error !== null ? null : `${driveUrl ?? FALLBACK_DRIVE_URL}/`}
|
||||
externalHref={`${driveUrl ?? FALLBACK_DRIVE_URL}/`}
|
||||
error={error}
|
||||
notice={notice}
|
||||
onRetry={drive.error !== null ? drive.reload : undefined}
|
||||
allow="fullscreen; clipboard-read; clipboard-write"
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
|
@ -8,7 +8,7 @@ import { FilesTabs } from "../../../components/files/files-tabs";
|
|||
* Files shell. Like the wiki, the language lives in localStorage per browser
|
||||
* (lang-context explains why it is not an account setting) and is provided here
|
||||
* so every Files screen and sub-component reads it through one context. The tab
|
||||
* strip switches between the group Library and the embedded oCIS Drive.
|
||||
* strip switches between the group Library views.
|
||||
*/
|
||||
export default function FilesLayout({ children }: { children: ReactNode }): React.ReactElement {
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -10,10 +10,13 @@ import { api, ApiUnreachableError, errorMessage } from "../../../lib/api-client"
|
|||
import { useAuth } from "../../../lib/auth-store";
|
||||
import { useApiResource } from "../../../lib/use-api";
|
||||
import { FileRow } from "../../../components/files/file-row";
|
||||
import { MediaThumb } from "../../../components/files/media-thumb";
|
||||
import { MediaViewer } from "../../../components/files/media-viewer";
|
||||
import { UploadDropzone } from "../../../components/files/upload-dropzone";
|
||||
import { ShareDialog } from "../../../components/files/share-dialog";
|
||||
import { RetentionDialog } from "../../../components/files/retention-dialog";
|
||||
import { downloadGroupFile } from "../../../components/files/files-client";
|
||||
import { downloadFileDirect, isDirectObject, saveBytes } from "../../../lib/files-direct";
|
||||
import { useFilesLang } from "../../../components/files/i18n";
|
||||
import { MIME_CLASS_LABEL } from "../../../components/files/labels";
|
||||
import type {
|
||||
|
|
@ -81,6 +84,8 @@ export default function FilesPage(): React.ReactElement {
|
|||
const [qInput, setQInput] = useState("");
|
||||
const [q, setQ] = useState("");
|
||||
const [mimeClass, setMimeClass] = useState<"" | MimeClass>("");
|
||||
const [view, setView] = useState<"list" | "grid">("grid");
|
||||
const [viewerIndex, setViewerIndex] = useState<number | null>(null);
|
||||
|
||||
const [actionError, setActionError] = useState<string | null>(null);
|
||||
const [notice, setNotice] = useState<string | null>(null);
|
||||
|
|
@ -171,8 +176,23 @@ export default function FilesPage(): React.ReactElement {
|
|||
async function handleDownload(item: GroupFile): Promise<void> {
|
||||
setActionError(null);
|
||||
try {
|
||||
if (isDirectObject(item)) {
|
||||
const bytes = await downloadFileDirect(item.id);
|
||||
saveBytes(bytes, item.name);
|
||||
return;
|
||||
}
|
||||
await downloadGroupFile(item.id, item.name);
|
||||
} catch (err) {
|
||||
// Direct path is best-effort: fall back to streaming via the api once.
|
||||
if (isDirectObject(item)) {
|
||||
try {
|
||||
await downloadGroupFile(item.id, item.name);
|
||||
return;
|
||||
} catch (fallbackErr) {
|
||||
setActionError(errorMessage(fallbackErr));
|
||||
return;
|
||||
}
|
||||
}
|
||||
setActionError(errorMessage(err));
|
||||
}
|
||||
}
|
||||
|
|
@ -352,6 +372,34 @@ export default function FilesPage(): React.ReactElement {
|
|||
)}
|
||||
|
||||
{showList && items.length > 0 && (
|
||||
<div className="mb-3 flex items-center gap-1" role="tablist" aria-label={t({ en: "View", es: "Vista" })}>
|
||||
<Button size="sm" variant={view === "grid" ? "primary" : "outline"} aria-pressed={view === "grid"} onClick={() => setView("grid")}>{t({ en: "Media", es: "Medios" })}</Button>
|
||||
<Button size="sm" variant={view === "list" ? "primary" : "outline"} aria-pressed={view === "list"} onClick={() => setView("list")}>{t({ en: "List", es: "Lista" })}</Button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{showList && view === "grid" && items.length > 0 && (() => {
|
||||
const mediaItems = items.filter((item) => item.mimeClass === "image" || item.mimeClass === "video");
|
||||
return (
|
||||
<>
|
||||
<div className="grid grid-cols-2 gap-3 sm:grid-cols-3 lg:grid-cols-4 xl:grid-cols-5">
|
||||
{mediaItems.map((item) => (
|
||||
<MediaThumb key={item.id} item={item} onOpen={() => setViewerIndex(mediaItems.findIndex((m) => m.id === item.id))} />
|
||||
))}
|
||||
</div>
|
||||
{items.some((item) => item.mimeClass !== "image" && item.mimeClass !== "video") && (
|
||||
<p className="mt-3 text-2xs text-text-faint">
|
||||
{t({ en: "Media view shows images and videos — switch to List for documents and archives.", es: "La vista de medios muestra imágenes y vídeos — cambia a Lista para documentos y archivos." })}
|
||||
</p>
|
||||
)}
|
||||
{viewerIndex !== null && mediaItems[viewerIndex] !== undefined && (
|
||||
<MediaViewer items={mediaItems} index={viewerIndex} onClose={() => setViewerIndex(null)} onIndex={setViewerIndex} />
|
||||
)}
|
||||
</>
|
||||
);
|
||||
})()}
|
||||
|
||||
{showList && view === "list" && items.length > 0 && (
|
||||
<div className="space-y-2.5">
|
||||
{items.map((item) => (
|
||||
<FileRow
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ import { useIsMobile } from "../../hooks/use-is-mobile";
|
|||
const titles: Record<string, string> = {
|
||||
streams: "Home", search: "Search", inbox: "Inbox", approvals: "Approvals", more: "More", compose: "New post",
|
||||
calendar: "Calendar", video: "Video", analytics: "Analytics", connections: "Connections", team: "Team",
|
||||
audit: "Audit log", onboarding: "Welcome", settings: "Settings", archive: "Archive", files: "Files", chat: "Chat",
|
||||
audit: "Audit log", onboarding: "Welcome", settings: "Settings", archive: "Archive", files: "Files", chat: "Chat", outreach: "Outreach",
|
||||
};
|
||||
const rootTabs = new Set(["/streams", "/search", "/inbox", "/more"]);
|
||||
const scrollPositions = new Map<string, { main: number; nested: number[] }>();
|
||||
|
|
|
|||
|
|
@ -71,14 +71,14 @@ export default function MorePage(): React.ReactElement {
|
|||
<h2 className="px-4 pt-6 pb-2 text-xs font-bold uppercase tracking-wider text-text-faint">Settings</h2>
|
||||
<div className="border-y border-border-subtle bg-surface">
|
||||
<Row href="/connections" title="Connections" detail="Accounts and health" />
|
||||
<Row href="/outreach" title="Outreach" detail="Contacts to write to by hand" />
|
||||
<Row href="/settings" title="Settings" />
|
||||
<Row href="/settings" title="Security & sessions" />
|
||||
<Row href="/groups" title="Groups" detail="Collective moderation — submissions, approvals" />
|
||||
<Row href="/calendar" title="Calendar" />
|
||||
<Row href="/analytics" title="Analytics" />
|
||||
<Row href="/video" title="Video library" />
|
||||
<Row href="/files" title="Files" detail="Group file library" />
|
||||
<Row href="/chat" title="Chat" detail="Matrix chat" />
|
||||
<Row href="/onboarding" title="Onboarding" />
|
||||
<button onClick={() => void logout()} className="flex min-h-[56px] w-full items-center px-4 text-left text-sm font-medium text-danger">Sign out</button>
|
||||
</div>
|
||||
</div>;
|
||||
|
|
|
|||
|
|
@ -271,7 +271,7 @@ export default function OnboardingPage(): React.ReactElement {
|
|||
onChange={(e) => setUsername(e.target.value.toLowerCase())}
|
||||
placeholder="Username — this is how you sign in"
|
||||
autoComplete="username"
|
||||
pattern="[a-z0-9][a-z0-9_.-]{1,31}"
|
||||
pattern="[a-z0-9][a-z0-9._-]{1,31}"
|
||||
title="2-32 characters: lowercase letters, digits, dot, underscore or hyphen"
|
||||
/>
|
||||
<Input
|
||||
|
|
|
|||
325
apps/web/src/app/(main)/outreach/page.tsx
Normal file
325
apps/web/src/app/(main)/outreach/page.tsx
Normal file
|
|
@ -0,0 +1,325 @@
|
|||
"use client";
|
||||
|
||||
import { useMemo, useState } from "react";
|
||||
import { Badge, Button, Card } from "../../../components/ui/primitives";
|
||||
import { Loading } from "../../../components/brand/loading";
|
||||
import { EmptyState } from "../../../components/brand/empty-state";
|
||||
import { OfflineBadge } from "../../../components/ui/offline-badge";
|
||||
import { api, errorMessage } from "../../../lib/api-client";
|
||||
import { useApiResource } from "../../../lib/use-api";
|
||||
import { useAuth } from "../../../lib/auth-store";
|
||||
import {
|
||||
OUTREACH_DAILY_SOFT_CAP,
|
||||
OUTREACH_STATUSES,
|
||||
fillTemplate,
|
||||
parseImport,
|
||||
sentToday,
|
||||
type OutreachContact,
|
||||
type OutreachStatus,
|
||||
} from "../../../lib/outreach-types";
|
||||
|
||||
/**
|
||||
* Outreach — write to artists, collectives and public figures one by one.
|
||||
*
|
||||
* The list lives in GET /outreach/contacts. For each contact the screen keeps
|
||||
* a drafted message; "Copy & open" puts it on the clipboard and opens the
|
||||
* profile so a person sends it from the platform's own DM box, then marks the
|
||||
* contact sent. Nothing is sent by the Hub: unsolicited bulk DMs get accounts
|
||||
* suspended, and a personal message is what gets an answer.
|
||||
*/
|
||||
|
||||
const SAMPLE: OutreachContact[] = [
|
||||
{
|
||||
id: "sample-1", name: "Sample collective", platform: "example", handle: "@sample@example.social",
|
||||
profileUrl: "https://example.social/@sample", category: "housing", language: "en", message: "", notes: "",
|
||||
status: "todo", sentAt: null, repliedAt: null, createdAt: "2026-10-15T00:00:00Z", updatedAt: "2026-10-15T00:00:00Z",
|
||||
},
|
||||
];
|
||||
|
||||
const identity = (contacts: OutreachContact[]): OutreachContact[] => contacts;
|
||||
|
||||
const STATUS_TONE: Record<OutreachStatus, "neutral" | "accent" | "live" | "danger"> = {
|
||||
todo: "neutral", sent: "accent", replied: "live", declined: "danger",
|
||||
};
|
||||
|
||||
const TEXTAREA = "w-full rounded-md border border-border bg-surface-2 p-2 text-sm focus:border-accent focus:outline-none";
|
||||
|
||||
export default function OutreachPage(): React.ReactElement {
|
||||
const { can } = useAuth();
|
||||
const contacts = useApiResource<OutreachContact[], OutreachContact[]>("/outreach/contacts", identity, SAMPLE);
|
||||
const live = contacts.source === "live";
|
||||
const canManage = live && can("outreach.manage");
|
||||
const [filter, setFilter] = useState<OutreachStatus | "all">("todo");
|
||||
const [category, setCategory] = useState<string>("all");
|
||||
const [openId, setOpenId] = useState<string | null>(null);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [notice, setNotice] = useState<string | null>(null);
|
||||
|
||||
const categories = useMemo(
|
||||
() => [...new Set(contacts.data.map((contact) => contact.category).filter((value): value is string => value !== null))].sort(),
|
||||
[contacts.data],
|
||||
);
|
||||
const visible = contacts.data.filter(
|
||||
(contact) => (filter === "all" || contact.status === filter) && (category === "all" || contact.category === category),
|
||||
);
|
||||
const counts = Object.fromEntries(
|
||||
OUTREACH_STATUSES.map((status) => [status, contacts.data.filter((contact) => contact.status === status).length]),
|
||||
) as Record<OutreachStatus, number>;
|
||||
const today = sentToday(contacts.data);
|
||||
|
||||
function replace(updated: OutreachContact): void {
|
||||
contacts.setData((current) => current.map((contact) => (contact.id === updated.id ? updated : contact)));
|
||||
}
|
||||
|
||||
async function patch(id: string, body: Partial<Pick<OutreachContact, "message" | "notes" | "status">>): Promise<void> {
|
||||
try {
|
||||
replace(await api.patch<OutreachContact>(`/outreach/contacts/${id}`, body));
|
||||
} catch (err) {
|
||||
setNotice(errorMessage(err));
|
||||
}
|
||||
}
|
||||
|
||||
async function copyAndOpen(contact: OutreachContact): Promise<void> {
|
||||
if (contact.message.trim() !== "") {
|
||||
try {
|
||||
await navigator.clipboard.writeText(contact.message);
|
||||
setNotice(`Message for ${contact.name} copied. Paste it in their DMs, then mark it sent.`);
|
||||
} catch {
|
||||
setNotice("Could not copy to the clipboard. Select the text and copy it by hand.");
|
||||
}
|
||||
}
|
||||
window.open(contact.profileUrl, "_blank", "noopener,noreferrer");
|
||||
}
|
||||
|
||||
async function remove(contact: OutreachContact): Promise<void> {
|
||||
if (!window.confirm(`Remove ${contact.name} from the list?`)) return;
|
||||
try {
|
||||
await api.del(`/outreach/contacts/${contact.id}`);
|
||||
contacts.setData((current) => current.filter((row) => row.id !== contact.id));
|
||||
} catch (err) {
|
||||
setNotice(errorMessage(err));
|
||||
}
|
||||
}
|
||||
|
||||
async function applyTemplate(template: string): Promise<void> {
|
||||
const targets = visible.filter((contact) => contact.status === "todo" && contact.message.trim() === "");
|
||||
if (targets.length === 0) {
|
||||
setNotice("Every pending contact in this view already has a message.");
|
||||
return;
|
||||
}
|
||||
setBusy(true);
|
||||
for (const contact of targets) await patch(contact.id, { message: fillTemplate(template, contact) });
|
||||
setBusy(false);
|
||||
setNotice(`Template applied to ${targets.length} contacts. Review each one before sending.`);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="h-full overflow-y-auto p-4 pb-24 scrollbar-thin lg:p-6">
|
||||
<div className="mb-3 flex flex-wrap items-center gap-3">
|
||||
<h1 className="text-lg font-semibold">Outreach</h1>
|
||||
<OfflineBadge source={contacts.source} />
|
||||
{!live && <Badge tone="warn">read-only sample</Badge>}
|
||||
</div>
|
||||
<p className="mb-4 max-w-2xl text-xs leading-relaxed text-text-faint">
|
||||
Public accounts to write to by hand. Draft a message per contact, copy it, open the profile and send it from
|
||||
the platform. Mark who answered so nobody writes twice.
|
||||
</p>
|
||||
|
||||
<div className="mb-4 flex flex-wrap items-center gap-2">
|
||||
{(["all", ...OUTREACH_STATUSES] as const).map((status) => (
|
||||
<Button key={status} size="sm" variant={filter === status ? "outline" : "ghost"} onClick={() => setFilter(status)}>
|
||||
{status} {status === "all" ? contacts.data.length : counts[status]}
|
||||
</Button>
|
||||
))}
|
||||
{categories.length > 0 && (
|
||||
<select
|
||||
value={category}
|
||||
onChange={(event) => setCategory(event.target.value)}
|
||||
className="rounded-md border border-border bg-surface-2 px-2 py-1 text-2xs"
|
||||
aria-label="Category"
|
||||
>
|
||||
<option value="all">all categories</option>
|
||||
{categories.map((value) => <option key={value} value={value}>{value}</option>)}
|
||||
</select>
|
||||
)}
|
||||
<span className="ml-auto text-2xs text-text-faint">
|
||||
Sent today: <span className={today >= OUTREACH_DAILY_SOFT_CAP ? "text-warn" : "text-text-dim"}>{today}</span>
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{today >= OUTREACH_DAILY_SOFT_CAP && (
|
||||
<Card className="mb-4 p-3">
|
||||
<p className="text-xs text-warn">
|
||||
{today} first messages today from this collective. Spread the rest over the next days: platforms
|
||||
restrict accounts that send many first messages in a short time.
|
||||
</p>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
{(notice !== null || contacts.error !== null) && (
|
||||
<Card className="mb-4 flex items-start justify-between gap-3 p-3">
|
||||
<p className={`text-xs ${contacts.error !== null ? "text-danger" : "text-text-dim"}`}>{contacts.error ?? notice}</p>
|
||||
{notice !== null && <Button size="sm" onClick={() => setNotice(null)}>Dismiss</Button>}
|
||||
</Card>
|
||||
)}
|
||||
|
||||
{canManage && <Tools busy={busy} onImported={contacts.reload} onTemplate={applyTemplate} onNotice={setNotice} />}
|
||||
|
||||
{contacts.source === "loading" ? (
|
||||
<div className="flex justify-center py-10"><Loading size={28} label="Loading contacts" /></div>
|
||||
) : visible.length === 0 ? (
|
||||
<EmptyState title="No contacts here" hint={canManage ? "Import a list above or change the filter." : "Change the filter."} />
|
||||
) : (
|
||||
<div className="flex flex-col gap-2">
|
||||
{visible.map((contact) => (
|
||||
<ContactRow
|
||||
key={contact.id}
|
||||
contact={contact}
|
||||
open={openId === contact.id}
|
||||
canManage={canManage}
|
||||
onToggle={() => setOpenId(openId === contact.id ? null : contact.id)}
|
||||
onPatch={(body) => patch(contact.id, body)}
|
||||
onCopyOpen={() => copyAndOpen(contact)}
|
||||
onRemove={() => remove(contact)}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ContactRow({ contact, open, canManage, onToggle, onPatch, onCopyOpen, onRemove }: {
|
||||
contact: OutreachContact;
|
||||
open: boolean;
|
||||
canManage: boolean;
|
||||
onToggle: () => void;
|
||||
onPatch: (body: Partial<Pick<OutreachContact, "message" | "notes" | "status">>) => Promise<void>;
|
||||
onCopyOpen: () => Promise<void>;
|
||||
onRemove: () => Promise<void>;
|
||||
}): React.ReactElement {
|
||||
const [message, setMessage] = useState(contact.message);
|
||||
const [notes, setNotes] = useState(contact.notes);
|
||||
|
||||
return (
|
||||
<Card className="p-3">
|
||||
<button type="button" onClick={onToggle} className="flex w-full flex-wrap items-center gap-2 text-left">
|
||||
<span className="text-sm font-medium">{contact.name}</span>
|
||||
<span className="text-2xs text-text-faint">{contact.platform} · {contact.handle}</span>
|
||||
{contact.category !== null && <Badge>{contact.category}</Badge>}
|
||||
{contact.language !== null && <Badge>{contact.language}</Badge>}
|
||||
<span className="ml-auto"><Badge tone={STATUS_TONE[contact.status]}>{contact.status}</Badge></span>
|
||||
</button>
|
||||
{open && (
|
||||
<div className="mt-3 flex flex-col gap-2">
|
||||
<textarea
|
||||
value={message}
|
||||
onChange={(event) => setMessage(event.target.value)}
|
||||
onBlur={() => { if (canManage && message !== contact.message) void onPatch({ message }); }}
|
||||
rows={5}
|
||||
maxLength={4000}
|
||||
readOnly={!canManage}
|
||||
placeholder="Message for this contact"
|
||||
className={TEXTAREA}
|
||||
/>
|
||||
<textarea
|
||||
value={notes}
|
||||
onChange={(event) => setNotes(event.target.value)}
|
||||
onBlur={() => { if (canManage && notes !== contact.notes) void onPatch({ notes }); }}
|
||||
rows={2}
|
||||
maxLength={2000}
|
||||
readOnly={!canManage}
|
||||
placeholder="Notes (who wrote, what they said)"
|
||||
className={TEXTAREA}
|
||||
/>
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Button variant="primary" size="sm" onClick={() => void onCopyOpen()}>Copy & open profile</Button>
|
||||
{canManage && contact.status !== "sent" && <Button variant="outline" size="sm" onClick={() => void onPatch({ status: "sent" })}>Mark sent</Button>}
|
||||
{canManage && contact.status !== "replied" && <Button size="sm" onClick={() => void onPatch({ status: "replied" })}>Replied</Button>}
|
||||
{canManage && contact.status !== "declined" && <Button size="sm" onClick={() => void onPatch({ status: "declined" })}>Declined</Button>}
|
||||
{canManage && contact.status !== "todo" && <Button size="sm" onClick={() => void onPatch({ status: "todo" })}>Back to pending</Button>}
|
||||
{canManage && <Button variant="danger" size="sm" className="ml-auto" onClick={() => void onRemove()}>Remove</Button>}
|
||||
</div>
|
||||
{contact.sentAt !== null && (
|
||||
<p className="text-2xs text-text-faint">
|
||||
Sent {new Date(contact.sentAt).toLocaleString()}
|
||||
{contact.repliedAt !== null && ` · replied ${new Date(contact.repliedAt).toLocaleString()}`}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
function Tools({ busy, onImported, onTemplate, onNotice }: {
|
||||
busy: boolean;
|
||||
onImported: () => void;
|
||||
onTemplate: (template: string) => Promise<void>;
|
||||
onNotice: (message: string) => void;
|
||||
}): React.ReactElement {
|
||||
const [csv, setCsv] = useState("");
|
||||
const [template, setTemplate] = useState("");
|
||||
const [importing, setImporting] = useState(false);
|
||||
|
||||
async function submit(): Promise<void> {
|
||||
const { rows, errors } = parseImport(csv);
|
||||
if (errors.length > 0) {
|
||||
onNotice(errors.slice(0, 3).join("; "));
|
||||
return;
|
||||
}
|
||||
if (rows.length === 0) return;
|
||||
setImporting(true);
|
||||
try {
|
||||
let added = 0;
|
||||
let skipped = 0;
|
||||
for (let start = 0; start < rows.length; start += 500) {
|
||||
const result = await api.post<{ added: number; skipped: number }>("/outreach/contacts", { contacts: rows.slice(start, start + 500) });
|
||||
added += result.added;
|
||||
skipped += result.skipped;
|
||||
}
|
||||
onNotice(`${added} contacts added, ${skipped} already on the list.`);
|
||||
setCsv("");
|
||||
onImported();
|
||||
} catch (err) {
|
||||
onNotice(errorMessage(err));
|
||||
} finally {
|
||||
setImporting(false);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="mb-4 grid gap-3 lg:grid-cols-2">
|
||||
<Card className="p-3">
|
||||
<h2 className="mb-1 text-xs font-semibold">Import</h2>
|
||||
<p className="mb-2 text-2xs text-text-faint">One per line: name, platform, handle, profileUrl, category, language</p>
|
||||
<textarea
|
||||
value={csv}
|
||||
onChange={(event) => setCsv(event.target.value)}
|
||||
rows={4}
|
||||
placeholder="Name,platform,handle,https://profile-url,category,es"
|
||||
className={`${TEXTAREA} font-mono text-2xs`}
|
||||
/>
|
||||
<Button variant="outline" size="sm" className="mt-2" disabled={importing || csv.trim() === ""} onClick={() => void submit()}>
|
||||
{importing ? "Importing…" : "Import"}
|
||||
</Button>
|
||||
</Card>
|
||||
<Card className="p-3">
|
||||
<h2 className="mb-1 text-xs font-semibold">Template</h2>
|
||||
<p className="mb-2 text-2xs text-text-faint">
|
||||
Fills pending contacts in the current view that have no message yet. Use {"{name}"} and {"{handle}"}.
|
||||
</p>
|
||||
<textarea
|
||||
value={template}
|
||||
onChange={(event) => setTemplate(event.target.value)}
|
||||
rows={4}
|
||||
placeholder="Hola {name}, os escribimos desde la Acampada 26S…"
|
||||
className={TEXTAREA}
|
||||
/>
|
||||
<Button variant="outline" size="sm" className="mt-2" disabled={busy || template.trim() === ""} onClick={() => void onTemplate(template)}>
|
||||
{busy ? "Applying…" : "Apply to pending"}
|
||||
</Button>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -119,7 +119,7 @@ export default function VideoPage(): React.ReactElement {
|
|||
const applyStatus = useCallback((status: VideoStatusDto): void => {
|
||||
const entry: VideoLibraryEntryDto = {
|
||||
videoAsset: status.videoAsset,
|
||||
processingState: status.processingJob.state,
|
||||
processingState: status.processingJob?.state ?? "queued",
|
||||
publications: [],
|
||||
};
|
||||
setItems((current) => [entry, ...(current ?? []).filter((e) => e.videoAsset.id !== entry.videoAsset.id)]);
|
||||
|
|
|
|||
|
|
@ -104,9 +104,15 @@ function ManualLanding(): React.ReactElement {
|
|||
}
|
||||
}
|
||||
|
||||
// files.* host: middleware normally rewrites "/" to the explorer; this is the
|
||||
// fallback when a proxy hid the Host header from it.
|
||||
useEffect(() => {
|
||||
if (window.location.hostname.startsWith("files.")) router.replace("/files-app");
|
||||
}, [router]);
|
||||
|
||||
// Signed-in visitors go straight to the app; the manual stays reachable at /wiki.
|
||||
useEffect(() => {
|
||||
if (status === "authenticated") router.replace("/streams");
|
||||
if (status === "authenticated" && !window.location.hostname.startsWith("files.")) router.replace("/streams");
|
||||
}, [status, router]);
|
||||
|
||||
const t = <T extends { es: string; en: string }>(text: T): string => text[lang];
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"use client";
|
||||
|
||||
import Link from "next/link";
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { useAuth } from "../../lib/auth-store";
|
||||
|
||||
/**
|
||||
|
|
@ -59,7 +60,16 @@ export function NoCollectivesHome(): React.ReactElement {
|
|||
const { session } = useAuth();
|
||||
// Spanish-first product; the landing's ES/EN preference carries when the
|
||||
// account stores one (P2).
|
||||
const t = <T extends { es: string; en: string }>(x: T): string => x.es;
|
||||
// Respect the reading language the user picked on the landing page (tmbh.lang
|
||||
// in localStorage); Spanish stays the fallback for a fresh visitor.
|
||||
const [lang, setLang] = useState<"es" | "en">("es");
|
||||
useEffect(() => {
|
||||
try {
|
||||
const stored = window.localStorage.getItem("tmbh.manual.lang");
|
||||
if (stored === "en" || stored === "es") setLang(stored);
|
||||
} catch { /* private mode: keep the default */ }
|
||||
}, []);
|
||||
const t = <T extends { es: string; en: string }>(x: T): string => x[lang];
|
||||
const name = session?.user.displayName?.trim() || session?.user.username || "";
|
||||
|
||||
return (
|
||||
|
|
|
|||
155
apps/web/src/components/files/explorer/details-drawer.tsx
Normal file
155
apps/web/src/components/files/explorer/details-drawer.tsx
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
"use client";
|
||||
|
||||
import { Badge, Button } from "../../ui/primitives";
|
||||
import { formatBytes } from "../../video/format";
|
||||
import { MimeIcon } from "../mime-icon";
|
||||
import { LEGAL_HOLD_LABEL, mimeClassLabel, ORIGIN_LABEL, ORIGIN_TONE, RETENTION_LABEL, RETENTION_TONE } from "../labels";
|
||||
import { useFilesLang } from "../i18n";
|
||||
import type { ExplorerFile } from "./explorer-api";
|
||||
|
||||
/**
|
||||
* Details drawer for the selected file: right rail on desktop, bottom sheet on
|
||||
* mobile. Metadata on top, then every action the caller may take. Actions whose
|
||||
* endpoint is not live yet render disabled with a "coming soon" note; Delete is
|
||||
* not rendered at all unless the caller is permitted.
|
||||
*/
|
||||
|
||||
export type DrawerFeature = "move" | "copy" | "shareToGroup" | "delete";
|
||||
|
||||
export function DetailsDrawer({
|
||||
item,
|
||||
live,
|
||||
location,
|
||||
personal,
|
||||
viewable,
|
||||
canRetention,
|
||||
canDelete,
|
||||
hasActiveShare,
|
||||
busy,
|
||||
unavailable,
|
||||
onClose,
|
||||
onView,
|
||||
onDownload,
|
||||
onShare,
|
||||
onRevoke,
|
||||
onRetention,
|
||||
onMove,
|
||||
onCopy,
|
||||
onShareToGroup,
|
||||
onDelete,
|
||||
}: {
|
||||
item: ExplorerFile;
|
||||
live: boolean;
|
||||
/** Human path of the folder the item sits in. */
|
||||
location: string;
|
||||
personal: boolean;
|
||||
viewable: boolean;
|
||||
canRetention: boolean;
|
||||
canDelete: boolean;
|
||||
hasActiveShare: boolean;
|
||||
busy: boolean;
|
||||
unavailable: ReadonlySet<DrawerFeature>;
|
||||
onClose: () => void;
|
||||
onView: () => void;
|
||||
onDownload: () => void;
|
||||
onShare: () => void;
|
||||
onRevoke: () => void;
|
||||
onRetention: () => void;
|
||||
onMove: () => void;
|
||||
onCopy: () => void;
|
||||
onShareToGroup: () => void;
|
||||
onDelete: () => void;
|
||||
}): React.ReactElement {
|
||||
const { t } = useFilesLang();
|
||||
const disabled = busy || !live || item.status !== "available";
|
||||
const soon = t({ en: "coming soon", es: "próximamente" });
|
||||
const retentionLabel =
|
||||
item.retention.mode === "until" && item.retention.expiresAt !== null
|
||||
? `${t(RETENTION_LABEL.until)} ${new Date(item.retention.expiresAt).toLocaleDateString()}`
|
||||
: t(RETENTION_LABEL[item.retention.mode]);
|
||||
|
||||
const action = (label: string, onClick: () => void, opts: { feature?: DrawerFeature; variant?: "outline" | "ghost" | "primary" | "danger"; off?: boolean } = {}): React.ReactElement => {
|
||||
const missing = opts.feature !== undefined && unavailable.has(opts.feature);
|
||||
return (
|
||||
<Button
|
||||
variant={opts.variant ?? "outline"}
|
||||
disabled={(opts.off ?? disabled) || missing}
|
||||
onClick={onClick}
|
||||
className="min-h-11 w-full justify-center lg:min-h-9"
|
||||
title={missing ? soon : undefined}
|
||||
>
|
||||
{label}
|
||||
{missing && <span className="text-2xs font-normal text-text-faint">· {soon}</span>}
|
||||
</Button>
|
||||
);
|
||||
};
|
||||
|
||||
const rows: Array<[string, React.ReactNode]> = [
|
||||
[t({ en: "Type", es: "Tipo" }), `${t(mimeClassLabel(item.mimeClass))} · ${item.mimeType}`],
|
||||
[t({ en: "Size", es: "Tamaño" }), formatBytes(item.sizeBytes)],
|
||||
[t({ en: "Folder", es: "Carpeta" }), location],
|
||||
[t({ en: "Added by", es: "Añadido por" }), item.senderId],
|
||||
[t({ en: "Added", es: "Añadido" }), <time key="created" dateTime={item.createdAt}>{new Date(item.createdAt).toLocaleString()}</time>],
|
||||
];
|
||||
|
||||
return (
|
||||
<aside
|
||||
aria-label={t({ en: "File details", es: "Detalles del archivo" })}
|
||||
className="fixed inset-x-0 bottom-0 z-40 flex max-h-[72dvh] flex-col rounded-t-xl border-t border-border bg-surface shadow-2xl lg:static lg:z-auto lg:max-h-none lg:w-80 lg:shrink-0 lg:rounded-none lg:border-l lg:border-t-0 lg:shadow-none"
|
||||
>
|
||||
<div className="flex items-start gap-3 border-b border-border-subtle p-4">
|
||||
<MimeIcon mimeClass={item.mimeClass} size={30} className="shrink-0 text-accent" />
|
||||
<div className="min-w-0 flex-1">
|
||||
<h2 className="break-all text-sm font-semibold">{item.name}</h2>
|
||||
<div className="mt-1 flex flex-wrap gap-1">
|
||||
<Badge tone={ORIGIN_TONE[item.origin.kind]}>{t(ORIGIN_LABEL[item.origin.kind])}</Badge>
|
||||
<Badge tone={RETENTION_TONE[item.retention.mode]}>{retentionLabel}</Badge>
|
||||
{item.retention.legalHold && <Badge tone="danger">{t(LEGAL_HOLD_LABEL)}</Badge>}
|
||||
{personal && <Badge tone="accent">{t({ en: "Personal", es: "Personal" })}</Badge>}
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onClose}
|
||||
aria-label={t({ en: "Close details", es: "Cerrar detalles" })}
|
||||
className="flex min-h-11 min-w-11 shrink-0 items-center justify-center rounded-md text-text-faint hover:bg-surface-2 hover:text-text lg:min-h-8 lg:min-w-8"
|
||||
>
|
||||
✕
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="min-h-0 flex-1 overflow-y-auto p-4 scrollbar-thin">
|
||||
<dl className="mb-4 grid grid-cols-[auto_1fr] gap-x-3 gap-y-1.5 text-2xs">
|
||||
{rows.map(([label, value]) => (
|
||||
<div key={label} className="contents">
|
||||
<dt className="text-text-faint">{label}</dt>
|
||||
<dd className="min-w-0 break-all text-text-dim">{value}</dd>
|
||||
</div>
|
||||
))}
|
||||
</dl>
|
||||
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
{viewable && action(t({ en: "View", es: "Ver" }), onView, { variant: "primary" })}
|
||||
{action(t({ en: "Download", es: "Descargar" }), onDownload, viewable ? {} : { variant: "primary" })}
|
||||
{action(t({ en: "Share link", es: "Enlace" }), onShare)}
|
||||
{hasActiveShare && action(t({ en: "Revoke link", es: "Revocar enlace" }), onRevoke, { variant: "ghost", off: busy || !live })}
|
||||
{action(t({ en: "Move", es: "Mover" }), onMove, { feature: "move" })}
|
||||
{action(t({ en: "Copy", es: "Copiar" }), onCopy, { feature: "copy" })}
|
||||
{personal && action(t({ en: "Share to collective", es: "Compartir con colectivo" }), onShareToGroup, { feature: "shareToGroup" })}
|
||||
{canRetention && action(t({ en: "Retention", es: "Retención" }), onRetention, { variant: "ghost", off: busy || !live })}
|
||||
</div>
|
||||
|
||||
{canDelete && (
|
||||
<div className="mt-4 border-t border-border-subtle pt-4">
|
||||
{action(t({ en: "Delete…", es: "Borrar…" }), onDelete, { feature: "delete", variant: "danger", off: busy || !live || item.retention.legalHold })}
|
||||
{item.retention.legalHold && (
|
||||
<p className="mt-1.5 text-2xs text-text-faint">
|
||||
{t({ en: "Legal hold blocks deletion until an admin releases it.", es: "La retención legal bloquea el borrado hasta que un admin la libere." })}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</aside>
|
||||
);
|
||||
}
|
||||
161
apps/web/src/components/files/explorer/explorer-api.ts
Normal file
161
apps/web/src/components/files/explorer/explorer-api.ts
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
import { api, ApiError } from "../../../lib/api-client";
|
||||
import type { GroupFile, GroupFilePage } from "../types";
|
||||
|
||||
/**
|
||||
* Wire calls for the files.* explorer: folders, move/copy, share-to-group,
|
||||
* delete and the per-group member-delete setting. The folder/move/copy
|
||||
* endpoints were landing in parallel with this screen, so every call here
|
||||
* reports a 404 (or 405) as `"unavailable"` instead of throwing — the UI then
|
||||
* shows a "coming soon" state for that one feature rather than an error.
|
||||
*/
|
||||
|
||||
export interface Folder {
|
||||
id: string;
|
||||
name: string;
|
||||
parentId: string | null;
|
||||
groupId?: string | null;
|
||||
personal: boolean;
|
||||
}
|
||||
|
||||
/** Where the center pane is pointed: a group (root or folder) or the caller's personal space. */
|
||||
export type FilesLocation =
|
||||
| { kind: "group"; groupId: string; folderId: string | null }
|
||||
| { kind: "personal"; folderId: string | null };
|
||||
|
||||
/** A library row as the explorer sees it: the served GroupFile plus the folder fields. */
|
||||
export type ExplorerFile = GroupFile & { folderId?: string | null; personal?: boolean };
|
||||
|
||||
export type Maybe<T> = { ok: true; value: T } | { ok: false; unavailable: true };
|
||||
|
||||
export function isMissingEndpoint(err: unknown): boolean {
|
||||
return err instanceof ApiError && (err.status === 404 || err.status === 405 || err.status === 501);
|
||||
}
|
||||
|
||||
async function maybe<T>(call: () => Promise<T>): Promise<Maybe<T>> {
|
||||
try {
|
||||
return { ok: true, value: await call() };
|
||||
} catch (err) {
|
||||
if (isMissingEndpoint(err)) return { ok: false, unavailable: true };
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export function sameLocation(a: FilesLocation, b: FilesLocation): boolean {
|
||||
if (a.kind !== b.kind || a.folderId !== b.folderId) return false;
|
||||
return a.kind === "personal" || (b.kind === "group" && a.groupId === b.groupId);
|
||||
}
|
||||
|
||||
export function locationKey(location: FilesLocation): string {
|
||||
return location.kind === "group"
|
||||
? `g:${location.groupId}:${location.folderId ?? ""}`
|
||||
: `p:${location.folderId ?? ""}`;
|
||||
}
|
||||
|
||||
export function listGroupFolders(groupId: string, signal?: AbortSignal): Promise<Maybe<Folder[]>> {
|
||||
return maybe(() => api.get<Folder[]>("/files/folders", { scope: "group", groupId }, signal));
|
||||
}
|
||||
|
||||
export function listPersonalFolders(signal?: AbortSignal): Promise<Maybe<Folder[]>> {
|
||||
return maybe(() => api.get<Folder[]>("/files/folders", { scope: "personal" }, signal));
|
||||
}
|
||||
|
||||
export function createFolder(name: string, location: FilesLocation): Promise<Maybe<Folder>> {
|
||||
const scope = location.kind === "group" ? { groupId: location.groupId } : { personal: true };
|
||||
const parent = location.folderId === null ? {} : { parentId: location.folderId };
|
||||
return maybe(() => api.post<Folder>("/files/folders", { name, ...scope, ...parent }));
|
||||
}
|
||||
|
||||
export function renameFolder(id: string, name: string): Promise<Maybe<Folder>> {
|
||||
return maybe(() => api.patch<Folder>(`/files/folders/${encodeURIComponent(id)}`, { name }));
|
||||
}
|
||||
|
||||
export function deleteFolder(id: string): Promise<Maybe<unknown>> {
|
||||
return maybe(() => api.del<unknown>(`/files/folders/${encodeURIComponent(id)}`));
|
||||
}
|
||||
|
||||
/** One page of the library at a location (the existing GroupFilePage shape). */
|
||||
export function listFiles(
|
||||
location: FilesLocation,
|
||||
filters: { name: string; mimeClass: string; limit: number; cursor?: string },
|
||||
signal?: AbortSignal,
|
||||
): Promise<Maybe<GroupFilePage>> {
|
||||
const scope = location.kind === "group" ? { groupId: location.groupId } : { scope: "personal" };
|
||||
const folder = location.folderId === null ? {} : { folderId: location.folderId };
|
||||
return maybe(() => api.get<GroupFilePage>("/files", { ...scope, ...folder, ...filters }, signal));
|
||||
}
|
||||
|
||||
/** `folderId: null` moves the item back to its scope's root. */
|
||||
export function moveFile(id: string, folderId: string | null): Promise<Maybe<unknown>> {
|
||||
return maybe(() => api.post<unknown>(`/files/${encodeURIComponent(id)}/move`, { folderId }));
|
||||
}
|
||||
|
||||
export type CopyTarget = { targetFolderId: string } | { targetGroupId: string };
|
||||
|
||||
export function copyFile(id: string, target: CopyTarget): Promise<Maybe<unknown>> {
|
||||
return maybe(() => api.post<unknown>(`/files/${encodeURIComponent(id)}/copy`, target));
|
||||
}
|
||||
|
||||
export function shareToGroup(id: string, groupId: string): Promise<Maybe<unknown>> {
|
||||
return maybe(() => api.post<unknown>(`/files/${encodeURIComponent(id)}/share-to-group`, { groupId }));
|
||||
}
|
||||
|
||||
/** DELETE /files/:id — a 403 means the delete guard refused (hold, retention, role). */
|
||||
export function deleteFile(id: string): Promise<Maybe<unknown>> {
|
||||
return maybe(() => api.del<unknown>(`/files/${encodeURIComponent(id)}`));
|
||||
}
|
||||
|
||||
/**
|
||||
* groups.member_delete_own for one group. The setting rides the groups
|
||||
* settings API; when the endpoint or the field is absent the documented
|
||||
* default is `true` (members may delete what they uploaded).
|
||||
*/
|
||||
export async function fetchMemberDeleteOwn(groupId: string, signal?: AbortSignal): Promise<boolean> {
|
||||
const id = encodeURIComponent(groupId);
|
||||
// Dedicated settings route first; until it lands, the group detail may carry the field.
|
||||
for (const path of [`/groups/${id}/settings`, `/groups/${id}`]) {
|
||||
try {
|
||||
const body = await api.get<Record<string, unknown> | null>(path, undefined, signal);
|
||||
const nested = body?.settings;
|
||||
const settings = typeof nested === "object" && nested !== null ? (nested as Record<string, unknown>) : body;
|
||||
const raw = settings?.member_delete_own ?? settings?.memberDeleteOwn ?? body?.member_delete_own ?? body?.memberDeleteOwn;
|
||||
if (typeof raw === "boolean") return raw;
|
||||
} catch {
|
||||
// Missing route or no access: try the next source, then fall back to the default.
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Uploads are stamped `@<username>:<server>`; the localpart identifies the caller's own files. */
|
||||
export function isOwnFile(item: GroupFile, username: string | undefined): boolean {
|
||||
if (username === undefined || username === "") return false;
|
||||
const localpart = item.senderId.replace(/^@/, "").split(":")[0] ?? "";
|
||||
return localpart.toLowerCase() === username.toLowerCase();
|
||||
}
|
||||
|
||||
/** Flat folder list → children lookup keyed by parent id ("" = root). */
|
||||
export function childrenByParent(folders: Folder[]): Map<string, Folder[]> {
|
||||
const map = new Map<string, Folder[]>();
|
||||
for (const folder of folders) {
|
||||
const key = folder.parentId ?? "";
|
||||
const list = map.get(key) ?? [];
|
||||
list.push(folder);
|
||||
map.set(key, list);
|
||||
}
|
||||
for (const list of map.values()) list.sort((a, b) => a.name.localeCompare(b.name));
|
||||
return map;
|
||||
}
|
||||
|
||||
/** Root → folder chain for breadcrumbs; stops on a broken or cyclic parent link. */
|
||||
export function folderPath(folders: Folder[], folderId: string | null): Folder[] {
|
||||
const byId = new Map(folders.map((folder) => [folder.id, folder]));
|
||||
const path: Folder[] = [];
|
||||
const seen = new Set<string>();
|
||||
let current = folderId === null ? undefined : byId.get(folderId);
|
||||
while (current !== undefined && !seen.has(current.id)) {
|
||||
seen.add(current.id);
|
||||
path.unshift(current);
|
||||
current = current.parentId === null ? undefined : byId.get(current.parentId);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue